Merge pull request #1185 from guardicore/mongo_db_launch_from_island

Mongo db launch from island

CHANGELOG.md

@@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Improved the structure of unit tests by scoping fixtures only to relevant modules
   instead of having a one huge fixture file, improved and renamed the directory
   structure of unit tests and unit test infrastructure. #1178
+- MongoDb now gets launched by the Island via python. #1148
 - Create/check data directory on Island init. #1170
 
 ### Removed

deployment_scripts/deploy_windows.ps1

@@ -176,7 +176,6 @@ function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName,
         } | Select-Object -ExpandProperty Name
         # Move all files from extracted folder to mongodb folder
         New-Item -ItemType directory -Path (Join-Path -Path $binDir -ChildPath "mongodb")
-        New-Item -ItemType directory -Path (Join-Path -Path $monkey_home -ChildPath $MONKEY_ISLAND_DIR | Join-Path -ChildPath "db")
         "Moving extracted files"
         Move-Item -Path (Join-Path -Path $binDir -ChildPath $mongodb_folder | Join-Path -ChildPath "\bin\*") -Destination (Join-Path -Path $binDir -ChildPath "mongodb\")
         "Removing zip file"

monkey/monkey_island/cc/environment/data_dir_generator.py

@@ -1,30 +0,0 @@
-import logging
-import os
-
-from monkey_island.cc.environment.utils import is_windows_os
-from monkey_island.cc.environment.windows_permissions import set_full_folder_access
-
-LOG = logging.getLogger(__name__)
-
-
-def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None:
-    if not os.path.isdir(data_dir):
-        try:
-            if create_parent_dirs:
-                os.makedirs(data_dir, mode=0o700)
-            else:
-                os.mkdir(data_dir, mode=0o700)
-        except Exception as ex:
-            LOG.error(
-                f'Could not create data directory at "{data_dir}" (maybe `$HOME` could not be '
-                f"resolved?): {str(ex)}"
-            )
-
-        if is_windows_os():  # `mode=0o700` doesn't work on Windows
-            try:
-                set_full_folder_access(folder_path=data_dir)
-            except Exception as ex:
-                LOG.error(
-                    f'Data directory was created at "{data_dir}" but permissions could not be '
-                    f"set successfully: {str(ex)}"
-                )

monkey/monkey_island/cc/environment/linux_permissions.py

@@ -0,0 +1,7 @@
+import os
+import stat
+
+
+def set_perms_to_owner_only(path: str):
+    # Read, write, and execute by owner
+    os.chmod(path, stat.S_IRWXU)

monkey/monkey_island/cc/environment/utils.py

@@ -1,5 +1,46 @@
-import sys
+import logging
+import os
+import platform
 
 
 def is_windows_os() -> bool:
-    return sys.platform.startswith("win")
+    return platform.system() == "Windows"
+
+
+if is_windows_os():
+    import monkey_island.cc.environment.windows_permissions as windows_permissions
+else:
+    import monkey_island.cc.environment.linux_permissions as linux_permissions  # noqa: E402
+
+LOG = logging.getLogger(__name__)
+
+
+def create_secure_directory(path: str, create_parent_dirs: bool):
+    if not os.path.isdir(path):
+        create_directory(path, create_parent_dirs)
+        set_secure_permissions(path)
+
+
+def create_directory(path: str, create_parent_dirs: bool):
+    try:
+        if create_parent_dirs:
+            os.makedirs(path)
+        else:
+            os.mkdir(path)
+    except Exception as ex:
+        LOG.error(
+            f'Could not create a directory at "{path}" (maybe environmental variables could not be '
+            f"resolved?): {str(ex)}"
+        )
+        raise ex
+
+
+def set_secure_permissions(dir_path: str):
+    try:
+        if is_windows_os():
+            windows_permissions.set_perms_to_owner_only(folder_path=dir_path)
+        else:
+            linux_permissions.set_perms_to_owner_only(path=dir_path)
+    except Exception as ex:
+        LOG.error(f"Permissions could not be set successfully for {dir_path}: {str(ex)}")
+        raise ex

monkey/monkey_island/cc/environment/windows_permissions.py

@@ -1,13 +1,10 @@
-from monkey_island.cc.environment.utils import is_windows_os
-
-if is_windows_os():
-    import ntsecuritycon
-    import win32api
-    import win32con
-    import win32security
+import ntsecuritycon
+import win32api
+import win32con
+import win32security
 
 
-def set_full_folder_access(folder_path: str) -> None:
+def set_perms_to_owner_only(folder_path: str) -> None:
     user = get_user_pySID_object()
 
     security_descriptor = win32security.GetFileSecurity(

monkey/monkey_island/cc/main.py

@@ -9,6 +9,7 @@ from threading import Thread
 # "monkey_island." work.
 from gevent.pywsgi import WSGIServer
 
+from monkey_island.cc.setup.database_initializer import init_collections
 from monkey_island.setup.island_config_options import IslandConfigOptions
 
 MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent)
@@ -24,13 +25,13 @@ from common.version import get_version  # noqa: E402
 from monkey_island.cc.app import init_app  # noqa: E402
 from monkey_island.cc.database import get_db_version  # noqa: E402
 from monkey_island.cc.database import is_db_server_up  # noqa: E402
-from monkey_island.cc.mongo_setup import init_collections, launch_mongodb  # noqa: E402
 from monkey_island.cc.resources.monkey_download import MonkeyDownload  # noqa: E402
 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer  # noqa: E402
 from monkey_island.cc.server_utils.encryptor import initialize_encryptor  # noqa: E402
 from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list  # noqa: E402
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses  # noqa: E402
+from monkey_island.cc.setup.mongo_process_runner import MongoDbRunner  # noqa: E402
 
 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
 
@@ -51,7 +52,9 @@ def main(setup_only: bool, config_options: IslandConfigOptions):
 
 def start_island_server(should_setup_only, config_options: IslandConfigOptions):
     if config_options.start_mongodb:
-        launch_mongodb()
+        MongoDbRunner(
+            db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir
+        ).launch_mongodb()
     mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
     wait_for_mongo_db_server(mongo_url)
     assert_mongo_db_version(mongo_url)

monkey/monkey_island/cc/server_utils/consts.py

@@ -20,6 +20,13 @@ DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())
 
 DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5
 
+_MONGO_BINARY_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "bin", "mongodb")
+_MONGO_EXECUTABLE_PATH_WIN = os.path.join(_MONGO_BINARY_DIR, "mongod.exe")
+_MONGO_EXECUTABLE_PATH_LINUX = os.path.join(_MONGO_BINARY_DIR, "bin", "mongod")
+MONGO_EXECUTABLE_PATH = (
+    _MONGO_EXECUTABLE_PATH_WIN if is_windows_os() else _MONGO_EXECUTABLE_PATH_LINUX
+)
+
 DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars(
     os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME)
 )

monkey/monkey_island/cc/setup/database_initializer.py

@@ -9,17 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa
 logger = logging.getLogger(__name__)
 
 
-def launch_mongodb():
-    # TODO: Implement the launch of mongodb process
-    pass
-
-
 def init_collections():
     logger.info("Setting up the Monkey Island, this might take a while...")
-    try_store_mitigations_on_mongo()
+    _try_store_mitigations_on_mongo()
 
 
-def try_store_mitigations_on_mongo():
+def _try_store_mitigations_on_mongo():
     mitigation_collection_name = AttackMitigations.COLLECTION_NAME
     try:
         mongo.db.validate_collection(mitigation_collection_name)
@@ -33,10 +28,10 @@ def try_store_mitigations_on_mongo():
         except errors.CollectionInvalid:
             pass
         finally:
-            store_mitigations_on_mongo()
+            _store_mitigations_on_mongo()
 
 
-def store_mitigations_on_mongo():
+def _store_mitigations_on_mongo():
     stix2_mitigations = MitreApiInterface.get_all_mitigations()
     mongo_mitigations = AttackMitigations.dict_from_stix2_attack_patterns(
         MitreApiInterface.get_all_attack_techniques()

monkey/monkey_island/cc/setup/mongo_process_runner.py

@@ -0,0 +1,50 @@
+import logging
+import os
+import subprocess
+from typing import List
+
+from monkey_island.cc.environment.utils import create_secure_directory
+from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH
+
+logger = logging.getLogger(__name__)
+
+DB_DIR_NAME = "db"
+DB_DIR_PARAM = "--dbpath"
+MONGO_LOG_FILENAME = "mongo_log.txt"
+
+
+class MongoDbRunner:
+    def __init__(self, db_dir_parent_path: str, logging_dir_path: str):
+        """
+        @param db_dir_parent_path: Path where a folder for database contents will be created
+        @param logging_dir_path: Path to a folder where mongodb logs will be created
+        """
+        self.db_dir_parent_path = db_dir_parent_path
+        self.logging_dir_path = logging_dir_path
+
+    def launch_mongodb(self):
+        db_path = self._create_db_dir()
+        self._start_mongodb_process(db_path)
+
+    def _create_db_dir(self) -> str:
+        db_path = os.path.join(self.db_dir_parent_path, DB_DIR_NAME)
+        logger.info(f"Database content directory: {db_path}.")
+        create_secure_directory(db_path, create_parent_dirs=False)
+        return db_path
+
+    def _start_mongodb_process(self, db_dir_path: str):
+        logger.info("Starting MongoDb process.")
+
+        mongo_run_cmd = MongoDbRunner._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, db_dir_path)
+        logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.")
+
+        mongo_log_path = os.path.join(self.logging_dir_path, MONGO_LOG_FILENAME)
+        logger.info(f"Mongodb log will be available at {mongo_log_path}.")
+
+        with open(mongo_log_path, "w") as log:
+            subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log)
+        logger.info("MongoDb launched successfully!")
+
+    @staticmethod
+    def _build_mongo_launch_cmd(exec_path: str, db_path: str) -> List[str]:
+        return [exec_path, DB_DIR_PARAM, db_path]

monkey/monkey_island/linux/install_mongo.sh

@@ -58,7 +58,6 @@ popd || {
 }
 
 mkdir -p "${MONGODB_DIR}"/bin
-mkdir -p "${MONGODB_DIR}"/db
 cp "${TEMP_MONGO}"/mongodb-*/bin/mongod "${MONGODB_DIR}"/bin/mongod
 cp "${TEMP_MONGO}"/mongodb-*/LICENSE-Community.txt "${MONGODB_DIR}"/
 chmod a+x "${MONGODB_DIR}"/bin/mongod

monkey/monkey_island/linux/run.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-start_mongo() {
-    # TODO: Handle starting and cleaning up mongo inside monkey_island.py or
-    # monkey_island/main.py.
-    ./bin/mongodb/bin/mongod --dbpath ./bin/mongodb/db &
-}
-
-cd_to_monkey() {
-    # Pipenv must be run from monkey/monkey/monkey_island, but monkey_island.py
-    # must be executed from monkey/monkey.
-    cd ..
-}
-
-start_monkey_island() {
-    cd_to_monkey
-    python ./monkey_island.py
-}
-
-start_mongo
-start_monkey_island

monkey/monkey_island/readme.md

@@ -28,7 +28,6 @@
     - Place portable version of mongodb
        1. Download from: <https://fastdl.mongodb.org/win32/mongodb-win32-x86_64-2012plus-4.2.1.zip>
        2. Extract contents of bin folder to \monkey\monkey_island\bin\mongodb.
-       3. Create monkey_island\db folder.
 
     OR
     - Use already running instance of mongodb
@@ -88,12 +87,8 @@
     - `pipenv sync --dev`
     - `cd ..`
 
-1. Set the linux `run.sh` to be executible:
-    - `chmod u+x monkey_island/linux/run.sh`
-
 1. Create the following directories in monkey island folder (execute from ./monkey):
     - `mkdir -p ./monkey_island/bin/mongodb`
-    - `mkdir -p ./monkey_island/db`
     - `mkdir -p ./monkey_island/cc/binaries`
 
 1. Put monkey binaries in /monkey_island/cc/binaries (binaries can be found in releases on github).
@@ -136,4 +131,4 @@
 
 #### How to run
 
-1. From the `monkey/monkey_island` directory, run `pipenv run ./linux/run.sh`
+1. From the `monkey` directory, run `python3.7 ./monkey_island.py`

monkey/monkey_island/setup/config_setup.py

@@ -2,7 +2,7 @@ import os
 from typing import Tuple
 
 from monkey_island.cc.environment import server_config_handler
-from monkey_island.cc.environment.data_dir_generator import create_data_dir  # noqa: E402
+from monkey_island.cc.environment.utils import create_secure_directory
 from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH
 from monkey_island.setup.island_config_options import IslandConfigOptions
 
@@ -10,14 +10,13 @@ from monkey_island.setup.island_config_options import IslandConfigOptions
 def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]:
     server_config_path = os.path.expandvars(os.path.expanduser(server_config_path))
     config = server_config_handler.load_server_config_from_file(server_config_path)
-
-    create_data_dir(config.data_dir, create_parent_dirs=True)
+    create_secure_directory(config.data_dir, create_parent_dirs=True)
     return config, server_config_path
 
 
 def setup_default_config() -> Tuple[IslandConfigOptions, str]:
     server_config_path = DEFAULT_SERVER_CONFIG_PATH
-    create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False)
+    create_secure_directory(DEFAULT_DATA_DIR, create_parent_dirs=False)
     server_config_handler.create_default_server_config_file()
     config = server_config_handler.load_server_config_from_file(server_config_path)
     return config, server_config_path

monkey/monkey_island/setup/island_config_options.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import os
+
 from monkey_island.cc.server_utils.consts import (
     DEFAULT_DATA_DIR,
     DEFAULT_LOG_LEVEL,
@@ -9,7 +11,7 @@ from monkey_island.cc.server_utils.consts import (
 
 class IslandConfigOptions:
     def __init__(self, config_contents: dict):
-        self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR)
+        self.data_dir = os.path.expanduser(config_contents.get("data_dir", DEFAULT_DATA_DIR))
 
         self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL)
 

monkey/monkey_island/windows/clear_db.bat

@@ -1,4 +0,0 @@
-@echo Are you sure? (Press Any Key)
-@pause
-@rmdir /s /q db
-@mkdir db

monkey/monkey_island/windows/run_mongodb.bat

@@ -1,3 +0,0 @@
-REM - Runs MongoDB Server -
-@title MongoDB
-@bin\mongodb\mongod.exe --dbpath db --bind_ip 127.0.0.1

monkey/monkey_island/windows/run_server.bat

@@ -1,5 +1,3 @@
 REM - Runs MongoDB Server & Monkey Island Server using built pyinstaller EXE -
-if not exist db mkdir db
-start windows\run_mongodb.bat
 start windows\run_cc_exe.bat
-start https://localhost:5000
+start https://localhost:5000

monkey/monkey_island/windows/run_server_py.bat

@@ -1,5 +1,3 @@
 REM - Runs MongoDB Server & Monkey Island Server using python -
-if not exist db mkdir db
-start windows\run_mongodb.bat
 pipenv run windows\run_cc.bat
 start https://localhost:5000

monkey/tests/unit_tests/monkey_island/cc/environment/test_utils.py

@@ -0,0 +1,52 @@
+import os
+import shutil
+import stat
+
+import pytest
+
+from monkey_island.cc.environment.utils import create_secure_directory, is_windows_os
+
+
+@pytest.fixture
+def test_path_nested(tmpdir):
+    nested_path = "test1/test2/test3"
+    path = os.path.join(tmpdir, nested_path)
+    yield path
+    try:
+        shutil.rmtree(os.path.join(tmpdir, "test1"))
+    except Exception:
+        pass
+
+
+@pytest.fixture
+def test_path(tmpdir):
+    test_path = "test1"
+    path = os.path.join(tmpdir, test_path)
+    yield path
+    try:
+        shutil.rmtree(path)
+    except Exception:
+        pass
+
+
+def test_create_secure_directory__parent_dirs(test_path_nested):
+    create_secure_directory(test_path_nested, create_parent_dirs=True)
+    assert os.path.isdir(test_path_nested)
+
+
+def test_create_secure_directory__already_created(test_path):
+    os.mkdir(test_path)
+    assert os.path.isdir(test_path)
+    create_secure_directory(test_path, create_parent_dirs=False)
+
+
+def test_create_secure_directory__no_parent_dir(test_path_nested):
+    with pytest.raises(Exception):
+        create_secure_directory(test_path_nested, create_parent_dirs=False)
+
+
+@pytest.mark.skipif(is_windows_os(), reason="Tests Posix (not Windows) permissions.")
+def test_create_secure_directory__perm_linux(test_path_nested):
+    create_secure_directory(test_path_nested, create_parent_dirs=True)
+    st = os.stat(test_path_nested)
+    return bool(st.st_mode & stat.S_IRWXU)

whitelist.py

@@ -166,6 +166,8 @@ IBM  # unused variable (monkey/common/cloud/environment_names.py:11)
 DigitalOcean  # unused variable (monkey/common/cloud/environment_names.py:12)
 _.aws_info  # unused attribute (monkey/monkey_island/cc/environment/aws.py:13)
 build_from_config_file_contents  # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18)
+fake_db_dir  # unused function 'fake_db_dir' (monkey/tests/unit_tests/monkey_island/cc/setup/test_process_runner.py:10)
+
 
 # these are not needed for it to work, but may be useful extra information to understand what's going on
 WINDOWS_PBA_TYPE  # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)