diff --git a/CHANGELOG.md b/CHANGELOG.md index c6e3bfc45..6ab9cfb1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Improved the structure of unit tests by scoping fixtures only to relevant modules instead of having a one huge fixture file, improved and renamed the directory structure of unit tests and unit test infrastructure. #1178 +- MongoDb now gets launched by the Island via python. #1148 - Create/check data directory on Island init. #1170 ### Removed diff --git a/deployment_scripts/deploy_windows.ps1 b/deployment_scripts/deploy_windows.ps1 index 46f2fb0f4..9ce8480cb 100644 --- a/deployment_scripts/deploy_windows.ps1 +++ b/deployment_scripts/deploy_windows.ps1 @@ -176,7 +176,6 @@ function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName, } | Select-Object -ExpandProperty Name # Move all files from extracted folder to mongodb folder New-Item -ItemType directory -Path (Join-Path -Path $binDir -ChildPath "mongodb") - New-Item -ItemType directory -Path (Join-Path -Path $monkey_home -ChildPath $MONKEY_ISLAND_DIR | Join-Path -ChildPath "db") "Moving extracted files" Move-Item -Path (Join-Path -Path $binDir -ChildPath $mongodb_folder | Join-Path -ChildPath "\bin\*") -Destination (Join-Path -Path $binDir -ChildPath "mongodb\") "Removing zip file" diff --git a/monkey/monkey_island/cc/environment/data_dir_generator.py b/monkey/monkey_island/cc/environment/data_dir_generator.py deleted file mode 100644 index 58e16d4b7..000000000 --- a/monkey/monkey_island/cc/environment/data_dir_generator.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging -import os - -from monkey_island.cc.environment.utils import is_windows_os -from monkey_island.cc.environment.windows_permissions import set_full_folder_access - -LOG = logging.getLogger(__name__) - - -def create_data_dir(data_dir: str, create_parent_dirs: bool) -> None: - if not os.path.isdir(data_dir): - try: - if create_parent_dirs: - os.makedirs(data_dir, mode=0o700) - else: - os.mkdir(data_dir, mode=0o700) - except Exception as ex: - LOG.error( - f'Could not create data directory at "{data_dir}" (maybe `$HOME` could not be ' - f"resolved?): {str(ex)}" - ) - - if is_windows_os(): # `mode=0o700` doesn't work on Windows - try: - set_full_folder_access(folder_path=data_dir) - except Exception as ex: - LOG.error( - f'Data directory was created at "{data_dir}" but permissions could not be ' - f"set successfully: {str(ex)}" - ) diff --git a/monkey/monkey_island/cc/environment/linux_permissions.py b/monkey/monkey_island/cc/environment/linux_permissions.py new file mode 100644 index 000000000..2280c7637 --- /dev/null +++ b/monkey/monkey_island/cc/environment/linux_permissions.py @@ -0,0 +1,7 @@ +import os +import stat + + +def set_perms_to_owner_only(path: str): + # Read, write, and execute by owner + os.chmod(path, stat.S_IRWXU) diff --git a/monkey/monkey_island/cc/environment/utils.py b/monkey/monkey_island/cc/environment/utils.py index cbb8a1d6f..907e30d47 100644 --- a/monkey/monkey_island/cc/environment/utils.py +++ b/monkey/monkey_island/cc/environment/utils.py @@ -1,5 +1,46 @@ -import sys +import logging +import os +import platform def is_windows_os() -> bool: - return sys.platform.startswith("win") + return platform.system() == "Windows" + + +if is_windows_os(): + import monkey_island.cc.environment.windows_permissions as windows_permissions +else: + import monkey_island.cc.environment.linux_permissions as linux_permissions # noqa: E402 + +LOG = logging.getLogger(__name__) + + +def create_secure_directory(path: str, create_parent_dirs: bool): + if not os.path.isdir(path): + create_directory(path, create_parent_dirs) + set_secure_permissions(path) + + +def create_directory(path: str, create_parent_dirs: bool): + try: + if create_parent_dirs: + os.makedirs(path) + else: + os.mkdir(path) + except Exception as ex: + LOG.error( + f'Could not create a directory at "{path}" (maybe environmental variables could not be ' + f"resolved?): {str(ex)}" + ) + raise ex + + +def set_secure_permissions(dir_path: str): + try: + if is_windows_os(): + windows_permissions.set_perms_to_owner_only(folder_path=dir_path) + else: + linux_permissions.set_perms_to_owner_only(path=dir_path) + except Exception as ex: + LOG.error(f"Permissions could not be set successfully for {dir_path}: {str(ex)}") + raise ex diff --git a/monkey/monkey_island/cc/environment/windows_permissions.py b/monkey/monkey_island/cc/environment/windows_permissions.py index d17947a2e..225e52370 100644 --- a/monkey/monkey_island/cc/environment/windows_permissions.py +++ b/monkey/monkey_island/cc/environment/windows_permissions.py @@ -1,13 +1,10 @@ -from monkey_island.cc.environment.utils import is_windows_os - -if is_windows_os(): - import ntsecuritycon - import win32api - import win32con - import win32security +import ntsecuritycon +import win32api +import win32con +import win32security -def set_full_folder_access(folder_path: str) -> None: +def set_perms_to_owner_only(folder_path: str) -> None: user = get_user_pySID_object() security_descriptor = win32security.GetFileSecurity( diff --git a/monkey/monkey_island/cc/main.py b/monkey/monkey_island/cc/main.py index df015863b..af84666e8 100644 --- a/monkey/monkey_island/cc/main.py +++ b/monkey/monkey_island/cc/main.py @@ -9,6 +9,7 @@ from threading import Thread # "monkey_island." work. from gevent.pywsgi import WSGIServer +from monkey_island.cc.setup.database_initializer import init_collections from monkey_island.setup.island_config_options import IslandConfigOptions MONKEY_ISLAND_DIR_BASE_PATH = str(Path(__file__).parent.parent) @@ -24,13 +25,13 @@ from common.version import get_version # noqa: E402 from monkey_island.cc.app import init_app # noqa: E402 from monkey_island.cc.database import get_db_version # noqa: E402 from monkey_island.cc.database import is_db_server_up # noqa: E402 -from monkey_island.cc.mongo_setup import init_collections, launch_mongodb # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 +from monkey_island.cc.setup.mongo_process_runner import MongoDbRunner # noqa: E402 MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0" @@ -51,7 +52,9 @@ def main(setup_only: bool, config_options: IslandConfigOptions): def start_island_server(should_setup_only, config_options: IslandConfigOptions): if config_options.start_mongodb: - launch_mongodb() + MongoDbRunner( + db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir + ).launch_mongodb() mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url()) wait_for_mongo_db_server(mongo_url) assert_mongo_db_version(mongo_url) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index bc99b4394..5cc9a0dd1 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -20,6 +20,13 @@ DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 +_MONGO_BINARY_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "bin", "mongodb") +_MONGO_EXECUTABLE_PATH_WIN = os.path.join(_MONGO_BINARY_DIR, "mongod.exe") +_MONGO_EXECUTABLE_PATH_LINUX = os.path.join(_MONGO_BINARY_DIR, "bin", "mongod") +MONGO_EXECUTABLE_PATH = ( + _MONGO_EXECUTABLE_PATH_WIN if is_windows_os() else _MONGO_EXECUTABLE_PATH_LINUX +) + DEFAULT_SERVER_CONFIG_PATH = os.path.expandvars( os.path.join(DEFAULT_DATA_DIR, SERVER_CONFIG_FILENAME) ) diff --git a/monkey/monkey_island/cc/setup/__init__.py b/monkey/monkey_island/cc/setup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/cc/mongo_setup.py b/monkey/monkey_island/cc/setup/database_initializer.py similarity index 88% rename from monkey/monkey_island/cc/mongo_setup.py rename to monkey/monkey_island/cc/setup/database_initializer.py index 74cb29fc2..34914c7ce 100644 --- a/monkey/monkey_island/cc/mongo_setup.py +++ b/monkey/monkey_island/cc/setup/database_initializer.py @@ -9,17 +9,12 @@ from monkey_island.cc.services.attack.mitre_api_interface import MitreApiInterfa logger = logging.getLogger(__name__) -def launch_mongodb(): - # TODO: Implement the launch of mongodb process - pass - - def init_collections(): logger.info("Setting up the Monkey Island, this might take a while...") - try_store_mitigations_on_mongo() + _try_store_mitigations_on_mongo() -def try_store_mitigations_on_mongo(): +def _try_store_mitigations_on_mongo(): mitigation_collection_name = AttackMitigations.COLLECTION_NAME try: mongo.db.validate_collection(mitigation_collection_name) @@ -33,10 +28,10 @@ def try_store_mitigations_on_mongo(): except errors.CollectionInvalid: pass finally: - store_mitigations_on_mongo() + _store_mitigations_on_mongo() -def store_mitigations_on_mongo(): +def _store_mitigations_on_mongo(): stix2_mitigations = MitreApiInterface.get_all_mitigations() mongo_mitigations = AttackMitigations.dict_from_stix2_attack_patterns( MitreApiInterface.get_all_attack_techniques() diff --git a/monkey/monkey_island/cc/setup/mongo_process_runner.py b/monkey/monkey_island/cc/setup/mongo_process_runner.py new file mode 100644 index 000000000..d03b62913 --- /dev/null +++ b/monkey/monkey_island/cc/setup/mongo_process_runner.py @@ -0,0 +1,50 @@ +import logging +import os +import subprocess +from typing import List + +from monkey_island.cc.environment.utils import create_secure_directory +from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH + +logger = logging.getLogger(__name__) + +DB_DIR_NAME = "db" +DB_DIR_PARAM = "--dbpath" +MONGO_LOG_FILENAME = "mongo_log.txt" + + +class MongoDbRunner: + def __init__(self, db_dir_parent_path: str, logging_dir_path: str): + """ + @param db_dir_parent_path: Path where a folder for database contents will be created + @param logging_dir_path: Path to a folder where mongodb logs will be created + """ + self.db_dir_parent_path = db_dir_parent_path + self.logging_dir_path = logging_dir_path + + def launch_mongodb(self): + db_path = self._create_db_dir() + self._start_mongodb_process(db_path) + + def _create_db_dir(self) -> str: + db_path = os.path.join(self.db_dir_parent_path, DB_DIR_NAME) + logger.info(f"Database content directory: {db_path}.") + create_secure_directory(db_path, create_parent_dirs=False) + return db_path + + def _start_mongodb_process(self, db_dir_path: str): + logger.info("Starting MongoDb process.") + + mongo_run_cmd = MongoDbRunner._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, db_dir_path) + logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.") + + mongo_log_path = os.path.join(self.logging_dir_path, MONGO_LOG_FILENAME) + logger.info(f"Mongodb log will be available at {mongo_log_path}.") + + with open(mongo_log_path, "w") as log: + subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log) + logger.info("MongoDb launched successfully!") + + @staticmethod + def _build_mongo_launch_cmd(exec_path: str, db_path: str) -> List[str]: + return [exec_path, DB_DIR_PARAM, db_path] diff --git a/monkey/monkey_island/linux/install_mongo.sh b/monkey/monkey_island/linux/install_mongo.sh index 2bf2d43d4..825daaf5a 100755 --- a/monkey/monkey_island/linux/install_mongo.sh +++ b/monkey/monkey_island/linux/install_mongo.sh @@ -58,7 +58,6 @@ popd || { } mkdir -p "${MONGODB_DIR}"/bin -mkdir -p "${MONGODB_DIR}"/db cp "${TEMP_MONGO}"/mongodb-*/bin/mongod "${MONGODB_DIR}"/bin/mongod cp "${TEMP_MONGO}"/mongodb-*/LICENSE-Community.txt "${MONGODB_DIR}"/ chmod a+x "${MONGODB_DIR}"/bin/mongod diff --git a/monkey/monkey_island/linux/run.sh b/monkey/monkey_island/linux/run.sh deleted file mode 100755 index a284ffa83..000000000 --- a/monkey/monkey_island/linux/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -start_mongo() { - # TODO: Handle starting and cleaning up mongo inside monkey_island.py or - # monkey_island/main.py. - ./bin/mongodb/bin/mongod --dbpath ./bin/mongodb/db & -} - -cd_to_monkey() { - # Pipenv must be run from monkey/monkey/monkey_island, but monkey_island.py - # must be executed from monkey/monkey. - cd .. -} - -start_monkey_island() { - cd_to_monkey - python ./monkey_island.py -} - -start_mongo -start_monkey_island diff --git a/monkey/monkey_island/readme.md b/monkey/monkey_island/readme.md index 0882aecfe..4351dacff 100644 --- a/monkey/monkey_island/readme.md +++ b/monkey/monkey_island/readme.md @@ -28,7 +28,6 @@ - Place portable version of mongodb 1. Download from: 2. Extract contents of bin folder to \monkey\monkey_island\bin\mongodb. - 3. Create monkey_island\db folder. OR - Use already running instance of mongodb @@ -88,12 +87,8 @@ - `pipenv sync --dev` - `cd ..` -1. Set the linux `run.sh` to be executible: - - `chmod u+x monkey_island/linux/run.sh` - 1. Create the following directories in monkey island folder (execute from ./monkey): - `mkdir -p ./monkey_island/bin/mongodb` - - `mkdir -p ./monkey_island/db` - `mkdir -p ./monkey_island/cc/binaries` 1. Put monkey binaries in /monkey_island/cc/binaries (binaries can be found in releases on github). @@ -136,4 +131,4 @@ #### How to run -1. From the `monkey/monkey_island` directory, run `pipenv run ./linux/run.sh` +1. From the `monkey` directory, run `python3.7 ./monkey_island.py` diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 5c9625ac4..50330aea3 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -2,7 +2,7 @@ import os from typing import Tuple from monkey_island.cc.environment import server_config_handler -from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 +from monkey_island.cc.environment.utils import create_secure_directory from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH from monkey_island.setup.island_config_options import IslandConfigOptions @@ -10,14 +10,13 @@ from monkey_island.setup.island_config_options import IslandConfigOptions def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]: server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) config = server_config_handler.load_server_config_from_file(server_config_path) - - create_data_dir(config.data_dir, create_parent_dirs=True) + create_secure_directory(config.data_dir, create_parent_dirs=True) return config, server_config_path def setup_default_config() -> Tuple[IslandConfigOptions, str]: server_config_path = DEFAULT_SERVER_CONFIG_PATH - create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False) + create_secure_directory(DEFAULT_DATA_DIR, create_parent_dirs=False) server_config_handler.create_default_server_config_file() config = server_config_handler.load_server_config_from_file(server_config_path) return config, server_config_path diff --git a/monkey/monkey_island/setup/island_config_options.py b/monkey/monkey_island/setup/island_config_options.py index bf1c06e1b..938c840a0 100644 --- a/monkey/monkey_island/setup/island_config_options.py +++ b/monkey/monkey_island/setup/island_config_options.py @@ -1,5 +1,7 @@ from __future__ import annotations +import os + from monkey_island.cc.server_utils.consts import ( DEFAULT_DATA_DIR, DEFAULT_LOG_LEVEL, @@ -9,7 +11,7 @@ from monkey_island.cc.server_utils.consts import ( class IslandConfigOptions: def __init__(self, config_contents: dict): - self.data_dir = config_contents.get("data_dir", DEFAULT_DATA_DIR) + self.data_dir = os.path.expanduser(config_contents.get("data_dir", DEFAULT_DATA_DIR)) self.log_level = config_contents.get("log_level", DEFAULT_LOG_LEVEL) diff --git a/monkey/monkey_island/windows/clear_db.bat b/monkey/monkey_island/windows/clear_db.bat deleted file mode 100644 index 8597f3d32..000000000 --- a/monkey/monkey_island/windows/clear_db.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo Are you sure? (Press Any Key) -@pause -@rmdir /s /q db -@mkdir db \ No newline at end of file diff --git a/monkey/monkey_island/windows/run_mongodb.bat b/monkey/monkey_island/windows/run_mongodb.bat deleted file mode 100644 index 106b5f00a..000000000 --- a/monkey/monkey_island/windows/run_mongodb.bat +++ /dev/null @@ -1,3 +0,0 @@ -REM - Runs MongoDB Server - -@title MongoDB -@bin\mongodb\mongod.exe --dbpath db --bind_ip 127.0.0.1 \ No newline at end of file diff --git a/monkey/monkey_island/windows/run_server.bat b/monkey/monkey_island/windows/run_server.bat index ab2ad274c..5e5331a2e 100644 --- a/monkey/monkey_island/windows/run_server.bat +++ b/monkey/monkey_island/windows/run_server.bat @@ -1,5 +1,3 @@ REM - Runs MongoDB Server & Monkey Island Server using built pyinstaller EXE - -if not exist db mkdir db -start windows\run_mongodb.bat start windows\run_cc_exe.bat -start https://localhost:5000 \ No newline at end of file +start https://localhost:5000 diff --git a/monkey/monkey_island/windows/run_server_py.bat b/monkey/monkey_island/windows/run_server_py.bat index 90d81c9b7..a727211ea 100644 --- a/monkey/monkey_island/windows/run_server_py.bat +++ b/monkey/monkey_island/windows/run_server_py.bat @@ -1,5 +1,3 @@ REM - Runs MongoDB Server & Monkey Island Server using python - -if not exist db mkdir db -start windows\run_mongodb.bat pipenv run windows\run_cc.bat start https://localhost:5000 diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/test_utils.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_utils.py new file mode 100644 index 000000000..fa2c4202b --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/environment/test_utils.py @@ -0,0 +1,52 @@ +import os +import shutil +import stat + +import pytest + +from monkey_island.cc.environment.utils import create_secure_directory, is_windows_os + + +@pytest.fixture +def test_path_nested(tmpdir): + nested_path = "test1/test2/test3" + path = os.path.join(tmpdir, nested_path) + yield path + try: + shutil.rmtree(os.path.join(tmpdir, "test1")) + except Exception: + pass + + +@pytest.fixture +def test_path(tmpdir): + test_path = "test1" + path = os.path.join(tmpdir, test_path) + yield path + try: + shutil.rmtree(path) + except Exception: + pass + + +def test_create_secure_directory__parent_dirs(test_path_nested): + create_secure_directory(test_path_nested, create_parent_dirs=True) + assert os.path.isdir(test_path_nested) + + +def test_create_secure_directory__already_created(test_path): + os.mkdir(test_path) + assert os.path.isdir(test_path) + create_secure_directory(test_path, create_parent_dirs=False) + + +def test_create_secure_directory__no_parent_dir(test_path_nested): + with pytest.raises(Exception): + create_secure_directory(test_path_nested, create_parent_dirs=False) + + +@pytest.mark.skipif(is_windows_os(), reason="Tests Posix (not Windows) permissions.") +def test_create_secure_directory__perm_linux(test_path_nested): + create_secure_directory(test_path_nested, create_parent_dirs=True) + st = os.stat(test_path_nested) + return bool(st.st_mode & stat.S_IRWXU) diff --git a/whitelist.py b/whitelist.py index 51d4c22b8..6739d4791 100644 --- a/whitelist.py +++ b/whitelist.py @@ -166,6 +166,8 @@ IBM # unused variable (monkey/common/cloud/environment_names.py:11) DigitalOcean # unused variable (monkey/common/cloud/environment_names.py:12) _.aws_info # unused attribute (monkey/monkey_island/cc/environment/aws.py:13) build_from_config_file_contents # unused method 'build_from_config_file_contents' (\monkey_island\setup\island_config_options.py:18) +fake_db_dir # unused function 'fake_db_dir' (monkey/tests/unit_tests/monkey_island/cc/setup/test_process_runner.py:10) + # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)