Merge pull request #1160 from guardicore/no-global-data-dir

No global data dir

monkey/common/config_value_paths.py

@@ -11,3 +11,5 @@ SUBNET_SCAN_LIST_PATH = ["basic_network", "scope", "subnet_scan_list"]
 LOCAL_NETWORK_SCAN_PATH = ["basic_network", "scope", "local_network_scan"]
 LM_HASH_LIST_PATH = ["internal", "exploits", "exploit_lm_hash_list"]
 NTLM_HASH_LIST_PATH = ["internal", "exploits", "exploit_ntlm_hash_list"]
+PBA_LINUX_FILENAME_PATH = ["monkey", "post_breach", "PBA_linux_filename"]
+PBA_WINDOWS_FILENAME_PATH = ["monkey", "post_breach", "PBA_windows_filename"]

monkey/monkey_island/cc/app.py

@@ -150,7 +150,7 @@ def init_api_resources(api):
     api.add_resource(TelemetryFeed, "/api/telemetry-feed", "/api/telemetry-feed/")
     api.add_resource(Log, "/api/log", "/api/log/")
     api.add_resource(IslandLog, "/api/log/island/download", "/api/log/island/download/")
-    api.add_resource(PBAFileDownload, "/api/pba/download/<string:path>")
+    api.add_resource(PBAFileDownload, "/api/pba/download/<string:filename>")
     api.add_resource(T1216PBAFileDownload, T1216_PBA_FILE_DOWNLOAD_PATH)
     api.add_resource(
         FileUpload,

monkey/monkey_island/cc/main.py

@@ -26,6 +26,7 @@ from monkey_island.cc.resources.monkey_download import MonkeyDownload  # noqa: E
 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer  # noqa: E402
 from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH  # noqa: E402
 from monkey_island.cc.server_utils.encryptor import initialize_encryptor  # noqa: E402
+from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list  # noqa: E402
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses  # noqa: E402
 from monkey_island.cc.setup import setup  # noqa: E402
@@ -35,8 +36,11 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
 
 def main(should_setup_only=False, server_config_filename=DEFAULT_SERVER_CONFIG_PATH):
     logger.info("Starting bootloader server")
+
+    data_dir = env_singleton.env.get_config().data_dir_abs_path
     env_singleton.initialize_from_file(server_config_filename)
-    initialize_encryptor(env_singleton.env.get_config().data_dir_abs_path)
+    initialize_encryptor(data_dir)
+    initialize_services(data_dir)
 
     mongo_url = os.environ.get("MONGO_URL", env_singleton.env.get_mongo_url())
     bootloader_server_thread = Thread(

monkey/monkey_island/cc/resources/local_run.py

@@ -1,62 +1,12 @@
 import json
-import logging
-import os
-import sys
-from shutil import copyfile
 
 import flask_restful
 from flask import jsonify, make_response, request
 
-import monkey_island.cc.environment.environment_singleton as env_singleton
 from monkey_island.cc.models import Monkey
 from monkey_island.cc.resources.auth.auth import jwt_required
-from monkey_island.cc.resources.monkey_download import get_monkey_executable
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
 from monkey_island.cc.services.node import NodeService
-from monkey_island.cc.services.utils.network_utils import local_ip_addresses
+from monkey_island.cc.services.run_local_monkey import LocalMonkeyRunService
-
-__author__ = "Barak"
-
-logger = logging.getLogger(__name__)
-
-
-def run_local_monkey():
-    import platform
-    import stat
-    import subprocess
-
-    # get the monkey executable suitable to run on the server
-    result = get_monkey_executable(platform.system().lower(), platform.machine().lower())
-    if not result:
-        return False, "OS Type not found"
-
-    src_path = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "binaries", result["filename"])
-    dest_dir = env_singleton.env.get_config().data_dir_abs_path
-    dest_path = os.path.join(dest_dir, result["filename"])
-
-    # copy the executable to temp path (don't run the monkey from its current location as it may
-    # delete itself)
-    try:
-        copyfile(src_path, dest_path)
-        os.chmod(dest_path, stat.S_IRWXU | stat.S_IRWXG)
-    except Exception as exc:
-        logger.error("Copy file failed", exc_info=True)
-        return False, "Copy file failed: %s" % exc
-
-    # run the monkey
-    try:
-        args = [
-            '"%s" m0nk3y -s %s:%s'
-            % (dest_path, local_ip_addresses()[0], env_singleton.env.get_island_port())
-        ]
-        if sys.platform == "win32":
-            args = "".join(args)
-        subprocess.Popen(args, cwd=dest_dir, shell=True).pid
-    except Exception as exc:
-        logger.error("popen failed", exc_info=True)
-        return False, "popen failed: %s" % exc
-
-    return True, ""
 
 
 class LocalRun(flask_restful.Resource):
@@ -75,7 +25,7 @@ class LocalRun(flask_restful.Resource):
     def post(self):
         body = json.loads(request.data)
         if body.get("action") == "run":
-            local_run = run_local_monkey()
+            local_run = LocalMonkeyRunService.run_local_monkey()
             return jsonify(is_running=local_run[0], error_text=local_run[1])
 
         # default action

monkey/monkey_island/cc/resources/pba_file_download.py

@@ -1,7 +1,7 @@
 import flask_restful
 from flask import send_from_directory
 
-import monkey_island.cc.environment.environment_singleton as env_singleton
+from monkey_island.cc.services.post_breach_files import PostBreachFilesService
 
 __author__ = "VakarisZ"
 
@@ -12,5 +12,6 @@ class PBAFileDownload(flask_restful.Resource):
     """
 
     # Used by monkey. can't secure.
-    def get(self, path):
+    def get(self, filename):
-        return send_from_directory(env_singleton.env.get_config().data_dir_abs_path, path)
+        custom_pba_dir = PostBreachFilesService.get_custom_pba_directory()
+        return send_from_directory(custom_pba_dir, filename)

monkey/monkey_island/cc/resources/pba_file_upload.py

@@ -1,19 +1,15 @@
 import copy
 import logging
-import os
-from pathlib import Path
 
 import flask_restful
 from flask import Response, request, send_from_directory
+from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
 
-import monkey_island.cc.environment.environment_singleton as env_singleton
+from common.config_value_paths import PBA_LINUX_FILENAME_PATH, PBA_WINDOWS_FILENAME_PATH
 from monkey_island.cc.resources.auth.auth import jwt_required
 from monkey_island.cc.services.config import ConfigService
-from monkey_island.cc.services.post_breach_files import (
+from monkey_island.cc.services.post_breach_files import PostBreachFilesService
-    PBA_LINUX_FILENAME_PATH,
-    PBA_WINDOWS_FILENAME_PATH,
-)
 
 __author__ = "VakarisZ"
 
@@ -28,10 +24,6 @@ class FileUpload(flask_restful.Resource):
     File upload endpoint used to exchange files with filepond component on the front-end
     """
 
-    def __init__(self):
-        # Create all directories on the way if they don't exist
-        Path(env_singleton.env.get_config().data_dir_abs_path).mkdir(parents=True, exist_ok=True)
-
     @jwt_required
     def get(self, file_type):
         """
@@ -44,7 +36,7 @@ class FileUpload(flask_restful.Resource):
             filename = ConfigService.get_config_value(copy.deepcopy(PBA_LINUX_FILENAME_PATH))
         else:
             filename = ConfigService.get_config_value(copy.deepcopy(PBA_WINDOWS_FILENAME_PATH))
-        return send_from_directory(env_singleton.env.get_config().data_dir_abs_path, filename)
+        return send_from_directory(PostBreachFilesService.get_custom_pba_directory(), filename)
 
     @jwt_required
     def post(self, file_type):
@@ -53,27 +45,30 @@ class FileUpload(flask_restful.Resource):
         :param file_type: Type indicates which file was received, linux or windows
         :return: Returns flask response object with uploaded file's filename
         """
-        filename = FileUpload.upload_pba_file(request, (file_type == LINUX_PBA_TYPE))
+        filename = FileUpload.upload_pba_file(
+            request.files["filepond"], (file_type == LINUX_PBA_TYPE)
+        )
 
         response = Response(response=filename, status=200, mimetype="text/plain")
         return response
 
     @staticmethod
-    def upload_pba_file(request_, is_linux=True):
+    def upload_pba_file(file_storage: FileStorage, is_linux=True):
         """
         Uploads PBA file to island's file system
         :param request_: Request object containing PBA file
         :param is_linux: Boolean indicating if this file is for windows or for linux
         :return: filename string
         """
-        filename = secure_filename(request_.files["filepond"].filename)
+        filename = secure_filename(file_storage.filename)
-        file_path = (
+        file_contents = file_storage.read()
-            Path(env_singleton.env.get_config().data_dir_abs_path).joinpath(filename).absolute()
+
-        )
+        PostBreachFilesService.save_file(filename, file_contents)
-        request_.files["filepond"].save(str(file_path))
+
         ConfigService.set_config_value(
             (PBA_LINUX_FILENAME_PATH if is_linux else PBA_WINDOWS_FILENAME_PATH), filename
         )
+
         return filename
 
     @jwt_required
@@ -88,16 +83,7 @@ class FileUpload(flask_restful.Resource):
         )
         filename = ConfigService.get_config_value(filename_path)
         if filename:
-            file_path = Path(env_singleton.env.get_config().data_dir_abs_path).joinpath(filename)
+            PostBreachFilesService.remove_file(filename)
-            FileUpload._delete_file(file_path)
             ConfigService.set_config_value(filename_path, "")
 
         return {}
-
-    @staticmethod
-    def _delete_file(file_path):
-        try:
-            if os.path.exists(file_path):
-                os.remove(file_path)
-        except OSError as e:
-            LOG.error("Couldn't remove previously uploaded post breach files: %s" % e)

monkey/monkey_island/cc/services/config.py

@@ -6,10 +6,10 @@ import logging
 from jsonschema import Draft4Validator, validators
 
 import monkey_island.cc.environment.environment_singleton as env_singleton
-import monkey_island.cc.services.post_breach_files
 from monkey_island.cc.database import mongo
 from monkey_island.cc.server_utils.encryptor import get_encryptor
 from monkey_island.cc.services.config_schema.config_schema import SCHEMA
+from monkey_island.cc.services.post_breach_files import PostBreachFilesService
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses
 
 __author__ = "itay.mizeretz"
@@ -20,6 +20,8 @@ from common.config_value_paths import (
     LM_HASH_LIST_PATH,
     NTLM_HASH_LIST_PATH,
     PASSWORD_LIST_PATH,
+    PBA_LINUX_FILENAME_PATH,
+    PBA_WINDOWS_FILENAME_PATH,
     SSH_KEYS_PATH,
     STARTED_ON_ISLAND_PATH,
     USER_LIST_PATH,
@@ -191,7 +193,7 @@ class ConfigService:
         # PBA file upload happens on pba_file_upload endpoint and corresponding config options
         # are set there
         config_json = ConfigService._filter_none_values(config_json)
-        monkey_island.cc.services.post_breach_files.set_config_PBA_files(config_json)
+        ConfigService.set_config_PBA_files(config_json)
         if should_encrypt:
             try:
                 ConfigService.encrypt_config(config_json)
@@ -202,6 +204,19 @@ class ConfigService:
         logger.info("monkey config was updated")
         return True
 
+    @staticmethod
+    def set_config_PBA_files(config_json):
+        """
+        Sets PBA file info in config_json to current config's PBA file info values.
+        :param config_json: config_json that will be modified
+        """
+        if ConfigService.get_config():
+            linux_filename = ConfigService.get_config_value(PBA_LINUX_FILENAME_PATH)
+            windows_filename = ConfigService.get_config_value(PBA_WINDOWS_FILENAME_PATH)
+
+            ConfigService.set_config_value(PBA_LINUX_FILENAME_PATH, linux_filename)
+            ConfigService.set_config_value(PBA_WINDOWS_FILENAME_PATH, windows_filename)
+
     @staticmethod
     def init_default_config():
         if ConfigService.default_config is None:
@@ -229,7 +244,7 @@ class ConfigService:
 
     @staticmethod
     def reset_config():
-        monkey_island.cc.services.post_breach_files.remove_PBA_files()
+        PostBreachFilesService.remove_PBA_files()
         config = ConfigService.get_default_config(True)
         ConfigService.set_server_ips_in_config(config)
         ConfigService.update_config(config, should_encrypt=False)

monkey/monkey_island/cc/services/database.py

@@ -6,7 +6,6 @@ from monkey_island.cc.database import mongo
 from monkey_island.cc.models.attack.attack_mitigations import AttackMitigations
 from monkey_island.cc.services.attack.attack_config import AttackConfig
 from monkey_island.cc.services.config import ConfigService
-from monkey_island.cc.services.post_breach_files import remove_PBA_files
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +17,6 @@ class Database(object):
     @staticmethod
     def reset_db():
         logger.info("Resetting database")
-        remove_PBA_files()
         # We can't drop system collections.
         [
             Database.drop_collection(x)

monkey/monkey_island/cc/services/initialize.py

@@ -0,0 +1,7 @@
+from monkey_island.cc.services.post_breach_files import PostBreachFilesService
+from monkey_island.cc.services.run_local_monkey import LocalMonkeyRunService
+
+
+def initialize_services(data_dir):
+    PostBreachFilesService.initialize(data_dir)
+    LocalMonkeyRunService.initialize(data_dir)

monkey/monkey_island/cc/services/post_breach_files.py

@@ -1,53 +1,44 @@
 import logging
 import os
-
+from pathlib import Path
-import monkey_island.cc.services.config
-
-__author__ = "VakarisZ"
-
-import monkey_island.cc.environment.environment_singleton as env_singleton
 
 logger = logging.getLogger(__name__)
 
-# Where to find file names in config
-PBA_WINDOWS_FILENAME_PATH = ["monkey", "post_breach", "PBA_windows_filename"]
-PBA_LINUX_FILENAME_PATH = ["monkey", "post_breach", "PBA_linux_filename"]
 
+class PostBreachFilesService:
+    DATA_DIR = None
+    CUSTOM_PBA_DIRNAME = "custom_pbas"
 
-def remove_PBA_files():
+    # TODO: A number of these services should be instance objects instead of
-    if monkey_island.cc.services.config.ConfigService.get_config():
+    # static/singleton hybrids. At the moment, this requires invasive refactoring that's
-        windows_filename = monkey_island.cc.services.config.ConfigService.get_config_value(
+    # not a priority.
-            PBA_WINDOWS_FILENAME_PATH
+    @classmethod
+    def initialize(cls, data_dir):
+        cls.DATA_DIR = data_dir
+        Path(cls.get_custom_pba_directory()).mkdir(mode=0o0700, parents=True, exist_ok=True)
+
+    @staticmethod
+    def save_file(filename: str, file_contents: bytes):
+        file_path = os.path.join(PostBreachFilesService.get_custom_pba_directory(), filename)
+        with open(file_path, "wb") as f:
+            f.write(file_contents)
+
+    @staticmethod
+    def remove_PBA_files():
+        for f in os.listdir(PostBreachFilesService.get_custom_pba_directory()):
+            PostBreachFilesService.remove_file(f)
+
+    @staticmethod
+    def remove_file(file_name):
+        file_path = os.path.join(PostBreachFilesService.get_custom_pba_directory(), file_name)
+        try:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+        except OSError as e:
+            logger.error("Can't remove previously uploaded post breach files: %s" % e)
+
+    @staticmethod
+    def get_custom_pba_directory():
+        return os.path.join(
+            PostBreachFilesService.DATA_DIR, PostBreachFilesService.CUSTOM_PBA_DIRNAME
         )
-        linux_filename = monkey_island.cc.services.config.ConfigService.get_config_value(
-            PBA_LINUX_FILENAME_PATH
-        )
-        if linux_filename:
-            remove_file(linux_filename)
-        if windows_filename:
-            remove_file(windows_filename)
-
-
-def remove_file(file_name):
-    file_path = os.path.join(env_singleton.env.get_config().data_dir_abs_path, file_name)
-    try:
-        if os.path.exists(file_path):
-            os.remove(file_path)
-    except OSError as e:
-        logger.error("Can't remove previously uploaded post breach files: %s" % e)
-
-
-def set_config_PBA_files(config_json):
-    """
-    Sets PBA file info in config_json to current config's PBA file info values.
-    :param config_json: config_json that will be modified
-    """
-    if monkey_island.cc.services.config.ConfigService.get_config():
-        linux_filename = monkey_island.cc.services.config.ConfigService.get_config_value(
-            PBA_LINUX_FILENAME_PATH
-        )
-        windows_filename = monkey_island.cc.services.config.ConfigService.get_config_value(
-            PBA_WINDOWS_FILENAME_PATH
-        )
-        config_json["monkey"]["post_breach"]["PBA_linux_filename"] = linux_filename
-        config_json["monkey"]["post_breach"]["PBA_windows_filename"] = windows_filename

monkey/monkey_island/cc/services/run_local_monkey.py

@@ -0,0 +1,56 @@
+import logging
+import os
+import platform
+import stat
+import subprocess
+from shutil import copyfile
+
+import monkey_island.cc.environment.environment_singleton as env_singleton
+from monkey_island.cc.resources.monkey_download import get_monkey_executable
+from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
+from monkey_island.cc.services.utils.network_utils import local_ip_addresses
+
+logger = logging.getLogger(__name__)
+
+
+class LocalMonkeyRunService:
+    DATA_DIR = None
+
+    # TODO: A number of these services should be instance objects instead of
+    # static/singleton hybrids. At the moment, this requires invasive refactoring that's
+    # not a priority.
+    @classmethod
+    def initialize(cls, data_dir):
+        cls.DATA_DIR = data_dir
+
+    @staticmethod
+    def run_local_monkey():
+        # get the monkey executable suitable to run on the server
+        result = get_monkey_executable(platform.system().lower(), platform.machine().lower())
+        if not result:
+            return False, "OS Type not found"
+
+        src_path = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "binaries", result["filename"])
+        dest_path = os.path.join(LocalMonkeyRunService.DATA_DIR, result["filename"])
+
+        # copy the executable to temp path (don't run the monkey from its current location as it may
+        # delete itself)
+        try:
+            copyfile(src_path, dest_path)
+            os.chmod(dest_path, stat.S_IRWXU | stat.S_IRWXG)
+        except Exception as exc:
+            logger.error("Copy file failed", exc_info=True)
+            return False, "Copy file failed: %s" % exc
+
+        # run the monkey
+        try:
+            ip = local_ip_addresses()[0]
+            port = env_singleton.env.get_island_port()
+
+            args = [dest_path, "m0nk3y", "-s", f"{ip}:{port}"]
+            subprocess.Popen(args, cwd=LocalMonkeyRunService.DATA_DIR)
+        except Exception as exc:
+            logger.error("popen failed", exc_info=True)
+            return False, "popen failed: %s" % exc
+
+        return True, ""

monkey/tests/monkey_island/cc/services/test_post_breach_files.py

@@ -0,0 +1,69 @@
+import os
+
+import pytest
+
+from monkey_island.cc.services.post_breach_files import PostBreachFilesService
+
+
+def raise_(ex):
+    raise ex
+
+
+@pytest.fixture(autouse=True)
+def custom_pba_directory(tmpdir):
+    PostBreachFilesService.initialize(tmpdir)
+
+
+def create_custom_pba_file(filename):
+    PostBreachFilesService.save_file(filename, b"")
+
+
+def test_remove_pba_files():
+    create_custom_pba_file("linux_file")
+    create_custom_pba_file("windows_file")
+
+    assert not dir_is_empty(PostBreachFilesService.get_custom_pba_directory())
+    PostBreachFilesService.remove_PBA_files()
+    assert dir_is_empty(PostBreachFilesService.get_custom_pba_directory())
+
+
+def dir_is_empty(dir_path):
+    dir_contents = os.listdir(dir_path)
+    return len(dir_contents) == 0
+
+
+@pytest.mark.skipif(os.name != "posix", reason="Tests Posix (not Windows) permissions.")
+def test_custom_pba_dir_permissions():
+    st = os.stat(PostBreachFilesService.get_custom_pba_directory())
+
+    assert st.st_mode == 0o40700
+
+
+def test_remove_failure(monkeypatch):
+    monkeypatch.setattr(os, "remove", lambda x: raise_(OSError("Permission denied")))
+
+    try:
+        create_custom_pba_file("windows_file")
+        PostBreachFilesService.remove_PBA_files()
+    except Exception as ex:
+        pytest.fail(f"Unxepected exception: {ex}")
+
+
+def test_remove_nonexistant_file(monkeypatch):
+    monkeypatch.setattr(os, "remove", lambda x: raise_(FileNotFoundError("FileNotFound")))
+
+    try:
+        PostBreachFilesService.remove_file("/nonexistant/file")
+    except Exception as ex:
+        pytest.fail(f"Unxepected exception: {ex}")
+
+
+def test_save_file():
+    FILE_NAME = "test_file"
+    FILE_CONTENTS = b"hello"
+    PostBreachFilesService.save_file(FILE_NAME, FILE_CONTENTS)
+
+    expected_file_path = os.path.join(PostBreachFilesService.get_custom_pba_directory(), FILE_NAME)
+
+    assert os.path.isfile(expected_file_path)
+    assert FILE_CONTENTS == open(expected_file_path, "rb").read()

whitelist.py

@@ -20,6 +20,7 @@ set_os_windows  # unused variable (monkey/tests/infection_monkey/post_breach/act
 patch_new_user_classes  # unused variable (monkey/tests/infection_monkey/utils/test_auto_new_user_factory.py:25)
 patch_new_user_classes  # unused variable (monkey/tests/infection_monkey/utils/test_auto_new_user_factory.py:31)
 mock_home_env  # unused variable (monkey/tests/monkey_island/cc/server_utils/test_island_logger.py:20)
+custom_pba_directory  # unused variable (monkey/tests/monkey_island/cc/services/test_post_breach_files.py:20)
 configure_resources  # unused function (monkey/tests/monkey_island/cc/environment/test_environment.py:26)
 change_to_mongo_mock  # unused function (monkey/monkey_island/cc/test_common/fixtures/mongomock_fixtures.py:9)
 uses_database  # unused function (monkey/monkey_island/cc/test_common/fixtures/mongomock_fixtures.py:16)