diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index 418074557..e98c4f652 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -3,6 +3,8 @@ import json import logging import sys from pathlib import Path +from threading import Thread +from typing import Sequence, Tuple import gevent.hub import requests @@ -11,6 +13,7 @@ from gevent.pywsgi import WSGIServer from monkey_island.cc import Version from monkey_island.cc.deployment import Deployment from monkey_island.cc.server_utils.consts import ISLAND_PORT +from monkey_island.cc.server_utils.island_logger import get_log_file_path from monkey_island.cc.setup.config_setup import get_server_config # Add the monkey_island directory to the path, to make sure imports that don't start with @@ -27,6 +30,7 @@ from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402 from monkey_island.cc.server_utils.consts import ( # noqa: E402 GEVENT_EXCEPTION_LOG, MONGO_CONNECTION_TIMEOUT, + MONKEY_ISLAND_ABS_PATH, ) from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 @@ -45,44 +49,41 @@ logger = logging.getLogger(__name__) def run_monkey_island(): island_args = parse_cli_args() config_options = _extract_config(island_args) - _setup_data_dir(config_options.data_dir) - _exit_on_invalid_config_options(config_options) + _setup_data_dir(config_options.data_dir) _configure_logging(config_options) - container = _initialize_di_container(config_options.data_dir) - mongo_db_process = None - if config_options.start_mongodb: - mongo_db_process = _start_mongodb(config_options.data_dir) + ip_addresses, deployment, version = _collect_system_info() - _connect_to_mongodb(mongo_db_process) + _send_analytics(deployment, version) + container = _initialize_di_container(ip_addresses, version, config_options.data_dir) - _configure_gevent_exception_handling(config_options.data_dir) - _start_island_server(island_args.setup_only, config_options, container) + _initialize_mongodb_connection(config_options.start_mongodb, config_options.data_dir) + _start_island_server(ip_addresses, island_args.setup_only, config_options, container) def _extract_config(island_args: IslandCmdArgs) -> IslandConfigOptions: try: return get_server_config(island_args) - except json.JSONDecodeError as ex: - print(f"Error loading server config: {ex}") + except json.JSONDecodeError as err: + print(f"Error loading server config: {err}") sys.exit(1) def _setup_data_dir(data_dir_path: Path): try: setup_data_dir(data_dir_path) - except IncompatibleDataDirectory as ex: - print(f"Incompatible data directory: {ex}") + except IncompatibleDataDirectory as err: + print(f"Incompatible data directory: {err}") sys.exit(1) def _exit_on_invalid_config_options(config_options: IslandConfigOptions): try: island_config_options_validator.raise_on_invalid_options(config_options) - except Exception as ex: - print(f"Configuration error: {ex}") + except Exception as err: + print(f"Configuration error: {err}") sys.exit(1) @@ -91,8 +92,48 @@ def _configure_logging(config_options): setup_logging(config_options.data_dir, config_options.log_level) -def _initialize_di_container(data_dir: Path) -> DIContainer: - return initialize_services(data_dir) +def _collect_system_info() -> Tuple[Sequence[str], Deployment, Version]: + deployment = _get_deployment() + version = Version(get_version(), deployment) + return (get_ip_addresses(), deployment, version) + + +def _get_deployment() -> Deployment: + deployment_file_path = Path(MONKEY_ISLAND_ABS_PATH) / "cc" / "deployment.json" + try: + with open(deployment_file_path, "r") as deployment_info_file: + deployment_info = json.load(deployment_info_file) + return Deployment[deployment_info["deployment"].upper()] + except KeyError as err: + raise Exception( + f"The deployment file ({deployment_file_path}) did not contain the expected data: " + f"missing key {err}" + ) + except Exception as err: + raise Exception(f"Failed to fetch the deployment from {deployment_file_path}: {err}") + + +def _initialize_di_container( + ip_addresses: Sequence[str], version: Version, data_dir: Path +) -> DIContainer: + container = DIContainer() + + container.register_convention(Sequence[str], "ip_addresses", ip_addresses) + container.register_instance(Version, version) + container.register_convention(Path, "data_dir", data_dir) + container.register_convention(Path, "island_log_file_path", get_log_file_path(data_dir)) + + initialize_services(container, data_dir) + + return container + + +def _initialize_mongodb_connection(start_mongodb: bool, data_dir: Path): + mongo_db_process = None + if start_mongodb: + mongo_db_process = _start_mongodb(data_dir) + + _connect_to_mongodb(mongo_db_process) def _start_mongodb(data_dir: Path) -> MongoDbProcess: @@ -105,34 +146,27 @@ def _start_mongodb(data_dir: Path) -> MongoDbProcess: def _connect_to_mongodb(mongo_db_process: MongoDbProcess): try: mongo_setup.connect_to_mongodb(MONGO_CONNECTION_TIMEOUT) - except mongo_setup.MongoDBTimeOutError as ex: + except mongo_setup.MongoDBTimeOutError as err: if mongo_db_process and not mongo_db_process.is_running(): logger.error( f"Failed to start MongoDB process. Check log at {mongo_db_process.log_file}." ) else: - logger.error(ex) + logger.error(err) sys.exit(1) - except mongo_setup.MongoDBVersionError as ex: - logger.error(ex) + except mongo_setup.MongoDBVersionError as err: + logger.error(err) sys.exit(1) -def _configure_gevent_exception_handling(data_dir): - hub = gevent.hub.get_hub() - - gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "w+", buffering=1) - atexit.register(gevent_exception_log.close) - - # Send gevent's exception output to a log file. - # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream - hub.exception_stream = gevent_exception_log - hub.handle_error = GeventHubErrorHandler(hub, logger) - - def _start_island_server( - should_setup_only: bool, config_options: IslandConfigOptions, container: DIContainer + ip_addresses: Sequence[str], + should_setup_only: bool, + config_options: IslandConfigOptions, + container: DIContainer, ): + _configure_gevent_exception_handling(config_options.data_dir) + app = init_app(mongo_setup.MONGO_URL, container) if should_setup_only: @@ -152,11 +186,22 @@ def _start_island_server( log=_get_wsgi_server_logger(), error_log=logger, ) - _log_init_info() - _send_analytics(container) + _log_init_info(ip_addresses) http_server.serve_forever() +def _configure_gevent_exception_handling(data_dir: Path): + hub = gevent.hub.get_hub() + + gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "w+", buffering=1) + atexit.register(gevent_exception_log.close) + + # Send gevent's exception output to a log file. + # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream + hub.exception_stream = gevent_exception_log + hub.handle_error = GeventHubErrorHandler(hub, logger) + + def _get_wsgi_server_logger() -> logging.Logger: wsgi_server_logger = logger.getChild("wsgi") wsgi_server_logger.addFilter(PyWSGILoggingFilter()) @@ -164,38 +209,37 @@ def _get_wsgi_server_logger() -> logging.Logger: return wsgi_server_logger -def _log_init_info(): +def _log_init_info(ip_addresses: Sequence[str]): logger.info("Monkey Island Server is running!") logger.info(f"version: {get_version()}") - _log_web_interface_access_urls() + _log_web_interface_access_urls(ip_addresses) -def _log_web_interface_access_urls(): - web_interface_urls = ", ".join([f"https://{ip}:{ISLAND_PORT}" for ip in get_ip_addresses()]) +def _log_web_interface_access_urls(ip_addresses: Sequence[str]): + web_interface_urls = ", ".join([f"https://{ip}:{ISLAND_PORT}" for ip in ip_addresses]) logger.info( "To access the web interface, navigate to one of the the following URLs using your " f"browser: {web_interface_urls}" ) -ANALYTICS_URL = ( - "https://m15mjynko3.execute-api.us-east-1.amazonaws.com/default?version={" - "version}&deployment={deployment}" -) - - -def _send_analytics(di_container): - version = di_container.resolve(Version) - deployment = di_container.resolve(Deployment) - url = ANALYTICS_URL.format(deployment=deployment.value, version=version.version_number) - try: - response = requests.get(url).json() - logger.info( - f"Version number and deployment type was sent to analytics server. " - f"The response is: {response}" - ) - except requests.exceptions.ConnectionError as err: - logger.info( - f"Failed to send deployment type and version number to the analytics server: {err}" +def _send_analytics(deployment: Deployment, version: Version): + def _inner(deployment: Deployment, version: Version): + url = ( + "https://m15mjynko3.execute-api.us-east-1.amazonaws.com/default" + f"?version={version.version_number}&deployment={deployment.value}" ) + + try: + response = requests.get(url).json() + logger.info( + f"Version number and deployment type was sent to analytics server. " + f"The response is: {response}" + ) + except requests.exceptions.ConnectionError as err: + logger.info( + f"Failed to send deployment type and version number to the analytics server: {err}" + ) + + Thread(target=_inner, args=(deployment, version), daemon=True).start() diff --git a/monkey/monkey_island/cc/services/initialize.py b/monkey/monkey_island/cc/services/initialize.py index fc7b6e432..3ceb489c2 100644 --- a/monkey/monkey_island/cc/services/initialize.py +++ b/monkey/monkey_island/cc/services/initialize.py @@ -1,7 +1,5 @@ -import json import logging from pathlib import Path -from typing import Sequence from pubsub.core import Publisher from pymongo import MongoClient @@ -16,9 +14,6 @@ from common.aws import AWSInstance from common.common_consts.telem_categories import TelemCategoryEnum from common.event_queue import IEventQueue, PyPubSubEventQueue from common.utils.file_utils import get_binary_io_sha256_hash -from common.version import get_version -from monkey_island.cc import Version -from monkey_island.cc.deployment import Deployment from monkey_island.cc.repository import ( AgentBinaryRepository, FileAgentConfigurationRepository, @@ -39,7 +34,6 @@ from monkey_island.cc.repository import ( ) from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.server_utils.encryption import ILockableEncryptor, RepositoryEncryptor -from monkey_island.cc.server_utils.island_logger import get_log_file_path from monkey_island.cc.services import AWSService, IslandModeService, RepositoryService from monkey_island.cc.services.attack.technique_reports.T1003 import T1003, T1003GetReportData from monkey_island.cc.services.run_local_monkey import LocalMonkeyRunService @@ -49,7 +43,6 @@ from monkey_island.cc.services.telemetry.processing.credentials.credentials_pars from monkey_island.cc.services.telemetry.processing.processing import ( TELEMETRY_CATEGORY_TO_PROCESSING_FUNC, ) -from monkey_island.cc.services.utils.network_utils import get_ip_addresses from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL from . import AuthenticationService @@ -58,21 +51,17 @@ from .reporting.report import ReportService logger = logging.getLogger(__name__) AGENT_BINARIES_PATH = Path(MONKEY_ISLAND_ABS_PATH) / "cc" / "binaries" -DEPLOYMENT_FILE_PATH = Path(MONKEY_ISLAND_ABS_PATH) / "cc" / "deployment.json" REPOSITORY_KEY_FILE_NAME = "repository_key.bin" -def initialize_services(data_dir: Path) -> DIContainer: - container = DIContainer() - _register_conventions(container, data_dir) +def initialize_services(container: DIContainer, data_dir: Path): + _register_conventions(container) - container.register_instance(Deployment, _get_depyloyment_from_file(DEPLOYMENT_FILE_PATH)) container.register_instance(AWSInstance, AWSInstance()) container.register_instance(MongoClient, MongoClient(MONGO_URL, serverSelectionTimeoutMS=100)) container.register_instance( ILockableEncryptor, RepositoryEncryptor(data_dir / REPOSITORY_KEY_FILE_NAME) ) - container.register_instance(Version, container.resolve(Version)) container.register(Publisher, Publisher) container.register_instance(IEventQueue, container.resolve(PyPubSubEventQueue)) @@ -88,11 +77,8 @@ def initialize_services(data_dir: Path) -> DIContainer: container.resolve(ICredentialsRepository), ) - return container - -def _register_conventions(container: DIContainer, data_dir: Path): - container.register_convention(Path, "data_dir", data_dir) +def _register_conventions(container: DIContainer): container.register_convention( AgentConfiguration, "default_agent_configuration", DEFAULT_AGENT_CONFIGURATION ) @@ -101,9 +87,6 @@ def _register_conventions(container: DIContainer, data_dir: Path): "default_ransomware_agent_configuration", DEFAULT_RANSOMWARE_AGENT_CONFIGURATION, ) - container.register_convention(Path, "island_log_file_path", get_log_file_path(data_dir)) - container.register_convention(str, "version_number", get_version()) - container.register_convention(Sequence[str], "ip_addresses", get_ip_addresses()) def _register_repositories(container: DIContainer, data_dir: Path): @@ -161,22 +144,6 @@ def _log_agent_binary_hashes(agent_binary_repository: IAgentBinaryRepository): logger.info(f"{os} agent: SHA-256 hash: {binary_sha256_hash}") -# TODO: The deployment should probably be passed into initialize_services(), but we can rework that -# when we refactor this file. -def _get_depyloyment_from_file(file_path: Path) -> Deployment: - try: - with open(file_path, "r") as deployment_info_file: - deployment_info = json.load(deployment_info_file) - return Deployment[deployment_info["deployment"].upper()] - except KeyError as err: - raise Exception( - f"The deployment file ({file_path}) did not contain the expected data: " - f"missing key {err}" - ) - except Exception as err: - raise Exception(f"Failed to fetch the deployment from {file_path}: {err}") - - def _register_services(container: DIContainer): container.register_instance(AWSService, container.resolve(AWSService)) container.register_instance(LocalMonkeyRunService, container.resolve(LocalMonkeyRunService))