diff --git a/CHANGELOG.md b/CHANGELOG.md index 093cf7398..8461bc731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Attempted to delete a directory when monkey config reset was called. #1054 - An errant space in the windows commands to run monkey manually. #1153 +- gevent tracebacks in console output. #859 ### Security - Address minor issues discovered by Dlint. #1075 diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index adbf87050..a72cea033 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -1,3 +1,4 @@ +import atexit import json import logging import sys @@ -5,6 +6,7 @@ from pathlib import Path from threading import Thread from typing import Tuple +import gevent.hub from gevent.pywsgi import WSGIServer # Add the monkey_island directory to the path, to make sure imports that don't start with @@ -21,12 +23,14 @@ from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402 from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 +from monkey_island.cc.server_utils.consts import GEVENT_EXCEPTION_LOG # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.setup import island_config_options_validator # noqa: E402 +from monkey_island.cc.setup.gevent_hub_error_handler import GeventHubErrorHandler # noqa: E402 from monkey_island.cc.setup.island_config_options import IslandConfigOptions # noqa: E402 from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402 from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 @@ -54,6 +58,7 @@ def run_monkey_island(): connect_to_mongodb() + _configure_gevent_exception_handling(Path(config_options.data_dir)) _start_island_server(island_args.setup_only, config_options) @@ -88,6 +93,18 @@ def _initialize_globals(config_options: IslandConfigOptions, server_config_path: initialize_services(config_options.data_dir) +def _configure_gevent_exception_handling(data_dir): + hub = gevent.hub.get_hub() + + gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "w+", buffering=1) + atexit.register(gevent_exception_log.close) + + # Send gevent's exception output to a log file. + # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream + hub.exception_stream = gevent_exception_log + hub.handle_error = GeventHubErrorHandler(hub, logger) + + def _start_island_server(should_setup_only, config_options: IslandConfigOptions): populate_exporter_list() app = init_app(MONGO_URL) @@ -117,6 +134,8 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions) app, certfile=config_options.crt_path, keyfile=config_options.key_path, + log=logger, + error_log=logger, ) _log_init_info() http_server.serve_forever() diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 13e0c66e4..28391279b 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -52,3 +52,5 @@ DEFAULT_CERTIFICATE_PATHS = { "ssl_certificate_file": DEFAULT_CRT_PATH, "ssl_certificate_key_file": DEFAULT_KEY_PATH, } + +GEVENT_EXCEPTION_LOG = "gevent_exceptions.log" diff --git a/monkey/monkey_island/cc/setup/gevent_hub_error_handler.py b/monkey/monkey_island/cc/setup/gevent_hub_error_handler.py new file mode 100644 index 000000000..c4c37b88d --- /dev/null +++ b/monkey/monkey_island/cc/setup/gevent_hub_error_handler.py @@ -0,0 +1,26 @@ +import traceback + +import gevent.hub + + +class GeventHubErrorHandler: + """ + Wraps gevent.hub.Hub's handle_error() method so that the exception can be + logged but the traceback can be stored in a separate file. This preserves + the default gevent functionality and adds a useful, concise log message to + the Monkey Island logs. + + For more information, see + https://github.com/guardicore/monkey/issues/859, + https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.handle_error + https://github.com/gevent/gevent/issues/1482 + """ + + def __init__(self, hub: gevent.hub.Hub, logger): + self._original_handle_error = hub.handle_error + self._logger = logger + + def __call__(self, context, type_, value, tb): + exception_msg = traceback.format_exception_only(type_, value) + self._logger.warning(f"gevent caught an exception: {exception_msg}") + self._original_handle_error(context, type_, value, tb) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 564f238ab..618fabaa6 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -170,6 +170,7 @@ ISLAND # unused variable (monkey/monkey_island/cc/services/utils/node_states.py MONKEY_LINUX_RUNNING # unused variable (monkey/monkey_island/cc/services/utils/node_states.py:26) import_status # monkey_island\cc\resources\configuration_import.py:19 config_schema # monkey_island\cc\resources\configuration_import.py:25 +exception_stream # unused attribute (monkey_island/cc/server_setup.py:104) # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)