Merge pull request #1298 from guardicore/gevent-ssl-traceback

Gevent ssl traceback

CHANGELOG.md

@@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Fixed
 - Attempted to delete a directory when monkey config reset was called. #1054
 - An errant space in the windows commands to run monkey manually. #1153
+- gevent tracebacks in console output. #859
 
 ### Security
 - Address minor issues discovered by Dlint. #1075

monkey/monkey_island/cc/server_setup.py

@@ -1,3 +1,4 @@
+import atexit
 import json
 import logging
 import sys
@@ -5,6 +6,7 @@ from pathlib import Path
 from threading import Thread
 from typing import Tuple
 
+import gevent.hub
 from gevent.pywsgi import WSGIServer
 
 # Add the monkey_island directory to the path, to make sure imports that don't start with
@@ -21,12 +23,14 @@ from monkey_island.cc.arg_parser import IslandCmdArgs  # noqa: E402
 from monkey_island.cc.arg_parser import parse_cli_args  # noqa: E402
 from monkey_island.cc.resources.monkey_download import MonkeyDownload  # noqa: E402
 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer  # noqa: E402
+from monkey_island.cc.server_utils.consts import GEVENT_EXCEPTION_LOG  # noqa: E402
 from monkey_island.cc.server_utils.encryptor import initialize_encryptor  # noqa: E402
 from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging  # noqa: E402
 from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list  # noqa: E402
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses  # noqa: E402
 from monkey_island.cc.setup import island_config_options_validator  # noqa: E402
+from monkey_island.cc.setup.gevent_hub_error_handler import GeventHubErrorHandler  # noqa: E402
 from monkey_island.cc.setup.island_config_options import IslandConfigOptions  # noqa: E402
 from monkey_island.cc.setup.mongo.database_initializer import init_collections  # noqa: E402
 from monkey_island.cc.setup.mongo.mongo_setup import (  # noqa: E402
@@ -54,6 +58,7 @@ def run_monkey_island():
 
     connect_to_mongodb()
 
+    _configure_gevent_exception_handling(Path(config_options.data_dir))
     _start_island_server(island_args.setup_only, config_options)
 
 
@@ -88,6 +93,18 @@ def _initialize_globals(config_options: IslandConfigOptions, server_config_path:
     initialize_services(config_options.data_dir)
 
 
+def _configure_gevent_exception_handling(data_dir):
+    hub = gevent.hub.get_hub()
+
+    gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "w+", buffering=1)
+    atexit.register(gevent_exception_log.close)
+
+    # Send gevent's exception output to a log file.
+    # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream
+    hub.exception_stream = gevent_exception_log
+    hub.handle_error = GeventHubErrorHandler(hub, logger)
+
+
 def _start_island_server(should_setup_only, config_options: IslandConfigOptions):
     populate_exporter_list()
     app = init_app(MONGO_URL)
@@ -117,6 +134,8 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions)
             app,
             certfile=config_options.crt_path,
             keyfile=config_options.key_path,
+            log=logger,
+            error_log=logger,
         )
         _log_init_info()
         http_server.serve_forever()

monkey/monkey_island/cc/server_utils/consts.py

@@ -52,3 +52,5 @@ DEFAULT_CERTIFICATE_PATHS = {
     "ssl_certificate_file": DEFAULT_CRT_PATH,
     "ssl_certificate_key_file": DEFAULT_KEY_PATH,
 }
+
+GEVENT_EXCEPTION_LOG = "gevent_exceptions.log"

monkey/monkey_island/cc/setup/gevent_hub_error_handler.py

@@ -0,0 +1,26 @@
+import traceback
+
+import gevent.hub
+
+
+class GeventHubErrorHandler:
+    """
+    Wraps gevent.hub.Hub's handle_error() method so that the exception can be
+    logged but the traceback can be stored in a separate file. This preserves
+    the default gevent functionality and adds a useful, concise log message to
+    the Monkey Island logs.
+
+    For more information, see
+        https://github.com/guardicore/monkey/issues/859,
+        https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.handle_error
+        https://github.com/gevent/gevent/issues/1482
+    """
+
+    def __init__(self, hub: gevent.hub.Hub, logger):
+        self._original_handle_error = hub.handle_error
+        self._logger = logger
+
+    def __call__(self, context, type_, value, tb):
+        exception_msg = traceback.format_exception_only(type_, value)
+        self._logger.warning(f"gevent caught an exception: {exception_msg}")
+        self._original_handle_error(context, type_, value, tb)

vulture_allowlist.py

@@ -170,6 +170,7 @@ ISLAND  # unused variable (monkey/monkey_island/cc/services/utils/node_states.py
 MONKEY_LINUX_RUNNING  # unused variable (monkey/monkey_island/cc/services/utils/node_states.py:26)
 import_status  # monkey_island\cc\resources\configuration_import.py:19
 config_schema  # monkey_island\cc\resources\configuration_import.py:25
+exception_stream  # unused attribute (monkey_island/cc/server_setup.py:104)
 
 # these are not needed for it to work, but may be useful extra information to understand what's going on
 WINDOWS_PBA_TYPE  # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)