From 96fc33025ebbcf4f699c8d72a02a7188b3737b99 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Jul 2021 11:55:56 -0400 Subject: [PATCH] Island: Redirect gevent tracebacks to file and log exceptions By default, gevent prints exceptions and tracebacks to stderr. This is obnoxious as it results in large tracebacks intermixed with the output that the logger prints to the console. This commit redirects this data to {DATA_DIR}/gevent_exceptions.log. Unfortunately, this would mean that the user might be left without any indication these exceptions had occurred, unless they take the time to inspect the gevent_exceptions.log. Therefore, when an excepion occurs, a message with just the exception (not the traceback) is logged to WARNING. Fixes #859 --- CHANGELOG.md | 1 + monkey/monkey_island/cc/server_setup.py | 17 +++++++++++++++++ monkey/monkey_island/cc/server_utils/consts.py | 2 ++ vulture_allowlist.py | 1 + 4 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 093cf7398..8461bc731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Attempted to delete a directory when monkey config reset was called. #1054 - An errant space in the windows commands to run monkey manually. #1153 +- gevent tracebacks in console output. #859 ### Security - Address minor issues discovered by Dlint. #1075 diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index fc23cd7f8..a72cea033 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -1,3 +1,4 @@ +import atexit import json import logging import sys @@ -5,6 +6,7 @@ from pathlib import Path from threading import Thread from typing import Tuple +import gevent.hub from gevent.pywsgi import WSGIServer # Add the monkey_island directory to the path, to make sure imports that don't start with @@ -21,12 +23,14 @@ from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402 from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 +from monkey_island.cc.server_utils.consts import GEVENT_EXCEPTION_LOG # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.setup import island_config_options_validator # noqa: E402 +from monkey_island.cc.setup.gevent_hub_error_handler import GeventHubErrorHandler # noqa: E402 from monkey_island.cc.setup.island_config_options import IslandConfigOptions # noqa: E402 from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402 from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 @@ -54,6 +58,7 @@ def run_monkey_island(): connect_to_mongodb() + _configure_gevent_exception_handling(Path(config_options.data_dir)) _start_island_server(island_args.setup_only, config_options) @@ -88,6 +93,18 @@ def _initialize_globals(config_options: IslandConfigOptions, server_config_path: initialize_services(config_options.data_dir) +def _configure_gevent_exception_handling(data_dir): + hub = gevent.hub.get_hub() + + gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "w+", buffering=1) + atexit.register(gevent_exception_log.close) + + # Send gevent's exception output to a log file. + # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream + hub.exception_stream = gevent_exception_log + hub.handle_error = GeventHubErrorHandler(hub, logger) + + def _start_island_server(should_setup_only, config_options: IslandConfigOptions): populate_exporter_list() app = init_app(MONGO_URL) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 13e0c66e4..28391279b 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -52,3 +52,5 @@ DEFAULT_CERTIFICATE_PATHS = { "ssl_certificate_file": DEFAULT_CRT_PATH, "ssl_certificate_key_file": DEFAULT_KEY_PATH, } + +GEVENT_EXCEPTION_LOG = "gevent_exceptions.log" diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 564f238ab..618fabaa6 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -170,6 +170,7 @@ ISLAND # unused variable (monkey/monkey_island/cc/services/utils/node_states.py MONKEY_LINUX_RUNNING # unused variable (monkey/monkey_island/cc/services/utils/node_states.py:26) import_status # monkey_island\cc\resources\configuration_import.py:19 config_schema # monkey_island\cc\resources\configuration_import.py:25 +exception_stream # unused attribute (monkey_island/cc/server_setup.py:104) # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)