From f86ff4fbd74aa584401b84d6cc910fda84459d99 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Jul 2021 11:38:32 -0400 Subject: [PATCH 1/3] Island: Set log and error_log parameters on WSGIServer constructor Provides WSGIServer with a logger for INFO log messages and ERROR log messages. https://www.gevent.org/api/gevent.pywsgi.html#gevent.pywsgi.WSGIServer --- monkey/monkey_island/cc/server_setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index adbf87050..fc23cd7f8 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -117,6 +117,8 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions) app, certfile=config_options.crt_path, keyfile=config_options.key_path, + log=logger, + error_log=logger, ) _log_init_info() http_server.serve_forever() From ebbdbc8dcb831bb43a97f43870e980f1c088dae2 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Jul 2021 11:40:34 -0400 Subject: [PATCH 2/3] Island: Add GeventHubErrorHandler to log gevent exceptions --- .../cc/setup/gevent_hub_error_handler.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 monkey/monkey_island/cc/setup/gevent_hub_error_handler.py diff --git a/monkey/monkey_island/cc/setup/gevent_hub_error_handler.py b/monkey/monkey_island/cc/setup/gevent_hub_error_handler.py new file mode 100644 index 000000000..c4c37b88d --- /dev/null +++ b/monkey/monkey_island/cc/setup/gevent_hub_error_handler.py @@ -0,0 +1,26 @@ +import traceback + +import gevent.hub + + +class GeventHubErrorHandler: + """ + Wraps gevent.hub.Hub's handle_error() method so that the exception can be + logged but the traceback can be stored in a separate file. This preserves + the default gevent functionality and adds a useful, concise log message to + the Monkey Island logs. + + For more information, see + https://github.com/guardicore/monkey/issues/859, + https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.handle_error + https://github.com/gevent/gevent/issues/1482 + """ + + def __init__(self, hub: gevent.hub.Hub, logger): + self._original_handle_error = hub.handle_error + self._logger = logger + + def __call__(self, context, type_, value, tb): + exception_msg = traceback.format_exception_only(type_, value) + self._logger.warning(f"gevent caught an exception: {exception_msg}") + self._original_handle_error(context, type_, value, tb) From 96fc33025ebbcf4f699c8d72a02a7188b3737b99 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Jul 2021 11:55:56 -0400 Subject: [PATCH 3/3] Island: Redirect gevent tracebacks to file and log exceptions By default, gevent prints exceptions and tracebacks to stderr. This is obnoxious as it results in large tracebacks intermixed with the output that the logger prints to the console. This commit redirects this data to {DATA_DIR}/gevent_exceptions.log. Unfortunately, this would mean that the user might be left without any indication these exceptions had occurred, unless they take the time to inspect the gevent_exceptions.log. Therefore, when an excepion occurs, a message with just the exception (not the traceback) is logged to WARNING. Fixes #859 --- CHANGELOG.md | 1 + monkey/monkey_island/cc/server_setup.py | 17 +++++++++++++++++ monkey/monkey_island/cc/server_utils/consts.py | 2 ++ vulture_allowlist.py | 1 + 4 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 093cf7398..8461bc731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Attempted to delete a directory when monkey config reset was called. #1054 - An errant space in the windows commands to run monkey manually. #1153 +- gevent tracebacks in console output. #859 ### Security - Address minor issues discovered by Dlint. #1075 diff --git a/monkey/monkey_island/cc/server_setup.py b/monkey/monkey_island/cc/server_setup.py index fc23cd7f8..a72cea033 100644 --- a/monkey/monkey_island/cc/server_setup.py +++ b/monkey/monkey_island/cc/server_setup.py @@ -1,3 +1,4 @@ +import atexit import json import logging import sys @@ -5,6 +6,7 @@ from pathlib import Path from threading import Thread from typing import Tuple +import gevent.hub from gevent.pywsgi import WSGIServer # Add the monkey_island directory to the path, to make sure imports that don't start with @@ -21,12 +23,14 @@ from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402 from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402 from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402 +from monkey_island.cc.server_utils.consts import GEVENT_EXCEPTION_LOG # noqa: E402 from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402 from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging # noqa: E402 from monkey_island.cc.services.initialize import initialize_services # noqa: E402 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402 from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402 from monkey_island.cc.setup import island_config_options_validator # noqa: E402 +from monkey_island.cc.setup.gevent_hub_error_handler import GeventHubErrorHandler # noqa: E402 from monkey_island.cc.setup.island_config_options import IslandConfigOptions # noqa: E402 from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402 from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402 @@ -54,6 +58,7 @@ def run_monkey_island(): connect_to_mongodb() + _configure_gevent_exception_handling(Path(config_options.data_dir)) _start_island_server(island_args.setup_only, config_options) @@ -88,6 +93,18 @@ def _initialize_globals(config_options: IslandConfigOptions, server_config_path: initialize_services(config_options.data_dir) +def _configure_gevent_exception_handling(data_dir): + hub = gevent.hub.get_hub() + + gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "w+", buffering=1) + atexit.register(gevent_exception_log.close) + + # Send gevent's exception output to a log file. + # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream + hub.exception_stream = gevent_exception_log + hub.handle_error = GeventHubErrorHandler(hub, logger) + + def _start_island_server(should_setup_only, config_options: IslandConfigOptions): populate_exporter_list() app = init_app(MONGO_URL) diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 13e0c66e4..28391279b 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -52,3 +52,5 @@ DEFAULT_CERTIFICATE_PATHS = { "ssl_certificate_file": DEFAULT_CRT_PATH, "ssl_certificate_key_file": DEFAULT_KEY_PATH, } + +GEVENT_EXCEPTION_LOG = "gevent_exceptions.log" diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 564f238ab..618fabaa6 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -170,6 +170,7 @@ ISLAND # unused variable (monkey/monkey_island/cc/services/utils/node_states.py MONKEY_LINUX_RUNNING # unused variable (monkey/monkey_island/cc/services/utils/node_states.py:26) import_status # monkey_island\cc\resources\configuration_import.py:19 config_schema # monkey_island\cc\resources\configuration_import.py:25 +exception_stream # unused attribute (monkey_island/cc/server_setup.py:104) # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)