diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c95dfa50..b8c062648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ Changelog](https://keepachangelog.com/en/1.0.0/). - "/api/client-monkey" endpoint. #1889 - "+dev" from version numbers. #1553 - agent's "--config" argument. #906 +- Option to export monkey telemetries. #1998 ### Fixed - A bug in network map page that caused delay of telemetry log loading. #1545 diff --git a/monkey/common/config_value_paths.py b/monkey/common/config_value_paths.py index 8aaf16d55..e65444147 100644 --- a/monkey/common/config_value_paths.py +++ b/monkey/common/config_value_paths.py @@ -1,4 +1,3 @@ -EXPORT_MONKEY_TELEMS_PATH = ["internal", "testing", "export_monkey_telems"] CURRENT_SERVER_PATH = ["internal", "island_server", "current_server"] SSH_KEYS_PATH = ["internal", "exploits", "exploit_ssh_keys"] INACCESSIBLE_SUBNETS_PATH = ["basic_network", "network_analysis", "inaccessible_subnets"] diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index b26762fc5..9bdb35d76 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -69,10 +69,5 @@ class Configuration(object): keep_tunnel_open_time = 30 - ########################### - # testing configuration - ########################### - export_monkey_telems = False - WormConfiguration = Configuration() diff --git a/monkey/monkey_island/cc/models/exported_telem.py b/monkey/monkey_island/cc/models/exported_telem.py deleted file mode 100644 index 6df2296fb..000000000 --- a/monkey/monkey_island/cc/models/exported_telem.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Define a Document Schema for the TelemForExport document. -""" -from mongoengine import DateTimeField, Document, StringField - - -# This document describes exported telemetry. -# These telemetries are used to mock monkeys sending telemetries to the island. -# This way we can replicate island state without running monkeys. -class ExportedTelem(Document): - # SCHEMA - name = StringField(required=True) - time = DateTimeField(required=True) - method = StringField(required=True) - endpoint = StringField(required=True) - content = StringField(required=True) diff --git a/monkey/monkey_island/cc/resources/blackbox/utils/telem_store.py b/monkey/monkey_island/cc/resources/blackbox/utils/telem_store.py deleted file mode 100644 index f6e9a8406..000000000 --- a/monkey/monkey_island/cc/resources/blackbox/utils/telem_store.py +++ /dev/null @@ -1,84 +0,0 @@ -import logging -import shutil -from datetime import datetime -from functools import wraps -from os import mkdir, path - -from flask import request - -from monkey_island.cc.models.exported_telem import ExportedTelem -from monkey_island.cc.services.config import ConfigService - -TELEM_SAMPLE_DIR = "./telem_sample" -MAX_SAME_CATEGORY_TELEMS = 10000 - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -# TODO this will break with the IRepository implementation. Remove it -class TestTelemStore: - TELEMS_EXPORTED = False - - @staticmethod - def store_exported_telem(f): - @wraps(f) - def decorated_function(*args, **kwargs): - if ConfigService.is_test_telem_export_enabled(): - time = datetime.now() - method = request.method - content = request.data.decode() - endpoint = request.path - name = ( - str(request.url_rule) - .replace("/", "_") - .replace("<", "_") - .replace(">", "_") - .replace(":", "_") - ) - ExportedTelem( - name=name, method=method, endpoint=endpoint, content=content, time=time - ).save() - return f(*args, **kwargs) - - return decorated_function - - @staticmethod - def export_telems(): - logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}") - try: - mkdir(TELEM_SAMPLE_DIR) - except FileExistsError: - logger.info("Deleting all previous telemetries.") - shutil.rmtree(TELEM_SAMPLE_DIR) - mkdir(TELEM_SAMPLE_DIR) - for test_telem in ExportedTelem.objects(): - with open( - TestTelemStore.get_unique_file_path_for_export_telem(TELEM_SAMPLE_DIR, test_telem), - "w", - ) as file: - file.write(test_telem.to_json(indent=2)) - TestTelemStore.TELEMS_EXPORTED = True - logger.info("Telemetries exported!") - - # Should be private - @staticmethod - def get_unique_file_path_for_export_telem(target_dir: str, test_telem: ExportedTelem): - telem_filename = TestTelemStore._get_filename_by_export_telem(test_telem) - for i in range(MAX_SAME_CATEGORY_TELEMS): - potential_filepath = path.join(target_dir, (telem_filename + str(i))) - if path.exists(potential_filepath): - continue - return potential_filepath - raise Exception( - f"Too many telemetries of the same category. Max amount {MAX_SAME_CATEGORY_TELEMS}" - ) - - @staticmethod - def _get_filename_by_export_telem(test_telem: ExportedTelem): - endpoint_part = test_telem.name - return endpoint_part + "_" + test_telem.method - - -if __name__ == "__main__": - TestTelemStore.export_telems() diff --git a/monkey/monkey_island/cc/resources/log.py b/monkey/monkey_island/cc/resources/log.py index 432b27c95..d56bc84bb 100644 --- a/monkey/monkey_island/cc/resources/log.py +++ b/monkey/monkey_island/cc/resources/log.py @@ -5,7 +5,6 @@ from flask import request from monkey_island.cc.database import mongo from monkey_island.cc.resources.AbstractResource import AbstractResource -from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.resources.request_authentication import jwt_required from monkey_island.cc.services.log import LogService from monkey_island.cc.services.node import NodeService @@ -25,7 +24,6 @@ class Log(AbstractResource): return LogService.log_exists(ObjectId(exists_monkey_id)) # Used by monkey. can't secure. - @TestTelemStore.store_exported_telem def post(self): telemetry_json = json.loads(request.data) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 411c383d2..dad1a8e73 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -6,7 +6,6 @@ from flask import request from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey_ttl import create_monkey_ttl_document from monkey_island.cc.resources.AbstractResource import AbstractResource -from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.resources.utils.semaphores import agent_killing_mutex from monkey_island.cc.server_utils.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS from monkey_island.cc.services.config import ConfigService @@ -29,7 +28,6 @@ class Monkey(AbstractResource): # Used by monkey. can't secure. # Called on monkey wakeup to initialize local configuration - @TestTelemStore.store_exported_telem def post(self, **kw): # TODO: Why is it the registration of an agent coupled to exploit telemetry? It's hard to diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index 1d16c3ad9..8b6bf790a 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -9,7 +9,6 @@ from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey import Monkey from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.resources.AbstractResource import AbstractResource -from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.resources.request_authentication import jwt_required from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.processing import process_telemetry @@ -45,7 +44,6 @@ class Telemetry(AbstractResource): return result # Used by monkey. can't secure. - @TestTelemStore.store_exported_telem def post(self): telemetry_json = json.loads(request.data) telemetry_json["data"] = json.loads(telemetry_json["data"]) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 3b1e44d01..2220bf38b 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -9,7 +9,6 @@ from typing import Any, Dict, List from jsonschema import Draft4Validator, validators from common.config_value_paths import ( - EXPORT_MONKEY_TELEMS_PATH, LM_HASH_LIST_PATH, NTLM_HASH_LIST_PATH, PASSWORD_LIST_PATH, @@ -355,10 +354,6 @@ class ConfigService: else get_datastore_encryptor().encrypt(config_arr) ) - @staticmethod - def is_test_telem_export_enabled(): - return ConfigService.get_config_value(EXPORT_MONKEY_TELEMS_PATH) - @staticmethod def get_config_propagation_credentials_from_flat_config(config) -> Dict[str, List[str]]: return { diff --git a/monkey/monkey_island/cc/services/config_schema/internal.py b/monkey/monkey_island/cc/services/config_schema/internal.py index a0a2bde52..b54370ac9 100644 --- a/monkey/monkey_island/cc/services/config_schema/internal.py +++ b/monkey/monkey_island/cc/services/config_schema/internal.py @@ -153,19 +153,5 @@ INTERNAL = { }, }, }, - "testing": { - "title": "Testing", - "type": "object", - "properties": { - "export_monkey_telems": { - "title": "Export monkey telemetries", - "type": "boolean", - "default": False, - "description": "Exports unencrypted telemetries that " - "can be used for tests in development." - " Do not turn on!", - } - }, - }, }, } diff --git a/monkey/monkey_island/cc/services/infection_lifecycle.py b/monkey/monkey_island/cc/services/infection_lifecycle.py index 84faa1953..937a3abeb 100644 --- a/monkey/monkey_island/cc/services/infection_lifecycle.py +++ b/monkey/monkey_island/cc/services/infection_lifecycle.py @@ -2,8 +2,6 @@ import logging from monkey_island.cc.models import Monkey from monkey_island.cc.models.agent_controls import AgentControls -from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore -from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.reporting.report import ReportService from monkey_island.cc.services.reporting.report_generation_synchronisation import ( @@ -73,5 +71,3 @@ def _on_finished_infection(): # we want to skip and reply. if not is_report_being_generated() and not ReportService.is_latest_report_exists(): safe_generate_reports() - if ConfigService.is_test_telem_export_enabled() and not TestTelemStore.TELEMS_EXPORTED: - TestTelemStore.export_telems() diff --git a/monkey/tests/data_for_tests/monkey_configs/flat_config.json b/monkey/tests/data_for_tests/monkey_configs/flat_config.json index 0f89e9477..59f8602ed 100644 --- a/monkey/tests/data_for_tests/monkey_configs/flat_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/flat_config.json @@ -47,7 +47,6 @@ "PowerShellExploiter", "Log4ShellExploiter" ], - "export_monkey_telems": false, "finger_classes": [ "SMBFinger", "SSHFinger", diff --git a/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json b/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json index 47aef8d69..0abd3924b 100644 --- a/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json +++ b/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json @@ -92,9 +92,6 @@ "exploit_lm_hash_list": [], "exploit_ntlm_hash_list": [], "exploit_ssh_keys": [] - }, - "testing": { - "export_monkey_telems": false } }, "monkey": { diff --git a/vulture_allowlist.py b/vulture_allowlist.py index edfebe4cb..ab09127b3 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -129,7 +129,6 @@ opnum # unused variable (monkey/infection_monkey/exploit/zerologon.py:466) structure # unused variable (monkey/infection_monkey/exploit/zerologon.py:467) structure # unused variable (monkey/infection_monkey/exploit/zerologon.py:478) oid_set # unused variable (monkey/infection_monkey/exploit/tools/wmi_tools.py:96) -export_monkey_telems # unused variable (monkey/infection_monkey/config.py:282) NoInternetError # unused class (monkey/common/utils/exceptions.py:33) _.__isabstractmethod__ # unused attribute (monkey/common/utils/code_utils.py:11) MIMIKATZ # unused variable (monkey/common/utils/attack_utils.py:21)