Merge pull request #2013 from guardicore/1998-remove-export_monkey_telems

Remove `export_monkey_telems`

CHANGELOG.md

@@ -76,6 +76,7 @@ Changelog](https://keepachangelog.com/en/1.0.0/).
 - "/api/client-monkey" endpoint. #1889
 - "+dev" from version numbers. #1553
 - agent's "--config" argument. #906
+- Option to export monkey telemetries. #1998
 
 ### Fixed
 - A bug in network map page that caused delay of telemetry log loading. #1545

monkey/common/config_value_paths.py

@@ -1,4 +1,3 @@
-EXPORT_MONKEY_TELEMS_PATH = ["internal", "testing", "export_monkey_telems"]
 CURRENT_SERVER_PATH = ["internal", "island_server", "current_server"]
 SSH_KEYS_PATH = ["internal", "exploits", "exploit_ssh_keys"]
 INACCESSIBLE_SUBNETS_PATH = ["basic_network", "network_analysis", "inaccessible_subnets"]

monkey/infection_monkey/config.py

@@ -69,10 +69,5 @@ class Configuration(object):
 
     keep_tunnel_open_time = 30
 
-    ###########################
-    # testing configuration
-    ###########################
-    export_monkey_telems = False
-
 
 WormConfiguration = Configuration()

monkey/monkey_island/cc/models/exported_telem.py

@@ -1,16 +0,0 @@
-"""
-Define a Document Schema for the TelemForExport document.
-"""
-from mongoengine import DateTimeField, Document, StringField
-
-
-# This document describes exported telemetry.
-# These telemetries are used to mock monkeys sending telemetries to the island.
-# This way we can replicate island state without running monkeys.
-class ExportedTelem(Document):
-    # SCHEMA
-    name = StringField(required=True)
-    time = DateTimeField(required=True)
-    method = StringField(required=True)
-    endpoint = StringField(required=True)
-    content = StringField(required=True)

monkey/monkey_island/cc/resources/blackbox/utils/telem_store.py

@@ -1,84 +0,0 @@
-import logging
-import shutil
-from datetime import datetime
-from functools import wraps
-from os import mkdir, path
-
-from flask import request
-
-from monkey_island.cc.models.exported_telem import ExportedTelem
-from monkey_island.cc.services.config import ConfigService
-
-TELEM_SAMPLE_DIR = "./telem_sample"
-MAX_SAME_CATEGORY_TELEMS = 10000
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-# TODO this will break with the IRepository implementation. Remove it
-class TestTelemStore:
-    TELEMS_EXPORTED = False
-
-    @staticmethod
-    def store_exported_telem(f):
-        @wraps(f)
-        def decorated_function(*args, **kwargs):
-            if ConfigService.is_test_telem_export_enabled():
-                time = datetime.now()
-                method = request.method
-                content = request.data.decode()
-                endpoint = request.path
-                name = (
-                    str(request.url_rule)
-                    .replace("/", "_")
-                    .replace("<", "_")
-                    .replace(">", "_")
-                    .replace(":", "_")
-                )
-                ExportedTelem(
-                    name=name, method=method, endpoint=endpoint, content=content, time=time
-                ).save()
-            return f(*args, **kwargs)
-
-        return decorated_function
-
-    @staticmethod
-    def export_telems():
-        logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}")
-        try:
-            mkdir(TELEM_SAMPLE_DIR)
-        except FileExistsError:
-            logger.info("Deleting all previous telemetries.")
-            shutil.rmtree(TELEM_SAMPLE_DIR)
-            mkdir(TELEM_SAMPLE_DIR)
-        for test_telem in ExportedTelem.objects():
-            with open(
-                TestTelemStore.get_unique_file_path_for_export_telem(TELEM_SAMPLE_DIR, test_telem),
-                "w",
-            ) as file:
-                file.write(test_telem.to_json(indent=2))
-        TestTelemStore.TELEMS_EXPORTED = True
-        logger.info("Telemetries exported!")
-
-    # Should be private
-    @staticmethod
-    def get_unique_file_path_for_export_telem(target_dir: str, test_telem: ExportedTelem):
-        telem_filename = TestTelemStore._get_filename_by_export_telem(test_telem)
-        for i in range(MAX_SAME_CATEGORY_TELEMS):
-            potential_filepath = path.join(target_dir, (telem_filename + str(i)))
-            if path.exists(potential_filepath):
-                continue
-            return potential_filepath
-        raise Exception(
-            f"Too many telemetries of the same category. Max amount {MAX_SAME_CATEGORY_TELEMS}"
-        )
-
-    @staticmethod
-    def _get_filename_by_export_telem(test_telem: ExportedTelem):
-        endpoint_part = test_telem.name
-        return endpoint_part + "_" + test_telem.method
-
-
-if __name__ == "__main__":
-    TestTelemStore.export_telems()

monkey/monkey_island/cc/resources/log.py

@@ -5,7 +5,6 @@ from flask import request
 
 from monkey_island.cc.database import mongo
 from monkey_island.cc.resources.AbstractResource import AbstractResource
-from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore
 from monkey_island.cc.resources.request_authentication import jwt_required
 from monkey_island.cc.services.log import LogService
 from monkey_island.cc.services.node import NodeService
@@ -25,7 +24,6 @@ class Log(AbstractResource):
             return LogService.log_exists(ObjectId(exists_monkey_id))
 
     # Used by monkey. can't secure.
-    @TestTelemStore.store_exported_telem
     def post(self):
         telemetry_json = json.loads(request.data)
 

monkey/monkey_island/cc/resources/monkey.py

@@ -6,7 +6,6 @@ from flask import request
 from monkey_island.cc.database import mongo
 from monkey_island.cc.models.monkey_ttl import create_monkey_ttl_document
 from monkey_island.cc.resources.AbstractResource import AbstractResource
-from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore
 from monkey_island.cc.resources.utils.semaphores import agent_killing_mutex
 from monkey_island.cc.server_utils.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS
 from monkey_island.cc.services.config import ConfigService
@@ -29,7 +28,6 @@ class Monkey(AbstractResource):
 
     # Used by monkey. can't secure.
     # Called on monkey wakeup to initialize local configuration
-    @TestTelemStore.store_exported_telem
     def post(self, **kw):
 
         # TODO: Why is it the registration of an agent coupled to exploit telemetry? It's hard to

monkey/monkey_island/cc/resources/telemetry.py

@@ -9,7 +9,6 @@ from monkey_island.cc.database import mongo
 from monkey_island.cc.models.monkey import Monkey
 from monkey_island.cc.models.telemetries import get_telemetry_by_query
 from monkey_island.cc.resources.AbstractResource import AbstractResource
-from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore
 from monkey_island.cc.resources.request_authentication import jwt_required
 from monkey_island.cc.services.node import NodeService
 from monkey_island.cc.services.telemetry.processing.processing import process_telemetry
@@ -45,7 +44,6 @@ class Telemetry(AbstractResource):
         return result
 
     # Used by monkey. can't secure.
-    @TestTelemStore.store_exported_telem
     def post(self):
         telemetry_json = json.loads(request.data)
         telemetry_json["data"] = json.loads(telemetry_json["data"])

monkey/monkey_island/cc/services/config.py

@@ -9,7 +9,6 @@ from typing import Any, Dict, List
 from jsonschema import Draft4Validator, validators
 
 from common.config_value_paths import (
-    EXPORT_MONKEY_TELEMS_PATH,
     LM_HASH_LIST_PATH,
     NTLM_HASH_LIST_PATH,
     PASSWORD_LIST_PATH,
@@ -355,10 +354,6 @@ class ConfigService:
                     else get_datastore_encryptor().encrypt(config_arr)
                 )
 
-    @staticmethod
-    def is_test_telem_export_enabled():
-        return ConfigService.get_config_value(EXPORT_MONKEY_TELEMS_PATH)
-
     @staticmethod
     def get_config_propagation_credentials_from_flat_config(config) -> Dict[str, List[str]]:
         return {

monkey/monkey_island/cc/services/config_schema/internal.py

@@ -153,19 +153,5 @@ INTERNAL = {
                 },
             },
         },
-        "testing": {
-            "title": "Testing",
-            "type": "object",
-            "properties": {
-                "export_monkey_telems": {
-                    "title": "Export monkey telemetries",
-                    "type": "boolean",
-                    "default": False,
-                    "description": "Exports unencrypted telemetries that "
-                    "can be used for tests in development."
-                    " Do not turn on!",
-                }
-            },
-        },
     },
 }

monkey/monkey_island/cc/services/infection_lifecycle.py

@@ -2,8 +2,6 @@ import logging
 
 from monkey_island.cc.models import Monkey
 from monkey_island.cc.models.agent_controls import AgentControls
-from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore
-from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.node import NodeService
 from monkey_island.cc.services.reporting.report import ReportService
 from monkey_island.cc.services.reporting.report_generation_synchronisation import (
@@ -73,5 +71,3 @@ def _on_finished_infection():
     # we want to skip and reply.
     if not is_report_being_generated() and not ReportService.is_latest_report_exists():
         safe_generate_reports()
-    if ConfigService.is_test_telem_export_enabled() and not TestTelemStore.TELEMS_EXPORTED:
-        TestTelemStore.export_telems()

monkey/tests/data_for_tests/monkey_configs/flat_config.json

@@ -47,7 +47,6 @@
         "PowerShellExploiter",
         "Log4ShellExploiter"
     ],
-    "export_monkey_telems": false,
     "finger_classes": [
         "SMBFinger",
         "SSHFinger",

monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json

@@ -92,9 +92,6 @@
         "exploit_lm_hash_list": [],
         "exploit_ntlm_hash_list": [],
         "exploit_ssh_keys": []
-      },
-      "testing": {
-        "export_monkey_telems": false
       }
     },
     "monkey": {

vulture_allowlist.py

@@ -129,7 +129,6 @@ opnum  # unused variable (monkey/infection_monkey/exploit/zerologon.py:466)
 structure  # unused variable (monkey/infection_monkey/exploit/zerologon.py:467)
 structure  # unused variable (monkey/infection_monkey/exploit/zerologon.py:478)
 oid_set  # unused variable (monkey/infection_monkey/exploit/tools/wmi_tools.py:96)
-export_monkey_telems  # unused variable (monkey/infection_monkey/config.py:282)
 NoInternetError  # unused class (monkey/common/utils/exceptions.py:33)
 _.__isabstractmethod__  # unused attribute (monkey/common/utils/code_utils.py:11)
 MIMIKATZ  # unused variable (monkey/common/utils/attack_utils.py:21)