From fb0fea6f6af99e10f6edbc40a505ae29d4710727 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 1 Jan 2020 15:33:02 +0200 Subject: [PATCH 01/29] Improved the monkey start function structure a bit, extracted to functions Prep work for changing system info collection to modular system --- monkey/infection_monkey/monkey.py | 269 ++++++++++-------- .../post_breach/post_breach_handler.py | 2 +- 2 files changed, 147 insertions(+), 124 deletions(-) diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index 80d2d8642..8543505a7 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -32,11 +32,17 @@ from infection_monkey.telemetry.attack.t1106_telem import T1106Telem from common.utils.attack_utils import ScanStatus, UsageEnum from infection_monkey.exploit.HostExploiter import HostExploiter +MAX_DEPTH_REACHED_MESSAGE = "Reached max depth, shutting down" + __author__ = 'itamar' LOG = logging.getLogger(__name__) +class PlannedShutdown(Exception): + pass + + class InfectionMonkey(object): def __init__(self, args): self._keep_running = False @@ -87,143 +93,158 @@ class InfectionMonkey(object): LOG.debug("Default server: %s is already in command servers list" % self._default_server) def start(self): - LOG.info("Monkey is running...") + try: + LOG.info("Monkey is starting...") - # Sets island's IP and port for monkey to communicate to - if not self.set_default_server(): - return - self.set_default_port() + LOG.debug("Starting the setup phase.") + # Sets island's IP and port for monkey to communicate to + self.set_default_server() + self.set_default_port() - # Create a dir for monkey files if there isn't one - create_monkey_dir() + # Create a dir for monkey files if there isn't one + create_monkey_dir() - if WindowsUpgrader.should_upgrade(): - self._upgrading_to_64 = True - self._singleton.unlock() - LOG.info("32bit monkey running on 64bit Windows. Upgrading.") - WindowsUpgrader.upgrade(self._opts) - return + self.upgrade_to_64_if_needed() - ControlClient.wakeup(parent=self._parent) - ControlClient.load_control_config() + ControlClient.wakeup(parent=self._parent) + ControlClient.load_control_config() - if is_windows_os(): - T1106Telem(ScanStatus.USED, UsageEnum.SINGLETON_WINAPI).send() + if is_windows_os(): + T1106Telem(ScanStatus.USED, UsageEnum.SINGLETON_WINAPI).send() - if not WormConfiguration.alive: - LOG.info("Marked not alive from configuration") - return + self.shutdown_by_not_alive_config() - if firewall.is_enabled(): - firewall.add_firewall_rule() + if firewall.is_enabled(): + firewall.add_firewall_rule() - monkey_tunnel = ControlClient.create_control_tunnel() - if monkey_tunnel: - monkey_tunnel.start() + monkey_tunnel = ControlClient.create_control_tunnel() + if monkey_tunnel: + monkey_tunnel.start() - StateTelem(is_done=False).send() - TunnelTelem().send() + StateTelem(is_done=False).send() + TunnelTelem().send() + LOG.debug("Starting the post-breach phase.") + self.collect_system_info_if_configured() + PostBreach().execute_all_configured() + + LOG.debug("Starting the propagation phase.") + self.shutdown_by_max_depth_reached() + + for iteration_index in range(WormConfiguration.max_iterations): + ControlClient.keepalive() + ControlClient.load_control_config() + + self._network.initialize() + + self._fingerprint = HostFinger.get_instances() + + self._exploiters = HostExploiter.get_classes() + + if not self._keep_running or not WormConfiguration.alive: + break + + machines = self._network.get_victim_machines(max_find=WormConfiguration.victims_max_find, + stop_callback=ControlClient.check_for_stop) + is_empty = True + for machine in machines: + if ControlClient.check_for_stop(): + break + + is_empty = False + for finger in self._fingerprint: + LOG.info("Trying to get OS fingerprint from %r with module %s", + machine, finger.__class__.__name__) + finger.get_host_fingerprint(machine) + + ScanTelem(machine).send() + + # skip machines that we've already exploited + if machine in self._exploited_machines: + LOG.debug("Skipping %r - already exploited", + machine) + continue + elif machine in self._fail_exploitation_machines: + if WormConfiguration.retry_failed_explotation: + LOG.debug("%r - exploitation failed before, trying again", machine) + else: + LOG.debug("Skipping %r - exploitation failed before", machine) + continue + + if monkey_tunnel: + monkey_tunnel.set_tunnel_for_host(machine) + if self._default_server: + if self._network.on_island(self._default_server): + machine.set_default_server(get_interface_to_target(machine.ip_addr) + + (':' + self._default_server_port if self._default_server_port else '')) + else: + machine.set_default_server(self._default_server) + LOG.debug("Default server for machine: %r set to %s" % (machine, machine.default_server)) + + # Order exploits according to their type + if WormConfiguration.should_exploit: + self._exploiters = sorted(self._exploiters, key=lambda exploiter_: exploiter_.EXPLOIT_TYPE.value) + host_exploited = False + for exploiter in [exploiter(machine) for exploiter in self._exploiters]: + if self.try_exploiting(machine, exploiter): + host_exploited = True + VictimHostTelem('T1210', ScanStatus.USED, machine=machine).send() + break + if not host_exploited: + self._fail_exploitation_machines.add(machine) + VictimHostTelem('T1210', ScanStatus.SCANNED, machine=machine).send() + if not self._keep_running: + break + + if (not is_empty) and (WormConfiguration.max_iterations > iteration_index + 1): + time_to_sleep = WormConfiguration.timeout_between_iterations + LOG.info("Sleeping %d seconds before next life cycle iteration", time_to_sleep) + time.sleep(time_to_sleep) + + if self._keep_running and WormConfiguration.alive: + LOG.info("Reached max iterations (%d)", WormConfiguration.max_iterations) + elif not WormConfiguration.alive: + LOG.info("Marked not alive from configuration") + + # if host was exploited, before continue to closing the tunnel ensure the exploited host had its chance to + # connect to the tunnel + if len(self._exploited_machines) > 0: + time_to_sleep = WormConfiguration.keep_tunnel_open_time + LOG.info("Sleeping %d seconds for exploited machines to connect to tunnel", time_to_sleep) + time.sleep(time_to_sleep) + + if monkey_tunnel: + monkey_tunnel.stop() + monkey_tunnel.join() + except PlannedShutdown: + LOG.info("A planned shutdown of the Monkey occurred. Logging the reason and finishing execution.") + LOG.exception("Planned shutdown, reason:") + + def shutdown_by_max_depth_reached(self): + if 0 == WormConfiguration.depth: + TraceTelem(MAX_DEPTH_REACHED_MESSAGE).send() + raise PlannedShutdown(MAX_DEPTH_REACHED_MESSAGE) + else: + LOG.debug("Running with depth: %d" % WormConfiguration.depth) + + def collect_system_info_if_configured(self): if WormConfiguration.collect_system_info: LOG.debug("Calling system info collection") system_info_collector = SystemInfoCollector() system_info = system_info_collector.get_info() SystemInfoTelem(system_info).send() - # Executes post breach actions - PostBreach().execute() + def shutdown_by_not_alive_config(self): + if not WormConfiguration.alive: + raise PlannedShutdown("Marked 'not alive' from configuration.") - if 0 == WormConfiguration.depth: - TraceTelem("Reached max depth, shutting down").send() - return - else: - LOG.debug("Running with depth: %d" % WormConfiguration.depth) - - for iteration_index in range(WormConfiguration.max_iterations): - ControlClient.keepalive() - ControlClient.load_control_config() - - self._network.initialize() - - self._fingerprint = HostFinger.get_instances() - - self._exploiters = HostExploiter.get_classes() - - if not self._keep_running or not WormConfiguration.alive: - break - - machines = self._network.get_victim_machines(max_find=WormConfiguration.victims_max_find, - stop_callback=ControlClient.check_for_stop) - is_empty = True - for machine in machines: - if ControlClient.check_for_stop(): - break - - is_empty = False - for finger in self._fingerprint: - LOG.info("Trying to get OS fingerprint from %r with module %s", - machine, finger.__class__.__name__) - finger.get_host_fingerprint(machine) - - ScanTelem(machine).send() - - # skip machines that we've already exploited - if machine in self._exploited_machines: - LOG.debug("Skipping %r - already exploited", - machine) - continue - elif machine in self._fail_exploitation_machines: - if WormConfiguration.retry_failed_explotation: - LOG.debug("%r - exploitation failed before, trying again", machine) - else: - LOG.debug("Skipping %r - exploitation failed before", machine) - continue - - if monkey_tunnel: - monkey_tunnel.set_tunnel_for_host(machine) - if self._default_server: - if self._network.on_island(self._default_server): - machine.set_default_server(get_interface_to_target(machine.ip_addr) + - (':' + self._default_server_port if self._default_server_port else '')) - else: - machine.set_default_server(self._default_server) - LOG.debug("Default server for machine: %r set to %s" % (machine, machine.default_server)) - - # Order exploits according to their type - if WormConfiguration.should_exploit: - self._exploiters = sorted(self._exploiters, key=lambda exploiter_: exploiter_.EXPLOIT_TYPE.value) - host_exploited = False - for exploiter in [exploiter(machine) for exploiter in self._exploiters]: - if self.try_exploiting(machine, exploiter): - host_exploited = True - VictimHostTelem('T1210', ScanStatus.USED, machine=machine).send() - break - if not host_exploited: - self._fail_exploitation_machines.add(machine) - VictimHostTelem('T1210', ScanStatus.SCANNED, machine=machine).send() - if not self._keep_running: - break - - if (not is_empty) and (WormConfiguration.max_iterations > iteration_index + 1): - time_to_sleep = WormConfiguration.timeout_between_iterations - LOG.info("Sleeping %d seconds before next life cycle iteration", time_to_sleep) - time.sleep(time_to_sleep) - - if self._keep_running and WormConfiguration.alive: - LOG.info("Reached max iterations (%d)", WormConfiguration.max_iterations) - elif not WormConfiguration.alive: - LOG.info("Marked not alive from configuration") - - # if host was exploited, before continue to closing the tunnel ensure the exploited host had its chance to - # connect to the tunnel - if len(self._exploited_machines) > 0: - time_to_sleep = WormConfiguration.keep_tunnel_open_time - LOG.info("Sleeping %d seconds for exploited machines to connect to tunnel", time_to_sleep) - time.sleep(time_to_sleep) - - if monkey_tunnel: - monkey_tunnel.stop() - monkey_tunnel.join() + def upgrade_to_64_if_needed(self): + if WindowsUpgrader.should_upgrade(): + self._upgrading_to_64 = True + self._singleton.unlock() + LOG.info("32bit monkey running on 64bit Windows. Upgrading.") + WindowsUpgrader.upgrade(self._opts) + raise PlannedShutdown("Finished upgrading from 32bit to 64bit.") def cleanup(self): LOG.info("Monkey cleanup started") @@ -346,9 +367,11 @@ class InfectionMonkey(object): self._default_server_port = '' def set_default_server(self): + """ + Sets the default server for the Monkey to communicate back to. + :raises PlannedShutdown if couldn't find the server. + """ if not ControlClient.find_server(default_tunnel=self._default_tunnel): - LOG.info("Monkey couldn't find server. Going down.") - return False + raise PlannedShutdown("Monkey couldn't find server with {} default tunnel.".format(self._default_tunnel)) self._default_server = WormConfiguration.current_server LOG.debug("default server set to: %s" % self._default_server) - return True diff --git a/monkey/infection_monkey/post_breach/post_breach_handler.py b/monkey/infection_monkey/post_breach/post_breach_handler.py index 7474c8ef1..d700bac62 100644 --- a/monkey/infection_monkey/post_breach/post_breach_handler.py +++ b/monkey/infection_monkey/post_breach/post_breach_handler.py @@ -20,7 +20,7 @@ class PostBreach(object): self.os_is_linux = not is_windows_os() self.pba_list = self.config_to_pba_list() - def execute(self): + def execute_all_configured(self): """ Executes all post breach actions. """ From 81b44f0ebbe7bc8d8ac70fd24178cae2ae420771 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 1 Jan 2020 17:01:06 +0200 Subject: [PATCH 02/29] WIP - created azure instance class --- monkey/common/cloud/azure/__init__.py | 0 monkey/common/cloud/azure/azure_instance.py | 27 +++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 monkey/common/cloud/azure/__init__.py create mode 100644 monkey/common/cloud/azure/azure_instance.py diff --git a/monkey/common/cloud/azure/__init__.py b/monkey/common/cloud/azure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py new file mode 100644 index 000000000..2c949aa2c --- /dev/null +++ b/monkey/common/cloud/azure/azure_instance.py @@ -0,0 +1,27 @@ +import requests + +AZURE_METADATA_SERVICE_URL = "http://169.254.169.254/metadata/instance?api-version=2019-06-04" + + +class AzureInstance(object): + """ + Access to useful information about the current machine if it's an Azure VM. + """ + + def __init__(self): + try: + response = requests.get(AZURE_METADATA_SERVICE_URL, headers={"Metadata": "true"}) + if response: + self.on_azure = True + self.try_parse_response(response) + else: + self.on_azure = False + except ConnectionError: + self.on_azure = False + + def try_parse_response(self, response): + # TODO implement - get fields from metadata like region etc. + pass + + def is_azure_instance(self): + return self.on_azure From 718291d5734f2e7e32451458b0c1e09edba83d84 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Thu, 2 Jan 2020 12:16:48 +0200 Subject: [PATCH 03/29] Tested the AzureInstance class Tested on Azure instance and non-cloud instace. Seems to work :leo:. Unit tests aren't relevant here --- monkey/common/cloud/azure/azure_instance.py | 35 +++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py index 2c949aa2c..a58e0e126 100644 --- a/monkey/common/cloud/azure/azure_instance.py +++ b/monkey/common/cloud/azure/azure_instance.py @@ -1,27 +1,50 @@ +import logging import requests -AZURE_METADATA_SERVICE_URL = "http://169.254.169.254/metadata/instance?api-version=2019-06-04" +LATEST_AZURE_METADATA_API_VERSION = "2019-06-04" +AZURE_METADATA_SERVICE_URL = "http://169.254.169.254/metadata/instance?api-version=%s" % LATEST_AZURE_METADATA_API_VERSION + +logger = logging.getLogger(__name__) class AzureInstance(object): """ Access to useful information about the current machine if it's an Azure VM. + Based on Azure metadata service: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service """ def __init__(self): + """ + Determines if on Azure and if so, gets some basic metadata on this instance. + """ + self.instance_name = None + self.instance_id = None + self.location = None + self.on_azure = False + try: response = requests.get(AZURE_METADATA_SERVICE_URL, headers={"Metadata": "true"}) + self.on_azure = True + + # If not on cloud, the metadata URL is non-routable and the connection will fail. + # If on AWS, should get 404 since the metadata service URL is different, so bool(response) will be false. if response: - self.on_azure = True + logger.debug("On Azure. Trying to parse metadata.") self.try_parse_response(response) else: - self.on_azure = False - except ConnectionError: + logger.warning("On Azure, but metadata response not ok: {}".format(response.status_code)) + except requests.RequestException: + logger.debug("Failed to get response from Azure metadata service: This instance is not on Azure.") self.on_azure = False def try_parse_response(self, response): - # TODO implement - get fields from metadata like region etc. - pass + try: + response_data = response.json() + self.instance_name = response_data["compute"]["name"] + self.instance_id = response_data["compute"]["vmId"] + self.location = response_data["compute"]["location"] + except KeyError: + logger.exception("Error while parsing response from Azure metadata service.") def is_azure_instance(self): return self.on_azure From 723b5b47a534235892b5c8ed03fed9a3babd89bd Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Thu, 2 Jan 2020 14:58:40 +0200 Subject: [PATCH 04/29] WIP - adding the environment collector as a plugin, still some work to do --- monkey/common/cloud/environment_names.py | 3 ++ .../system_info/collectors/__init__.py | 0 .../collectors/environment_collector.py | 21 ++++++++++++ .../system_info/system_info_collector.py | 30 ++++++++++++++++ .../system_info/system_info_handler.py | 34 +++++++++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 monkey/common/cloud/environment_names.py create mode 100644 monkey/infection_monkey/system_info/collectors/__init__.py create mode 100644 monkey/infection_monkey/system_info/collectors/environment_collector.py create mode 100644 monkey/infection_monkey/system_info/system_info_collector.py create mode 100644 monkey/infection_monkey/system_info/system_info_handler.py diff --git a/monkey/common/cloud/environment_names.py b/monkey/common/cloud/environment_names.py new file mode 100644 index 000000000..f9e881a5a --- /dev/null +++ b/monkey/common/cloud/environment_names.py @@ -0,0 +1,3 @@ +ON_PREMISE = "On Premise" +AZURE = "Azure" +AWS = "AWS" diff --git a/monkey/infection_monkey/system_info/collectors/__init__.py b/monkey/infection_monkey/system_info/collectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py new file mode 100644 index 000000000..1459b5633 --- /dev/null +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -0,0 +1,21 @@ +from common.cloud.aws_instance import AwsInstance +from common.cloud.azure.azure_instance import AzureInstance +from common.cloud.environment_names import ON_PREMISE, AZURE, AWS +from infection_monkey.system_info.system_info_collector import SystemInfoCollector + + +class EnvironmentCollector(SystemInfoCollector): + def __init__(self): + super(EnvironmentCollector, self).__init__(name="EnvironmentCollector") + + def collect(self) -> dict: + # Check if on any cloud env. Default is on prem. + if AwsInstance().is_aws_instance(): + env = AWS + elif AzureInstance().is_azure_instance(): + env = AZURE + # TODO: elif GcpInstance().is_gcp_instance(): + else: + env = ON_PREMISE + + return {"environment": env} diff --git a/monkey/infection_monkey/system_info/system_info_collector.py b/monkey/infection_monkey/system_info/system_info_collector.py new file mode 100644 index 000000000..863e0e0ab --- /dev/null +++ b/monkey/infection_monkey/system_info/system_info_collector.py @@ -0,0 +1,30 @@ +from config import WormConfiguration +from infection_monkey.utils.plugins.plugin import Plugin + + +import infection_monkey.system_info.collectors + + +class SystemInfoCollector(Plugin): + def __init__(self, name="unknown"): + self.name = name + + @staticmethod + def should_run(class_name) -> bool: + return class_name in WormConfiguration.system_info_collectors + + @staticmethod + def base_package_file(): + return infection_monkey.system_info.collectors.__file__ + + @staticmethod + def base_package_name(): + return infection_monkey.system_info.collectors.__package__ + + def collect(self) -> dict: + """ + Collect the relevant information and return it in a dictionary. + To be implemented by each collector. + TODO should this be an abstractmethod, or will that ruin the plugin system somehow? if can be abstract should add UT + """ + raise NotImplementedError() diff --git a/monkey/infection_monkey/system_info/system_info_handler.py b/monkey/infection_monkey/system_info/system_info_handler.py new file mode 100644 index 000000000..dd085300f --- /dev/null +++ b/monkey/infection_monkey/system_info/system_info_handler.py @@ -0,0 +1,34 @@ +import logging +from typing import Sequence + +from infection_monkey.system_info.system_info_collector import SystemInfoCollector +from telemetry.system_info_telem import SystemInfoTelem + +LOG = logging.getLogger(__name__) + +PATH_TO_COLLECTORS = "infection_monkey.system_info.collectors." + + +# TODO Add new collectors to config and config schema +class SystemInfo(object): + def __init__(self): + self.collectors_list = self.config_to_collectors_list() + + def execute_all_configured(self): + successful_collections = 0 + system_info_telemetry = {} + for collector in self.collectors_list: + try: + LOG.debug("Executing system info collector: '{}'".format(collector.name)) + collected_info = collector.collect() + system_info_telemetry[collector.name] = collected_info + successful_collections += 1 + except Exception as e: + LOG.error("Collector {} failed. Error info: {}".format(collector.name, e)) + LOG.info("All system info collectors executed. Total {} executed, out of which {} collected successfully.". + format(len(self.collectors_list), successful_collections)) + # TODO Send SystemInfoTelem() + + @staticmethod + def config_to_collectors_list() -> Sequence[SystemInfoCollector]: + return SystemInfoCollector.get_instances() From 974e2205d1cff8c5383cccc2c89eed940fe2e3c5 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 5 Jan 2020 15:47:37 +0200 Subject: [PATCH 05/29] Bugfix in error handling - func_name does not exist --- .../cc/services/telemetry/processing/system_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 9ab0b45f0..8df189655 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -34,7 +34,7 @@ def safe_process_telemetry(processing_function, telemetry_json): processing_function(telemetry_json) except Exception as err: logger.error( - "Error {} while in {} stage of processing telemetry.".format(str(err), processing_function.func_name), + "Error {} while in {} stage of processing telemetry.".format(str(err), processing_function.__name__), exc_info=True) From c0331f84ffc7a11b276e8088756a58f99fcb75a7 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 5 Jan 2020 15:49:05 +0200 Subject: [PATCH 06/29] Added system info collectors as plugins and the first plugin - EnvironmentCollector --- monkey/common/cloud/environment_names.py | 4 +++ monkey/infection_monkey/config.py | 1 + .../infection_monkey/system_info/__init__.py | 5 ++++ .../system_info/collectors/__init__.py | 3 +++ .../system_info/system_info_collector.py | 2 +- ...r.py => system_info_collectors_handler.py} | 5 ++-- monkey/monkey_island/cc/models/monkey.py | 7 ++++- .../cc/services/config_schema.py | 26 +++++++++++++++++++ .../telemetry/processing/system_info.py | 2 ++ .../system_info_collectors/__init__.py | 0 .../system_info_collectors/environment.py | 14 ++++++++++ 11 files changed, 65 insertions(+), 4 deletions(-) rename monkey/infection_monkey/system_info/{system_info_handler.py => system_info_collectors_handler.py} (92%) create mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/__init__.py create mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py diff --git a/monkey/common/cloud/environment_names.py b/monkey/common/cloud/environment_names.py index f9e881a5a..1745eed62 100644 --- a/monkey/common/cloud/environment_names.py +++ b/monkey/common/cloud/environment_names.py @@ -1,3 +1,7 @@ +UNKNOWN = "Unknown" ON_PREMISE = "On Premise" AZURE = "Azure" AWS = "AWS" +GCP = "GCP" + +ALL_ENV_NAMES = [UNKNOWN, ON_PREMISE, AZURE, AWS, GCP] diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index 2d2a93939..e76ed8101 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -125,6 +125,7 @@ class Configuration(object): finger_classes = [] exploiter_classes = [] + system_info_collectors_classes = ["EnvironmentCollector"] # how many victims to look for in a single scan iteration victims_max_find = 100 diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index 7d4395af7..ccec45cde 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -9,6 +9,7 @@ from infection_monkey.network.info import get_host_subnets from infection_monkey.system_info.aws_collector import AwsCollector from infection_monkey.system_info.azure_cred_collector import AzureCollector from infection_monkey.system_info.netstat_collector import NetstatCollector +from system_info.system_info_collectors_handler import SystemInfoCollectorsHandler LOG = logging.getLogger(__name__) @@ -61,12 +62,16 @@ class InfoCollector(object): self.info = {} def get_info(self): + # Collect all hardcoded self.get_hostname() self.get_process_list() self.get_network_info() self.get_azure_info() self.get_aws_info() + # Collect all plugins + SystemInfoCollectorsHandler().execute_all_configured() + def get_hostname(self): """ Adds the fully qualified computer hostname to the system information. diff --git a/monkey/infection_monkey/system_info/collectors/__init__.py b/monkey/infection_monkey/system_info/collectors/__init__.py index e69de29bb..f5b7166e9 100644 --- a/monkey/infection_monkey/system_info/collectors/__init__.py +++ b/monkey/infection_monkey/system_info/collectors/__init__.py @@ -0,0 +1,3 @@ +""" +This package holds all the dynamic (plugin) collectors +""" diff --git a/monkey/infection_monkey/system_info/system_info_collector.py b/monkey/infection_monkey/system_info/system_info_collector.py index 863e0e0ab..3a977fcde 100644 --- a/monkey/infection_monkey/system_info/system_info_collector.py +++ b/monkey/infection_monkey/system_info/system_info_collector.py @@ -11,7 +11,7 @@ class SystemInfoCollector(Plugin): @staticmethod def should_run(class_name) -> bool: - return class_name in WormConfiguration.system_info_collectors + return class_name in WormConfiguration.system_info_collectors_classes @staticmethod def base_package_file(): diff --git a/monkey/infection_monkey/system_info/system_info_handler.py b/monkey/infection_monkey/system_info/system_info_collectors_handler.py similarity index 92% rename from monkey/infection_monkey/system_info/system_info_handler.py rename to monkey/infection_monkey/system_info/system_info_collectors_handler.py index dd085300f..1fdc74cfa 100644 --- a/monkey/infection_monkey/system_info/system_info_handler.py +++ b/monkey/infection_monkey/system_info/system_info_collectors_handler.py @@ -10,7 +10,7 @@ PATH_TO_COLLECTORS = "infection_monkey.system_info.collectors." # TODO Add new collectors to config and config schema -class SystemInfo(object): +class SystemInfoCollectorsHandler(object): def __init__(self): self.collectors_list = self.config_to_collectors_list() @@ -27,7 +27,8 @@ class SystemInfo(object): LOG.error("Collector {} failed. Error info: {}".format(collector.name, e)) LOG.info("All system info collectors executed. Total {} executed, out of which {} collected successfully.". format(len(self.collectors_list), successful_collections)) - # TODO Send SystemInfoTelem() + + SystemInfoTelem({"collectors": system_info_telemetry}).send() @staticmethod def config_to_collectors_list() -> Sequence[SystemInfoCollector]: diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index 07b5ba3fe..65dda0850 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -9,6 +9,7 @@ from monkey_island.cc.models.monkey_ttl import MonkeyTtl, create_monkey_ttl_docu from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS from monkey_island.cc.models.command_control_channel import CommandControlChannel from monkey_island.cc.utils import local_ip_addresses +from common.cloud import environment_names MAX_MONKEYS_AMOUNT_TO_CACHE = 100 @@ -42,6 +43,9 @@ class Monkey(Document): ttl_ref = ReferenceField(MonkeyTtl) tunnel = ReferenceField("self") command_control_channel = EmbeddedDocumentField(CommandControlChannel) + + # Environment related fields + environment = StringField(default=environment_names.UNKNOWN, choices=environment_names.ALL_ENV_NAMES) aws_instance_id = StringField(required=False) # This field only exists when the monkey is running on an AWS # instance. See https://github.com/guardicore/monkey/issues/426. @@ -55,7 +59,8 @@ class Monkey(Document): raise MonkeyNotFoundError("info: {0} | id: {1}".format(ex, str(db_id))) @staticmethod - def get_single_monkey_by_guid(monkey_guid): + # See https://www.python.org/dev/peps/pep-0484/#forward-references + def get_single_monkey_by_guid(monkey_guid) -> 'Monkey': try: return Monkey.objects.get(guid=monkey_guid) except DoesNotExist as ex: diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index 32ee13b12..5de57e26b 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -99,6 +99,20 @@ SCHEMA = { } ] }, + "system_info_collectors_classes": { + "title": "System Information Collectors", + "type": "string", + "anyOf": [ + { + "type": "string", + "enum": [ + "EnvironmentCollector" + ], + "title": "Which Environment this machine is on (on prem/cloud)", + "attack_techniques": [] + }, + ], + }, "post_breach_acts": { "title": "Post breach actions", "type": "string", @@ -433,6 +447,18 @@ SCHEMA = { "attack_techniques": ["T1003"], "description": "Determines whether to use Mimikatz" }, + "system_info_collectors_classes": { + "title": "System info collectors", + "type": "array", + "uniqueItems": True, + "items": { + "$ref": "#/definitions/system_info_collectors_classes" + }, + "default": [ + "EnvironmentCollector" + ], + "description": "Determines which system information collectors will collect information." + }, } }, "life_cycle": { diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 8df189655..04ab27d95 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -5,6 +5,7 @@ from monkey_island.cc.models import Monkey from monkey_island.cc.services import mimikatz_utils from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence from monkey_island.cc.services.wmi_handler import WMIHandler from monkey_island.cc.encryptor import encryptor @@ -20,6 +21,7 @@ def process_system_info_telemetry(telemetry_json): process_aws_data, update_db_with_new_hostname, test_antivirus_existence, + process_environment_telemetry ] # Calling safe_process_telemetry so if one of the stages fail, we log and move on instead of failing the rest of diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/__init__.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py new file mode 100644 index 000000000..d66019b39 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py @@ -0,0 +1,14 @@ +import logging + +from monkey_island.cc.models.monkey import Monkey + +logger = logging.getLogger(__name__) + + +def process_environment_telemetry(telemetry_json): + if "EnvironmentCollector" in telemetry_json["data"]["collectors"]: + env = telemetry_json["data"]["collectors"]["EnvironmentCollector"]["environment"] + relevant_monkey = Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']) + relevant_monkey.environment = env + relevant_monkey.save() + logger.debug("Updated Monkey {} with env {}".format(str(relevant_monkey), env)) From fdb54f6b8d9b3269a361b65cf1bac61e4e9897e8 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 5 Jan 2020 16:23:22 +0200 Subject: [PATCH 07/29] Extracted function in EnvCollector for reuse in other parts of the Monkey --- .../collectors/environment_collector.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index 1459b5633..523989f61 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -4,18 +4,21 @@ from common.cloud.environment_names import ON_PREMISE, AZURE, AWS from infection_monkey.system_info.system_info_collector import SystemInfoCollector +def get_monkey_environment(): + # Check if on any cloud env. Default is on prem. + if AwsInstance().is_aws_instance(): + env = AWS + elif AzureInstance().is_azure_instance(): + env = AZURE + # TODO: elif GcpInstance().is_gcp_instance(): + else: + env = ON_PREMISE + return env + + class EnvironmentCollector(SystemInfoCollector): def __init__(self): super(EnvironmentCollector, self).__init__(name="EnvironmentCollector") def collect(self) -> dict: - # Check if on any cloud env. Default is on prem. - if AwsInstance().is_aws_instance(): - env = AWS - elif AzureInstance().is_azure_instance(): - env = AZURE - # TODO: elif GcpInstance().is_gcp_instance(): - else: - env = ON_PREMISE - - return {"environment": env} + return {"environment": get_monkey_environment()} From b9d2614271689d3752cbf0f57b3e2f4f07f37a9c Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 8 Jan 2020 11:09:52 +0200 Subject: [PATCH 08/29] CR: Moved AWS classes to own package, create generic CloudInstance class --- monkey/common/cloud/aws/__init__.py | 0 monkey/common/cloud/{ => aws}/aws_instance.py | 6 ++++-- monkey/common/cloud/{ => aws}/aws_service.py | 2 +- monkey/common/cloud/{ => aws}/aws_service_test.py | 0 monkey/common/cloud/azure/azure_instance.py | 8 +++++--- monkey/common/cloud/environment_names.py | 7 ++++++- monkey/common/cloud/instance.py | 3 +++ monkey/common/cmd/aws/aws_cmd_runner.py | 2 +- monkey/infection_monkey/system_info/aws_collector.py | 4 ++-- .../system_info/collectors/environment_collector.py | 6 +++--- monkey/monkey_island/cc/environment/aws.py | 2 +- monkey/monkey_island/cc/resources/remote_run.py | 2 +- monkey/monkey_island/cc/services/remote_run_aws.py | 6 +++--- .../monkey_island/cc/services/reporting/aws_exporter.py | 2 +- 14 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 monkey/common/cloud/aws/__init__.py rename monkey/common/cloud/{ => aws}/aws_instance.py (96%) rename monkey/common/cloud/{ => aws}/aws_service.py (98%) rename monkey/common/cloud/{ => aws}/aws_service_test.py (100%) create mode 100644 monkey/common/cloud/instance.py diff --git a/monkey/common/cloud/aws/__init__.py b/monkey/common/cloud/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/common/cloud/aws_instance.py b/monkey/common/cloud/aws/aws_instance.py similarity index 96% rename from monkey/common/cloud/aws_instance.py rename to monkey/common/cloud/aws/aws_instance.py index 4339fbcf4..301881894 100644 --- a/monkey/common/cloud/aws_instance.py +++ b/monkey/common/cloud/aws/aws_instance.py @@ -6,6 +6,8 @@ import logging __author__ = 'itay.mizeretz' +from common.cloud.instance import CloudInstance + AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS = "169.254.169.254" AWS_LATEST_METADATA_URI_PREFIX = 'http://{0}/latest/'.format(AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS) ACCOUNT_ID_KEY = "accountId" @@ -13,7 +15,7 @@ ACCOUNT_ID_KEY = "accountId" logger = logging.getLogger(__name__) -class AwsInstance(object): +class AwsInstance(CloudInstance): """ Class which gives useful information about the current instance you're on. """ @@ -57,7 +59,7 @@ class AwsInstance(object): def get_region(self): return self.region - def is_aws_instance(self): + def is_instance(self): return self.instance_id is not None @staticmethod diff --git a/monkey/common/cloud/aws_service.py b/monkey/common/cloud/aws/aws_service.py similarity index 98% rename from monkey/common/cloud/aws_service.py rename to monkey/common/cloud/aws/aws_service.py index 6ef385542..a42c2e1dd 100644 --- a/monkey/common/cloud/aws_service.py +++ b/monkey/common/cloud/aws/aws_service.py @@ -4,7 +4,7 @@ import boto3 import botocore from botocore.exceptions import ClientError -from common.cloud.aws_instance import AwsInstance +from common.cloud.aws.aws_instance import AwsInstance __author__ = ['itay.mizeretz', 'shay.nehmad'] diff --git a/monkey/common/cloud/aws_service_test.py b/monkey/common/cloud/aws/aws_service_test.py similarity index 100% rename from monkey/common/cloud/aws_service_test.py rename to monkey/common/cloud/aws/aws_service_test.py diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py index a58e0e126..5222c7620 100644 --- a/monkey/common/cloud/azure/azure_instance.py +++ b/monkey/common/cloud/azure/azure_instance.py @@ -1,13 +1,15 @@ import logging import requests -LATEST_AZURE_METADATA_API_VERSION = "2019-06-04" +from common.cloud.instance import CloudInstance + +LATEST_AZURE_METADATA_API_VERSION = "2019-04-30" AZURE_METADATA_SERVICE_URL = "http://169.254.169.254/metadata/instance?api-version=%s" % LATEST_AZURE_METADATA_API_VERSION logger = logging.getLogger(__name__) -class AzureInstance(object): +class AzureInstance(CloudInstance): """ Access to useful information about the current machine if it's an Azure VM. Based on Azure metadata service: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service @@ -46,5 +48,5 @@ class AzureInstance(object): except KeyError: logger.exception("Error while parsing response from Azure metadata service.") - def is_azure_instance(self): + def is_instance(self): return self.on_azure diff --git a/monkey/common/cloud/environment_names.py b/monkey/common/cloud/environment_names.py index 1745eed62..0c8655753 100644 --- a/monkey/common/cloud/environment_names.py +++ b/monkey/common/cloud/environment_names.py @@ -1,7 +1,12 @@ +# When adding a new environment to this file, make sure to add it to ALL_ENV_NAMES as well! + UNKNOWN = "Unknown" ON_PREMISE = "On Premise" AZURE = "Azure" AWS = "AWS" GCP = "GCP" +ALIBABA = "Alibaba Cloud" +IBM = "IBM Cloud" +DigitalOcean = "Digital Ocean" -ALL_ENV_NAMES = [UNKNOWN, ON_PREMISE, AZURE, AWS, GCP] +ALL_ENV_NAMES = [UNKNOWN, ON_PREMISE, AZURE, AWS, GCP, ALIBABA, IBM, DigitalOcean] diff --git a/monkey/common/cloud/instance.py b/monkey/common/cloud/instance.py new file mode 100644 index 000000000..52dd56b02 --- /dev/null +++ b/monkey/common/cloud/instance.py @@ -0,0 +1,3 @@ +class CloudInstance(object): + def is_instance(self) -> bool: + raise NotImplementedError() diff --git a/monkey/common/cmd/aws/aws_cmd_runner.py b/monkey/common/cmd/aws/aws_cmd_runner.py index 459a42129..1ab680c4d 100644 --- a/monkey/common/cmd/aws/aws_cmd_runner.py +++ b/monkey/common/cmd/aws/aws_cmd_runner.py @@ -1,6 +1,6 @@ import logging -from common.cloud.aws_service import AwsService +from common.cloud.aws.aws_service import AwsService from common.cmd.aws.aws_cmd_result import AwsCmdResult from common.cmd.cmd_runner import CmdRunner from common.cmd.cmd_status import CmdStatus diff --git a/monkey/infection_monkey/system_info/aws_collector.py b/monkey/infection_monkey/system_info/aws_collector.py index df90e5913..f39662d13 100644 --- a/monkey/infection_monkey/system_info/aws_collector.py +++ b/monkey/infection_monkey/system_info/aws_collector.py @@ -1,6 +1,6 @@ import logging -from common.cloud.aws_instance import AwsInstance +from common.cloud.aws.aws_instance import AwsInstance __author__ = 'itay.mizeretz' @@ -17,7 +17,7 @@ class AwsCollector(object): LOG.info("Collecting AWS info") aws = AwsInstance() info = {} - if aws.is_aws_instance(): + if aws.is_instance(): LOG.info("Machine is an AWS instance") info = \ { diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index 523989f61..208bbfa42 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -1,4 +1,4 @@ -from common.cloud.aws_instance import AwsInstance +from common.cloud.aws.aws_instance import AwsInstance from common.cloud.azure.azure_instance import AzureInstance from common.cloud.environment_names import ON_PREMISE, AZURE, AWS from infection_monkey.system_info.system_info_collector import SystemInfoCollector @@ -6,9 +6,9 @@ from infection_monkey.system_info.system_info_collector import SystemInfoCollect def get_monkey_environment(): # Check if on any cloud env. Default is on prem. - if AwsInstance().is_aws_instance(): + if AwsInstance().is_instance(): env = AWS - elif AzureInstance().is_azure_instance(): + elif AzureInstance().is_instance(): env = AZURE # TODO: elif GcpInstance().is_gcp_instance(): else: diff --git a/monkey/monkey_island/cc/environment/aws.py b/monkey/monkey_island/cc/environment/aws.py index 18db5c376..5608bddcd 100644 --- a/monkey/monkey_island/cc/environment/aws.py +++ b/monkey/monkey_island/cc/environment/aws.py @@ -1,6 +1,6 @@ import monkey_island.cc.auth from monkey_island.cc.environment import Environment -from common.cloud.aws_instance import AwsInstance +from common.cloud.aws.aws_instance import AwsInstance __author__ = 'itay.mizeretz' diff --git a/monkey/monkey_island/cc/resources/remote_run.py b/monkey/monkey_island/cc/resources/remote_run.py index c41699add..98d3694bf 100644 --- a/monkey/monkey_island/cc/resources/remote_run.py +++ b/monkey/monkey_island/cc/resources/remote_run.py @@ -6,7 +6,7 @@ import flask_restful from monkey_island.cc.auth import jwt_required from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService -from common.cloud.aws_service import AwsService +from common.cloud.aws.aws_service import AwsService CLIENT_ERROR_FORMAT = "ClientError, error message: '{}'. Probably, the IAM role that has been associated with the " \ "instance doesn't permit SSM calls. " diff --git a/monkey/monkey_island/cc/services/remote_run_aws.py b/monkey/monkey_island/cc/services/remote_run_aws.py index 9627bf74c..0ba6fa4ef 100644 --- a/monkey/monkey_island/cc/services/remote_run_aws.py +++ b/monkey/monkey_island/cc/services/remote_run_aws.py @@ -1,7 +1,7 @@ import logging -from common.cloud.aws_instance import AwsInstance -from common.cloud.aws_service import AwsService +from common.cloud.aws.aws_instance import AwsInstance +from common.cloud.aws.aws_service import AwsService from common.cmd.aws.aws_cmd_runner import AwsCmdRunner from common.cmd.cmd import Cmd from common.cmd.cmd_runner import CmdRunner @@ -54,7 +54,7 @@ class RemoteRunAwsService: @staticmethod def is_running_on_aws(): - return RemoteRunAwsService.aws_instance.is_aws_instance() + return RemoteRunAwsService.aws_instance.is_instance() @staticmethod def update_aws_region_authless(): diff --git a/monkey/monkey_island/cc/services/reporting/aws_exporter.py b/monkey/monkey_island/cc/services/reporting/aws_exporter.py index 1df12e2eb..86486b9ba 100644 --- a/monkey/monkey_island/cc/services/reporting/aws_exporter.py +++ b/monkey/monkey_island/cc/services/reporting/aws_exporter.py @@ -5,7 +5,7 @@ from datetime import datetime import boto3 from botocore.exceptions import UnknownServiceError -from common.cloud.aws_instance import AwsInstance +from common.cloud.aws.aws_instance import AwsInstance from monkey_island.cc.environment.environment import load_server_configuration_from_file from monkey_island.cc.services.reporting.exporter import Exporter From 676d46307bf4c761bf59f56226e53dc314022155 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 8 Jan 2020 11:20:49 +0200 Subject: [PATCH 09/29] Using the generic CloudInstance class to determine environment --- monkey/common/cloud/aws/aws_instance.py | 9 ++++++--- monkey/common/cloud/azure/azure_instance.py | 9 ++++++--- monkey/common/cloud/instance.py | 15 +++++++++++++++ .../collectors/environment_collector.py | 15 ++++++--------- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/monkey/common/cloud/aws/aws_instance.py b/monkey/common/cloud/aws/aws_instance.py index 301881894..c77220d17 100644 --- a/monkey/common/cloud/aws/aws_instance.py +++ b/monkey/common/cloud/aws/aws_instance.py @@ -6,6 +6,7 @@ import logging __author__ = 'itay.mizeretz' +from common.cloud.environment_names import AWS from common.cloud.instance import CloudInstance AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS = "169.254.169.254" @@ -19,6 +20,11 @@ class AwsInstance(CloudInstance): """ Class which gives useful information about the current instance you're on. """ + def is_instance(self): + return self.instance_id is not None + + def get_cloud_provider_name(self) -> str: + return AWS def __init__(self): self.instance_id = None @@ -59,9 +65,6 @@ class AwsInstance(CloudInstance): def get_region(self): return self.region - def is_instance(self): - return self.instance_id is not None - @staticmethod def _extract_account_id(instance_identity_document_response): """ diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py index 5222c7620..f0d5a8044 100644 --- a/monkey/common/cloud/azure/azure_instance.py +++ b/monkey/common/cloud/azure/azure_instance.py @@ -1,6 +1,7 @@ import logging import requests +from common.cloud.environment_names import AZURE from common.cloud.instance import CloudInstance LATEST_AZURE_METADATA_API_VERSION = "2019-04-30" @@ -14,6 +15,11 @@ class AzureInstance(CloudInstance): Access to useful information about the current machine if it's an Azure VM. Based on Azure metadata service: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service """ + def is_instance(self): + return self.on_azure + + def get_cloud_provider_name(self) -> str: + return AZURE def __init__(self): """ @@ -47,6 +53,3 @@ class AzureInstance(CloudInstance): self.location = response_data["compute"]["location"] except KeyError: logger.exception("Error while parsing response from Azure metadata service.") - - def is_instance(self): - return self.on_azure diff --git a/monkey/common/cloud/instance.py b/monkey/common/cloud/instance.py index 52dd56b02..2e702b867 100644 --- a/monkey/common/cloud/instance.py +++ b/monkey/common/cloud/instance.py @@ -1,3 +1,18 @@ +from typing import List + +from common.cloud.aws.aws_instance import AwsInstance +from common.cloud.azure.azure_instance import AzureInstance + + class CloudInstance(object): def is_instance(self) -> bool: raise NotImplementedError() + + def get_cloud_provider_name(self) -> str: + raise NotImplementedError() + + all_cloud_instances = [AwsInstance(), AzureInstance()] + + @staticmethod + def get_all_cloud_instances() -> List['CloudInstance']: + return CloudInstance.all_cloud_instances diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index 208bbfa42..56df5906b 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -1,18 +1,15 @@ from common.cloud.aws.aws_instance import AwsInstance from common.cloud.azure.azure_instance import AzureInstance from common.cloud.environment_names import ON_PREMISE, AZURE, AWS +from common.cloud.instance import CloudInstance from infection_monkey.system_info.system_info_collector import SystemInfoCollector -def get_monkey_environment(): - # Check if on any cloud env. Default is on prem. - if AwsInstance().is_instance(): - env = AWS - elif AzureInstance().is_instance(): - env = AZURE - # TODO: elif GcpInstance().is_gcp_instance(): - else: - env = ON_PREMISE +def get_monkey_environment() -> str: + env = ON_PREMISE + for instance in CloudInstance.get_all_cloud_instances(): + if instance.is_instance(): + env = instance.get_cloud_provider_name() return env From 875cf3318d4a6373922c99ba6f58a88d0af1b234 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 8 Jan 2020 12:21:38 +0200 Subject: [PATCH 10/29] Fixed circular import --- monkey/common/cloud/all_instances.py | 11 +++++++++++ monkey/common/cloud/instance.py | 17 +++++------------ .../collectors/environment_collector.py | 3 ++- 3 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 monkey/common/cloud/all_instances.py diff --git a/monkey/common/cloud/all_instances.py b/monkey/common/cloud/all_instances.py new file mode 100644 index 000000000..986bf1a80 --- /dev/null +++ b/monkey/common/cloud/all_instances.py @@ -0,0 +1,11 @@ +from typing import List + +from common.cloud.aws.aws_instance import AwsInstance +from common.cloud.azure.azure_instance import AzureInstance +from common.cloud.instance import CloudInstance + +all_cloud_instances = [AwsInstance(), AzureInstance()] + + +def get_all_cloud_instances() -> List[CloudInstance]: + return all_cloud_instances diff --git a/monkey/common/cloud/instance.py b/monkey/common/cloud/instance.py index 2e702b867..61ab4c734 100644 --- a/monkey/common/cloud/instance.py +++ b/monkey/common/cloud/instance.py @@ -1,18 +1,11 @@ -from typing import List - -from common.cloud.aws.aws_instance import AwsInstance -from common.cloud.azure.azure_instance import AzureInstance - - class CloudInstance(object): + """ + This is an abstract class which represents a cloud instance. + + The current machine can be a cloud instance (for example EC2 instance or Azure VM). + """ def is_instance(self) -> bool: raise NotImplementedError() def get_cloud_provider_name(self) -> str: raise NotImplementedError() - - all_cloud_instances = [AwsInstance(), AzureInstance()] - - @staticmethod - def get_all_cloud_instances() -> List['CloudInstance']: - return CloudInstance.all_cloud_instances diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index 56df5906b..c679e04f7 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -1,3 +1,4 @@ +from common.cloud.all_instances import get_all_cloud_instances from common.cloud.aws.aws_instance import AwsInstance from common.cloud.azure.azure_instance import AzureInstance from common.cloud.environment_names import ON_PREMISE, AZURE, AWS @@ -7,7 +8,7 @@ from infection_monkey.system_info.system_info_collector import SystemInfoCollect def get_monkey_environment() -> str: env = ON_PREMISE - for instance in CloudInstance.get_all_cloud_instances(): + for instance in get_all_cloud_instances(): if instance.is_instance(): env = instance.get_cloud_provider_name() return env From a3d81a00864c4f8b33bd158b4eb11a1d68921a33 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 8 Jan 2020 14:00:12 +0200 Subject: [PATCH 11/29] Renamed PlannedShutdown to PlannedShutdownException --- monkey/infection_monkey/monkey.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index 8543505a7..06a08f131 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -39,7 +39,7 @@ __author__ = 'itamar' LOG = logging.getLogger(__name__) -class PlannedShutdown(Exception): +class PlannedShutdownException(Exception): pass @@ -216,14 +216,14 @@ class InfectionMonkey(object): if monkey_tunnel: monkey_tunnel.stop() monkey_tunnel.join() - except PlannedShutdown: + except PlannedShutdownException: LOG.info("A planned shutdown of the Monkey occurred. Logging the reason and finishing execution.") LOG.exception("Planned shutdown, reason:") def shutdown_by_max_depth_reached(self): if 0 == WormConfiguration.depth: TraceTelem(MAX_DEPTH_REACHED_MESSAGE).send() - raise PlannedShutdown(MAX_DEPTH_REACHED_MESSAGE) + raise PlannedShutdownException(MAX_DEPTH_REACHED_MESSAGE) else: LOG.debug("Running with depth: %d" % WormConfiguration.depth) @@ -236,7 +236,7 @@ class InfectionMonkey(object): def shutdown_by_not_alive_config(self): if not WormConfiguration.alive: - raise PlannedShutdown("Marked 'not alive' from configuration.") + raise PlannedShutdownException("Marked 'not alive' from configuration.") def upgrade_to_64_if_needed(self): if WindowsUpgrader.should_upgrade(): @@ -244,7 +244,7 @@ class InfectionMonkey(object): self._singleton.unlock() LOG.info("32bit monkey running on 64bit Windows. Upgrading.") WindowsUpgrader.upgrade(self._opts) - raise PlannedShutdown("Finished upgrading from 32bit to 64bit.") + raise PlannedShutdownException("Finished upgrading from 32bit to 64bit.") def cleanup(self): LOG.info("Monkey cleanup started") @@ -369,9 +369,9 @@ class InfectionMonkey(object): def set_default_server(self): """ Sets the default server for the Monkey to communicate back to. - :raises PlannedShutdown if couldn't find the server. + :raises PlannedShutdownException if couldn't find the server. """ if not ControlClient.find_server(default_tunnel=self._default_tunnel): - raise PlannedShutdown("Monkey couldn't find server with {} default tunnel.".format(self._default_tunnel)) + raise PlannedShutdownException("Monkey couldn't find server with {} default tunnel.".format(self._default_tunnel)) self._default_server = WormConfiguration.current_server LOG.debug("default server set to: %s" % self._default_server) From 41fa1d3e3fe92e96f8c8de99640e06ec910b0303 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 8 Jan 2020 14:08:53 +0200 Subject: [PATCH 12/29] Made collect an abstract method --- .../infection_monkey/system_info/system_info_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/system_info/system_info_collector.py b/monkey/infection_monkey/system_info/system_info_collector.py index 3a977fcde..f65b4c080 100644 --- a/monkey/infection_monkey/system_info/system_info_collector.py +++ b/monkey/infection_monkey/system_info/system_info_collector.py @@ -1,11 +1,11 @@ from config import WormConfiguration from infection_monkey.utils.plugins.plugin import Plugin - +from abc import ABCMeta, abstractmethod import infection_monkey.system_info.collectors -class SystemInfoCollector(Plugin): +class SystemInfoCollector(Plugin, metaclass=ABCMeta): def __init__(self, name="unknown"): self.name = name @@ -21,10 +21,10 @@ class SystemInfoCollector(Plugin): def base_package_name(): return infection_monkey.system_info.collectors.__package__ + @abstractmethod def collect(self) -> dict: """ Collect the relevant information and return it in a dictionary. To be implemented by each collector. - TODO should this be an abstractmethod, or will that ruin the plugin system somehow? if can be abstract should add UT """ raise NotImplementedError() From 26355540bd70081f32563c6041696a46c8bf3a80 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 8 Jan 2020 21:06:02 +0200 Subject: [PATCH 13/29] Update system_info_collectors_handler.py --- .../system_info/system_info_collectors_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/monkey/infection_monkey/system_info/system_info_collectors_handler.py b/monkey/infection_monkey/system_info/system_info_collectors_handler.py index 1fdc74cfa..792024b12 100644 --- a/monkey/infection_monkey/system_info/system_info_collectors_handler.py +++ b/monkey/infection_monkey/system_info/system_info_collectors_handler.py @@ -2,14 +2,11 @@ import logging from typing import Sequence from infection_monkey.system_info.system_info_collector import SystemInfoCollector -from telemetry.system_info_telem import SystemInfoTelem +from infection_monkey.telemetry.system_info_telem import SystemInfoTelem LOG = logging.getLogger(__name__) -PATH_TO_COLLECTORS = "infection_monkey.system_info.collectors." - -# TODO Add new collectors to config and config schema class SystemInfoCollectorsHandler(object): def __init__(self): self.collectors_list = self.config_to_collectors_list() From 422fe6ff06e45d7d91164af4090140f70247bafb Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 19 Jan 2020 16:22:28 +0200 Subject: [PATCH 14/29] Added GCP instance as well --- monkey/common/cloud/all_instances.py | 3 +- monkey/common/cloud/gcp/__init__.py | 0 monkey/common/cloud/gcp/gcp_instance.py | 41 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 monkey/common/cloud/gcp/__init__.py create mode 100644 monkey/common/cloud/gcp/gcp_instance.py diff --git a/monkey/common/cloud/all_instances.py b/monkey/common/cloud/all_instances.py index 986bf1a80..6387730f6 100644 --- a/monkey/common/cloud/all_instances.py +++ b/monkey/common/cloud/all_instances.py @@ -2,9 +2,10 @@ from typing import List from common.cloud.aws.aws_instance import AwsInstance from common.cloud.azure.azure_instance import AzureInstance +from common.cloud.gcp.gcp_instance import GcpInstance from common.cloud.instance import CloudInstance -all_cloud_instances = [AwsInstance(), AzureInstance()] +all_cloud_instances = [AwsInstance(), AzureInstance(), GcpInstance()] def get_all_cloud_instances() -> List[CloudInstance]: diff --git a/monkey/common/cloud/gcp/__init__.py b/monkey/common/cloud/gcp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/common/cloud/gcp/gcp_instance.py b/monkey/common/cloud/gcp/gcp_instance.py new file mode 100644 index 000000000..5f04aabd0 --- /dev/null +++ b/monkey/common/cloud/gcp/gcp_instance.py @@ -0,0 +1,41 @@ +import logging +import requests + +from common.cloud.environment_names import GCP +from common.cloud.instance import CloudInstance + +logger = logging.getLogger(__name__) +GCP_METADATA_SERVICE_URL = "http://metadata.google.internal/" + + +class GcpInstance(CloudInstance): + def is_instance(self): + return self.on_gcp + + def get_cloud_provider_name(self) -> str: + return GCP + + def __init__(self): + """ + Determines if on GCP. + """ + self.on_gcp = False + + try: + # If not on GCP, this domain shouldn't resolve. + response = requests.get(GCP_METADATA_SERVICE_URL) + + if response: + logger.debug("Got response, so probably on GCP. Trying to parse.") + self.on_gcp = True + + if "Metadata-Flavor" not in response.headers: + logger.warning("Got unexpected GCP Metadata format") + else: + if not response.headers["Metadata-Flavor"] == "Google": + logger.warning("Got unexpected Metadata flavor: {}".format(response.headers["Metadata-Flavor"])) + else: + logger.warning("On GCP, but metadata response not ok: {}".format(response.status_code)) + except requests.RequestException: + logger.debug("Failed to get response from GCP metadata service: This instance is not on GCP.") + self.on_gcp = False From d52672f4d765d77eb5b602fc99c9287eb4a84e6b Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 19 Jan 2020 16:28:04 +0200 Subject: [PATCH 15/29] Added some documentation --- monkey/common/cloud/gcp/gcp_instance.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/monkey/common/cloud/gcp/gcp_instance.py b/monkey/common/cloud/gcp/gcp_instance.py index 5f04aabd0..26738db43 100644 --- a/monkey/common/cloud/gcp/gcp_instance.py +++ b/monkey/common/cloud/gcp/gcp_instance.py @@ -5,10 +5,15 @@ from common.cloud.environment_names import GCP from common.cloud.instance import CloudInstance logger = logging.getLogger(__name__) + + GCP_METADATA_SERVICE_URL = "http://metadata.google.internal/" class GcpInstance(CloudInstance): + """ + Used to determine if on GCP. See https://cloud.google.com/compute/docs/storing-retrieving-metadata#runninggce + """ def is_instance(self): return self.on_gcp @@ -16,9 +21,6 @@ class GcpInstance(CloudInstance): return GCP def __init__(self): - """ - Determines if on GCP. - """ self.on_gcp = False try: @@ -26,7 +28,7 @@ class GcpInstance(CloudInstance): response = requests.get(GCP_METADATA_SERVICE_URL) if response: - logger.debug("Got response, so probably on GCP. Trying to parse.") + logger.debug("Got ok metadata response: on GCP") self.on_gcp = True if "Metadata-Flavor" not in response.headers: @@ -37,5 +39,5 @@ class GcpInstance(CloudInstance): else: logger.warning("On GCP, but metadata response not ok: {}".format(response.status_code)) except requests.RequestException: - logger.debug("Failed to get response from GCP metadata service: This instance is not on GCP.") + logger.debug("Failed to get response from GCP metadata service: This instance is not on GCP") self.on_gcp = False From 9583956683676c2a559b4c88714bbdbdc669f188 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 19 Jan 2020 18:14:59 +0200 Subject: [PATCH 16/29] Optimised imports and added some documentation --- .../system_info/collectors/environment_collector.py | 5 +---- .../infection_monkey/system_info/system_info_collector.py | 8 ++++++++ .../system_info/system_info_collectors_handler.py | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index c679e04f7..ac5a5433d 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -1,8 +1,5 @@ from common.cloud.all_instances import get_all_cloud_instances -from common.cloud.aws.aws_instance import AwsInstance -from common.cloud.azure.azure_instance import AzureInstance -from common.cloud.environment_names import ON_PREMISE, AZURE, AWS -from common.cloud.instance import CloudInstance +from common.cloud.environment_names import ON_PREMISE from infection_monkey.system_info.system_info_collector import SystemInfoCollector diff --git a/monkey/infection_monkey/system_info/system_info_collector.py b/monkey/infection_monkey/system_info/system_info_collector.py index f65b4c080..c511c1e86 100644 --- a/monkey/infection_monkey/system_info/system_info_collector.py +++ b/monkey/infection_monkey/system_info/system_info_collector.py @@ -6,6 +6,14 @@ import infection_monkey.system_info.collectors class SystemInfoCollector(Plugin, metaclass=ABCMeta): + """ + ABC for system info collection. See system_info_collector_handler for more info. Basically, to implement a new system info + collector, inherit from this class in an implementation in the infection_monkey.system_info.collectors class, and override + the 'collect' method. Don't forget to parse your results in the Monkey Island and to add the collector to the configuration + as well - see monkey_island.cc.services.processing.system_info_collectors for examples. + + See the Wiki page "How to add a new System Info Collector to the Monkey?" for a detailed guide. + """ def __init__(self, name="unknown"): self.name = name diff --git a/monkey/infection_monkey/system_info/system_info_collectors_handler.py b/monkey/infection_monkey/system_info/system_info_collectors_handler.py index 792024b12..cc007ff86 100644 --- a/monkey/infection_monkey/system_info/system_info_collectors_handler.py +++ b/monkey/infection_monkey/system_info/system_info_collectors_handler.py @@ -21,6 +21,7 @@ class SystemInfoCollectorsHandler(object): system_info_telemetry[collector.name] = collected_info successful_collections += 1 except Exception as e: + # If we failed one collector, no need to stop execution. Log and continue. LOG.error("Collector {} failed. Error info: {}".format(collector.name, e)) LOG.info("All system info collectors executed. Total {} executed, out of which {} collected successfully.". format(len(self.collectors_list), successful_collections)) From 3496a78f6c0fdf2b0ee4128da2c14c9a77b72d6e Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 19 Jan 2020 21:36:01 +0200 Subject: [PATCH 17/29] Added generic collector processing functions, a dispatcher (name to function) with unit tests, and moved AWS to collector from regular sysinfo --- monkey/infection_monkey/config.py | 2 +- .../infection_monkey/system_info/__init__.py | 10 --- .../system_info/collectors/aws_collector.py | 30 +++++++++ monkey/monkey_island/cc/server_config.json | 2 +- .../cc/services/config_schema.py | 11 +++- .../telemetry/processing/system_info.py | 11 +--- .../processing/system_info_collectors/aws.py | 15 +++++ .../system_info_collectors/environment.py | 12 ++-- .../system_info_telemetry_dispatcher.py | 36 ++++++++++ .../test_system_info_telemetry_dispatcher.py | 65 +++++++++++++++++++ 10 files changed, 165 insertions(+), 29 deletions(-) create mode 100644 monkey/infection_monkey/system_info/collectors/aws_collector.py create mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/aws.py create mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py create mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index e76ed8101..e1b1ece83 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -125,7 +125,7 @@ class Configuration(object): finger_classes = [] exploiter_classes = [] - system_info_collectors_classes = ["EnvironmentCollector"] + system_info_collectors_classes = ["EnvironmentCollector", "AwsCollector"] # how many victims to look for in a single scan iteration victims_max_find = 100 diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index ccec45cde..fc4aa1caf 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -6,7 +6,6 @@ import psutil from enum import IntEnum from infection_monkey.network.info import get_host_subnets -from infection_monkey.system_info.aws_collector import AwsCollector from infection_monkey.system_info.azure_cred_collector import AzureCollector from infection_monkey.system_info.netstat_collector import NetstatCollector from system_info.system_info_collectors_handler import SystemInfoCollectorsHandler @@ -67,7 +66,6 @@ class InfoCollector(object): self.get_process_list() self.get_network_info() self.get_azure_info() - self.get_aws_info() # Collect all plugins SystemInfoCollectorsHandler().execute_all_configured() @@ -155,11 +153,3 @@ class InfoCollector(object): except Exception: # If we failed to collect azure info, no reason to fail all the collection. Log and continue. LOG.error("Failed collecting Azure info.", exc_info=True) - - def get_aws_info(self): - # noinspection PyBroadException - try: - self.info['aws'] = AwsCollector().get_aws_info() - except Exception: - # If we failed to collect aws info, no reason to fail all the collection. Log and continue. - LOG.error("Failed collecting AWS info.", exc_info=True) diff --git a/monkey/infection_monkey/system_info/collectors/aws_collector.py b/monkey/infection_monkey/system_info/collectors/aws_collector.py new file mode 100644 index 000000000..71f9e58c1 --- /dev/null +++ b/monkey/infection_monkey/system_info/collectors/aws_collector.py @@ -0,0 +1,30 @@ +import logging + +from common.cloud.aws.aws_instance import AwsInstance +from infection_monkey.system_info.system_info_collector import SystemInfoCollector + + +logger = logging.getLogger(__name__) + + +class AwsCollector(SystemInfoCollector): + """ + Extract info from AWS machines. + """ + def __init__(self): + super(AwsCollector, self).__init__(name="AwsCollector") + + def collect(self) -> dict: + logger.info("Collecting AWS info") + aws = AwsInstance() + info = {} + if aws.is_instance(): + logger.info("Machine is an AWS instance") + info = \ + { + 'instance_id': aws.get_instance_id() + } + else: + logger.info("Machine is NOT an AWS instance") + + return info diff --git a/monkey/monkey_island/cc/server_config.json b/monkey/monkey_island/cc/server_config.json index 420f1b303..7bf106194 100644 --- a/monkey/monkey_island/cc/server_config.json +++ b/monkey/monkey_island/cc/server_config.json @@ -1,4 +1,4 @@ { - "server_config": "standard", + "server_config": "testing", "deployment": "develop" } diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index 5de57e26b..d5e015866 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -111,6 +111,14 @@ SCHEMA = { "title": "Which Environment this machine is on (on prem/cloud)", "attack_techniques": [] }, + { + "type": "string", + "enum": [ + "AwsCollector" + ], + "title": "If on AWS, collect more information about the instance", + "attack_techniques": [] + }, ], }, "post_breach_acts": { @@ -455,7 +463,8 @@ SCHEMA = { "$ref": "#/definitions/system_info_collectors_classes" }, "default": [ - "EnvironmentCollector" + "EnvironmentCollector", + "AwsCollector" ], "description": "Determines which system information collectors will collect information." }, diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 04ab27d95..915fa7a25 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -5,6 +5,7 @@ from monkey_island.cc.models import Monkey from monkey_island.cc.services import mimikatz_utils from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import process_aws_telemetry from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence from monkey_island.cc.services.wmi_handler import WMIHandler @@ -18,7 +19,7 @@ def process_system_info_telemetry(telemetry_json): process_ssh_info, process_credential_info, process_mimikatz_and_wmi_info, - process_aws_data, + process_aws_telemetry, update_db_with_new_hostname, test_antivirus_existence, process_environment_telemetry @@ -116,13 +117,5 @@ def process_mimikatz_and_wmi_info(telemetry_json): wmi_handler.process_and_handle_wmi_info() -def process_aws_data(telemetry_json): - if 'aws' in telemetry_json['data']: - if 'instance_id' in telemetry_json['data']['aws']: - monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') - mongo.db.monkey.update_one({'_id': monkey_id}, - {'$set': {'aws_instance_id': telemetry_json['data']['aws']['instance_id']}}) - - def update_db_with_new_hostname(telemetry_json): Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']).set_hostname(telemetry_json['data']['hostname']) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/aws.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/aws.py new file mode 100644 index 000000000..2b4d8085e --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/aws.py @@ -0,0 +1,15 @@ +import logging + +from monkey_island.cc.models.monkey import Monkey + +logger = logging.getLogger(__name__) + + +def process_aws_telemetry(collector_results, monkey_guid): + relevant_monkey = Monkey.get_single_monkey_by_guid(monkey_guid) + + if "instance_id" in collector_results: + instance_id = collector_results["instance_id"] + relevant_monkey.aws_instance_id = instance_id + relevant_monkey.save() + logger.debug("Updated Monkey {} with aws instance id {}".format(str(relevant_monkey), instance_id)) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py index d66019b39..9ddee70ce 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py @@ -5,10 +5,8 @@ from monkey_island.cc.models.monkey import Monkey logger = logging.getLogger(__name__) -def process_environment_telemetry(telemetry_json): - if "EnvironmentCollector" in telemetry_json["data"]["collectors"]: - env = telemetry_json["data"]["collectors"]["EnvironmentCollector"]["environment"] - relevant_monkey = Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']) - relevant_monkey.environment = env - relevant_monkey.save() - logger.debug("Updated Monkey {} with env {}".format(str(relevant_monkey), env)) +def process_environment_telemetry(collector_results, monkey_guid): + relevant_monkey = Monkey.get_single_monkey_by_guid(monkey_guid) + relevant_monkey.environment = collector_results + relevant_monkey.save() + logger.debug("Updated Monkey {} with env {}".format(str(relevant_monkey), collector_results)) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py new file mode 100644 index 000000000..661034efb --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -0,0 +1,36 @@ +import logging + +from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import process_aws_telemetry +from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry + +logger = logging.getLogger(__name__) + +SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSOR = { + "AwsCollector": process_aws_telemetry, + "EnvironmentCollector": process_environment_telemetry, +} + + +class SystemInfoTelemetryDispatcher(object): + def __init__(self, collector_to_parsing_function=None): + if collector_to_parsing_function is None: + collector_to_parsing_function = SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSOR + self.collector_to_parsing_function = collector_to_parsing_function + + def dispatch_to_relevant_collector(self, telemetry_json): + if "collectors" in telemetry_json["data"]: + self.send_each_result_to_relevant_processor(telemetry_json) + + def send_each_result_to_relevant_processor(self, telemetry_json): + relevant_monkey_guid = telemetry_json['monkey_guid'] + for collector_name, collector_results in telemetry_json["data"]["collectors"].items(): + if collector_name in self.collector_to_parsing_function: + # noinspection PyBroadException + try: + self.collector_to_parsing_function[collector_name](collector_results, relevant_monkey_guid) + except Exception as e: + logger.error( + "Error {} while processing {} system info telemetry".format(str(e), collector_name), + exc_info=True) + else: + logger.warning("Unknown system info collector name: {}".format(collector_name)) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py new file mode 100644 index 000000000..db36cd5bb --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py @@ -0,0 +1,65 @@ +from importlib import reload +from unittest import mock +from unittest.mock import MagicMock + +import uuid + +from monkey_island.cc.models import Monkey +from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ + SystemInfoTelemetryDispatcher +from monkey_island.cc.testing.IslandTestCase import IslandTestCase +from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ + process_aws_telemetry + +TEST_SYS_INFO_TO_PROCESSING = { + "AwsCollector": process_aws_telemetry, +} + + +def do_nothing(x, y): + pass + + +class SystemInfoTelemetryDispatcherTest(IslandTestCase): + def test_dispatch_to_relevant_collector_bad_inputs(self): + self.fail_if_not_testing_env() + + dispatcher = SystemInfoTelemetryDispatcher(TEST_SYS_INFO_TO_PROCESSING) + + # Bad format telem JSONs - throws + bad_empty_telem_json = {} + self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collector, bad_empty_telem_json) + bad_no_data_telem_json = {"monkey_guid": "bla"} + self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collector, bad_no_data_telem_json) + bad_no_monkey_telem_json = {"data": {"collectors": {"AwsCollector": "Bla"}}} + self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collector, bad_no_monkey_telem_json) + + # Telem JSON with no collectors - nothing gets dispatched + good_telem_no_collectors = {"monkey_guid": "bla", "data": {"bla": "bla"}} + good_telem_empty_collectors = {"monkey_guid": "bla", "data": {"bla": "bla", "collectors": {}}} + + dispatcher.dispatch_to_relevant_collector(good_telem_no_collectors) + dispatcher.dispatch_to_relevant_collector(good_telem_empty_collectors) + + def test_dispatch_to_relevant_collector(self): + self.fail_if_not_testing_env() + self.clean_monkey_db() + + a_monkey = Monkey(guid=str(uuid.uuid4())) + a_monkey.save() + + dispatcher = SystemInfoTelemetryDispatcher() + + # JSON with results - make sure functions are called + instance_id = "i-0bd2c14bd4c7d703f" + telem_json = { + "data": { + "collectors": { + "AwsCollector": {"instance_id": instance_id}, + } + }, + "monkey_guid": a_monkey.guid + } + dispatcher.dispatch_to_relevant_collector(telem_json) + + self.assertEquals(Monkey.get_single_monkey_by_guid(a_monkey.guid).aws_instance_id, instance_id) From 6815433a85d493778f085c908cbd79f1d4c58f0b Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 19 Jan 2020 21:39:36 +0200 Subject: [PATCH 18/29] Using the dispatcher instead of naming the functions one by one + optimize imports --- .../telemetry/processing/system_info.py | 13 +++++------ .../system_info_telemetry_dispatcher.py | 2 +- .../test_system_info_telemetry_dispatcher.py | 22 ++++++------------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 915fa7a25..6734c6725 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,28 +1,27 @@ import logging -from monkey_island.cc.database import mongo +from monkey_island.cc.encryptor import encryptor from monkey_island.cc.models import Monkey from monkey_island.cc.services import mimikatz_utils -from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.config import ConfigService -from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import process_aws_telemetry -from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry +from monkey_island.cc.services.node import NodeService +from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ + SystemInfoTelemetryDispatcher from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence from monkey_island.cc.services.wmi_handler import WMIHandler -from monkey_island.cc.encryptor import encryptor logger = logging.getLogger(__name__) def process_system_info_telemetry(telemetry_json): + dispatcher = SystemInfoTelemetryDispatcher() telemetry_processing_stages = [ process_ssh_info, process_credential_info, process_mimikatz_and_wmi_info, - process_aws_telemetry, update_db_with_new_hostname, test_antivirus_existence, - process_environment_telemetry + dispatcher.dispatch_to_relevant_collectors ] # Calling safe_process_telemetry so if one of the stages fail, we log and move on instead of failing the rest of diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index 661034efb..64fb146ab 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -17,7 +17,7 @@ class SystemInfoTelemetryDispatcher(object): collector_to_parsing_function = SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSOR self.collector_to_parsing_function = collector_to_parsing_function - def dispatch_to_relevant_collector(self, telemetry_json): + def dispatch_to_relevant_collectors(self, telemetry_json): if "collectors" in telemetry_json["data"]: self.send_each_result_to_relevant_processor(telemetry_json) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py index db36cd5bb..4db5352c8 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py @@ -1,25 +1,17 @@ -from importlib import reload -from unittest import mock -from unittest.mock import MagicMock - import uuid from monkey_island.cc.models import Monkey from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ SystemInfoTelemetryDispatcher -from monkey_island.cc.testing.IslandTestCase import IslandTestCase from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ process_aws_telemetry +from monkey_island.cc.testing.IslandTestCase import IslandTestCase TEST_SYS_INFO_TO_PROCESSING = { "AwsCollector": process_aws_telemetry, } -def do_nothing(x, y): - pass - - class SystemInfoTelemetryDispatcherTest(IslandTestCase): def test_dispatch_to_relevant_collector_bad_inputs(self): self.fail_if_not_testing_env() @@ -28,18 +20,18 @@ class SystemInfoTelemetryDispatcherTest(IslandTestCase): # Bad format telem JSONs - throws bad_empty_telem_json = {} - self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collector, bad_empty_telem_json) + self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collectors, bad_empty_telem_json) bad_no_data_telem_json = {"monkey_guid": "bla"} - self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collector, bad_no_data_telem_json) + self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collectors, bad_no_data_telem_json) bad_no_monkey_telem_json = {"data": {"collectors": {"AwsCollector": "Bla"}}} - self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collector, bad_no_monkey_telem_json) + self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collectors, bad_no_monkey_telem_json) # Telem JSON with no collectors - nothing gets dispatched good_telem_no_collectors = {"monkey_guid": "bla", "data": {"bla": "bla"}} good_telem_empty_collectors = {"monkey_guid": "bla", "data": {"bla": "bla", "collectors": {}}} - dispatcher.dispatch_to_relevant_collector(good_telem_no_collectors) - dispatcher.dispatch_to_relevant_collector(good_telem_empty_collectors) + dispatcher.dispatch_to_relevant_collectors(good_telem_no_collectors) + dispatcher.dispatch_to_relevant_collectors(good_telem_empty_collectors) def test_dispatch_to_relevant_collector(self): self.fail_if_not_testing_env() @@ -60,6 +52,6 @@ class SystemInfoTelemetryDispatcherTest(IslandTestCase): }, "monkey_guid": a_monkey.guid } - dispatcher.dispatch_to_relevant_collector(telem_json) + dispatcher.dispatch_to_relevant_collectors(telem_json) self.assertEquals(Monkey.get_single_monkey_by_guid(a_monkey.guid).aws_instance_id, instance_id) From 2a09d54ed12587eaf6ed6c69397a500d638b49ae Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 19 Jan 2020 21:45:31 +0200 Subject: [PATCH 19/29] Fixed dict bugs + server config --- monkey/monkey_island/cc/server_config.json | 2 +- .../cc/services/telemetry/processing/system_info.py | 3 ++- .../telemetry/processing/system_info_collectors/environment.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/server_config.json b/monkey/monkey_island/cc/server_config.json index 7bf106194..420f1b303 100644 --- a/monkey/monkey_island/cc/server_config.json +++ b/monkey/monkey_island/cc/server_config.json @@ -1,4 +1,4 @@ { - "server_config": "testing", + "server_config": "standard", "deployment": "develop" } diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 6734c6725..d4368469e 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -117,4 +117,5 @@ def process_mimikatz_and_wmi_info(telemetry_json): def update_db_with_new_hostname(telemetry_json): - Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']).set_hostname(telemetry_json['data']['hostname']) + if 'hostname' in telemetry_json['data']: + Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']).set_hostname(telemetry_json['data']['hostname']) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py index 9ddee70ce..4c685a01b 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py @@ -7,6 +7,6 @@ logger = logging.getLogger(__name__) def process_environment_telemetry(collector_results, monkey_guid): relevant_monkey = Monkey.get_single_monkey_by_guid(monkey_guid) - relevant_monkey.environment = collector_results + relevant_monkey.environment = collector_results["environment"] relevant_monkey.save() logger.debug("Updated Monkey {} with env {}".format(str(relevant_monkey), collector_results)) From ed138de8c45673e1016b4be8bfa88386fff6aa7d Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 11:57:19 +0200 Subject: [PATCH 20/29] Deleted the old (unused) aws collector --- .../system_info/aws_collector.py | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 monkey/infection_monkey/system_info/aws_collector.py diff --git a/monkey/infection_monkey/system_info/aws_collector.py b/monkey/infection_monkey/system_info/aws_collector.py deleted file mode 100644 index f39662d13..000000000 --- a/monkey/infection_monkey/system_info/aws_collector.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging - -from common.cloud.aws.aws_instance import AwsInstance - -__author__ = 'itay.mizeretz' - -LOG = logging.getLogger(__name__) - - -class AwsCollector(object): - """ - Extract info from AWS machines - """ - - @staticmethod - def get_aws_info(): - LOG.info("Collecting AWS info") - aws = AwsInstance() - info = {} - if aws.is_instance(): - LOG.info("Machine is an AWS instance") - info = \ - { - 'instance_id': aws.get_instance_id() - } - else: - LOG.info("Machine is NOT an AWS instance") - - return info From 99785236720bfe4c4912832298573b24e8dccd6b Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 15:58:06 +0200 Subject: [PATCH 21/29] Fixed configuration bug - didn't use the same instance of WormConfiguration --- monkey/infection_monkey/config.py | 4 ++-- monkey/infection_monkey/system_info/system_info_collector.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index e1b1ece83..49aa38426 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -1,6 +1,6 @@ import hashlib -import os import json +import os import sys import uuid from abc import ABCMeta @@ -125,7 +125,7 @@ class Configuration(object): finger_classes = [] exploiter_classes = [] - system_info_collectors_classes = ["EnvironmentCollector", "AwsCollector"] + system_info_collectors_classes = [] # how many victims to look for in a single scan iteration victims_max_find = 100 diff --git a/monkey/infection_monkey/system_info/system_info_collector.py b/monkey/infection_monkey/system_info/system_info_collector.py index c511c1e86..8c0b6aa65 100644 --- a/monkey/infection_monkey/system_info/system_info_collector.py +++ b/monkey/infection_monkey/system_info/system_info_collector.py @@ -1,4 +1,4 @@ -from config import WormConfiguration +from infection_monkey.config import WormConfiguration from infection_monkey.utils.plugins.plugin import Plugin from abc import ABCMeta, abstractmethod From d584890dca5c9e8c0f6b99ba6013718e9ba262ce Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 15:58:28 +0200 Subject: [PATCH 22/29] Added hostname collector + moved collector names to common file --- .../common/data/system_info_collectors_names.py | 3 +++ .../system_info/collectors/hostname_collector.py | 16 ++++++++++++++++ .../monkey_island/cc/services/config_schema.py | 13 +++++++++++-- .../services/telemetry/processing/system_info.py | 6 ------ .../system_info_collectors/hostname.py | 9 +++++++++ .../system_info_telemetry_dispatcher.py | 7 +++++-- 6 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 monkey/common/data/system_info_collectors_names.py create mode 100644 monkey/infection_monkey/system_info/collectors/hostname_collector.py create mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/hostname.py diff --git a/monkey/common/data/system_info_collectors_names.py b/monkey/common/data/system_info_collectors_names.py new file mode 100644 index 000000000..8bdf757c7 --- /dev/null +++ b/monkey/common/data/system_info_collectors_names.py @@ -0,0 +1,3 @@ +AWS_COLLECTOR = "AwsCollector" +HOSTNAME_COLLECTOR = "HostnameCollector" +ENVIRONMENT_COLLECTOR = "EnvironmentCollector" diff --git a/monkey/infection_monkey/system_info/collectors/hostname_collector.py b/monkey/infection_monkey/system_info/collectors/hostname_collector.py new file mode 100644 index 000000000..92a522bf9 --- /dev/null +++ b/monkey/infection_monkey/system_info/collectors/hostname_collector.py @@ -0,0 +1,16 @@ +import logging +import socket + +from common.data.system_info_collectors_names import HOSTNAME_COLLECTOR +from infection_monkey.system_info.system_info_collector import SystemInfoCollector + + +logger = logging.getLogger(__name__) + + +class HostnameCollector(SystemInfoCollector): + def __init__(self): + super(HostnameCollector, self).__init__(name=HOSTNAME_COLLECTOR) + + def collect(self) -> dict: + return {"hostname": socket.getfqdn()} diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index d5e015866..86e6225e0 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -108,7 +108,7 @@ SCHEMA = { "enum": [ "EnvironmentCollector" ], - "title": "Which Environment this machine is on (on prem/cloud)", + "title": "Collect which environment this machine is on (on prem/cloud)", "attack_techniques": [] }, { @@ -119,6 +119,14 @@ SCHEMA = { "title": "If on AWS, collect more information about the instance", "attack_techniques": [] }, + { + "type": "string", + "enum": [ + "HostnameCollector" + ], + "title": "Collect the machine's hostname", + "attack_techniques": [] + }, ], }, "post_breach_acts": { @@ -464,7 +472,8 @@ SCHEMA = { }, "default": [ "EnvironmentCollector", - "AwsCollector" + "AwsCollector", + "HostnameCollector" ], "description": "Determines which system information collectors will collect information." }, diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index d4368469e..c490b1d69 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,7 +1,6 @@ import logging from monkey_island.cc.encryptor import encryptor -from monkey_island.cc.models import Monkey from monkey_island.cc.services import mimikatz_utils from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService @@ -19,7 +18,6 @@ def process_system_info_telemetry(telemetry_json): process_ssh_info, process_credential_info, process_mimikatz_and_wmi_info, - update_db_with_new_hostname, test_antivirus_existence, dispatcher.dispatch_to_relevant_collectors ] @@ -115,7 +113,3 @@ def process_mimikatz_and_wmi_info(telemetry_json): wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets) wmi_handler.process_and_handle_wmi_info() - -def update_db_with_new_hostname(telemetry_json): - if 'hostname' in telemetry_json['data']: - Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']).set_hostname(telemetry_json['data']['hostname']) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/hostname.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/hostname.py new file mode 100644 index 000000000..e2de4519c --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/hostname.py @@ -0,0 +1,9 @@ +import logging + +from monkey_island.cc.models.monkey import Monkey + +logger = logging.getLogger(__name__) + + +def process_hostname_telemetry(collector_results, monkey_guid): + Monkey.get_single_monkey_by_guid(monkey_guid).set_hostname(collector_results["hostname"]) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index 64fb146ab..6a3890491 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -1,13 +1,16 @@ import logging +from common.data.system_info_collectors_names import AWS_COLLECTOR, ENVIRONMENT_COLLECTOR, HOSTNAME_COLLECTOR from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import process_aws_telemetry from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry +from monkey_island.cc.services.telemetry.processing.system_info_collectors.hostname import process_hostname_telemetry logger = logging.getLogger(__name__) SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSOR = { - "AwsCollector": process_aws_telemetry, - "EnvironmentCollector": process_environment_telemetry, + AWS_COLLECTOR: process_aws_telemetry, + ENVIRONMENT_COLLECTOR: process_environment_telemetry, + HOSTNAME_COLLECTOR: process_hostname_telemetry, } From 476c6e7a4b69552c4a612aed7f3be6319b680866 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 16:43:25 +0200 Subject: [PATCH 23/29] Deleted hostname old collection, moved to collector --- monkey/infection_monkey/system_info/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index fc4aa1caf..66056828e 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -62,7 +62,6 @@ class InfoCollector(object): def get_info(self): # Collect all hardcoded - self.get_hostname() self.get_process_list() self.get_network_info() self.get_azure_info() @@ -70,14 +69,6 @@ class InfoCollector(object): # Collect all plugins SystemInfoCollectorsHandler().execute_all_configured() - def get_hostname(self): - """ - Adds the fully qualified computer hostname to the system information. - :return: None. Updates class information - """ - LOG.debug("Reading hostname") - self.info['hostname'] = socket.getfqdn() - def get_process_list(self): """ Adds process information from the host to the system information. From f8aff44e8b80331fc46b129231dbd086fc756308 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 16:44:30 +0200 Subject: [PATCH 24/29] Changed dispatcher to use a list of processing functions to support multiple processing functions --- .../system_info_telemetry_dispatcher.py | 54 +++++++++++++------ .../test_system_info_telemetry_dispatcher.py | 14 ++--- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index 6a3890491..e20231d8b 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -1,39 +1,61 @@ import logging +import typing -from common.data.system_info_collectors_names import AWS_COLLECTOR, ENVIRONMENT_COLLECTOR, HOSTNAME_COLLECTOR +from common.data.system_info_collectors_names import * from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import process_aws_telemetry from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry from monkey_island.cc.services.telemetry.processing.system_info_collectors.hostname import process_hostname_telemetry logger = logging.getLogger(__name__) -SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSOR = { - AWS_COLLECTOR: process_aws_telemetry, - ENVIRONMENT_COLLECTOR: process_environment_telemetry, - HOSTNAME_COLLECTOR: process_hostname_telemetry, +SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS = { + AWS_COLLECTOR: [process_aws_telemetry], + ENVIRONMENT_COLLECTOR: [process_environment_telemetry], + HOSTNAME_COLLECTOR: [process_hostname_telemetry], } class SystemInfoTelemetryDispatcher(object): - def __init__(self, collector_to_parsing_function=None): - if collector_to_parsing_function is None: - collector_to_parsing_function = SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSOR - self.collector_to_parsing_function = collector_to_parsing_function + def __init__(self, collector_to_parsing_functions: typing.Mapping[str, typing.List[typing.Callable]] = None): + """ + :param collector_to_parsing_functions: Map between collector names and a list of functions that process the output of + that collector. If `None` is supplied, uses the default one; This should be the normal flow, overriding the + collector->functions mapping is useful mostly for testing. + """ + if collector_to_parsing_functions is None: + collector_to_parsing_functions = SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS + self.collector_to_processing_functions = collector_to_parsing_functions - def dispatch_to_relevant_collectors(self, telemetry_json): + def dispatch_collector_results_to_relevant_processors(self, telemetry_json): + """ + If the telemetry has collectors' results, dispatches the results to the relevant processing functions. + :param telemetry_json: Telemetry sent from the Monkey + """ if "collectors" in telemetry_json["data"]: - self.send_each_result_to_relevant_processor(telemetry_json) + self.dispatch_each_result_to_relevant_processors(telemetry_json) - def send_each_result_to_relevant_processor(self, telemetry_json): + def dispatch_each_result_to_relevant_processors(self, telemetry_json): relevant_monkey_guid = telemetry_json['monkey_guid'] + for collector_name, collector_results in telemetry_json["data"]["collectors"].items(): - if collector_name in self.collector_to_parsing_function: + self.dispatch_result_of_single_collector_to_processing_functions( + collector_name, + collector_results, + relevant_monkey_guid) + + def dispatch_result_of_single_collector_to_processing_functions( + self, + collector_name, + collector_results, + relevant_monkey_guid): + if collector_name in self.collector_to_processing_functions: + for processing_function in self.collector_to_processing_functions[collector_name]: # noinspection PyBroadException try: - self.collector_to_parsing_function[collector_name](collector_results, relevant_monkey_guid) + processing_function(collector_results, relevant_monkey_guid) except Exception as e: logger.error( "Error {} while processing {} system info telemetry".format(str(e), collector_name), exc_info=True) - else: - logger.warning("Unknown system info collector name: {}".format(collector_name)) + else: + logger.warning("Unknown system info collector name: {}".format(collector_name)) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py index 4db5352c8..c5cc7aca2 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py @@ -8,7 +8,7 @@ from monkey_island.cc.services.telemetry.processing.system_info_collectors.syste from monkey_island.cc.testing.IslandTestCase import IslandTestCase TEST_SYS_INFO_TO_PROCESSING = { - "AwsCollector": process_aws_telemetry, + "AwsCollector": [process_aws_telemetry], } @@ -20,18 +20,18 @@ class SystemInfoTelemetryDispatcherTest(IslandTestCase): # Bad format telem JSONs - throws bad_empty_telem_json = {} - self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collectors, bad_empty_telem_json) + self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_empty_telem_json) bad_no_data_telem_json = {"monkey_guid": "bla"} - self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collectors, bad_no_data_telem_json) + self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_no_data_telem_json) bad_no_monkey_telem_json = {"data": {"collectors": {"AwsCollector": "Bla"}}} - self.assertRaises(KeyError, dispatcher.dispatch_to_relevant_collectors, bad_no_monkey_telem_json) + self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_no_monkey_telem_json) # Telem JSON with no collectors - nothing gets dispatched good_telem_no_collectors = {"monkey_guid": "bla", "data": {"bla": "bla"}} good_telem_empty_collectors = {"monkey_guid": "bla", "data": {"bla": "bla", "collectors": {}}} - dispatcher.dispatch_to_relevant_collectors(good_telem_no_collectors) - dispatcher.dispatch_to_relevant_collectors(good_telem_empty_collectors) + dispatcher.dispatch_collector_results_to_relevant_processors(good_telem_no_collectors) + dispatcher.dispatch_collector_results_to_relevant_processors(good_telem_empty_collectors) def test_dispatch_to_relevant_collector(self): self.fail_if_not_testing_env() @@ -52,6 +52,6 @@ class SystemInfoTelemetryDispatcherTest(IslandTestCase): }, "monkey_guid": a_monkey.guid } - dispatcher.dispatch_to_relevant_collectors(telem_json) + dispatcher.dispatch_collector_results_to_relevant_processors(telem_json) self.assertEquals(Monkey.get_single_monkey_by_guid(a_monkey.guid).aws_instance_id, instance_id) From 04b737057540435ba0d429588d77b2d610170fe4 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 17:11:30 +0200 Subject: [PATCH 25/29] Fixed bug in report generation, added lock release for exceptions in report generation --- .../cc/services/reporting/report.py | 5 ++-- .../report_generation_synchronisation.py | 26 ++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 6a44679a4..97e8fa4f1 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -386,10 +386,11 @@ class ReportService: @staticmethod def get_monkey_subnets(monkey_guid): network_info = mongo.db.telemetry.find_one( - {'telem_category': 'system_info', 'monkey_guid': monkey_guid}, + {'telem_category': 'system_info', + 'monkey_guid': monkey_guid}, {'data.network_info.networks': 1} ) - if network_info is None: + if network_info is None or not network_info["data"]: return [] return \ diff --git a/monkey/monkey_island/cc/services/reporting/report_generation_synchronisation.py b/monkey/monkey_island/cc/services/reporting/report_generation_synchronisation.py index 9025ff68f..1a041bb3b 100644 --- a/monkey/monkey_island/cc/services/reporting/report_generation_synchronisation.py +++ b/monkey/monkey_island/cc/services/reporting/report_generation_synchronisation.py @@ -15,28 +15,34 @@ __regular_report_generating_lock = threading.Semaphore() def safe_generate_reports(): # Entering the critical section; Wait until report generation is available. __report_generating_lock.acquire() - report = safe_generate_regular_report() - attack_report = safe_generate_attack_report() - # Leaving the critical section. - __report_generating_lock.release() + try: + report = safe_generate_regular_report() + attack_report = safe_generate_attack_report() + finally: + # Leaving the critical section. + __report_generating_lock.release() return report, attack_report def safe_generate_regular_report(): # Local import to avoid circular imports from monkey_island.cc.services.reporting.report import ReportService - __regular_report_generating_lock.acquire() - report = ReportService.generate_report() - __regular_report_generating_lock.release() + try: + __regular_report_generating_lock.acquire() + report = ReportService.generate_report() + finally: + __regular_report_generating_lock.release() return report def safe_generate_attack_report(): # Local import to avoid circular imports from monkey_island.cc.services.attack.attack_report import AttackReportService - __attack_report_generating_lock.acquire() - attack_report = AttackReportService.generate_new_report() - __attack_report_generating_lock.release() + try: + __attack_report_generating_lock.acquire() + attack_report = AttackReportService.generate_new_report() + finally: + __attack_report_generating_lock.release() return attack_report From 2286571a72030d4497470c3c9611c3ee1e5e9a44 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 20 Jan 2020 17:12:12 +0200 Subject: [PATCH 26/29] Refactored process list collector --- .../data/system_info_collectors_names.py | 1 + .../infection_monkey/system_info/__init__.py | 32 ------------ .../collectors/process_list_collector.py | 50 +++++++++++++++++++ .../cc/services/config_schema.py | 23 ++++++--- .../telemetry/processing/system_info.py | 4 +- .../system_info_telemetry_dispatcher.py | 6 ++- .../zero_trust_tests/antivirus_existence.py | 50 +++++++++---------- 7 files changed, 98 insertions(+), 68 deletions(-) create mode 100644 monkey/infection_monkey/system_info/collectors/process_list_collector.py diff --git a/monkey/common/data/system_info_collectors_names.py b/monkey/common/data/system_info_collectors_names.py index 8bdf757c7..831bbe142 100644 --- a/monkey/common/data/system_info_collectors_names.py +++ b/monkey/common/data/system_info_collectors_names.py @@ -1,3 +1,4 @@ AWS_COLLECTOR = "AwsCollector" HOSTNAME_COLLECTOR = "HostnameCollector" ENVIRONMENT_COLLECTOR = "EnvironmentCollector" +PROCESS_LIST_COLLECTOR = "ProcessListCollector" diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index 66056828e..889b558a1 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -62,44 +62,12 @@ class InfoCollector(object): def get_info(self): # Collect all hardcoded - self.get_process_list() self.get_network_info() self.get_azure_info() # Collect all plugins SystemInfoCollectorsHandler().execute_all_configured() - def get_process_list(self): - """ - Adds process information from the host to the system information. - Currently lists process name, ID, parent ID, command line - and the full image path of each process. - :return: None. Updates class information - """ - LOG.debug("Reading process list") - processes = {} - for process in psutil.process_iter(): - try: - processes[process.pid] = {"name": process.name(), - "pid": process.pid, - "ppid": process.ppid(), - "cmdline": " ".join(process.cmdline()), - "full_image_path": process.exe(), - } - except (psutil.AccessDenied, WindowsError): - # we may be running as non root - # and some processes are impossible to acquire in Windows/Linux - # in this case we'll just add what we can - processes[process.pid] = {"name": "null", - "pid": process.pid, - "ppid": process.ppid(), - "cmdline": "ACCESS DENIED", - "full_image_path": "null", - } - continue - - self.info['process_list'] = processes - def get_network_info(self): """ Adds network information from the host to the system information. diff --git a/monkey/infection_monkey/system_info/collectors/process_list_collector.py b/monkey/infection_monkey/system_info/collectors/process_list_collector.py new file mode 100644 index 000000000..93f836376 --- /dev/null +++ b/monkey/infection_monkey/system_info/collectors/process_list_collector.py @@ -0,0 +1,50 @@ +import logging +import psutil + +from common.data.system_info_collectors_names import PROCESS_LIST_COLLECTOR +from infection_monkey.system_info.system_info_collector import SystemInfoCollector + +logger = logging.getLogger(__name__) + +# Linux doesn't have WindowsError +try: + WindowsError +except NameError: + # noinspection PyShadowingBuiltins + WindowsError = psutil.AccessDenied + + +class ProcessListCollector(SystemInfoCollector): + def __init__(self): + super(ProcessListCollector, self).__init__(name=PROCESS_LIST_COLLECTOR) + + def collect(self) -> dict: + """ + Adds process information from the host to the system information. + Currently lists process name, ID, parent ID, command line + and the full image path of each process. + """ + logger.debug("Reading process list") + processes = {} + for process in psutil.process_iter(): + try: + processes[process.pid] = { + "name": process.name(), + "pid": process.pid, + "ppid": process.ppid(), + "cmdline": " ".join(process.cmdline()), + "full_image_path": process.exe(), + } + except (psutil.AccessDenied, WindowsError): + # we may be running as non root and some processes are impossible to acquire in Windows/Linux. + # In this case we'll just add what we know. + processes[process.pid] = { + "name": "null", + "pid": process.pid, + "ppid": process.ppid(), + "cmdline": "ACCESS DENIED", + "full_image_path": "null", + } + continue + + return {'process_list': processes} diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index 86e6225e0..e7d599cc5 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -1,3 +1,5 @@ +from common.data.system_info_collectors_names import * + WARNING_SIGN = " \u26A0" SCHEMA = { @@ -106,7 +108,7 @@ SCHEMA = { { "type": "string", "enum": [ - "EnvironmentCollector" + ENVIRONMENT_COLLECTOR ], "title": "Collect which environment this machine is on (on prem/cloud)", "attack_techniques": [] @@ -114,7 +116,7 @@ SCHEMA = { { "type": "string", "enum": [ - "AwsCollector" + AWS_COLLECTOR ], "title": "If on AWS, collect more information about the instance", "attack_techniques": [] @@ -122,11 +124,19 @@ SCHEMA = { { "type": "string", "enum": [ - "HostnameCollector" + HOSTNAME_COLLECTOR ], "title": "Collect the machine's hostname", "attack_techniques": [] }, +{ + "type": "string", + "enum": [ + PROCESS_LIST_COLLECTOR + ], + "title": "Collect running processes on the machine", + "attack_techniques": [] + }, ], }, "post_breach_acts": { @@ -471,9 +481,10 @@ SCHEMA = { "$ref": "#/definitions/system_info_collectors_classes" }, "default": [ - "EnvironmentCollector", - "AwsCollector", - "HostnameCollector" + ENVIRONMENT_COLLECTOR, + AWS_COLLECTOR, + HOSTNAME_COLLECTOR, + PROCESS_LIST_COLLECTOR ], "description": "Determines which system information collectors will collect information." }, diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index c490b1d69..b923b6395 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -6,7 +6,6 @@ from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ SystemInfoTelemetryDispatcher -from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence from monkey_island.cc.services.wmi_handler import WMIHandler logger = logging.getLogger(__name__) @@ -18,8 +17,7 @@ def process_system_info_telemetry(telemetry_json): process_ssh_info, process_credential_info, process_mimikatz_and_wmi_info, - test_antivirus_existence, - dispatcher.dispatch_to_relevant_collectors + dispatcher.dispatch_collector_results_to_relevant_processors ] # Calling safe_process_telemetry so if one of the stages fail, we log and move on instead of failing the rest of diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index e20231d8b..d67979e8d 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -5,6 +5,7 @@ from common.data.system_info_collectors_names import * from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import process_aws_telemetry from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import process_environment_telemetry from monkey_island.cc.services.telemetry.processing.system_info_collectors.hostname import process_hostname_telemetry +from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence logger = logging.getLogger(__name__) @@ -12,6 +13,7 @@ SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS = { AWS_COLLECTOR: [process_aws_telemetry], ENVIRONMENT_COLLECTOR: [process_environment_telemetry], HOSTNAME_COLLECTOR: [process_hostname_telemetry], + PROCESS_LIST_COLLECTOR: [test_antivirus_existence] } @@ -32,9 +34,9 @@ class SystemInfoTelemetryDispatcher(object): :param telemetry_json: Telemetry sent from the Monkey """ if "collectors" in telemetry_json["data"]: - self.dispatch_each_result_to_relevant_processors(telemetry_json) + self.dispatch_single_result_to_relevant_processor(telemetry_json) - def dispatch_each_result_to_relevant_processors(self, telemetry_json): + def dispatch_single_result_to_relevant_processor(self, telemetry_json): relevant_monkey_guid = telemetry_json['monkey_guid'] for collector_name, collector_results in telemetry_json["data"]["collectors"].items(): diff --git a/monkey/monkey_island/cc/services/telemetry/zero_trust_tests/antivirus_existence.py b/monkey/monkey_island/cc/services/telemetry/zero_trust_tests/antivirus_existence.py index ddc1af65b..1916291e2 100644 --- a/monkey/monkey_island/cc/services/telemetry/zero_trust_tests/antivirus_existence.py +++ b/monkey/monkey_island/cc/services/telemetry/zero_trust_tests/antivirus_existence.py @@ -7,36 +7,36 @@ from monkey_island.cc.models.zero_trust.event import Event from monkey_island.cc.services.telemetry.zero_trust_tests.known_anti_viruses import ANTI_VIRUS_KNOWN_PROCESS_NAMES -def test_antivirus_existence(telemetry_json): - current_monkey = Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']) - if 'process_list' in telemetry_json['data']: - process_list_event = Event.create_event( - title="Process list", - message="Monkey on {} scanned the process list".format(current_monkey.hostname), - event_type=zero_trust_consts.EVENT_TYPE_MONKEY_LOCAL) - events = [process_list_event] +def test_antivirus_existence(process_list_json, monkey_guid): + current_monkey = Monkey.get_single_monkey_by_guid(monkey_guid) - av_processes = filter_av_processes(telemetry_json) + process_list_event = Event.create_event( + title="Process list", + message="Monkey on {} scanned the process list".format(current_monkey.hostname), + event_type=zero_trust_consts.EVENT_TYPE_MONKEY_LOCAL) + events = [process_list_event] - for process in av_processes: - events.append(Event.create_event( - title="Found AV process", - message="The process '{}' was recognized as an Anti Virus process. Process " - "details: {}".format(process[1]['name'], json.dumps(process[1])), - event_type=zero_trust_consts.EVENT_TYPE_MONKEY_LOCAL - )) + av_processes = filter_av_processes(process_list_json["process_list"]) - if len(av_processes) > 0: - test_status = zero_trust_consts.STATUS_PASSED - else: - test_status = zero_trust_consts.STATUS_FAILED - AggregateFinding.create_or_add_to_existing( - test=zero_trust_consts.TEST_ENDPOINT_SECURITY_EXISTS, status=test_status, events=events - ) + for process in av_processes: + events.append(Event.create_event( + title="Found AV process", + message="The process '{}' was recognized as an Anti Virus process. Process " + "details: {}".format(process[1]['name'], json.dumps(process[1])), + event_type=zero_trust_consts.EVENT_TYPE_MONKEY_LOCAL + )) + + if len(av_processes) > 0: + test_status = zero_trust_consts.STATUS_PASSED + else: + test_status = zero_trust_consts.STATUS_FAILED + AggregateFinding.create_or_add_to_existing( + test=zero_trust_consts.TEST_ENDPOINT_SECURITY_EXISTS, status=test_status, events=events + ) -def filter_av_processes(telemetry_json): - all_processes = list(telemetry_json['data']['process_list'].items()) +def filter_av_processes(process_list): + all_processes = list(process_list.items()) av_processes = [] for process in all_processes: process_name = process[1]['name'] From ab330219d5ec322773abf9021b42a6fa4e4b0aa7 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 21 Jan 2020 15:27:41 +0200 Subject: [PATCH 27/29] Using new style `super` calles --- .../infection_monkey/system_info/collectors/aws_collector.py | 3 ++- .../system_info/collectors/environment_collector.py | 3 ++- .../system_info/collectors/hostname_collector.py | 2 +- .../system_info/collectors/process_list_collector.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/monkey/infection_monkey/system_info/collectors/aws_collector.py b/monkey/infection_monkey/system_info/collectors/aws_collector.py index 71f9e58c1..68d125279 100644 --- a/monkey/infection_monkey/system_info/collectors/aws_collector.py +++ b/monkey/infection_monkey/system_info/collectors/aws_collector.py @@ -1,6 +1,7 @@ import logging from common.cloud.aws.aws_instance import AwsInstance +from common.data.system_info_collectors_names import AWS_COLLECTOR from infection_monkey.system_info.system_info_collector import SystemInfoCollector @@ -12,7 +13,7 @@ class AwsCollector(SystemInfoCollector): Extract info from AWS machines. """ def __init__(self): - super(AwsCollector, self).__init__(name="AwsCollector") + super().__init__(name=AWS_COLLECTOR) def collect(self) -> dict: logger.info("Collecting AWS info") diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index ac5a5433d..5567477d3 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -1,5 +1,6 @@ from common.cloud.all_instances import get_all_cloud_instances from common.cloud.environment_names import ON_PREMISE +from common.data.system_info_collectors_names import ENVIRONMENT_COLLECTOR from infection_monkey.system_info.system_info_collector import SystemInfoCollector @@ -13,7 +14,7 @@ def get_monkey_environment() -> str: class EnvironmentCollector(SystemInfoCollector): def __init__(self): - super(EnvironmentCollector, self).__init__(name="EnvironmentCollector") + super().__init__(name=ENVIRONMENT_COLLECTOR) def collect(self) -> dict: return {"environment": get_monkey_environment()} diff --git a/monkey/infection_monkey/system_info/collectors/hostname_collector.py b/monkey/infection_monkey/system_info/collectors/hostname_collector.py index 92a522bf9..21d03aac7 100644 --- a/monkey/infection_monkey/system_info/collectors/hostname_collector.py +++ b/monkey/infection_monkey/system_info/collectors/hostname_collector.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) class HostnameCollector(SystemInfoCollector): def __init__(self): - super(HostnameCollector, self).__init__(name=HOSTNAME_COLLECTOR) + super().__init__(name=HOSTNAME_COLLECTOR) def collect(self) -> dict: return {"hostname": socket.getfqdn()} diff --git a/monkey/infection_monkey/system_info/collectors/process_list_collector.py b/monkey/infection_monkey/system_info/collectors/process_list_collector.py index 93f836376..c0610cc74 100644 --- a/monkey/infection_monkey/system_info/collectors/process_list_collector.py +++ b/monkey/infection_monkey/system_info/collectors/process_list_collector.py @@ -16,7 +16,7 @@ except NameError: class ProcessListCollector(SystemInfoCollector): def __init__(self): - super(ProcessListCollector, self).__init__(name=PROCESS_LIST_COLLECTOR) + super().__init__(name=PROCESS_LIST_COLLECTOR) def collect(self) -> dict: """ From db5c0f478626f3dc9c3ce1cd5db52d4f44f52f34 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 21 Jan 2020 15:29:46 +0200 Subject: [PATCH 28/29] Changed get_monkey_env logic to return as soon as a results is found and added docs --- .../system_info/collectors/environment_collector.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index 5567477d3..12a58b6d5 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -5,11 +5,15 @@ from infection_monkey.system_info.system_info_collector import SystemInfoCollect def get_monkey_environment() -> str: - env = ON_PREMISE + """ + Get the Monkey's running environment. + :return: One of the cloud providers if on cloud; otherwise, assumes "on premise". + """ for instance in get_all_cloud_instances(): if instance.is_instance(): - env = instance.get_cloud_provider_name() - return env + return instance.get_cloud_provider_name() + + return ON_PREMISE class EnvironmentCollector(SystemInfoCollector): From 6f289915fc404bfc68d1276eaedb8a8f3d443b90 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 21 Jan 2020 16:19:10 +0200 Subject: [PATCH 29/29] Made envs an enum --- monkey/common/cloud/aws/aws_instance.py | 6 ++--- monkey/common/cloud/azure/azure_instance.py | 6 ++--- monkey/common/cloud/environment_names.py | 23 +++++++++++-------- monkey/common/cloud/gcp/gcp_instance.py | 6 ++--- monkey/common/cloud/instance.py | 5 +++- .../collectors/environment_collector.py | 6 ++--- monkey/monkey_island/cc/models/monkey.py | 2 +- 7 files changed, 30 insertions(+), 24 deletions(-) diff --git a/monkey/common/cloud/aws/aws_instance.py b/monkey/common/cloud/aws/aws_instance.py index c77220d17..03c5482ba 100644 --- a/monkey/common/cloud/aws/aws_instance.py +++ b/monkey/common/cloud/aws/aws_instance.py @@ -6,7 +6,7 @@ import logging __author__ = 'itay.mizeretz' -from common.cloud.environment_names import AWS +from common.cloud.environment_names import Environment from common.cloud.instance import CloudInstance AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS = "169.254.169.254" @@ -23,8 +23,8 @@ class AwsInstance(CloudInstance): def is_instance(self): return self.instance_id is not None - def get_cloud_provider_name(self) -> str: - return AWS + def get_cloud_provider_name(self) -> Environment: + return Environment.AWS def __init__(self): self.instance_id = None diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py index f0d5a8044..ec910fb98 100644 --- a/monkey/common/cloud/azure/azure_instance.py +++ b/monkey/common/cloud/azure/azure_instance.py @@ -1,7 +1,7 @@ import logging import requests -from common.cloud.environment_names import AZURE +from common.cloud.environment_names import Environment from common.cloud.instance import CloudInstance LATEST_AZURE_METADATA_API_VERSION = "2019-04-30" @@ -18,8 +18,8 @@ class AzureInstance(CloudInstance): def is_instance(self): return self.on_azure - def get_cloud_provider_name(self) -> str: - return AZURE + def get_cloud_provider_name(self) -> Environment: + return Environment.AZURE def __init__(self): """ diff --git a/monkey/common/cloud/environment_names.py b/monkey/common/cloud/environment_names.py index 0c8655753..945d438ce 100644 --- a/monkey/common/cloud/environment_names.py +++ b/monkey/common/cloud/environment_names.py @@ -1,12 +1,15 @@ -# When adding a new environment to this file, make sure to add it to ALL_ENV_NAMES as well! +from enum import Enum -UNKNOWN = "Unknown" -ON_PREMISE = "On Premise" -AZURE = "Azure" -AWS = "AWS" -GCP = "GCP" -ALIBABA = "Alibaba Cloud" -IBM = "IBM Cloud" -DigitalOcean = "Digital Ocean" -ALL_ENV_NAMES = [UNKNOWN, ON_PREMISE, AZURE, AWS, GCP, ALIBABA, IBM, DigitalOcean] +class Environment(Enum): + UNKNOWN = "Unknown" + ON_PREMISE = "On Premise" + AZURE = "Azure" + AWS = "AWS" + GCP = "GCP" + ALIBABA = "Alibaba Cloud" + IBM = "IBM Cloud" + DigitalOcean = "Digital Ocean" + + +ALL_ENVIRONMENTS_NAMES = [x.value for x in Environment] diff --git a/monkey/common/cloud/gcp/gcp_instance.py b/monkey/common/cloud/gcp/gcp_instance.py index 26738db43..184465bf5 100644 --- a/monkey/common/cloud/gcp/gcp_instance.py +++ b/monkey/common/cloud/gcp/gcp_instance.py @@ -1,7 +1,7 @@ import logging import requests -from common.cloud.environment_names import GCP +from common.cloud.environment_names import Environment from common.cloud.instance import CloudInstance logger = logging.getLogger(__name__) @@ -17,8 +17,8 @@ class GcpInstance(CloudInstance): def is_instance(self): return self.on_gcp - def get_cloud_provider_name(self) -> str: - return GCP + def get_cloud_provider_name(self) -> Environment: + return Environment.GCP def __init__(self): self.on_gcp = False diff --git a/monkey/common/cloud/instance.py b/monkey/common/cloud/instance.py index 61ab4c734..abe0c7910 100644 --- a/monkey/common/cloud/instance.py +++ b/monkey/common/cloud/instance.py @@ -1,3 +1,6 @@ +from common.cloud.environment_names import Environment + + class CloudInstance(object): """ This is an abstract class which represents a cloud instance. @@ -7,5 +10,5 @@ class CloudInstance(object): def is_instance(self) -> bool: raise NotImplementedError() - def get_cloud_provider_name(self) -> str: + def get_cloud_provider_name(self) -> Environment: raise NotImplementedError() diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py index 12a58b6d5..7a953fce9 100644 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ b/monkey/infection_monkey/system_info/collectors/environment_collector.py @@ -1,10 +1,10 @@ from common.cloud.all_instances import get_all_cloud_instances -from common.cloud.environment_names import ON_PREMISE +from common.cloud.environment_names import Environment from common.data.system_info_collectors_names import ENVIRONMENT_COLLECTOR from infection_monkey.system_info.system_info_collector import SystemInfoCollector -def get_monkey_environment() -> str: +def get_monkey_environment() -> Environment: """ Get the Monkey's running environment. :return: One of the cloud providers if on cloud; otherwise, assumes "on premise". @@ -13,7 +13,7 @@ def get_monkey_environment() -> str: if instance.is_instance(): return instance.get_cloud_provider_name() - return ON_PREMISE + return Environment.ON_PREMISE class EnvironmentCollector(SystemInfoCollector): diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index 65dda0850..da6d880b4 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -45,7 +45,7 @@ class Monkey(Document): command_control_channel = EmbeddedDocumentField(CommandControlChannel) # Environment related fields - environment = StringField(default=environment_names.UNKNOWN, choices=environment_names.ALL_ENV_NAMES) + environment = StringField(default=environment_names.Environment.UNKNOWN, choices=environment_names.ALL_ENVIRONMENTS_NAMES) aws_instance_id = StringField(required=False) # This field only exists when the monkey is running on an AWS # instance. See https://github.com/guardicore/monkey/issues/426.