From 5724695181d82e5fecf462b88773022f2ca5836d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 13:18:26 -0500 Subject: [PATCH 01/25] Agent: Fix incorrect import in ControlChannel --- monkey/infection_monkey/master/control_channel.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/master/control_channel.py b/monkey/infection_monkey/master/control_channel.py index 17a2d3287..5fdd03942 100644 --- a/monkey/infection_monkey/master/control_channel.py +++ b/monkey/infection_monkey/master/control_channel.py @@ -6,7 +6,7 @@ import requests from common.common_consts.timeouts import SHORT_REQUEST_TIMEOUT from infection_monkey.config import WormConfiguration from infection_monkey.control import ControlClient -from monkey.infection_monkey.i_control_channel import IControlChannel +from infection_monkey.i_control_channel import IControlChannel requests.packages.urllib3.disable_warnings() @@ -23,8 +23,12 @@ class ControlChannel(IControlChannel): logger.error("Agent should stop because it can't connect to the C&C server.") return True try: + url = ( + f"https://{self._control_channel_server}/api/monkey_control" + f"/needs-to-stop/{self._agent_id}" + ) response = requests.get( # noqa: DUO123 - f"https://{self._control_channel_server}/api/monkey_control/needs-to-stop/{self._agent_id}", + url, verify=False, proxies=ControlClient.proxies, timeout=SHORT_REQUEST_TIMEOUT, From 05adf6bae666af5ba38dfa644c5351c612c5c690 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 13:26:05 -0500 Subject: [PATCH 02/25] Agent: Implement a preliminary propagation thread in AutomatedMaster --- .../master/automated_master.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 9c36dc17d..5ad31408c 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,6 +1,8 @@ import logging import threading import time +from queue import Queue +from threading import Thread from typing import Any, Callable, Dict, List, Tuple from infection_monkey.i_control_channel import IControlChannel @@ -149,6 +151,27 @@ class AutomatedMaster(IMaster): return True def _propagate(self, config: Dict): + logger.info("Attempting to propagate") + + hosts_to_exploit = Queue() + + scan_thread = _create_daemon_thread(target=self._scan_network) + exploit_thread = _create_daemon_thread( + target=self._exploit_targets, args=(hosts_to_exploit, scan_thread) + ) + + scan_thread.start() + exploit_thread.start() + + scan_thread.join() + exploit_thread.join() + + logger.info("Finished attempting to propagate") + + def _scan_network(self): + pass + + def _exploit_targets(self, hosts_to_exploit: Queue, scan_thread: Thread): pass def _run_payload(self, payload: Tuple[str, Dict]): @@ -175,4 +198,4 @@ class AutomatedMaster(IMaster): def _create_daemon_thread(target: Callable[[Any], None], args: Tuple[Any] = ()): - return threading.Thread(target=target, args=args, daemon=True) + return Thread(target=target, args=args, daemon=True) From 7b40996d6af9ed70d4121948ba074c0b63128c9d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 14:15:41 -0500 Subject: [PATCH 03/25] Agent: Implement preliminary network scanning thread --- .../master/automated_master.py | 82 +++++++++++++++++-- 1 file changed, 76 insertions(+), 6 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 5ad31408c..fb97e9978 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,4 +1,5 @@ import logging +import queue import threading import time from queue import Queue @@ -7,15 +8,18 @@ from typing import Any, Callable, Dict, List, Tuple from infection_monkey.i_control_channel import IControlChannel from infection_monkey.i_master import IMaster -from infection_monkey.i_puppet import IPuppet +from infection_monkey.i_puppet import IPuppet, PortStatus +from infection_monkey.model.host import VictimHost from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.post_breach_telem import PostBreachTelem +from infection_monkey.telemetry.scan_telem import ScanTelem from infection_monkey.telemetry.system_info_telem import SystemInfoTelem from infection_monkey.utils.timer import Timer CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC = 5 CHECK_FOR_TERMINATE_INTERVAL_SEC = CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC / 5 SHUTDOWN_TIMEOUT = 5 +NUM_SCAN_THREADS = 16 # TODO: Adjust this to the optimal number of scan threads logger = logging.getLogger() @@ -109,7 +113,7 @@ class AutomatedMaster(IMaster): # requires the output of PBAs, so we don't need to join on that thread here. We will join on # the PBA thread later in this function to prevent the simulation from ending while PBAs are # still running. - system_info_collector_thread.join() + # system_info_collector_thread.join() if self._can_propagate(): propagation_thread = _create_daemon_thread(target=self._propagate, args=(config,)) @@ -155,7 +159,9 @@ class AutomatedMaster(IMaster): hosts_to_exploit = Queue() - scan_thread = _create_daemon_thread(target=self._scan_network) + scan_thread = _create_daemon_thread( + target=self._scan_network, args=(config, hosts_to_exploit) + ) exploit_thread = _create_daemon_thread( target=self._exploit_targets, args=(hosts_to_exploit, scan_thread) ) @@ -168,12 +174,76 @@ class AutomatedMaster(IMaster): logger.info("Finished attempting to propagate") - def _scan_network(self): - pass - def _exploit_targets(self, hosts_to_exploit: Queue, scan_thread: Thread): pass + # TODO: Refactor this into its own class + def _scan_network(self, scan_config: Dict, hosts_to_exploit: Queue): + logger.info("Starting network scan") + + # TODO: Generate list of IPs to scan + ips_to_scan = Queue() + for i in range(1, 255): + ips_to_scan.put(f"10.0.0.{i}") + + scan_threads = [] + for i in range(0, NUM_SCAN_THREADS): + t = _create_daemon_thread( + target=self._scan_ips, args=(ips_to_scan, scan_config, hosts_to_exploit) + ) + t.start() + scan_threads.append(t) + + for t in scan_threads: + t.join() + + logger.info("Finished network scan") + + def _scan_ips(self, ips_to_scan: Queue, scan_config: Dict, hosts_to_exploit: Queue): + logger.debug(f"Starting scan thread -- Thread ID: {threading.get_ident()}") + try: + while not self._stop.is_set(): + ip = ips_to_scan.get_nowait() + logger.info(f"Scanning {ip}") + + victim_host = VictimHost(ip) + + self._ping_ip(ip, victim_host) + + # TODO: get ports from config + ports = [22, 445, 3389, 8008] + self._scan_tcp_ports(ip, ports, victim_host) + + hosts_to_exploit.put(hosts_to_exploit) + self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) + + except queue.Empty: + logger.debug( + f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting" + ) + + logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") + + def _ping_ip(self, ip: str, victim_host: VictimHost): + (response_received, os) = self._puppet.ping(ip) + + victim_host.icmp = response_received + if os is not None: + victim_host.os["type"] = os + + def _scan_tcp_ports(self, ip: str, ports: List[int], victim_host: VictimHost): + for p in ports: + if self._stop.is_set(): + break + + port_scan_data = self._puppet.scan_tcp_port(ip, p) + if port_scan_data.status == PortStatus.OPEN: + victim_host.services[port_scan_data.service] = {} + victim_host.services[port_scan_data.service]["display_name"] = "unknown(TCP)" + victim_host.services[port_scan_data.service]["port"] = port_scan_data.port + if port_scan_data.banner is not None: + victim_host.services[port_scan_data.service]["banner"] = port_scan_data.banner + def _run_payload(self, payload: Tuple[str, Dict]): name = payload[0] options = payload[1] From 56e71f3120545fab794f562981efc1b67b8073e3 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 14:59:35 -0500 Subject: [PATCH 04/25] Agent: Remove PingScanner from fingerprinter list The ping scanner is currently required by the monkey agent in order to determine the OS of the victim. In the future, scanning can be reworked to be more configurable under a variety of different scenarios. For the moment, it's not optional. --- monkey/infection_monkey/example.conf | 1 - .../services/config_schema/definitions/finger_classes.py | 7 ------- monkey/monkey_island/cc/services/config_schema/internal.py | 1 - .../tests/data_for_tests/monkey_configs/flat_config.json | 1 - .../monkey_configs/monkey_config_standard.json | 1 - 5 files changed, 11 deletions(-) diff --git a/monkey/infection_monkey/example.conf b/monkey/infection_monkey/example.conf index dcb3b3138..42b37ddf4 100644 --- a/monkey/infection_monkey/example.conf +++ b/monkey/infection_monkey/example.conf @@ -36,7 +36,6 @@ ], "finger_classes": [ "SSHFinger", - "PingScanner", "HTTPFinger", "SMBFinger", "MySQLFinger", diff --git a/monkey/monkey_island/cc/services/config_schema/definitions/finger_classes.py b/monkey/monkey_island/cc/services/config_schema/definitions/finger_classes.py index 6389f1b13..5daa90672 100644 --- a/monkey/monkey_island/cc/services/config_schema/definitions/finger_classes.py +++ b/monkey/monkey_island/cc/services/config_schema/definitions/finger_classes.py @@ -20,13 +20,6 @@ FINGER_CLASSES = { "info": "Figures out if SSH is running.", "attack_techniques": ["T1210"], }, - { - "type": "string", - "enum": ["PingScanner"], - "title": "Ping Scanner", - "safe": True, - "info": "Tries to identify if host is alive and which OS it's running by ping scan.", - }, { "type": "string", "enum": ["HTTPFinger"], diff --git a/monkey/monkey_island/cc/services/config_schema/internal.py b/monkey/monkey_island/cc/services/config_schema/internal.py index a145233f9..92bacf669 100644 --- a/monkey/monkey_island/cc/services/config_schema/internal.py +++ b/monkey/monkey_island/cc/services/config_schema/internal.py @@ -165,7 +165,6 @@ INTERNAL = { "default": [ "SMBFinger", "SSHFinger", - "PingScanner", "HTTPFinger", "MySQLFinger", "MSSQLFinger", diff --git a/monkey/tests/data_for_tests/monkey_configs/flat_config.json b/monkey/tests/data_for_tests/monkey_configs/flat_config.json index 8f024b9b9..8edb45a86 100644 --- a/monkey/tests/data_for_tests/monkey_configs/flat_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/flat_config.json @@ -65,7 +65,6 @@ "finger_classes": [ "SMBFinger", "SSHFinger", - "PingScanner", "HTTPFinger", "MySQLFinger", "MSSQLFinger", diff --git a/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json b/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json index ba16a75ae..107f17e5c 100644 --- a/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json +++ b/monkey/tests/data_for_tests/monkey_configs/monkey_config_standard.json @@ -100,7 +100,6 @@ "finger_classes": [ "SMBFinger", "SSHFinger", - "PingScanner", "HTTPFinger", "MySQLFinger", "MSSQLFinger", From c497962d9ee61298b39ce59ca9c540360a07c126 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 15:28:10 -0500 Subject: [PATCH 05/25] Island: Reformat network scan parameters before sending to agent --- monkey/monkey_island/cc/services/config.py | 86 ++++++++++++++++++- .../monkey_configs/flat_config.json | 6 +- .../monkey_island/cc/services/test_config.py | 44 ++++++++++ 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 1daec8a76..3bc0a4f16 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -2,7 +2,7 @@ import collections import copy import functools import logging -from typing import Dict +from typing import Dict, List from jsonschema import Draft4Validator, validators @@ -419,6 +419,7 @@ class ConfigService: ConfigService._remove_credentials_from_flat_config(config) ConfigService._format_payloads_from_flat_config(config) ConfigService._format_pbas_from_flat_config(config) + ConfigService._format_network_scan_from_flat_config(config) @staticmethod def _remove_credentials_from_flat_config(config: Dict): @@ -462,3 +463,86 @@ class ConfigService: config.pop(flat_linux_filename_field, None) config.pop(flat_windows_command_field, None) config.pop(flat_windows_filename_field, None) + + @staticmethod + def _format_network_scan_from_flat_config(config: Dict): + formatted_network_scan_config = {"tcp": {}, "icmp": {}, "targets": {}} + + formatted_network_scan_config["tcp"] = ConfigService._format_tcp_scan_from_flat_config( + config + ) + formatted_network_scan_config["icmp"] = ConfigService._format_icmp_scan_from_flat_config( + config + ) + formatted_network_scan_config[ + "targets" + ] = ConfigService._format_scan_targets_from_flat_config(config) + + config["network_scan"] = formatted_network_scan_config + + @staticmethod + def _format_tcp_scan_from_flat_config(config: Dict): + flat_http_ports_field = "HTTP_PORTS" + flat_tcp_timeout_field = "tcp_scan_timeout" + flat_tcp_ports_field = "tcp_target_ports" + + formatted_tcp_scan_config = {} + + formatted_tcp_scan_config["timeout"] = config[flat_tcp_timeout_field] + + ports = ConfigService._union_tcp_and_http_ports( + config[flat_tcp_ports_field], config[flat_http_ports_field] + ) + formatted_tcp_scan_config["ports"] = ports + + # Do not remove HTTP_PORTS field. Other components besides scanning need it. + config.pop(flat_tcp_timeout_field, None) + config.pop(flat_tcp_ports_field, None) + + return formatted_tcp_scan_config + + @staticmethod + def _union_tcp_and_http_ports(tcp_ports: List[int], http_ports: List[int]) -> List[int]: + combined_ports = list(set(tcp_ports) | set(http_ports)) + + return sorted(combined_ports) + + @staticmethod + def _format_icmp_scan_from_flat_config(config: Dict): + flat_ping_timeout_field = "ping_scan_timeout" + + formatted_icmp_scan_config = {} + formatted_icmp_scan_config["timeout"] = config[flat_ping_timeout_field] + + config.pop(flat_ping_timeout_field, None) + + return formatted_icmp_scan_config + + @staticmethod + def _format_scan_targets_from_flat_config(config: Dict): + flat_blocked_ips_field = "blocked_ips" + flat_inaccessible_subnets_field = "inaccessible_subnets" + flat_local_network_scan_field = "local_network_scan" + flat_subnet_scan_list_field = "subnet_scan_list" + + formatted_scan_targets_config = {} + + formatted_scan_targets_config[flat_blocked_ips_field] = config[ + flat_blocked_ips_field + ] + formatted_scan_targets_config[flat_inaccessible_subnets_field] = config[ + flat_inaccessible_subnets_field + ] + formatted_scan_targets_config[flat_local_network_scan_field] = config[ + flat_local_network_scan_field + ] + formatted_scan_targets_config[flat_subnet_scan_list_field] = config[ + flat_subnet_scan_list_field + ] + + config.pop(flat_blocked_ips_field, None) + config.pop(flat_inaccessible_subnets_field, None) + config.pop(flat_local_network_scan_field, None) + config.pop(flat_subnet_scan_list_field, None) + + return formatted_scan_targets_config diff --git a/monkey/tests/data_for_tests/monkey_configs/flat_config.json b/monkey/tests/data_for_tests/monkey_configs/flat_config.json index 8edb45a86..031dfd35a 100644 --- a/monkey/tests/data_for_tests/monkey_configs/flat_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/flat_config.json @@ -13,7 +13,7 @@ "aws_access_key_id": "", "aws_secret_access_key": "", "aws_session_token": "", - "blocked_ips": [], + "blocked_ips": ["192.168.1.1", "192.168.1.100"], "command_servers": [ "10.197.94.72:5000" ], @@ -70,7 +70,7 @@ "MSSQLFinger", "ElasticFinger" ], - "inaccessible_subnets": [], + "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], "keep_tunnel_open_time": 60, "local_network_scan": true, "max_depth": null, @@ -100,7 +100,7 @@ "skip_exploit_if_file_exist": false, "smb_download_timeout": 300, "smb_service_name": "InfectionMonkey", - "subnet_scan_list": [], + "subnet_scan_list": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], "system_info_collector_classes": [ "AwsCollector", "ProcessListCollector", diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index 1aece8180..ec78ad054 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -93,3 +93,47 @@ def test_get_config_propagation_credentials_from_flat_config(flat_monkey_config) creds = ConfigService.get_config_propagation_credentials_from_flat_config(flat_monkey_config) assert creds == expected_creds + + +def test_format_config_for_agent__network_scan(flat_monkey_config): + expected_network_scan_config = { + "tcp": { + "timeout": 3000, + "ports": [ + 22, + 80, + 135, + 443, + 445, + 2222, + 3306, + 3389, + 7001, + 8008, + 8080, + 8088, + 9200, + ], + }, + "icmp": { + "timeout": 1000, + }, + "targets": { + "blocked_ips": ["192.168.1.1", "192.168.1.100"], + "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], + "local_network_scan": True, + "subnet_scan_list": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], + }, + } + ConfigService.format_flat_config_for_agent(flat_monkey_config) + + assert "network_scan" in flat_monkey_config + assert flat_monkey_config["network_scan"] == expected_network_scan_config + + assert "tcp_scan_timeout" not in flat_monkey_config + assert "tcp_target_ports" not in flat_monkey_config + assert "ping_scan_timeout" not in flat_monkey_config + assert "blocked_ips" not in flat_monkey_config + assert "inaccessible_subnets" not in flat_monkey_config + assert "local_network_scan" not in flat_monkey_config + assert "subnet_scan_list" not in flat_monkey_config From 8c47d113c3322783b0e39baa76a2f1c10e78000f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 15:45:46 -0500 Subject: [PATCH 06/25] Agent: Add "options" parameter to IPuppet.ping() --- monkey/infection_monkey/i_puppet.py | 2 +- monkey/infection_monkey/puppet/mock_puppet.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/i_puppet.py b/monkey/infection_monkey/i_puppet.py index d9d225b7b..03ce3999f 100644 --- a/monkey/infection_monkey/i_puppet.py +++ b/monkey/infection_monkey/i_puppet.py @@ -35,7 +35,7 @@ class IPuppet(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def ping(self, host: str) -> Tuple[bool, Optional[str]]: + def ping(self, host: str, options: Dict) -> Tuple[bool, Optional[str]]: """ Sends a ping (ICMP packet) to a remote host :param str host: The domain name or IP address of a host diff --git a/monkey/infection_monkey/puppet/mock_puppet.py b/monkey/infection_monkey/puppet/mock_puppet.py index 3a32f3718..a606e7043 100644 --- a/monkey/infection_monkey/puppet/mock_puppet.py +++ b/monkey/infection_monkey/puppet/mock_puppet.py @@ -155,7 +155,7 @@ class MockPuppet(IPuppet): else: return PostBreachData("pba command 2", ["pba result 2", False]) - def ping(self, host: str) -> Tuple[bool, Optional[str]]: + def ping(self, host: str, options: Dict) -> Tuple[bool, Optional[str]]: logger.debug(f"run_ping({host})") if host == DOT_1: return (True, "windows") From 25410716d3a8f11aaa05c309a980c120cef40e99 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 8 Dec 2021 15:46:12 -0500 Subject: [PATCH 07/25] Agent: Integrate scan configuration with network scanning thread --- .../master/automated_master.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index fb97e9978..3809002aa 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -160,7 +160,7 @@ class AutomatedMaster(IMaster): hosts_to_exploit = Queue() scan_thread = _create_daemon_thread( - target=self._scan_network, args=(config, hosts_to_exploit) + target=self._scan_network, args=(config["network_scan"], hosts_to_exploit) ) exploit_thread = _create_daemon_thread( target=self._exploit_targets, args=(hosts_to_exploit, scan_thread) @@ -208,11 +208,8 @@ class AutomatedMaster(IMaster): victim_host = VictimHost(ip) - self._ping_ip(ip, victim_host) - - # TODO: get ports from config - ports = [22, 445, 3389, 8008] - self._scan_tcp_ports(ip, ports, victim_host) + self._ping_ip(ip, victim_host, scan_config["icmp"]) + self._scan_tcp_ports(ip, victim_host, scan_config["tcp"]) hosts_to_exploit.put(hosts_to_exploit) self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) @@ -224,19 +221,20 @@ class AutomatedMaster(IMaster): logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") - def _ping_ip(self, ip: str, victim_host: VictimHost): - (response_received, os) = self._puppet.ping(ip) + def _ping_ip(self, ip: str, victim_host: VictimHost, options: Dict): + (response_received, os) = self._puppet.ping(ip, options) victim_host.icmp = response_received if os is not None: victim_host.os["type"] = os - def _scan_tcp_ports(self, ip: str, ports: List[int], victim_host: VictimHost): - for p in ports: + def _scan_tcp_ports(self, ip: str, victim_host: VictimHost, options: Dict): + for p in options["ports"]: if self._stop.is_set(): break - port_scan_data = self._puppet.scan_tcp_port(ip, p) + # TODO: check units of timeout + port_scan_data = self._puppet.scan_tcp_port(ip, p, options["timeout"]) if port_scan_data.status == PortStatus.OPEN: victim_host.services[port_scan_data.service] = {} victim_host.services[port_scan_data.service]["display_name"] = "unknown(TCP)" From da8e814b95988f166d4ca513e9755320a9a54466 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 9 Dec 2021 14:29:05 -0500 Subject: [PATCH 08/25] Island: Add units TCP and ICMP timeout option The timeout option for TCP and ICMP scans is in milliseconds. Change "timeout" -> "timeout_ms" to avoid confusion. --- monkey/infection_monkey/master/automated_master.py | 2 +- monkey/monkey_island/cc/services/config.py | 8 +++----- .../unit_tests/monkey_island/cc/services/test_config.py | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 3809002aa..7d8258a6d 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -234,7 +234,7 @@ class AutomatedMaster(IMaster): break # TODO: check units of timeout - port_scan_data = self._puppet.scan_tcp_port(ip, p, options["timeout"]) + port_scan_data = self._puppet.scan_tcp_port(ip, p, options["timeout_ms"]) if port_scan_data.status == PortStatus.OPEN: victim_host.services[port_scan_data.service] = {} victim_host.services[port_scan_data.service]["display_name"] = "unknown(TCP)" diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 3bc0a4f16..3215af091 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -488,7 +488,7 @@ class ConfigService: formatted_tcp_scan_config = {} - formatted_tcp_scan_config["timeout"] = config[flat_tcp_timeout_field] + formatted_tcp_scan_config["timeout_ms"] = config[flat_tcp_timeout_field] ports = ConfigService._union_tcp_and_http_ports( config[flat_tcp_ports_field], config[flat_http_ports_field] @@ -512,7 +512,7 @@ class ConfigService: flat_ping_timeout_field = "ping_scan_timeout" formatted_icmp_scan_config = {} - formatted_icmp_scan_config["timeout"] = config[flat_ping_timeout_field] + formatted_icmp_scan_config["timeout_ms"] = config[flat_ping_timeout_field] config.pop(flat_ping_timeout_field, None) @@ -527,9 +527,7 @@ class ConfigService: formatted_scan_targets_config = {} - formatted_scan_targets_config[flat_blocked_ips_field] = config[ - flat_blocked_ips_field - ] + formatted_scan_targets_config[flat_blocked_ips_field] = config[flat_blocked_ips_field] formatted_scan_targets_config[flat_inaccessible_subnets_field] = config[ flat_inaccessible_subnets_field ] diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index ec78ad054..8537ee233 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -98,7 +98,7 @@ def test_get_config_propagation_credentials_from_flat_config(flat_monkey_config) def test_format_config_for_agent__network_scan(flat_monkey_config): expected_network_scan_config = { "tcp": { - "timeout": 3000, + "timeout_ms": 3000, "ports": [ 22, 80, @@ -116,7 +116,7 @@ def test_format_config_for_agent__network_scan(flat_monkey_config): ], }, "icmp": { - "timeout": 1000, + "timeout_ms": 1000, }, "targets": { "blocked_ips": ["192.168.1.1", "192.168.1.100"], From 86203c81383358fd2d16a238fe5c79500536362d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 9 Dec 2021 14:56:54 -0500 Subject: [PATCH 09/25] Agent: Add AutomatedMaster to master/__init__.py --- monkey/infection_monkey/master/__init__.py | 1 + .../infection_monkey/master/test_automated_master.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/__init__.py b/monkey/infection_monkey/master/__init__.py index e69de29bb..6d3942abd 100644 --- a/monkey/infection_monkey/master/__init__.py +++ b/monkey/infection_monkey/master/__init__.py @@ -0,0 +1 @@ +from .automated_master import AutomatedMaster diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py b/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py index 2ae9ae5d4..1610e752b 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py @@ -1,4 +1,5 @@ -from infection_monkey.master.automated_master import AutomatedMaster +from infection_monkey.master import AutomatedMaster + def test_terminate_without_start(): m = AutomatedMaster(None, None, None) From 3f7dbbccc204400f3b939ef2cea3d5cfb8751b55 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 9 Dec 2021 15:26:35 -0500 Subject: [PATCH 10/25] Agent: Move _create_daemon_thread to threading_utils.py --- .../master/automated_master.py | 24 +++++++++---------- .../master/threading_utils.py | 6 +++++ 2 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 monkey/infection_monkey/master/threading_utils.py diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 7d8258a6d..7a72faca0 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -16,6 +16,8 @@ from infection_monkey.telemetry.scan_telem import ScanTelem from infection_monkey.telemetry.system_info_telem import SystemInfoTelem from infection_monkey.utils.timer import Timer +from .threading_utils import create_daemon_thread + CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC = 5 CHECK_FOR_TERMINATE_INTERVAL_SEC = CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC / 5 SHUTDOWN_TIMEOUT = 5 @@ -36,8 +38,8 @@ class AutomatedMaster(IMaster): self._control_channel = control_channel self._stop = threading.Event() - self._master_thread = _create_daemon_thread(target=self._run_master_thread) - self._simulation_thread = _create_daemon_thread(target=self._run_simulation) + self._master_thread = create_daemon_thread(target=self._run_master_thread) + self._simulation_thread = create_daemon_thread(target=self._run_simulation) def start(self): logger.info("Starting automated breach and attack simulation") @@ -93,7 +95,7 @@ class AutomatedMaster(IMaster): def _run_simulation(self): config = self._control_channel.get_config() - system_info_collector_thread = _create_daemon_thread( + system_info_collector_thread = create_daemon_thread( target=self._run_plugins, args=( config["system_info_collector_classes"], @@ -101,7 +103,7 @@ class AutomatedMaster(IMaster): self._collect_system_info, ), ) - pba_thread = _create_daemon_thread( + pba_thread = create_daemon_thread( target=self._run_plugins, args=(config["post_breach_actions"].items(), "post-breach action", self._run_pba), ) @@ -116,11 +118,11 @@ class AutomatedMaster(IMaster): # system_info_collector_thread.join() if self._can_propagate(): - propagation_thread = _create_daemon_thread(target=self._propagate, args=(config,)) + propagation_thread = create_daemon_thread(target=self._propagate, args=(config,)) propagation_thread.start() propagation_thread.join() - payload_thread = _create_daemon_thread( + payload_thread = create_daemon_thread( target=self._run_plugins, args=(config["payloads"].items(), "payload", self._run_payload), ) @@ -159,10 +161,10 @@ class AutomatedMaster(IMaster): hosts_to_exploit = Queue() - scan_thread = _create_daemon_thread( + scan_thread = create_daemon_thread( target=self._scan_network, args=(config["network_scan"], hosts_to_exploit) ) - exploit_thread = _create_daemon_thread( + exploit_thread = create_daemon_thread( target=self._exploit_targets, args=(hosts_to_exploit, scan_thread) ) @@ -188,7 +190,7 @@ class AutomatedMaster(IMaster): scan_threads = [] for i in range(0, NUM_SCAN_THREADS): - t = _create_daemon_thread( + t = create_daemon_thread( target=self._scan_ips, args=(ips_to_scan, scan_config, hosts_to_exploit) ) t.start() @@ -263,7 +265,3 @@ class AutomatedMaster(IMaster): def cleanup(self): pass - - -def _create_daemon_thread(target: Callable[[Any], None], args: Tuple[Any] = ()): - return Thread(target=target, args=args, daemon=True) diff --git a/monkey/infection_monkey/master/threading_utils.py b/monkey/infection_monkey/master/threading_utils.py new file mode 100644 index 000000000..5c7da9363 --- /dev/null +++ b/monkey/infection_monkey/master/threading_utils.py @@ -0,0 +1,6 @@ +from threading import Thread +from typing import Any, Callable, Tuple + + +def create_daemon_thread(target: Callable[[Any], None], args: Tuple[Any] = ()): + return Thread(target=target, args=args, daemon=True) From 81d4afab5248845f7bdb0c79700b6f873636d85c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 9 Dec 2021 20:59:42 -0500 Subject: [PATCH 11/25] Agent: Extract network scanner into its own class --- monkey/infection_monkey/master/__init__.py | 1 + .../master/automated_master.py | 89 +++-------- monkey/infection_monkey/master/ip_scanner.py | 100 ++++++++++++ .../master/test_network_scanner.py | 146 ++++++++++++++++++ 4 files changed, 270 insertions(+), 66 deletions(-) create mode 100644 monkey/infection_monkey/master/ip_scanner.py create mode 100644 monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py diff --git a/monkey/infection_monkey/master/__init__.py b/monkey/infection_monkey/master/__init__.py index 6d3942abd..bf8e1775c 100644 --- a/monkey/infection_monkey/master/__init__.py +++ b/monkey/infection_monkey/master/__init__.py @@ -1 +1,2 @@ +from .ip_scanner import IPScanner from .automated_master import AutomatedMaster diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 7a72faca0..bc304d2d8 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,5 +1,4 @@ import logging -import queue import threading import time from queue import Queue @@ -8,7 +7,7 @@ from typing import Any, Callable, Dict, List, Tuple from infection_monkey.i_control_channel import IControlChannel from infection_monkey.i_master import IMaster -from infection_monkey.i_puppet import IPuppet, PortStatus +from infection_monkey.i_puppet import IPuppet from infection_monkey.model.host import VictimHost from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.post_breach_telem import PostBreachTelem @@ -16,6 +15,7 @@ from infection_monkey.telemetry.scan_telem import ScanTelem from infection_monkey.telemetry.system_info_telem import SystemInfoTelem from infection_monkey.utils.timer import Timer +from . import IPScanner from .threading_utils import create_daemon_thread CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC = 5 @@ -37,6 +37,9 @@ class AutomatedMaster(IMaster): self._telemetry_messenger = telemetry_messenger self._control_channel = control_channel + self._ip_scanner = IPScanner(self._puppet, NUM_SCAN_THREADS) + self._hosts_to_exploit = None + self._stop = threading.Event() self._master_thread = create_daemon_thread(target=self._run_master_thread) self._simulation_thread = create_daemon_thread(target=self._run_simulation) @@ -156,17 +159,16 @@ class AutomatedMaster(IMaster): def _can_propagate(self): return True + # TODO: Refactor propagation into its own class def _propagate(self, config: Dict): logger.info("Attempting to propagate") - hosts_to_exploit = Queue() + self._hosts_to_exploit = Queue() scan_thread = create_daemon_thread( - target=self._scan_network, args=(config["network_scan"], hosts_to_exploit) - ) - exploit_thread = create_daemon_thread( - target=self._exploit_targets, args=(hosts_to_exploit, scan_thread) + target=self._scan_network, args=(config["network_scan"],) ) + exploit_thread = create_daemon_thread(target=self._exploit_targets, args=(scan_thread,)) scan_thread.start() exploit_thread.start() @@ -176,73 +178,28 @@ class AutomatedMaster(IMaster): logger.info("Finished attempting to propagate") - def _exploit_targets(self, hosts_to_exploit: Queue, scan_thread: Thread): - pass - - # TODO: Refactor this into its own class - def _scan_network(self, scan_config: Dict, hosts_to_exploit: Queue): + def _scan_network(self, scan_config: Dict): logger.info("Starting network scan") # TODO: Generate list of IPs to scan - ips_to_scan = Queue() - for i in range(1, 255): - ips_to_scan.put(f"10.0.0.{i}") + ips_to_scan = [f"10.0.0.{i}" for i in range(1, 255)] - scan_threads = [] - for i in range(0, NUM_SCAN_THREADS): - t = create_daemon_thread( - target=self._scan_ips, args=(ips_to_scan, scan_config, hosts_to_exploit) - ) - t.start() - scan_threads.append(t) - - for t in scan_threads: - t.join() + self._ip_scanner.scan( + ips_to_scan, + scan_config["icmp"], + scan_config["tcp"], + self._handle_scanned_host, + self._stop, + ) logger.info("Finished network scan") - def _scan_ips(self, ips_to_scan: Queue, scan_config: Dict, hosts_to_exploit: Queue): - logger.debug(f"Starting scan thread -- Thread ID: {threading.get_ident()}") - try: - while not self._stop.is_set(): - ip = ips_to_scan.get_nowait() - logger.info(f"Scanning {ip}") + def _handle_scanned_host(self, host: VictimHost): + self._hosts_to_exploit.put(host) + self._telemetry_messenger.send_telemetry(ScanTelem(host)) - victim_host = VictimHost(ip) - - self._ping_ip(ip, victim_host, scan_config["icmp"]) - self._scan_tcp_ports(ip, victim_host, scan_config["tcp"]) - - hosts_to_exploit.put(hosts_to_exploit) - self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) - - except queue.Empty: - logger.debug( - f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting" - ) - - logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") - - def _ping_ip(self, ip: str, victim_host: VictimHost, options: Dict): - (response_received, os) = self._puppet.ping(ip, options) - - victim_host.icmp = response_received - if os is not None: - victim_host.os["type"] = os - - def _scan_tcp_ports(self, ip: str, victim_host: VictimHost, options: Dict): - for p in options["ports"]: - if self._stop.is_set(): - break - - # TODO: check units of timeout - port_scan_data = self._puppet.scan_tcp_port(ip, p, options["timeout_ms"]) - if port_scan_data.status == PortStatus.OPEN: - victim_host.services[port_scan_data.service] = {} - victim_host.services[port_scan_data.service]["display_name"] = "unknown(TCP)" - victim_host.services[port_scan_data.service]["port"] = port_scan_data.port - if port_scan_data.banner is not None: - victim_host.services[port_scan_data.service]["banner"] = port_scan_data.banner + def _exploit_targets(self, scan_thread: Thread): + pass def _run_payload(self, payload: Tuple[str, Dict]): name = payload[0] diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py new file mode 100644 index 000000000..61329ef5d --- /dev/null +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -0,0 +1,100 @@ +import logging +import queue +import threading +from queue import Queue +from threading import Event +from typing import Callable, Dict, List + +from infection_monkey.i_puppet import IPuppet, PortStatus +from infection_monkey.model.host import VictimHost + +from .threading_utils import create_daemon_thread + +logger = logging.getLogger() + +Callback = Callable[[VictimHost], None] + + +class IPScanner: + def __init__(self, puppet: IPuppet, num_workers: int): + self._puppet = puppet + self._num_workers = num_workers + + def scan( + self, + ips: List[str], + icmp_config: Dict, + tcp_config: Dict, + report_results_callback: Callback, + stop: Event, + ): + # Pre-fill a Queue with all IPs so that threads can safely exit when the queue is empty. + ips_to_scan = Queue() + for ip in ips: + ips_to_scan.put(ip) + + scan_ips_args = ( + ips_to_scan, + icmp_config, + tcp_config, + report_results_callback, + stop, + ) + scan_threads = [] + for i in range(0, self._num_workers): + t = create_daemon_thread(target=self._scan_ips, args=scan_ips_args) + t.start() + scan_threads.append(t) + + for t in scan_threads: + t.join() + + def _scan_ips( + self, + ips_to_scan: Queue, + icmp_config: Dict, + tcp_config: Dict, + report_results_callback: Callback, + stop: Event, + ): + logger.debug(f"Starting scan thread -- Thread ID: {threading.get_ident()}") + + try: + while not stop.is_set(): + ip = ips_to_scan.get_nowait() + logger.info(f"Scanning {ip}") + + victim_host = VictimHost(ip) + + self._ping_ip(ip, victim_host, icmp_config) + self._scan_tcp_ports(ip, victim_host, tcp_config, stop) + + report_results_callback(victim_host) + + except queue.Empty: + logger.debug( + f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting" + ) + return + + logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") + + def _ping_ip(self, ip: str, victim_host: VictimHost, options: Dict): + (response_received, os) = self._puppet.ping(ip, options) + + victim_host.icmp = response_received + if os is not None: + victim_host.os["type"] = os + + def _scan_tcp_ports(self, ip: str, victim_host: VictimHost, options: Dict, stop: Event): + for p in options["ports"]: + if stop.is_set(): + break + + port_scan_data = self._puppet.scan_tcp_port(ip, p, options["timeout_ms"]) + if port_scan_data.status == PortStatus.OPEN: + victim_host.services[port_scan_data.service] = {} + victim_host.services[port_scan_data.service]["display_name"] = "unknown(TCP)" + victim_host.services[port_scan_data.service]["port"] = port_scan_data.port + if port_scan_data.banner is not None: + victim_host.services[port_scan_data.service]["banner"] = port_scan_data.banner diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py new file mode 100644 index 000000000..f73b5f39a --- /dev/null +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -0,0 +1,146 @@ +from threading import Barrier, Event +from unittest.mock import MagicMock + +import pytest + +from infection_monkey.i_puppet import PortScanData +from infection_monkey.master import IPScanner +from infection_monkey.puppet.mock_puppet import MockPuppet + +WINDOWS_OS = {"type": "windows"} +LINUX_OS = {"type": "linux"} + + +class MockPuppet(MockPuppet): + def __init__(self): + self.ping = MagicMock(side_effect=super().ping) + self.scan_tcp_port = MagicMock(side_effect=super().scan_tcp_port) + + +@pytest.fixture +def tcp_scan_config(): + return { + "timeout_ms": 3000, + "ports": [ + 22, + 445, + 3389, + 443, + 8008, + 3306, + ], + } + + +@pytest.fixture +def icmp_scan_config(): + return { + "timeout_ms": 1000, + } + + +@pytest.fixture +def stop(): + return Event() + + +@pytest.fixture +def callback(): + return MagicMock() + + +def assert_dot_1(victim_host): + assert victim_host.icmp is True + assert victim_host.os == WINDOWS_OS + + assert len(victim_host.services.keys()) == 2 + assert "tcp-445" in victim_host.services + assert victim_host.services["tcp-445"]["port"] == 445 + assert victim_host.services["tcp-445"]["banner"] == "SMB BANNER" + assert "tcp-3389" in victim_host.services + assert victim_host.services["tcp-3389"]["port"] == 3389 + + +def assert_dot_3(victim_host): + assert victim_host.icmp is True + assert victim_host.os == LINUX_OS + + assert len(victim_host.services.keys()) == 2 + assert "tcp-22" in victim_host.services + assert victim_host.services["tcp-22"]["port"] == 22 + assert victim_host.services["tcp-22"]["banner"] == "SSH BANNER" + + assert "tcp-443" in victim_host.services + assert victim_host.services["tcp-443"]["port"] == 443 + assert victim_host.services["tcp-443"]["banner"] == "HTTPS BANNER" + + +def assert_host_down(victim_host): + assert victim_host.icmp is False + assert len(victim_host.services.keys()) == 0 + + +def test_scan_single_ip(callback, icmp_scan_config, tcp_scan_config, stop): + ips = ["10.0.0.1"] + + ns = IPScanner(MockPuppet(), num_workers=1) + ns.scan(ips, icmp_scan_config, tcp_scan_config, callback, stop) + + callback.assert_called_once() + + assert_dot_1(callback.call_args_list[0][0][0]) + + +def test_scan_multiple_ips(callback, icmp_scan_config, tcp_scan_config, stop): + ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] + + ns = IPScanner(MockPuppet(), num_workers=4) + ns.scan(ips, icmp_scan_config, tcp_scan_config, callback, stop) + + assert callback.call_count == 4 + + assert_dot_1(callback.call_args_list[0][0][0]) + assert_host_down(callback.call_args_list[1][0][0]) + assert_dot_3(callback.call_args_list[2][0][0]) + assert_host_down(callback.call_args_list[3][0][0]) + + +def test_stop_after_callback(icmp_scan_config, tcp_scan_config, stop): + def _callback(_): + # Block all threads here until 2 threads reach this barrier, then set stop + # and test that niether thread continues to scan. + _callback.barrier.wait() + stop.set() + + _callback.barrier = Barrier(2) + + stopable_callback = MagicMock(side_effect=_callback) + + ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] + + ns = IPScanner(MockPuppet(), num_workers=2) + ns.scan(ips, icmp_scan_config, tcp_scan_config, stopable_callback, stop) + + assert stopable_callback.call_count == 2 + + +def test_interrupt_port_scanning(callback, icmp_scan_config, tcp_scan_config, stop): + def stopable_scan_tcp_port(port, _, __): + # Block all threads here until 2 threads reach this barrier, then set stop + # and test that niether thread scans any more ports + stopable_scan_tcp_port.barrier.wait() + stop.set() + + return PortScanData(port, False, None, None) + + stopable_scan_tcp_port.barrier = Barrier(2) + + puppet = MockPuppet() + puppet.scan_tcp_port = MagicMock(side_effect=stopable_scan_tcp_port) + + ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] + + ns = IPScanner(puppet, num_workers=2) + ns.scan(ips, icmp_scan_config, tcp_scan_config, callback, stop) + + assert puppet.scan_tcp_port.call_count == 2 From 80707dac8e4b2e6c79a2a0d118ac4863f2ddf8de Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 09:31:07 -0500 Subject: [PATCH 12/25] Island: Reformat "propagation" config options before sending to Agent --- monkey/monkey_island/cc/services/config.py | 25 ++++++++---- .../monkey_island/cc/services/test_config.py | 40 +++++++++++++------ 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 3215af091..10fbde66d 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -419,7 +419,7 @@ class ConfigService: ConfigService._remove_credentials_from_flat_config(config) ConfigService._format_payloads_from_flat_config(config) ConfigService._format_pbas_from_flat_config(config) - ConfigService._format_network_scan_from_flat_config(config) + ConfigService._format_propagation_from_flat_config(config) @staticmethod def _remove_credentials_from_flat_config(config: Dict): @@ -464,9 +464,23 @@ class ConfigService: config.pop(flat_windows_command_field, None) config.pop(flat_windows_filename_field, None) + @staticmethod + def _format_propagation_from_flat_config(config: Dict): + formatted_propagation_config = {"network_scan": {}, "targets": {}} + + formatted_propagation_config[ + "network_scan" + ] = ConfigService._format_network_scan_from_flat_config(config) + + formatted_propagation_config["targets"] = ConfigService._format_targets_from_flat_config( + config + ) + + config["propagation"] = formatted_propagation_config + @staticmethod def _format_network_scan_from_flat_config(config: Dict): - formatted_network_scan_config = {"tcp": {}, "icmp": {}, "targets": {}} + formatted_network_scan_config = {"tcp": {}, "icmp": {}} formatted_network_scan_config["tcp"] = ConfigService._format_tcp_scan_from_flat_config( config @@ -474,11 +488,8 @@ class ConfigService: formatted_network_scan_config["icmp"] = ConfigService._format_icmp_scan_from_flat_config( config ) - formatted_network_scan_config[ - "targets" - ] = ConfigService._format_scan_targets_from_flat_config(config) - config["network_scan"] = formatted_network_scan_config + return formatted_network_scan_config @staticmethod def _format_tcp_scan_from_flat_config(config: Dict): @@ -519,7 +530,7 @@ class ConfigService: return formatted_icmp_scan_config @staticmethod - def _format_scan_targets_from_flat_config(config: Dict): + def _format_targets_from_flat_config(config: Dict): flat_blocked_ips_field = "blocked_ips" flat_inaccessible_subnets_field = "inaccessible_subnets" flat_local_network_scan_field = "local_network_scan" diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index 8537ee233..c10c77b42 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -95,6 +95,31 @@ def test_get_config_propagation_credentials_from_flat_config(flat_monkey_config) assert creds == expected_creds +def test_format_config_for_agent__propagation(flat_monkey_config): + ConfigService.format_flat_config_for_agent(flat_monkey_config) + + assert "propagation" in flat_monkey_config + assert "network_scan" in flat_monkey_config["propagation"] + assert "targets" in flat_monkey_config["propagation"] + + +def test_format_config_for_agent__propagation_targets(flat_monkey_config): + expected_targets = { + "blocked_ips": ["192.168.1.1", "192.168.1.100"], + "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], + "local_network_scan": True, + "subnet_scan_list": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], + } + + ConfigService.format_flat_config_for_agent(flat_monkey_config) + + assert flat_monkey_config["propagation"]["targets"] == expected_targets + assert "blocked_ips" not in flat_monkey_config + assert "inaccessible_subnets" not in flat_monkey_config + assert "local_network_scan" not in flat_monkey_config + assert "subnet_scan_list" not in flat_monkey_config + + def test_format_config_for_agent__network_scan(flat_monkey_config): expected_network_scan_config = { "tcp": { @@ -118,22 +143,13 @@ def test_format_config_for_agent__network_scan(flat_monkey_config): "icmp": { "timeout_ms": 1000, }, - "targets": { - "blocked_ips": ["192.168.1.1", "192.168.1.100"], - "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], - "local_network_scan": True, - "subnet_scan_list": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], - }, } ConfigService.format_flat_config_for_agent(flat_monkey_config) - assert "network_scan" in flat_monkey_config - assert flat_monkey_config["network_scan"] == expected_network_scan_config + assert "propagation" in flat_monkey_config + assert "network_scan" in flat_monkey_config["propagation"] + assert flat_monkey_config["propagation"]["network_scan"] == expected_network_scan_config assert "tcp_scan_timeout" not in flat_monkey_config assert "tcp_target_ports" not in flat_monkey_config assert "ping_scan_timeout" not in flat_monkey_config - assert "blocked_ips" not in flat_monkey_config - assert "inaccessible_subnets" not in flat_monkey_config - assert "local_network_scan" not in flat_monkey_config - assert "subnet_scan_list" not in flat_monkey_config From 75cfa252c9d287044fdd593e20a3633b97eab969 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 09:32:29 -0500 Subject: [PATCH 13/25] Agent: Modify AutomatedMaster to handle propagation config options --- .../master/automated_master.py | 21 +++---- monkey/infection_monkey/master/ip_scanner.py | 43 ++++---------- .../master/test_network_scanner.py | 57 +++++++++++-------- 3 files changed, 52 insertions(+), 69 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index bc304d2d8..b31c21550 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -121,7 +121,9 @@ class AutomatedMaster(IMaster): # system_info_collector_thread.join() if self._can_propagate(): - propagation_thread = create_daemon_thread(target=self._propagate, args=(config,)) + propagation_thread = create_daemon_thread( + target=self._propagate, args=(config["propagation"],) + ) propagation_thread.start() propagation_thread.join() @@ -160,14 +162,12 @@ class AutomatedMaster(IMaster): return True # TODO: Refactor propagation into its own class - def _propagate(self, config: Dict): + def _propagate(self, propagation_config: Dict): logger.info("Attempting to propagate") self._hosts_to_exploit = Queue() - scan_thread = create_daemon_thread( - target=self._scan_network, args=(config["network_scan"],) - ) + scan_thread = create_daemon_thread(target=self._scan_network, args=(propagation_config,)) exploit_thread = create_daemon_thread(target=self._exploit_targets, args=(scan_thread,)) scan_thread.start() @@ -178,19 +178,14 @@ class AutomatedMaster(IMaster): logger.info("Finished attempting to propagate") - def _scan_network(self, scan_config: Dict): + def _scan_network(self, propagation_config: Dict): logger.info("Starting network scan") # TODO: Generate list of IPs to scan ips_to_scan = [f"10.0.0.{i}" for i in range(1, 255)] - self._ip_scanner.scan( - ips_to_scan, - scan_config["icmp"], - scan_config["tcp"], - self._handle_scanned_host, - self._stop, - ) + scan_config = propagation_config["network_scan"] + self._ip_scanner.scan(ips_to_scan, scan_config, self._handle_scanned_host, self._stop) logger.info("Finished network scan") diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 61329ef5d..4f438ccf3 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -20,26 +20,14 @@ class IPScanner: self._puppet = puppet self._num_workers = num_workers - def scan( - self, - ips: List[str], - icmp_config: Dict, - tcp_config: Dict, - report_results_callback: Callback, - stop: Event, - ): - # Pre-fill a Queue with all IPs so that threads can safely exit when the queue is empty. - ips_to_scan = Queue() - for ip in ips: - ips_to_scan.put(ip) + def scan(self, ips_to_scan: List[str], options: Dict, results_callback: Callback, stop: Event): + # Pre-fill a Queue with all IPs to scan so that threads know they can safely exit when the + # queue is empty. + ips = Queue() + for ip in ips_to_scan: + ips.put(ip) - scan_ips_args = ( - ips_to_scan, - icmp_config, - tcp_config, - report_results_callback, - stop, - ) + scan_ips_args = (ips, options, results_callback, stop) scan_threads = [] for i in range(0, self._num_workers): t = create_daemon_thread(target=self._scan_ips, args=scan_ips_args) @@ -49,27 +37,20 @@ class IPScanner: for t in scan_threads: t.join() - def _scan_ips( - self, - ips_to_scan: Queue, - icmp_config: Dict, - tcp_config: Dict, - report_results_callback: Callback, - stop: Event, - ): + def _scan_ips(self, ips: Queue, options: Dict, results_callback: Callback, stop: Event): logger.debug(f"Starting scan thread -- Thread ID: {threading.get_ident()}") try: while not stop.is_set(): - ip = ips_to_scan.get_nowait() + ip = ips.get_nowait() logger.info(f"Scanning {ip}") victim_host = VictimHost(ip) - self._ping_ip(ip, victim_host, icmp_config) - self._scan_tcp_ports(ip, victim_host, tcp_config, stop) + self._ping_ip(ip, victim_host, options["icmp"]) + self._scan_tcp_ports(ip, victim_host, options["tcp"], stop) - report_results_callback(victim_host) + results_callback(victim_host) except queue.Empty: logger.debug( diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py index f73b5f39a..186d85be1 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -18,24 +18,22 @@ class MockPuppet(MockPuppet): @pytest.fixture -def tcp_scan_config(): +def scan_config(): return { - "timeout_ms": 3000, - "ports": [ - 22, - 445, - 3389, - 443, - 8008, - 3306, - ], - } - - -@pytest.fixture -def icmp_scan_config(): - return { - "timeout_ms": 1000, + "tcp": { + "timeout_ms": 3000, + "ports": [ + 22, + 445, + 3389, + 443, + 8008, + 3306, + ], + }, + "icmp": { + "timeout_ms": 1000, + }, } @@ -80,22 +78,22 @@ def assert_host_down(victim_host): assert len(victim_host.services.keys()) == 0 -def test_scan_single_ip(callback, icmp_scan_config, tcp_scan_config, stop): +def test_scan_single_ip(callback, scan_config, stop): ips = ["10.0.0.1"] ns = IPScanner(MockPuppet(), num_workers=1) - ns.scan(ips, icmp_scan_config, tcp_scan_config, callback, stop) + ns.scan(ips, scan_config, callback, stop) callback.assert_called_once() assert_dot_1(callback.call_args_list[0][0][0]) -def test_scan_multiple_ips(callback, icmp_scan_config, tcp_scan_config, stop): +def test_scan_multiple_ips(callback, scan_config, stop): ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] ns = IPScanner(MockPuppet(), num_workers=4) - ns.scan(ips, icmp_scan_config, tcp_scan_config, callback, stop) + ns.scan(ips, scan_config, callback, stop) assert callback.call_count == 4 @@ -105,7 +103,16 @@ def test_scan_multiple_ips(callback, icmp_scan_config, tcp_scan_config, stop): assert_host_down(callback.call_args_list[3][0][0]) -def test_stop_after_callback(icmp_scan_config, tcp_scan_config, stop): +def test_scan_lots_of_ips(callback, scan_config, stop): + ips = [f"10.0.0.{i}" for i in range(0, 255)] + + ns = IPScanner(MockPuppet(), num_workers=4) + ns.scan(ips, scan_config, callback, stop) + + assert callback.call_count == 255 + + +def test_stop_after_callback(scan_config, stop): def _callback(_): # Block all threads here until 2 threads reach this barrier, then set stop # and test that niether thread continues to scan. @@ -119,12 +126,12 @@ def test_stop_after_callback(icmp_scan_config, tcp_scan_config, stop): ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] ns = IPScanner(MockPuppet(), num_workers=2) - ns.scan(ips, icmp_scan_config, tcp_scan_config, stopable_callback, stop) + ns.scan(ips, scan_config, stopable_callback, stop) assert stopable_callback.call_count == 2 -def test_interrupt_port_scanning(callback, icmp_scan_config, tcp_scan_config, stop): +def test_interrupt_port_scanning(callback, scan_config, stop): def stopable_scan_tcp_port(port, _, __): # Block all threads here until 2 threads reach this barrier, then set stop # and test that niether thread scans any more ports @@ -141,6 +148,6 @@ def test_interrupt_port_scanning(callback, icmp_scan_config, tcp_scan_config, st ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] ns = IPScanner(puppet, num_workers=2) - ns.scan(ips, icmp_scan_config, tcp_scan_config, callback, stop) + ns.scan(ips, scan_config, callback, stop) assert puppet.scan_tcp_port.call_count == 2 From 8d361777bc53f1cfae32c433d04f55f7b9b1f597 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 09:46:13 -0500 Subject: [PATCH 14/25] Agent: Return PingScanData from IPuppet.ping() --- monkey/infection_monkey/i_puppet.py | 5 +++-- monkey/infection_monkey/master/ip_scanner.py | 8 ++++---- monkey/infection_monkey/master/mock_master.py | 8 ++++---- monkey/infection_monkey/puppet/mock_puppet.py | 15 ++++++++------- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/monkey/infection_monkey/i_puppet.py b/monkey/infection_monkey/i_puppet.py index 03ce3999f..49040dd9f 100644 --- a/monkey/infection_monkey/i_puppet.py +++ b/monkey/infection_monkey/i_puppet.py @@ -2,7 +2,7 @@ import abc import threading from collections import namedtuple from enum import Enum -from typing import Dict, Optional, Tuple +from typing import Dict class PortStatus(Enum): @@ -11,6 +11,7 @@ class PortStatus(Enum): ExploiterResultData = namedtuple("ExploiterResultData", ["result", "info", "attempts"]) +PingScanData = namedtuple("PingScanData", ["response_received", "os"]) PortScanData = namedtuple("PortScanData", ["port", "status", "banner", "service"]) PostBreachData = namedtuple("PostBreachData", ["command", "result"]) @@ -35,7 +36,7 @@ class IPuppet(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def ping(self, host: str, options: Dict) -> Tuple[bool, Optional[str]]: + def ping(self, host: str, options: Dict) -> PingScanData: """ Sends a ping (ICMP packet) to a remote host :param str host: The domain name or IP address of a host diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 4f438ccf3..419931064 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -61,11 +61,11 @@ class IPScanner: logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") def _ping_ip(self, ip: str, victim_host: VictimHost, options: Dict): - (response_received, os) = self._puppet.ping(ip, options) + ping_scan_data = self._puppet.ping(ip, options) - victim_host.icmp = response_received - if os is not None: - victim_host.os["type"] = os + victim_host.icmp = ping_scan_data.response_received + if ping_scan_data.os is not None: + victim_host.os["type"] = ping_scan_data.os def _scan_tcp_ports(self, ip: str, victim_host: VictimHost, options: Dict, stop: Event): for p in options["ports"]: diff --git a/monkey/infection_monkey/master/mock_master.py b/monkey/infection_monkey/master/mock_master.py index e78519a43..8c8ecebdd 100644 --- a/monkey/infection_monkey/master/mock_master.py +++ b/monkey/infection_monkey/master/mock_master.py @@ -66,10 +66,10 @@ class MockMaster(IMaster): for ip in ips: h = self._hosts[ip] - (response_received, os) = self._puppet.ping(ip) - h.icmp = response_received - if os is not None: - h.os["type"] = os + ping_scan_data = self._puppet.ping(ip, {}) + h.icmp = ping_scan_data.response_received + if ping_scan_data.os is not None: + h.os["type"] = ping_scan_data.os for p in ports: port_scan_data = self._puppet.scan_tcp_port(ip, p) diff --git a/monkey/infection_monkey/puppet/mock_puppet.py b/monkey/infection_monkey/puppet/mock_puppet.py index a606e7043..a7f7fa324 100644 --- a/monkey/infection_monkey/puppet/mock_puppet.py +++ b/monkey/infection_monkey/puppet/mock_puppet.py @@ -1,10 +1,11 @@ import logging import threading -from typing import Dict, Optional, Tuple +from typing import Dict, Tuple from infection_monkey.i_puppet import ( ExploiterResultData, IPuppet, + PingScanData, PortScanData, PortStatus, PostBreachData, @@ -155,21 +156,21 @@ class MockPuppet(IPuppet): else: return PostBreachData("pba command 2", ["pba result 2", False]) - def ping(self, host: str, options: Dict) -> Tuple[bool, Optional[str]]: + def ping(self, host: str, options: Dict) -> PingScanData: logger.debug(f"run_ping({host})") if host == DOT_1: - return (True, "windows") + return PingScanData(True, "windows") if host == DOT_2: - return (False, None) + return PingScanData(False, None) if host == DOT_3: - return (True, "linux") + return PingScanData(True, "linux") if host == DOT_4: - return (False, None) + return PingScanData(False, None) - return (False, None) + return PingScanData(False, None) def scan_tcp_port(self, host: str, port: int, timeout: int = 3) -> PortScanData: logger.debug(f"run_scan_tcp_port({host}, {port}, {timeout})") From b3c520f2725c37bb2d020fab25f122dcf750a771 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 10:11:49 -0500 Subject: [PATCH 15/25] Agent: Fix incorrect port status in MockPuppet --- monkey/infection_monkey/puppet/mock_puppet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/puppet/mock_puppet.py b/monkey/infection_monkey/puppet/mock_puppet.py index a7f7fa324..de89db172 100644 --- a/monkey/infection_monkey/puppet/mock_puppet.py +++ b/monkey/infection_monkey/puppet/mock_puppet.py @@ -279,4 +279,4 @@ class MockPuppet(IPuppet): def _get_empty_results(port: int): - return PortScanData(port, False, None, None) + return PortScanData(port, PortStatus.CLOSED, None, None) From 037d63c9f33cbd42efdf36364abecdd7e13badaa Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 10:34:19 -0500 Subject: [PATCH 16/25] Agent: Move VictimHost construction to AutomatedMaster --- .../master/automated_master.py | 31 +++++- monkey/infection_monkey/master/ip_scanner.py | 33 ++----- .../master/test_network_scanner.py | 96 ++++++++++++------- 3 files changed, 100 insertions(+), 60 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index b31c21550..721c1a243 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List, Tuple from infection_monkey.i_control_channel import IControlChannel from infection_monkey.i_master import IMaster -from infection_monkey.i_puppet import IPuppet +from infection_monkey.i_puppet import IPuppet, PingScanData, PortScanData, PortStatus from infection_monkey.model.host import VictimHost from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.post_breach_telem import PostBreachTelem @@ -185,13 +185,34 @@ class AutomatedMaster(IMaster): ips_to_scan = [f"10.0.0.{i}" for i in range(1, 255)] scan_config = propagation_config["network_scan"] - self._ip_scanner.scan(ips_to_scan, scan_config, self._handle_scanned_host, self._stop) + self._ip_scanner.scan(ips_to_scan, scan_config, self._process_scan_results, self._stop) logger.info("Finished network scan") - def _handle_scanned_host(self, host: VictimHost): - self._hosts_to_exploit.put(host) - self._telemetry_messenger.send_telemetry(ScanTelem(host)) + def _process_scan_results( + self, ip: str, ping_scan_data: PingScanData, port_scan_data: PortScanData + ): + victim_host = VictimHost(ip) + has_open_port = False + + victim_host.icmp = ping_scan_data.response_received + if ping_scan_data.os is not None: + victim_host.os["type"] = ping_scan_data.os + + for psd in port_scan_data.values(): + if psd.status == PortStatus.OPEN: + has_open_port = True + + victim_host.services[psd.service] = {} + victim_host.services[psd.service]["display_name"] = "unknown(TCP)" + victim_host.services[psd.service]["port"] = psd.port + if psd.banner is not None: + victim_host.services[psd.service]["banner"] = psd.banner + + if has_open_port: + self._hosts_to_exploit.put(victim_host) + + self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) def _exploit_targets(self, scan_thread: Thread): pass diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 419931064..8073abad3 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -5,14 +5,13 @@ from queue import Queue from threading import Event from typing import Callable, Dict, List -from infection_monkey.i_puppet import IPuppet, PortStatus -from infection_monkey.model.host import VictimHost +from infection_monkey.i_puppet import IPuppet, PingScanData, PortScanData from .threading_utils import create_daemon_thread logger = logging.getLogger() -Callback = Callable[[VictimHost], None] +Callback = Callable[[str, PingScanData, Dict[int, PortScanData]], None] class IPScanner: @@ -45,12 +44,10 @@ class IPScanner: ip = ips.get_nowait() logger.info(f"Scanning {ip}") - victim_host = VictimHost(ip) + ping_scan_data = self._puppet.ping(ip, options["icmp"]) + port_scan_data = self._scan_tcp_ports(ip, options["tcp"], stop) - self._ping_ip(ip, victim_host, options["icmp"]) - self._scan_tcp_ports(ip, victim_host, options["tcp"], stop) - - results_callback(victim_host) + results_callback(ip, ping_scan_data, port_scan_data) except queue.Empty: logger.debug( @@ -60,22 +57,12 @@ class IPScanner: logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") - def _ping_ip(self, ip: str, victim_host: VictimHost, options: Dict): - ping_scan_data = self._puppet.ping(ip, options) - - victim_host.icmp = ping_scan_data.response_received - if ping_scan_data.os is not None: - victim_host.os["type"] = ping_scan_data.os - - def _scan_tcp_ports(self, ip: str, victim_host: VictimHost, options: Dict, stop: Event): + def _scan_tcp_ports(self, ip: str, options: Dict, stop: Event): + port_scan_data = {} for p in options["ports"]: if stop.is_set(): break - port_scan_data = self._puppet.scan_tcp_port(ip, p, options["timeout_ms"]) - if port_scan_data.status == PortStatus.OPEN: - victim_host.services[port_scan_data.service] = {} - victim_host.services[port_scan_data.service]["display_name"] = "unknown(TCP)" - victim_host.services[port_scan_data.service]["port"] = port_scan_data.port - if port_scan_data.banner is not None: - victim_host.services[port_scan_data.service]["banner"] = port_scan_data.banner + port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, options["timeout_ms"]) + + return port_scan_data diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py index 186d85be1..078a47593 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -1,14 +1,15 @@ from threading import Barrier, Event +from typing import Set from unittest.mock import MagicMock import pytest -from infection_monkey.i_puppet import PortScanData +from infection_monkey.i_puppet import PortScanData, PortStatus from infection_monkey.master import IPScanner from infection_monkey.puppet.mock_puppet import MockPuppet -WINDOWS_OS = {"type": "windows"} -LINUX_OS = {"type": "linux"} +WINDOWS_OS = "windows" +LINUX_OS = "linux" class MockPuppet(MockPuppet): @@ -47,35 +48,65 @@ def callback(): return MagicMock() -def assert_dot_1(victim_host): - assert victim_host.icmp is True - assert victim_host.os == WINDOWS_OS - - assert len(victim_host.services.keys()) == 2 - assert "tcp-445" in victim_host.services - assert victim_host.services["tcp-445"]["port"] == 445 - assert victim_host.services["tcp-445"]["banner"] == "SMB BANNER" - assert "tcp-3389" in victim_host.services - assert victim_host.services["tcp-3389"]["port"] == 3389 +def assert_port_status(port_scan_data, expected_open_ports: Set[int]): + for psd in port_scan_data.values(): + if psd.port in expected_open_ports: + assert psd.status == PortStatus.OPEN + else: + assert psd.status == PortStatus.CLOSED -def assert_dot_3(victim_host): - assert victim_host.icmp is True - assert victim_host.os == LINUX_OS +def assert_dot_1(ip, ping_scan_data, port_scan_data): + assert ip == "10.0.0.1" - assert len(victim_host.services.keys()) == 2 - assert "tcp-22" in victim_host.services - assert victim_host.services["tcp-22"]["port"] == 22 - assert victim_host.services["tcp-22"]["banner"] == "SSH BANNER" + assert ping_scan_data.response_received is True + assert ping_scan_data.os == WINDOWS_OS - assert "tcp-443" in victim_host.services - assert victim_host.services["tcp-443"]["port"] == 443 - assert victim_host.services["tcp-443"]["banner"] == "HTTPS BANNER" + assert len(port_scan_data.keys()) == 6 + + psd_445 = port_scan_data[445] + psd_3389 = port_scan_data[3389] + + assert psd_445.status == PortStatus.OPEN + assert psd_445.port == 445 + assert psd_445.banner == "SMB BANNER" + assert psd_445.service == "tcp-445" + + assert psd_3389.status == PortStatus.OPEN + assert psd_3389.port == 3389 + assert psd_3389.banner == "" + assert psd_3389.service == "tcp-3389" + + assert_port_status(port_scan_data, {445, 3389}) -def assert_host_down(victim_host): - assert victim_host.icmp is False - assert len(victim_host.services.keys()) == 0 +def assert_dot_3(ip, ping_scan_data, port_scan_data): + assert ip == "10.0.0.3" + + assert ping_scan_data.response_received is True + assert ping_scan_data.os == LINUX_OS + assert len(port_scan_data.keys()) == 6 + + psd_443 = port_scan_data[443] + psd_22 = port_scan_data[22] + + assert psd_443.port == 443 + assert psd_443.banner == "HTTPS BANNER" + assert psd_443.service == "tcp-443" + + assert psd_22.port == 22 + assert psd_22.banner == "SSH BANNER" + assert psd_22.service == "tcp-22" + + assert_port_status(port_scan_data, {22, 443}) + + +def assert_host_down(ip, ping_scan_data, port_scan_data): + assert ip not in {"10.0.0.1", "10.0.0.3"} + + assert ping_scan_data.response_received is False + assert len(port_scan_data.keys()) == 6 + assert_port_status(port_scan_data, {}) def test_scan_single_ip(callback, scan_config, stop): @@ -86,7 +117,8 @@ def test_scan_single_ip(callback, scan_config, stop): callback.assert_called_once() - assert_dot_1(callback.call_args_list[0][0][0]) + print(type(callback.call_args_list[0][0])) + assert_dot_1(*(callback.call_args_list[0][0])) def test_scan_multiple_ips(callback, scan_config, stop): @@ -97,10 +129,10 @@ def test_scan_multiple_ips(callback, scan_config, stop): assert callback.call_count == 4 - assert_dot_1(callback.call_args_list[0][0][0]) - assert_host_down(callback.call_args_list[1][0][0]) - assert_dot_3(callback.call_args_list[2][0][0]) - assert_host_down(callback.call_args_list[3][0][0]) + assert_dot_1(*(callback.call_args_list[0][0])) + assert_host_down(*(callback.call_args_list[1][0])) + assert_dot_3(*(callback.call_args_list[2][0])) + assert_host_down(*(callback.call_args_list[3][0])) def test_scan_lots_of_ips(callback, scan_config, stop): @@ -113,7 +145,7 @@ def test_scan_lots_of_ips(callback, scan_config, stop): def test_stop_after_callback(scan_config, stop): - def _callback(_): + def _callback(_, __, ___): # Block all threads here until 2 threads reach this barrier, then set stop # and test that niether thread continues to scan. _callback.barrier.wait() From 8091a0c4a52f40a1c7a48213e1c22648392bf5e8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 10:41:02 -0500 Subject: [PATCH 17/25] Agent: Join on system info collector thread This was mistakenly commented out somewhere along the way. --- monkey/infection_monkey/master/automated_master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 721c1a243..9863b47d2 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -118,7 +118,7 @@ class AutomatedMaster(IMaster): # requires the output of PBAs, so we don't need to join on that thread here. We will join on # the PBA thread later in this function to prevent the simulation from ending while PBAs are # still running. - # system_info_collector_thread.join() + system_info_collector_thread.join() if self._can_propagate(): propagation_thread = create_daemon_thread( From abec851ed0b9c945f5924077d63d809afcc3c7db Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 11:45:20 -0500 Subject: [PATCH 18/25] Agent: Make minor code cleanliness changes --- monkey/infection_monkey/master/ip_scanner.py | 7 ++++--- .../infection_monkey/master/test_network_scanner.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 8073abad3..3e469ee9c 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -49,13 +49,14 @@ class IPScanner: results_callback(ip, ping_scan_data, port_scan_data) + logger.debug( + f"Detected the stop signal, scanning thread {threading.get_ident()} exiting" + ) + except queue.Empty: logger.debug( f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting" ) - return - - logger.debug(f"Detected the stop signal, scanning thread {threading.get_ident()} exiting") def _scan_tcp_ports(self, ip: str, options: Dict, stop: Event): port_scan_data = {} diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py index 078a47593..1ace7f67f 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -145,7 +145,7 @@ def test_scan_lots_of_ips(callback, scan_config, stop): def test_stop_after_callback(scan_config, stop): - def _callback(_, __, ___): + def _callback(*_): # Block all threads here until 2 threads reach this barrier, then set stop # and test that niether thread continues to scan. _callback.barrier.wait() @@ -164,7 +164,7 @@ def test_stop_after_callback(scan_config, stop): def test_interrupt_port_scanning(callback, scan_config, stop): - def stopable_scan_tcp_port(port, _, __): + def stopable_scan_tcp_port(port, *_): # Block all threads here until 2 threads reach this barrier, then set stop # and test that niether thread scans any more ports stopable_scan_tcp_port.barrier.wait() From 6147d635d62b5fb9c6b5043d1ada139e64156002 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 10 Dec 2021 12:48:45 -0500 Subject: [PATCH 19/25] Agent: Extract propagation logic into Propagator class --- monkey/infection_monkey/master/__init__.py | 1 + .../master/automated_master.py | 74 ++--------------- monkey/infection_monkey/master/propagator.py | 80 ++++++++++++++++++ .../master/test_propagator.py | 82 +++++++++++++++++++ 4 files changed, 168 insertions(+), 69 deletions(-) create mode 100644 monkey/infection_monkey/master/propagator.py create mode 100644 monkey/tests/unit_tests/infection_monkey/master/test_propagator.py diff --git a/monkey/infection_monkey/master/__init__.py b/monkey/infection_monkey/master/__init__.py index bf8e1775c..21ef8f9b6 100644 --- a/monkey/infection_monkey/master/__init__.py +++ b/monkey/infection_monkey/master/__init__.py @@ -1,2 +1,3 @@ from .ip_scanner import IPScanner +from .propagator import Propagator from .automated_master import AutomatedMaster diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 9863b47d2..784046323 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,21 +1,17 @@ import logging import threading import time -from queue import Queue -from threading import Thread from typing import Any, Callable, Dict, List, Tuple from infection_monkey.i_control_channel import IControlChannel from infection_monkey.i_master import IMaster -from infection_monkey.i_puppet import IPuppet, PingScanData, PortScanData, PortStatus -from infection_monkey.model.host import VictimHost +from infection_monkey.i_puppet import IPuppet from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.post_breach_telem import PostBreachTelem -from infection_monkey.telemetry.scan_telem import ScanTelem from infection_monkey.telemetry.system_info_telem import SystemInfoTelem from infection_monkey.utils.timer import Timer -from . import IPScanner +from . import IPScanner, Propagator from .threading_utils import create_daemon_thread CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC = 5 @@ -37,8 +33,8 @@ class AutomatedMaster(IMaster): self._telemetry_messenger = telemetry_messenger self._control_channel = control_channel - self._ip_scanner = IPScanner(self._puppet, NUM_SCAN_THREADS) - self._hosts_to_exploit = None + ip_scanner = IPScanner(self._puppet, NUM_SCAN_THREADS) + self._propagator = Propagator(self._telemetry_messenger, ip_scanner) self._stop = threading.Event() self._master_thread = create_daemon_thread(target=self._run_master_thread) @@ -121,11 +117,7 @@ class AutomatedMaster(IMaster): system_info_collector_thread.join() if self._can_propagate(): - propagation_thread = create_daemon_thread( - target=self._propagate, args=(config["propagation"],) - ) - propagation_thread.start() - propagation_thread.join() + self._propagator.propagate(config["propagation"], self._stop) payload_thread = create_daemon_thread( target=self._run_plugins, @@ -161,62 +153,6 @@ class AutomatedMaster(IMaster): def _can_propagate(self): return True - # TODO: Refactor propagation into its own class - def _propagate(self, propagation_config: Dict): - logger.info("Attempting to propagate") - - self._hosts_to_exploit = Queue() - - scan_thread = create_daemon_thread(target=self._scan_network, args=(propagation_config,)) - exploit_thread = create_daemon_thread(target=self._exploit_targets, args=(scan_thread,)) - - scan_thread.start() - exploit_thread.start() - - scan_thread.join() - exploit_thread.join() - - logger.info("Finished attempting to propagate") - - def _scan_network(self, propagation_config: Dict): - logger.info("Starting network scan") - - # TODO: Generate list of IPs to scan - ips_to_scan = [f"10.0.0.{i}" for i in range(1, 255)] - - scan_config = propagation_config["network_scan"] - self._ip_scanner.scan(ips_to_scan, scan_config, self._process_scan_results, self._stop) - - logger.info("Finished network scan") - - def _process_scan_results( - self, ip: str, ping_scan_data: PingScanData, port_scan_data: PortScanData - ): - victim_host = VictimHost(ip) - has_open_port = False - - victim_host.icmp = ping_scan_data.response_received - if ping_scan_data.os is not None: - victim_host.os["type"] = ping_scan_data.os - - for psd in port_scan_data.values(): - if psd.status == PortStatus.OPEN: - has_open_port = True - - victim_host.services[psd.service] = {} - victim_host.services[psd.service]["display_name"] = "unknown(TCP)" - victim_host.services[psd.service]["port"] = psd.port - if psd.banner is not None: - victim_host.services[psd.service]["banner"] = psd.banner - - if has_open_port: - self._hosts_to_exploit.put(victim_host) - - self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) - - def _exploit_targets(self, scan_thread: Thread): - pass - def _run_payload(self, payload: Tuple[str, Dict]): name = payload[0] options = payload[1] diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py new file mode 100644 index 000000000..ba0f5dccd --- /dev/null +++ b/monkey/infection_monkey/master/propagator.py @@ -0,0 +1,80 @@ +import logging +from queue import Queue +from threading import Event, Thread +from typing import Dict + +from infection_monkey.i_puppet import PingScanData, PortScanData, PortStatus +from infection_monkey.model.host import VictimHost +from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger +from infection_monkey.telemetry.scan_telem import ScanTelem + +from . import IPScanner +from .threading_utils import create_daemon_thread + +logger = logging.getLogger() + + +class Propagator: + def __init__(self, telemetry_messenger: ITelemetryMessenger, ip_scanner: IPScanner): + self._telemetry_messenger = telemetry_messenger + self._ip_scanner = ip_scanner + self._hosts_to_exploit = None + + def propagate(self, propagation_config: Dict, stop: Event): + logger.info("Attempting to propagate") + + self._hosts_to_exploit = Queue() + + scan_thread = create_daemon_thread( + target=self._scan_network, args=(propagation_config, stop) + ) + exploit_thread = create_daemon_thread( + target=self._exploit_targets, args=(scan_thread, stop) + ) + + scan_thread.start() + exploit_thread.start() + + scan_thread.join() + exploit_thread.join() + + logger.info("Finished attempting to propagate") + + def _scan_network(self, propagation_config: Dict, stop: Event): + logger.info("Starting network scan") + + # TODO: Generate list of IPs to scan from propagation targets config + ips_to_scan = propagation_config["targets"]["subnet_scan_list"] + + scan_config = propagation_config["network_scan"] + self._ip_scanner.scan(ips_to_scan, scan_config, self._process_scan_results, stop) + + logger.info("Finished network scan") + + def _process_scan_results( + self, ip: str, ping_scan_data: PingScanData, port_scan_data: PortScanData + ): + victim_host = VictimHost(ip) + has_open_port = False + + victim_host.icmp = ping_scan_data.response_received + if ping_scan_data.os is not None: + victim_host.os["type"] = ping_scan_data.os + + for psd in port_scan_data.values(): + if psd.status == PortStatus.OPEN: + has_open_port = True + + victim_host.services[psd.service] = {} + victim_host.services[psd.service]["display_name"] = "unknown(TCP)" + victim_host.services[psd.service]["port"] = psd.port + if psd.banner is not None: + victim_host.services[psd.service]["banner"] = psd.banner + + if has_open_port: + self._hosts_to_exploit.put(victim_host) + + self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) + + def _exploit_targets(self, scan_thread: Thread, stop: Event): + pass diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py new file mode 100644 index 000000000..b5c97760b --- /dev/null +++ b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py @@ -0,0 +1,82 @@ +from threading import Event + +from infection_monkey.i_puppet import PingScanData, PortScanData, PortStatus +from infection_monkey.master import Propagator + +dot_1_results = ( + PingScanData(True, "windows"), + { + 22: PortScanData(22, PortStatus.CLOSED, None, None), + 445: PortScanData(445, PortStatus.OPEN, "SMB BANNER", "tcp-445"), + 3389: PortScanData(3389, PortStatus.OPEN, "", "tcp-3389"), + }, +) + +dot_3_results = ( + PingScanData(True, "linux"), + { + 22: PortScanData(22, PortStatus.OPEN, "SSH BANNER", "tcp-22"), + 443: PortScanData(443, PortStatus.OPEN, "HTTPS BANNER", "tcp-443"), + 3389: PortScanData(3389, PortStatus.CLOSED, "", None), + }, +) + +dead_host_results = ( + PingScanData(False, None), + { + 22: PortScanData(22, PortStatus.CLOSED, None, None), + 443: PortScanData(443, PortStatus.CLOSED, None, None), + 3389: PortScanData(3389, PortStatus.CLOSED, "", None), + }, +) + +dot_1_services = { + "tcp-445": {"display_name": "unknown(TCP)", "port": 445, "banner": "SMB BANNER"}, + "tcp-3389": {"display_name": "unknown(TCP)", "port": 3389, "banner": ""}, +} + +dot_3_services = { + "tcp-22": {"display_name": "unknown(TCP)", "port": 22, "banner": "SSH BANNER"}, + "tcp-443": {"display_name": "unknown(TCP)", "port": 443, "banner": "HTTPS BANNER"}, +} + + +class MockIPScanner: + def scan(self, ips_to_scan, options, results_callback, stop): + for ip in ips_to_scan: + if ip.endswith(".1"): + results_callback(ip, *dot_1_results) + elif ip.endswith(".3"): + results_callback(ip, *dot_3_results) + else: + results_callback(ip, *dead_host_results) + + +def test_scan_result_processing(telemetry_messenger_spy): + p = Propagator(telemetry_messenger_spy, MockIPScanner()) + p.propagate( + {"targets": {"subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"]}, "network_scan": {}}, + Event(), + ) + + assert len(telemetry_messenger_spy.telemetries) == 3 + + for t in telemetry_messenger_spy.telemetries: + data = t.get_data() + ip = data["machine"]["ip_addr"] + + if ip.endswith(".1"): + assert data["service_count"] == 2 + assert data["machine"]["os"]["type"] == "windows" + assert data["machine"]["services"] == dot_1_services + assert data["machine"]["icmp"] is True + elif ip.endswith(".3"): + assert data["service_count"] == 2 + assert data["machine"]["os"]["type"] == "linux" + assert data["machine"]["services"] == dot_3_services + assert data["machine"]["icmp"] is True + else: + assert data["service_count"] == 0 + assert data["machine"]["os"] == {} + assert data["machine"]["services"] == {} + assert data["machine"]["icmp"] is False From 88608c1cf1de5c02b26b28c7c1c3716486ca6a10 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 07:05:10 -0500 Subject: [PATCH 20/25] Agent: Fix some type hints in automated master --- monkey/infection_monkey/master/propagator.py | 2 +- monkey/infection_monkey/master/threading_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index ba0f5dccd..da36ce5b9 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -52,7 +52,7 @@ class Propagator: logger.info("Finished network scan") def _process_scan_results( - self, ip: str, ping_scan_data: PingScanData, port_scan_data: PortScanData + self, ip: str, ping_scan_data: PingScanData, port_scan_data: Dict[int, PortScanData] ): victim_host = VictimHost(ip) has_open_port = False diff --git a/monkey/infection_monkey/master/threading_utils.py b/monkey/infection_monkey/master/threading_utils.py index 5c7da9363..56cf4a459 100644 --- a/monkey/infection_monkey/master/threading_utils.py +++ b/monkey/infection_monkey/master/threading_utils.py @@ -1,6 +1,6 @@ from threading import Thread -from typing import Any, Callable, Tuple +from typing import Callable, Tuple -def create_daemon_thread(target: Callable[[Any], None], args: Tuple[Any] = ()): +def create_daemon_thread(target: Callable[..., None], args: Tuple = ()): return Thread(target=target, args=args, daemon=True) From 11e3c5d6e4cc06e20ac0c8f91123db6fa9458e22 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 07:07:43 -0500 Subject: [PATCH 21/25] UT: Remove superfluous Asserts in test_network_scanner.py --- .../unit_tests/infection_monkey/master/test_network_scanner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py index 1ace7f67f..9447bdfc1 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -67,12 +67,10 @@ def assert_dot_1(ip, ping_scan_data, port_scan_data): psd_445 = port_scan_data[445] psd_3389 = port_scan_data[3389] - assert psd_445.status == PortStatus.OPEN assert psd_445.port == 445 assert psd_445.banner == "SMB BANNER" assert psd_445.service == "tcp-445" - assert psd_3389.status == PortStatus.OPEN assert psd_3389.port == 3389 assert psd_3389.banner == "" assert psd_3389.service == "tcp-3389" From 5a1e19391dbda99e3c6c6b7c33132fe56fef863b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 07:44:05 -0500 Subject: [PATCH 22/25] Agent: Make tcp/ping timeouts consistent * Ping takes a `timeout: float` instead of `options: Dict` the same way that `scan_tcp_port()` does. * Timeouts are floats instead of ints --- monkey/infection_monkey/i_puppet.py | 12 ++++++------ monkey/infection_monkey/master/ip_scanner.py | 7 +++++-- monkey/infection_monkey/master/mock_master.py | 2 +- monkey/infection_monkey/puppet/mock_puppet.py | 4 ++-- .../infection_monkey/master/test_network_scanner.py | 6 ------ 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/monkey/infection_monkey/i_puppet.py b/monkey/infection_monkey/i_puppet.py index 49040dd9f..f158be08c 100644 --- a/monkey/infection_monkey/i_puppet.py +++ b/monkey/infection_monkey/i_puppet.py @@ -36,22 +36,22 @@ class IPuppet(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def ping(self, host: str, options: Dict) -> PingScanData: + def ping(self, host: str, timeout: float) -> PingScanData: """ Sends a ping (ICMP packet) to a remote host :param str host: The domain name or IP address of a host - :return: A tuple that contains whether or not the host responded and the host's inferred - operating system - :rtype: Tuple[bool, Optional[str]] + :param float timeout: The maximum amount of time (in seconds) to wait for a response + :return: The data collected by attempting to ping the target host + :rtype: PingScanData """ @abc.abstractmethod - def scan_tcp_port(self, host: str, port: int, timeout: int) -> PortScanData: + def scan_tcp_port(self, host: str, port: int, timeout: float) -> PortScanData: """ Scans a TCP port on a remote host :param str host: The domain name or IP address of a host :param int port: A TCP port number to scan - :param int timeout: The maximum amount of time (in seconds) to wait for a response + :param float timeout: The maximum amount of time (in seconds) to wait for a response :return: The data collected by scanning the provided host:port combination :rtype: PortScanData """ diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 3e469ee9c..7933202f6 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -44,7 +44,8 @@ class IPScanner: ip = ips.get_nowait() logger.info(f"Scanning {ip}") - ping_scan_data = self._puppet.ping(ip, options["icmp"]) + icmp_timeout = options["icmp"]["timeout_ms"] / 1000 + ping_scan_data = self._puppet.ping(ip, icmp_timeout) port_scan_data = self._scan_tcp_ports(ip, options["tcp"], stop) results_callback(ip, ping_scan_data, port_scan_data) @@ -59,11 +60,13 @@ class IPScanner: ) def _scan_tcp_ports(self, ip: str, options: Dict, stop: Event): + tcp_timeout = options["timeout_ms"] / 1000 port_scan_data = {} + for p in options["ports"]: if stop.is_set(): break - port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, options["timeout_ms"]) + port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, tcp_timeout) return port_scan_data diff --git a/monkey/infection_monkey/master/mock_master.py b/monkey/infection_monkey/master/mock_master.py index 8c8ecebdd..551ff886c 100644 --- a/monkey/infection_monkey/master/mock_master.py +++ b/monkey/infection_monkey/master/mock_master.py @@ -66,7 +66,7 @@ class MockMaster(IMaster): for ip in ips: h = self._hosts[ip] - ping_scan_data = self._puppet.ping(ip, {}) + ping_scan_data = self._puppet.ping(ip, 1) h.icmp = ping_scan_data.response_received if ping_scan_data.os is not None: h.os["type"] = ping_scan_data.os diff --git a/monkey/infection_monkey/puppet/mock_puppet.py b/monkey/infection_monkey/puppet/mock_puppet.py index de89db172..8c6a39c65 100644 --- a/monkey/infection_monkey/puppet/mock_puppet.py +++ b/monkey/infection_monkey/puppet/mock_puppet.py @@ -156,8 +156,8 @@ class MockPuppet(IPuppet): else: return PostBreachData("pba command 2", ["pba result 2", False]) - def ping(self, host: str, options: Dict) -> PingScanData: - logger.debug(f"run_ping({host})") + def ping(self, host: str, timeout: float = 1) -> PingScanData: + logger.debug(f"run_ping({host}, {timeout})") if host == DOT_1: return PingScanData(True, "windows") diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py index 9447bdfc1..d302cbbfb 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -12,12 +12,6 @@ WINDOWS_OS = "windows" LINUX_OS = "linux" -class MockPuppet(MockPuppet): - def __init__(self): - self.ping = MagicMock(side_effect=super().ping) - self.scan_tcp_port = MagicMock(side_effect=super().scan_tcp_port) - - @pytest.fixture def scan_config(): return { From 0c180a455c308c402ca21357e6ed6b912835ef0e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 08:29:56 -0500 Subject: [PATCH 23/25] Agent: Improve "options" handling in IPScanner --- monkey/infection_monkey/master/ip_scanner.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 7933202f6..1c273fa22 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -46,7 +46,10 @@ class IPScanner: icmp_timeout = options["icmp"]["timeout_ms"] / 1000 ping_scan_data = self._puppet.ping(ip, icmp_timeout) - port_scan_data = self._scan_tcp_ports(ip, options["tcp"], stop) + + tcp_timeout = options["tcp"]["timeout_ms"] / 1000 + tcp_ports = options["tcp"]["ports"] + port_scan_data = self._scan_tcp_ports(ip, tcp_ports, tcp_timeout, stop) results_callback(ip, ping_scan_data, port_scan_data) @@ -59,14 +62,13 @@ class IPScanner: f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting" ) - def _scan_tcp_ports(self, ip: str, options: Dict, stop: Event): - tcp_timeout = options["timeout_ms"] / 1000 + def _scan_tcp_ports(self, ip: str, ports: List[int], timeout: float, stop: Event): port_scan_data = {} - for p in options["ports"]: + for p in ports: if stop.is_set(): break - port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, tcp_timeout) + port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, timeout) return port_scan_data From 1a7135e13f934f4588b32abad01c2ffa7a3cf035 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 08:37:00 -0500 Subject: [PATCH 24/25] Agent: Improve Callback type hint in IPScanner --- monkey/infection_monkey/master/ip_scanner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 1c273fa22..b54adfb4a 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -11,7 +11,9 @@ from .threading_utils import create_daemon_thread logger = logging.getLogger() -Callback = Callable[[str, PingScanData, Dict[int, PortScanData]], None] +IP = str +Port = int +Callback = Callable[[IP, PingScanData, Dict[Port, PortScanData]], None] class IPScanner: From 0058aa4f37180433d1bca2c0f8d2100382b81d4e Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 13 Dec 2021 16:23:04 +0200 Subject: [PATCH 25/25] UT: Improve readability of test_network_scanner.py --- .../master/test_network_scanner.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py index d302cbbfb..6d38097a7 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py @@ -50,7 +50,7 @@ def assert_port_status(port_scan_data, expected_open_ports: Set[int]): assert psd.status == PortStatus.CLOSED -def assert_dot_1(ip, ping_scan_data, port_scan_data): +def assert_scan_results_no_1(ip, ping_scan_data, port_scan_data): assert ip == "10.0.0.1" assert ping_scan_data.response_received is True @@ -72,7 +72,7 @@ def assert_dot_1(ip, ping_scan_data, port_scan_data): assert_port_status(port_scan_data, {445, 3389}) -def assert_dot_3(ip, ping_scan_data, port_scan_data): +def assert_scan_results_no_3(ip, ping_scan_data, port_scan_data): assert ip == "10.0.0.3" assert ping_scan_data.response_received is True @@ -93,12 +93,12 @@ def assert_dot_3(ip, ping_scan_data, port_scan_data): assert_port_status(port_scan_data, {22, 443}) -def assert_host_down(ip, ping_scan_data, port_scan_data): +def assert_scan_results_host_down(ip, ping_scan_data, port_scan_data): assert ip not in {"10.0.0.1", "10.0.0.3"} assert ping_scan_data.response_received is False assert len(port_scan_data.keys()) == 6 - assert_port_status(port_scan_data, {}) + assert_port_status(port_scan_data, set()) def test_scan_single_ip(callback, scan_config, stop): @@ -109,8 +109,8 @@ def test_scan_single_ip(callback, scan_config, stop): callback.assert_called_once() - print(type(callback.call_args_list[0][0])) - assert_dot_1(*(callback.call_args_list[0][0])) + (ip, ping_scan_data, port_scan_data) = callback.call_args_list[0][0] + assert_scan_results_no_1(ip, ping_scan_data, port_scan_data) def test_scan_multiple_ips(callback, scan_config, stop): @@ -121,10 +121,17 @@ def test_scan_multiple_ips(callback, scan_config, stop): assert callback.call_count == 4 - assert_dot_1(*(callback.call_args_list[0][0])) - assert_host_down(*(callback.call_args_list[1][0])) - assert_dot_3(*(callback.call_args_list[2][0])) - assert_host_down(*(callback.call_args_list[3][0])) + (ip, ping_scan_data, port_scan_data) = callback.call_args_list[0][0] + assert_scan_results_no_1(ip, ping_scan_data, port_scan_data) + + (ip, ping_scan_data, port_scan_data) = callback.call_args_list[1][0] + assert_scan_results_host_down(ip, ping_scan_data, port_scan_data) + + (ip, ping_scan_data, port_scan_data) = callback.call_args_list[2][0] + assert_scan_results_no_3(ip, ping_scan_data, port_scan_data) + + (ip, ping_scan_data, port_scan_data) = callback.call_args_list[3][0] + assert_scan_results_host_down(ip, ping_scan_data, port_scan_data) def test_scan_lots_of_ips(callback, scan_config, stop): @@ -139,7 +146,7 @@ def test_scan_lots_of_ips(callback, scan_config, stop): def test_stop_after_callback(scan_config, stop): def _callback(*_): # Block all threads here until 2 threads reach this barrier, then set stop - # and test that niether thread continues to scan. + # and test that neither thread continues to scan. _callback.barrier.wait() stop.set() @@ -158,7 +165,7 @@ def test_stop_after_callback(scan_config, stop): def test_interrupt_port_scanning(callback, scan_config, stop): def stopable_scan_tcp_port(port, *_): # Block all threads here until 2 threads reach this barrier, then set stop - # and test that niether thread scans any more ports + # and test that neither thread scans any more ports stopable_scan_tcp_port.barrier.wait() stop.set()