From 96f59cc628538130f5e86b1eefb23864ee112e4b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 10:20:33 -0500 Subject: [PATCH 1/9] Agent: Remove unused "os-version" from fingerprinters --- monkey/infection_monkey/network/smbfinger.py | 3 +-- monkey/infection_monkey/network/sshfinger.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/monkey/infection_monkey/network/smbfinger.py b/monkey/infection_monkey/network/smbfinger.py index f3301f33c..2c76f652a 100644 --- a/monkey/infection_monkey/network/smbfinger.py +++ b/monkey/infection_monkey/network/smbfinger.py @@ -181,8 +181,7 @@ class SMBFinger(HostFinger): host.services[SMB_SERVICE]["name"] = service_client if "version" not in host.os: host.os["version"] = os_version - else: - host.services[SMB_SERVICE]["os-version"] = os_version + return True except Exception as exc: logger.debug("Error getting smb fingerprint: %s", exc) diff --git a/monkey/infection_monkey/network/sshfinger.py b/monkey/infection_monkey/network/sshfinger.py index 59c0395a9..df21ef35b 100644 --- a/monkey/infection_monkey/network/sshfinger.py +++ b/monkey/infection_monkey/network/sshfinger.py @@ -28,8 +28,7 @@ class SSHFinger(HostFinger): os_version = banner.split(" ").pop().strip() if "version" not in host.os: host.os["version"] = os_version - else: - host.services[service]["os-version"] = os_version + break def get_host_fingerprint(self, host): From af338be41877d36dcf62e38b6c0de9d3263dff98 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 11:01:38 -0500 Subject: [PATCH 2/9] UT: Rename test_network_scanner.py -> test_ip_scanner.py --- .../master/{test_network_scanner.py => test_ip_scanner.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename monkey/tests/unit_tests/infection_monkey/master/{test_network_scanner.py => test_ip_scanner.py} (100%) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py similarity index 100% rename from monkey/tests/unit_tests/infection_monkey/master/test_network_scanner.py rename to monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py From 0ff45e3af1a1d7aa8cf5974eebd0c496c61794bd Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 12:00:18 -0500 Subject: [PATCH 3/9] Agent: Change return type of IPuppet.fingerprint() --- monkey/infection_monkey/i_puppet.py | 7 ++-- monkey/infection_monkey/puppet/mock_puppet.py | 33 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/monkey/infection_monkey/i_puppet.py b/monkey/infection_monkey/i_puppet.py index f158be08c..518b299b6 100644 --- a/monkey/infection_monkey/i_puppet.py +++ b/monkey/infection_monkey/i_puppet.py @@ -13,6 +13,7 @@ class PortStatus(Enum): ExploiterResultData = namedtuple("ExploiterResultData", ["result", "info", "attempts"]) PingScanData = namedtuple("PingScanData", ["response_received", "os"]) PortScanData = namedtuple("PortScanData", ["port", "status", "banner", "service"]) +FingerprintData = namedtuple("FingerprintData", ["os_type", "os_version", "services"]) PostBreachData = namedtuple("PostBreachData", ["command", "result"]) @@ -57,13 +58,13 @@ class IPuppet(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def fingerprint(self, name: str, host: str) -> Dict: + def fingerprint(self, name: str, host: str) -> FingerprintData: """ Runs a fingerprinter against a remote host :param str name: The name of the fingerprinter to run :param str host: The domain name or IP address of a host - :return: A dictionary containing the information collected by the fingerprinter - :rtype: Dict + :return: The data collected by running the fingerprinter on the specified host + :rtype: FingerprintData """ @abc.abstractmethod diff --git a/monkey/infection_monkey/puppet/mock_puppet.py b/monkey/infection_monkey/puppet/mock_puppet.py index 8c6a39c65..f8c76d843 100644 --- a/monkey/infection_monkey/puppet/mock_puppet.py +++ b/monkey/infection_monkey/puppet/mock_puppet.py @@ -4,6 +4,7 @@ from typing import Dict, Tuple from infection_monkey.i_puppet import ( ExploiterResultData, + FingerprintData, IPuppet, PingScanData, PortScanData, @@ -193,29 +194,37 @@ class MockPuppet(IPuppet): return _get_empty_results(port) - def fingerprint(self, name: str, host: str) -> Dict: + def fingerprint(self, name: str, host: str) -> FingerprintData: logger.debug(f"fingerprint({name}, {host})") + empty_fingerprint_data = FingerprintData(None, None, {}) + dot_1_results = { - "SMBFinger": { - "os": {"type": "windows", "version": "vista"}, - "services": {"tcp-445": {"name": "SSH", "os": "linux"}}, - } + "SMBFinger": FingerprintData( + "windows", "vista", {"tcp-445": {"name": "smb_service_name"}} + ) } dot_3_results = { - "SSHFinger": {"os": "linux", "services": {"tcp-22": {"name": "SSH"}}}, - "HTTPFinger": { - "services": {"tcp-https": {"name": "http", "data": ("SERVER_HEADERS", DOT_3)}} - }, + "SSHFinger": FingerprintData( + "linux", "ubuntu", {"tcp-22": {"name": "SSH", "banner": "SSH BANNER"}} + ), + "HTTPFinger": FingerprintData( + None, + None, + { + "tcp-80": {"name": "http", "data": ("SERVER_HEADERS", False)}, + "tcp-443": {"name": "http", "data": ("SERVER_HEADERS_2", True)}, + }, + ), } if host == DOT_1: - return dot_1_results.get(name, {}) + return dot_1_results.get(name, empty_fingerprint_data) if host == DOT_3: - return dot_3_results.get(name, {}) + return dot_3_results.get(name, empty_fingerprint_data) - return {} + return empty_fingerprint_data def exploit_host( self, name: str, host: str, options: Dict, interrupt: threading.Event From 438563af9ca027bc2fe7aecc68ef7472815ca759 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 12:04:08 -0500 Subject: [PATCH 4/9] Agent: Add fingerprinting to IPScanner --- monkey/infection_monkey/master/ip_scanner.py | 43 ++++++++- .../master/test_ip_scanner.py | 94 +++++++++++++++---- 2 files changed, 117 insertions(+), 20 deletions(-) diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index b54adfb4a..62bf9c7d8 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -5,7 +5,13 @@ from queue import Queue from threading import Event from typing import Callable, Dict, List -from infection_monkey.i_puppet import IPuppet, PingScanData, PortScanData +from infection_monkey.i_puppet import ( + FingerprintData, + IPuppet, + PingScanData, + PortScanData, + PortStatus, +) from .threading_utils import create_daemon_thread @@ -13,7 +19,10 @@ logger = logging.getLogger() IP = str Port = int -Callback = Callable[[IP, PingScanData, Dict[Port, PortScanData]], None] +FingerprinterName = str +Callback = Callable[ + [IP, PingScanData, Dict[Port, PortScanData], Dict[FingerprinterName, FingerprintData]], None +] class IPScanner: @@ -53,7 +62,12 @@ class IPScanner: tcp_ports = options["tcp"]["ports"] port_scan_data = self._scan_tcp_ports(ip, tcp_ports, tcp_timeout, stop) - results_callback(ip, ping_scan_data, port_scan_data) + fingerprint_data = {} + if IPScanner._found_open_port(port_scan_data): + fingerprinters = options["fingerprinters"] + fingerprint_data = self._run_fingerprinters(ip, fingerprinters, stop) + + results_callback(ip, ping_scan_data, port_scan_data, fingerprint_data) logger.debug( f"Detected the stop signal, scanning thread {threading.get_ident()} exiting" @@ -64,7 +78,9 @@ class IPScanner: f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting" ) - def _scan_tcp_ports(self, ip: str, ports: List[int], timeout: float, stop: Event): + def _scan_tcp_ports( + self, ip: str, ports: List[int], timeout: float, stop: Event + ) -> Dict[int, PortScanData]: port_scan_data = {} for p in ports: @@ -74,3 +90,22 @@ class IPScanner: port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, timeout) return port_scan_data + + @staticmethod + def _found_open_port(port_scan_data: Dict[int, PortScanData]): + for psd in port_scan_data.values(): + if psd.status == PortStatus.OPEN: + return True + + return False + + def _run_fingerprinters(self, ip: str, fingerprinters: List[str], stop: Event): + fingerprint_data = {} + + for f in fingerprinters: + if stop.is_set(): + break + + fingerprint_data[f] = self._puppet.fingerprint(f, ip) + + return fingerprint_data diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py index 6d38097a7..12e822fa3 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest -from infection_monkey.i_puppet import PortScanData, PortStatus +from infection_monkey.i_puppet import FingerprintData, PortScanData, PortStatus from infection_monkey.master import IPScanner from infection_monkey.puppet.mock_puppet import MockPuppet @@ -29,6 +29,7 @@ def scan_config(): "icmp": { "timeout_ms": 1000, }, + "fingerprinters": {"HTTPFinger", "SMBFinger", "SSHFinger"}, } @@ -50,9 +51,16 @@ def assert_port_status(port_scan_data, expected_open_ports: Set[int]): assert psd.status == PortStatus.CLOSED -def assert_scan_results_no_1(ip, ping_scan_data, port_scan_data): - assert ip == "10.0.0.1" +def assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data): + if ip == "10.0.0.1": + assert_scan_results_no_1(ping_scan_data, port_scan_data, fingerprint_data) + elif ip == "10.0.0.3": + assert_scan_results_no_3(ping_scan_data, port_scan_data, fingerprint_data) + else: + assert_scan_results_host_down(ip, ping_scan_data, port_scan_data, fingerprint_data) + +def assert_scan_results_no_1(ping_scan_data, port_scan_data, fingerprint_data): assert ping_scan_data.response_received is True assert ping_scan_data.os == WINDOWS_OS @@ -70,11 +78,22 @@ def assert_scan_results_no_1(ip, ping_scan_data, port_scan_data): assert psd_3389.service == "tcp-3389" assert_port_status(port_scan_data, {445, 3389}) + assert_fingerprint_results_no_1(fingerprint_data) -def assert_scan_results_no_3(ip, ping_scan_data, port_scan_data): - assert ip == "10.0.0.3" +def assert_fingerprint_results_no_1(fingerprint_data): + assert len(fingerprint_data.keys()) == 3 + assert fingerprint_data["SSHFinger"].services == {} + assert fingerprint_data["HTTPFinger"].services == {} + assert fingerprint_data["SMBFinger"].os_type == WINDOWS_OS + assert fingerprint_data["SMBFinger"].os_version == "vista" + + assert len(fingerprint_data["SMBFinger"].services.keys()) == 1 + assert fingerprint_data["SMBFinger"].services["tcp-445"]["name"] == "smb_service_name" + + +def assert_scan_results_no_3(ping_scan_data, port_scan_data, fingerprint_data): assert ping_scan_data.response_received is True assert ping_scan_data.os == LINUX_OS assert len(port_scan_data.keys()) == 6 @@ -91,15 +110,36 @@ def assert_scan_results_no_3(ip, ping_scan_data, port_scan_data): assert psd_22.service == "tcp-22" assert_port_status(port_scan_data, {22, 443}) + assert_fingerprint_results_no_3(fingerprint_data) -def assert_scan_results_host_down(ip, ping_scan_data, port_scan_data): +def assert_fingerprint_results_no_3(fingerprint_data): + assert len(fingerprint_data.keys()) == 3 + assert fingerprint_data["SMBFinger"].services == {} + + assert fingerprint_data["SSHFinger"].os_type == LINUX_OS + assert fingerprint_data["SSHFinger"].os_version == "ubuntu" + + assert len(fingerprint_data["SSHFinger"].services.keys()) == 1 + assert fingerprint_data["SSHFinger"].services["tcp-22"]["name"] == "SSH" + assert fingerprint_data["SSHFinger"].services["tcp-22"]["banner"] == "SSH BANNER" + + assert len(fingerprint_data["HTTPFinger"].services.keys()) == 2 + assert fingerprint_data["HTTPFinger"].services["tcp-80"]["name"] == "http" + assert fingerprint_data["HTTPFinger"].services["tcp-80"]["data"] == ("SERVER_HEADERS", False) + assert fingerprint_data["HTTPFinger"].services["tcp-443"]["name"] == "http" + assert fingerprint_data["HTTPFinger"].services["tcp-443"]["data"] == ("SERVER_HEADERS_2", True) + + +def assert_scan_results_host_down(ip, ping_scan_data, port_scan_data, fingerprint_data): assert ip not in {"10.0.0.1", "10.0.0.3"} assert ping_scan_data.response_received is False assert len(port_scan_data.keys()) == 6 assert_port_status(port_scan_data, set()) + assert fingerprint_data == {} + def test_scan_single_ip(callback, scan_config, stop): ips = ["10.0.0.1"] @@ -109,8 +149,8 @@ def test_scan_single_ip(callback, scan_config, stop): callback.assert_called_once() - (ip, ping_scan_data, port_scan_data) = callback.call_args_list[0][0] - assert_scan_results_no_1(ip, ping_scan_data, port_scan_data) + (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[0][0] + assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) def test_scan_multiple_ips(callback, scan_config, stop): @@ -121,17 +161,17 @@ def test_scan_multiple_ips(callback, scan_config, stop): assert callback.call_count == 4 - (ip, ping_scan_data, port_scan_data) = callback.call_args_list[0][0] - assert_scan_results_no_1(ip, ping_scan_data, port_scan_data) + (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[0][0] + assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) - (ip, ping_scan_data, port_scan_data) = callback.call_args_list[1][0] - assert_scan_results_host_down(ip, ping_scan_data, port_scan_data) + (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[1][0] + assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) - (ip, ping_scan_data, port_scan_data) = callback.call_args_list[2][0] - assert_scan_results_no_3(ip, ping_scan_data, port_scan_data) + (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[2][0] + assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) - (ip, ping_scan_data, port_scan_data) = callback.call_args_list[3][0] - assert_scan_results_host_down(ip, ping_scan_data, port_scan_data) + (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[3][0] + assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) def test_scan_lots_of_ips(callback, scan_config, stop): @@ -182,3 +222,25 @@ def test_interrupt_port_scanning(callback, scan_config, stop): ns.scan(ips, scan_config, callback, stop) assert puppet.scan_tcp_port.call_count == 2 + + +def test_interrupt_fingerprinting(callback, scan_config, stop): + def stopable_fingerprint(port, *_): + # Block all threads here until 2 threads reach this barrier, then set stop + # and test that neither thread scans any more ports + stopable_fingerprint.barrier.wait() + stop.set() + + return FingerprintData(None, None, {}) + + stopable_fingerprint.barrier = Barrier(2) + + puppet = MockPuppet() + puppet.fingerprint = MagicMock(side_effect=stopable_fingerprint) + + ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"] + + ns = IPScanner(puppet, num_workers=2) + ns.scan(ips, scan_config, callback, stop) + + assert puppet.fingerprint.call_count == 2 From 8067dc9ff86b5882656cf3a0be7b25228f6d501d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 13:06:44 -0500 Subject: [PATCH 5/9] Agent: Process fingerprinter results in Propagator --- monkey/infection_monkey/master/propagator.py | 39 +++++++++++--- .../master/test_propagator.py | 53 ++++++++++++++++--- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index da36ce5b9..0d63bc904 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -3,7 +3,7 @@ from queue import Queue from threading import Event, Thread from typing import Dict -from infection_monkey.i_puppet import PingScanData, PortScanData, PortStatus +from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData, PortStatus from infection_monkey.model.host import VictimHost from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.scan_telem import ScanTelem @@ -52,15 +52,33 @@ class Propagator: logger.info("Finished network scan") def _process_scan_results( - self, ip: str, ping_scan_data: PingScanData, port_scan_data: Dict[int, PortScanData] + self, + ip: str, + ping_scan_data: PingScanData, + port_scan_data: Dict[int, PortScanData], + fingerprint_data: Dict[str, FingerprintData], ): victim_host = VictimHost(ip) - has_open_port = False + Propagator._process_ping_scan_results(victim_host, ping_scan_data) + has_open_port = Propagator._process_tcp_scan_results(victim_host, port_scan_data) + Propagator._process_fingerprinter_results(victim_host, fingerprint_data) + + if has_open_port: + self._hosts_to_exploit.put(victim_host) + + self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) + + @staticmethod + def _process_ping_scan_results(victim_host: VictimHost, ping_scan_data: PingScanData): victim_host.icmp = ping_scan_data.response_received if ping_scan_data.os is not None: victim_host.os["type"] = ping_scan_data.os + @staticmethod + def _process_tcp_scan_results(victim_host: VictimHost, port_scan_data: PortScanData) -> bool: + has_open_port = False + for psd in port_scan_data.values(): if psd.status == PortStatus.OPEN: has_open_port = True @@ -71,10 +89,19 @@ class Propagator: if psd.banner is not None: victim_host.services[psd.service]["banner"] = psd.banner - if has_open_port: - self._hosts_to_exploit.put(victim_host) + return has_open_port - self._telemetry_messenger.send_telemetry(ScanTelem(victim_host)) + @staticmethod + def _process_fingerprinter_results(victim_host: VictimHost, fingerprint_data: FingerprintData): + for fd in fingerprint_data.values(): + if fd.os_type is not None: + victim_host.os["type"] = fd.os_type + + if ("version" not in victim_host.os) and (fd.os_version is not None): + victim_host.os["version"] = fd.os_version + + for service, details in fd.services.items(): + victim_host.services.setdefault(service, {}).update(details) def _exploit_targets(self, scan_thread: Thread, stop: Event): pass diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py index b5c97760b..cec779aa5 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py @@ -1,8 +1,10 @@ from threading import Event -from infection_monkey.i_puppet import PingScanData, PortScanData, PortStatus +from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData, PortStatus from infection_monkey.master import Propagator +empty_fingerprint_data = FingerprintData(None, None, {}) + dot_1_results = ( PingScanData(True, "windows"), { @@ -10,6 +12,11 @@ dot_1_results = ( 445: PortScanData(445, PortStatus.OPEN, "SMB BANNER", "tcp-445"), 3389: PortScanData(3389, PortStatus.OPEN, "", "tcp-3389"), }, + { + "SMBFinger": FingerprintData("windows", "vista", {"tcp-445": {"name": "smb_service_name"}}), + "SSHFinger": empty_fingerprint_data, + "HTTPFinger": empty_fingerprint_data, + }, ) dot_3_results = ( @@ -19,6 +26,20 @@ dot_3_results = ( 443: PortScanData(443, PortStatus.OPEN, "HTTPS BANNER", "tcp-443"), 3389: PortScanData(3389, PortStatus.CLOSED, "", None), }, + { + "SSHFinger": FingerprintData( + "linux", "ubuntu", {"tcp-22": {"name": "SSH", "banner": "SSH BANNER"}} + ), + "HTTPFinger": FingerprintData( + None, + None, + { + "tcp-80": {"name": "http", "data": ("SERVER_HEADERS", False)}, + "tcp-443": {"name": "http", "data": ("SERVER_HEADERS_2", True)}, + }, + ), + "SMBFinger": empty_fingerprint_data, + }, ) dead_host_results = ( @@ -28,21 +49,34 @@ dead_host_results = ( 443: PortScanData(443, PortStatus.CLOSED, None, None), 3389: PortScanData(3389, PortStatus.CLOSED, "", None), }, + {}, ) dot_1_services = { - "tcp-445": {"display_name": "unknown(TCP)", "port": 445, "banner": "SMB BANNER"}, + "tcp-445": { + "name": "smb_service_name", + "display_name": "unknown(TCP)", + "port": 445, + "banner": "SMB BANNER", + }, "tcp-3389": {"display_name": "unknown(TCP)", "port": 3389, "banner": ""}, } dot_3_services = { - "tcp-22": {"display_name": "unknown(TCP)", "port": 22, "banner": "SSH BANNER"}, - "tcp-443": {"display_name": "unknown(TCP)", "port": 443, "banner": "HTTPS BANNER"}, + "tcp-22": {"name": "SSH", "display_name": "unknown(TCP)", "port": 22, "banner": "SSH BANNER"}, + "tcp-80": {"name": "http", "data": ("SERVER_HEADERS", False)}, + "tcp-443": { + "name": "http", + "display_name": "unknown(TCP)", + "port": 443, + "banner": "HTTPS BANNER", + "data": ("SERVER_HEADERS_2", True), + }, } class MockIPScanner: - def scan(self, ips_to_scan, options, results_callback, stop): + def scan(self, ips_to_scan, _, results_callback, stop): for ip in ips_to_scan: if ip.endswith(".1"): results_callback(ip, *dot_1_results) @@ -55,7 +89,10 @@ class MockIPScanner: def test_scan_result_processing(telemetry_messenger_spy): p = Propagator(telemetry_messenger_spy, MockIPScanner()) p.propagate( - {"targets": {"subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"]}, "network_scan": {}}, + { + "targets": {"subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"]}, + "network_scan": {}, + }, Event(), ) @@ -68,11 +105,13 @@ def test_scan_result_processing(telemetry_messenger_spy): if ip.endswith(".1"): assert data["service_count"] == 2 assert data["machine"]["os"]["type"] == "windows" + assert data["machine"]["os"]["version"] == "vista" assert data["machine"]["services"] == dot_1_services assert data["machine"]["icmp"] is True elif ip.endswith(".3"): - assert data["service_count"] == 2 + assert data["service_count"] == 3 assert data["machine"]["os"]["type"] == "linux" + assert data["machine"]["os"]["version"] == "ubuntu" assert data["machine"]["services"] == dot_3_services assert data["machine"]["icmp"] is True else: From d51af8a5835347d426d4bece36d36f7586888c37 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 13:28:40 -0500 Subject: [PATCH 6/9] Agent: Add IPScanResults dataclass --- monkey/infection_monkey/master/__init__.py | 1 + .../master/ip_scan_results.py | 14 ++++++++++ monkey/infection_monkey/master/ip_scanner.py | 22 ++++++---------- monkey/infection_monkey/master/propagator.py | 18 +++++-------- .../master/test_ip_scanner.py | 26 +++++++++++-------- .../master/test_propagator.py | 14 +++++----- 6 files changed, 52 insertions(+), 43 deletions(-) create mode 100644 monkey/infection_monkey/master/ip_scan_results.py diff --git a/monkey/infection_monkey/master/__init__.py b/monkey/infection_monkey/master/__init__.py index 21ef8f9b6..fda536194 100644 --- a/monkey/infection_monkey/master/__init__.py +++ b/monkey/infection_monkey/master/__init__.py @@ -1,3 +1,4 @@ +from .ip_scan_results import IPScanResults from .ip_scanner import IPScanner from .propagator import Propagator from .automated_master import AutomatedMaster diff --git a/monkey/infection_monkey/master/ip_scan_results.py b/monkey/infection_monkey/master/ip_scan_results.py new file mode 100644 index 000000000..98f7b6646 --- /dev/null +++ b/monkey/infection_monkey/master/ip_scan_results.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from typing import Dict + +from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData + +Port = int +FingerprinterName = str + + +@dataclass +class IPScanResults: + ping_scan_data: PingScanData + port_scan_data: Dict[Port, PortScanData] + fingerprint_data: Dict[FingerprinterName, FingerprintData] diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 62bf9c7d8..26a321212 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -5,24 +5,15 @@ from queue import Queue from threading import Event from typing import Callable, Dict, List -from infection_monkey.i_puppet import ( - FingerprintData, - IPuppet, - PingScanData, - PortScanData, - PortStatus, -) +from infection_monkey.i_puppet import FingerprintData, IPuppet, PortScanData, PortStatus +from . import IPScanResults from .threading_utils import create_daemon_thread logger = logging.getLogger() IP = str -Port = int -FingerprinterName = str -Callback = Callable[ - [IP, PingScanData, Dict[Port, PortScanData], Dict[FingerprinterName, FingerprintData]], None -] +Callback = Callable[[IP, IPScanResults], None] class IPScanner: @@ -67,7 +58,8 @@ class IPScanner: fingerprinters = options["fingerprinters"] fingerprint_data = self._run_fingerprinters(ip, fingerprinters, stop) - results_callback(ip, ping_scan_data, port_scan_data, fingerprint_data) + scan_results = IPScanResults(ping_scan_data, port_scan_data, fingerprint_data) + results_callback(ip, scan_results) logger.debug( f"Detected the stop signal, scanning thread {threading.get_ident()} exiting" @@ -99,7 +91,9 @@ class IPScanner: return False - def _run_fingerprinters(self, ip: str, fingerprinters: List[str], stop: Event): + def _run_fingerprinters( + self, ip: str, fingerprinters: List[str], stop: Event + ) -> Dict[str, FingerprintData]: fingerprint_data = {} for f in fingerprinters: diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index 0d63bc904..1d6e4462e 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -8,7 +8,7 @@ from infection_monkey.model.host import VictimHost from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.scan_telem import ScanTelem -from . import IPScanner +from . import IPScanner, IPScanResults from .threading_utils import create_daemon_thread logger = logging.getLogger() @@ -51,18 +51,14 @@ class Propagator: logger.info("Finished network scan") - def _process_scan_results( - self, - ip: str, - ping_scan_data: PingScanData, - port_scan_data: Dict[int, PortScanData], - fingerprint_data: Dict[str, FingerprintData], - ): + def _process_scan_results(self, ip: str, scan_results: IPScanResults): victim_host = VictimHost(ip) - Propagator._process_ping_scan_results(victim_host, ping_scan_data) - has_open_port = Propagator._process_tcp_scan_results(victim_host, port_scan_data) - Propagator._process_fingerprinter_results(victim_host, fingerprint_data) + Propagator._process_ping_scan_results(victim_host, scan_results.ping_scan_data) + has_open_port = Propagator._process_tcp_scan_results( + victim_host, scan_results.port_scan_data + ) + Propagator._process_fingerprinter_results(victim_host, scan_results.fingerprint_data) if has_open_port: self._hosts_to_exploit.put(victim_host) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py index 12e822fa3..22c850837 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py @@ -51,7 +51,11 @@ def assert_port_status(port_scan_data, expected_open_ports: Set[int]): assert psd.status == PortStatus.CLOSED -def assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data): +def assert_scan_results(ip, scan_results): + ping_scan_data = scan_results.ping_scan_data + port_scan_data = scan_results.port_scan_data + fingerprint_data = scan_results.fingerprint_data + if ip == "10.0.0.1": assert_scan_results_no_1(ping_scan_data, port_scan_data, fingerprint_data) elif ip == "10.0.0.3": @@ -149,8 +153,8 @@ def test_scan_single_ip(callback, scan_config, stop): callback.assert_called_once() - (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[0][0] - assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) + (ip, scan_results) = callback.call_args_list[0][0] + assert_scan_results(ip, scan_results) def test_scan_multiple_ips(callback, scan_config, stop): @@ -161,17 +165,17 @@ def test_scan_multiple_ips(callback, scan_config, stop): assert callback.call_count == 4 - (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[0][0] - assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) + (ip, scan_results) = callback.call_args_list[0][0] + assert_scan_results(ip, scan_results) - (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[1][0] - assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) + (ip, scan_results) = callback.call_args_list[1][0] + assert_scan_results(ip, scan_results) - (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[2][0] - assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) + (ip, scan_results) = callback.call_args_list[2][0] + assert_scan_results(ip, scan_results) - (ip, ping_scan_data, port_scan_data, fingerprint_data) = callback.call_args_list[3][0] - assert_scan_results(ip, ping_scan_data, port_scan_data, fingerprint_data) + (ip, scan_results) = callback.call_args_list[3][0] + assert_scan_results(ip, scan_results) def test_scan_lots_of_ips(callback, scan_config, stop): diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py index cec779aa5..d8f65b54e 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py @@ -1,11 +1,11 @@ from threading import Event from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData, PortStatus -from infection_monkey.master import Propagator +from infection_monkey.master import IPScanResults, Propagator empty_fingerprint_data = FingerprintData(None, None, {}) -dot_1_results = ( +dot_1_results = IPScanResults( PingScanData(True, "windows"), { 22: PortScanData(22, PortStatus.CLOSED, None, None), @@ -19,7 +19,7 @@ dot_1_results = ( }, ) -dot_3_results = ( +dot_3_results = IPScanResults( PingScanData(True, "linux"), { 22: PortScanData(22, PortStatus.OPEN, "SSH BANNER", "tcp-22"), @@ -42,7 +42,7 @@ dot_3_results = ( }, ) -dead_host_results = ( +dead_host_results = IPScanResults( PingScanData(False, None), { 22: PortScanData(22, PortStatus.CLOSED, None, None), @@ -79,11 +79,11 @@ class MockIPScanner: def scan(self, ips_to_scan, _, results_callback, stop): for ip in ips_to_scan: if ip.endswith(".1"): - results_callback(ip, *dot_1_results) + results_callback(ip, dot_1_results) elif ip.endswith(".3"): - results_callback(ip, *dot_3_results) + results_callback(ip, dot_3_results) else: - results_callback(ip, *dead_host_results) + results_callback(ip, dead_host_results) def test_scan_result_processing(telemetry_messenger_spy): From e52471896039e8ef0e1f808435b42d5926bfe68f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 13:58:30 -0500 Subject: [PATCH 7/9] Island: Reformat "finger_classes" config options before sending to Agent --- monkey/monkey_island/cc/services/config.py | 12 ++++++++++++ .../data_for_tests/monkey_configs/flat_config.json | 1 - .../monkey_island/cc/services/test_config.py | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 10fbde66d..2e587444c 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -488,6 +488,9 @@ class ConfigService: formatted_network_scan_config["icmp"] = ConfigService._format_icmp_scan_from_flat_config( config ) + formatted_network_scan_config[ + "fingerprinters" + ] = ConfigService._format_fingerprinters_from_flat_config(config) return formatted_network_scan_config @@ -529,6 +532,15 @@ class ConfigService: return formatted_icmp_scan_config + @staticmethod + def _format_fingerprinters_from_flat_config(config: Dict): + flat_fingerprinter_classes_field = "finger_classes" + + formatted_fingerprinters = config[flat_fingerprinter_classes_field] + config.pop(flat_fingerprinter_classes_field) + + return formatted_fingerprinters + @staticmethod def _format_targets_from_flat_config(config: Dict): flat_blocked_ips_field = "blocked_ips" diff --git a/monkey/tests/data_for_tests/monkey_configs/flat_config.json b/monkey/tests/data_for_tests/monkey_configs/flat_config.json index 031dfd35a..0b9f63b84 100644 --- a/monkey/tests/data_for_tests/monkey_configs/flat_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/flat_config.json @@ -66,7 +66,6 @@ "SMBFinger", "SSHFinger", "HTTPFinger", - "MySQLFinger", "MSSQLFinger", "ElasticFinger" ], diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index c10c77b42..5cf5090a3 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -143,6 +143,13 @@ def test_format_config_for_agent__network_scan(flat_monkey_config): "icmp": { "timeout_ms": 1000, }, + "fingerprinters": [ + "SMBFinger", + "SSHFinger", + "HTTPFinger", + "MSSQLFinger", + "ElasticFinger", + ], } ConfigService.format_flat_config_for_agent(flat_monkey_config) @@ -153,3 +160,4 @@ def test_format_config_for_agent__network_scan(flat_monkey_config): assert "tcp_scan_timeout" not in flat_monkey_config assert "tcp_target_ports" not in flat_monkey_config assert "ping_scan_timeout" not in flat_monkey_config + assert "finger_classes" not in flat_monkey_config From 2dc6e0600da15e24819d03812ead64f23e235c7a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 14:13:10 -0500 Subject: [PATCH 8/9] Agent: Pass ping_scan_data and port_scan_data to IPuppet.fingerprint() Fingerprinters can reuse the port scan data to avoid unnecessarily rescanning the hosts' ports. --- monkey/infection_monkey/i_puppet.py | 11 +++++++++- monkey/infection_monkey/master/ip_scanner.py | 21 +++++++++++++++---- monkey/infection_monkey/master/mock_master.py | 6 +++--- monkey/infection_monkey/puppet/mock_puppet.py | 8 ++++++- .../master/test_ip_scanner.py | 2 +- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/monkey/infection_monkey/i_puppet.py b/monkey/infection_monkey/i_puppet.py index 518b299b6..285e32bca 100644 --- a/monkey/infection_monkey/i_puppet.py +++ b/monkey/infection_monkey/i_puppet.py @@ -58,11 +58,20 @@ class IPuppet(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def fingerprint(self, name: str, host: str) -> FingerprintData: + def fingerprint( + self, + name: str, + host: str, + ping_scan_data: PingScanData, + port_scan_data: Dict[int, PortScanData], + ) -> FingerprintData: """ Runs a fingerprinter against a remote host :param str name: The name of the fingerprinter to run :param str host: The domain name or IP address of a host + :param PingScanData ping_scan_data: Data retrieved from the target host via ICMP + :param Dict[int, PortScanData] port_scan_data: Data retrieved from the target host via a TCP + port scan :return: The data collected by running the fingerprinter on the specified host :rtype: FingerprintData """ diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 26a321212..cf77ea54d 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -5,7 +5,13 @@ from queue import Queue from threading import Event from typing import Callable, Dict, List -from infection_monkey.i_puppet import FingerprintData, IPuppet, PortScanData, PortStatus +from infection_monkey.i_puppet import ( + FingerprintData, + IPuppet, + PingScanData, + PortScanData, + PortStatus, +) from . import IPScanResults from .threading_utils import create_daemon_thread @@ -56,7 +62,9 @@ class IPScanner: fingerprint_data = {} if IPScanner._found_open_port(port_scan_data): fingerprinters = options["fingerprinters"] - fingerprint_data = self._run_fingerprinters(ip, fingerprinters, stop) + fingerprint_data = self._run_fingerprinters( + ip, fingerprinters, ping_scan_data, port_scan_data, stop + ) scan_results = IPScanResults(ping_scan_data, port_scan_data, fingerprint_data) results_callback(ip, scan_results) @@ -92,7 +100,12 @@ class IPScanner: return False def _run_fingerprinters( - self, ip: str, fingerprinters: List[str], stop: Event + self, + ip: str, + fingerprinters: List[str], + ping_scan_data: PingScanData, + port_scan_data: Dict[int, PortScanData], + stop: Event, ) -> Dict[str, FingerprintData]: fingerprint_data = {} @@ -100,6 +113,6 @@ class IPScanner: if stop.is_set(): break - fingerprint_data[f] = self._puppet.fingerprint(f, ip) + fingerprint_data[f] = self._puppet.fingerprint(f, ip, ping_scan_data, port_scan_data) return fingerprint_data diff --git a/monkey/infection_monkey/master/mock_master.py b/monkey/infection_monkey/master/mock_master.py index 551ff886c..3844ef590 100644 --- a/monkey/infection_monkey/master/mock_master.py +++ b/monkey/infection_monkey/master/mock_master.py @@ -88,13 +88,13 @@ class MockMaster(IMaster): machine_1 = self._hosts["10.0.0.1"] machine_3 = self._hosts["10.0.0.3"] - self._puppet.fingerprint("SMBFinger", machine_1) + self._puppet.fingerprint("SMBFinger", machine_1, None, None) self._telemetry_messenger.send_telemetry(ScanTelem(machine_1)) - self._puppet.fingerprint("SMBFinger", machine_3) + self._puppet.fingerprint("SMBFinger", machine_3, None, None) self._telemetry_messenger.send_telemetry(ScanTelem(machine_3)) - self._puppet.fingerprint("HTTPFinger", machine_3) + self._puppet.fingerprint("HTTPFinger", machine_3, None, None) self._telemetry_messenger.send_telemetry(ScanTelem(machine_3)) logger.info("Finished running fingerprinters on potential victims") diff --git a/monkey/infection_monkey/puppet/mock_puppet.py b/monkey/infection_monkey/puppet/mock_puppet.py index f8c76d843..d5c8fa2f8 100644 --- a/monkey/infection_monkey/puppet/mock_puppet.py +++ b/monkey/infection_monkey/puppet/mock_puppet.py @@ -194,7 +194,13 @@ class MockPuppet(IPuppet): return _get_empty_results(port) - def fingerprint(self, name: str, host: str) -> FingerprintData: + def fingerprint( + self, + name: str, + host: str, + ping_scan_data: PingScanData, + port_scan_data: Dict[int, PortScanData], + ) -> FingerprintData: logger.debug(f"fingerprint({name}, {host})") empty_fingerprint_data = FingerprintData(None, None, {}) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py index 22c850837..3b071eb9a 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py @@ -229,7 +229,7 @@ def test_interrupt_port_scanning(callback, scan_config, stop): def test_interrupt_fingerprinting(callback, scan_config, stop): - def stopable_fingerprint(port, *_): + def stopable_fingerprint(*_): # Block all threads here until 2 threads reach this barrier, then set stop # and test that neither thread scans any more ports stopable_fingerprint.barrier.wait() From 7e3945dd024aaa9819c829708149d416e41fb9b0 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 13 Dec 2021 14:21:04 -0500 Subject: [PATCH 9/9] Agent: Add TODO to Propagator --- monkey/infection_monkey/master/propagator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index 1d6e4462e..0c3acea1d 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -90,6 +90,10 @@ class Propagator: @staticmethod def _process_fingerprinter_results(victim_host: VictimHost, fingerprint_data: FingerprintData): for fd in fingerprint_data.values(): + # TODO: This logic preserves the existing behavior prior to introducing IMaster and + # IPuppet, but it is possibly flawed. Different fingerprinters may detect + # different os types or versions, and this logic isn't sufficient to handle those + # conflicts. Reevaluate this logic when we overhaul our scanners/fingerprinters. if fd.os_type is not None: victim_host.os["type"] = fd.os_type