Merge pull request #1653 from guardicore/1597-implement-fingerprinting

1597 implement fingerprinting
This commit is contained in:
Mike Salvatore 2021-12-14 07:14:29 -05:00 committed by GitHub
commit 44479ef49e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 298 additions and 67 deletions

View File

@ -13,6 +13,7 @@ class PortStatus(Enum):
ExploiterResultData = namedtuple("ExploiterResultData", ["result", "info", "attempts"])
PingScanData = namedtuple("PingScanData", ["response_received", "os"])
PortScanData = namedtuple("PortScanData", ["port", "status", "banner", "service"])
FingerprintData = namedtuple("FingerprintData", ["os_type", "os_version", "services"])
PostBreachData = namedtuple("PostBreachData", ["command", "result"])
@ -57,13 +58,22 @@ class IPuppet(metaclass=abc.ABCMeta):
"""
@abc.abstractmethod
def fingerprint(self, name: str, host: str) -> Dict:
def fingerprint(
self,
name: str,
host: str,
ping_scan_data: PingScanData,
port_scan_data: Dict[int, PortScanData],
) -> FingerprintData:
"""
Runs a fingerprinter against a remote host
:param str name: The name of the fingerprinter to run
:param str host: The domain name or IP address of a host
:return: A dictionary containing the information collected by the fingerprinter
:rtype: Dict
:param PingScanData ping_scan_data: Data retrieved from the target host via ICMP
:param Dict[int, PortScanData] port_scan_data: Data retrieved from the target host via a TCP
port scan
:return: The data collected by running the fingerprinter on the specified host
:rtype: FingerprintData
"""
@abc.abstractmethod

View File

@ -1,3 +1,4 @@
from .ip_scan_results import IPScanResults
from .ip_scanner import IPScanner
from .propagator import Propagator
from .automated_master import AutomatedMaster

View File

@ -0,0 +1,14 @@
from dataclasses import dataclass
from typing import Dict
from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData
Port = int
FingerprinterName = str
@dataclass
class IPScanResults:
ping_scan_data: PingScanData
port_scan_data: Dict[Port, PortScanData]
fingerprint_data: Dict[FingerprinterName, FingerprintData]

View File

@ -5,15 +5,21 @@ from queue import Queue
from threading import Event
from typing import Callable, Dict, List
from infection_monkey.i_puppet import IPuppet, PingScanData, PortScanData
from infection_monkey.i_puppet import (
FingerprintData,
IPuppet,
PingScanData,
PortScanData,
PortStatus,
)
from . import IPScanResults
from .threading_utils import create_daemon_thread
logger = logging.getLogger()
IP = str
Port = int
Callback = Callable[[IP, PingScanData, Dict[Port, PortScanData]], None]
Callback = Callable[[IP, IPScanResults], None]
class IPScanner:
@ -53,7 +59,15 @@ class IPScanner:
tcp_ports = options["tcp"]["ports"]
port_scan_data = self._scan_tcp_ports(ip, tcp_ports, tcp_timeout, stop)
results_callback(ip, ping_scan_data, port_scan_data)
fingerprint_data = {}
if IPScanner._found_open_port(port_scan_data):
fingerprinters = options["fingerprinters"]
fingerprint_data = self._run_fingerprinters(
ip, fingerprinters, ping_scan_data, port_scan_data, stop
)
scan_results = IPScanResults(ping_scan_data, port_scan_data, fingerprint_data)
results_callback(ip, scan_results)
logger.debug(
f"Detected the stop signal, scanning thread {threading.get_ident()} exiting"
@ -64,7 +78,9 @@ class IPScanner:
f"ips_to_scan queue is empty, scanning thread {threading.get_ident()} exiting"
)
def _scan_tcp_ports(self, ip: str, ports: List[int], timeout: float, stop: Event):
def _scan_tcp_ports(
self, ip: str, ports: List[int], timeout: float, stop: Event
) -> Dict[int, PortScanData]:
port_scan_data = {}
for p in ports:
@ -74,3 +90,29 @@ class IPScanner:
port_scan_data[p] = self._puppet.scan_tcp_port(ip, p, timeout)
return port_scan_data
@staticmethod
def _found_open_port(port_scan_data: Dict[int, PortScanData]):
for psd in port_scan_data.values():
if psd.status == PortStatus.OPEN:
return True
return False
def _run_fingerprinters(
self,
ip: str,
fingerprinters: List[str],
ping_scan_data: PingScanData,
port_scan_data: Dict[int, PortScanData],
stop: Event,
) -> Dict[str, FingerprintData]:
fingerprint_data = {}
for f in fingerprinters:
if stop.is_set():
break
fingerprint_data[f] = self._puppet.fingerprint(f, ip, ping_scan_data, port_scan_data)
return fingerprint_data

View File

@ -88,13 +88,13 @@ class MockMaster(IMaster):
machine_1 = self._hosts["10.0.0.1"]
machine_3 = self._hosts["10.0.0.3"]
self._puppet.fingerprint("SMBFinger", machine_1)
self._puppet.fingerprint("SMBFinger", machine_1, None, None)
self._telemetry_messenger.send_telemetry(ScanTelem(machine_1))
self._puppet.fingerprint("SMBFinger", machine_3)
self._puppet.fingerprint("SMBFinger", machine_3, None, None)
self._telemetry_messenger.send_telemetry(ScanTelem(machine_3))
self._puppet.fingerprint("HTTPFinger", machine_3)
self._puppet.fingerprint("HTTPFinger", machine_3, None, None)
self._telemetry_messenger.send_telemetry(ScanTelem(machine_3))
logger.info("Finished running fingerprinters on potential victims")

View File

@ -3,12 +3,12 @@ from queue import Queue
from threading import Event, Thread
from typing import Dict
from infection_monkey.i_puppet import PingScanData, PortScanData, PortStatus
from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData, PortStatus
from infection_monkey.model.host import VictimHost
from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger
from infection_monkey.telemetry.scan_telem import ScanTelem
from . import IPScanner
from . import IPScanner, IPScanResults
from .threading_utils import create_daemon_thread
logger = logging.getLogger()
@ -51,16 +51,30 @@ class Propagator:
logger.info("Finished network scan")
def _process_scan_results(
self, ip: str, ping_scan_data: PingScanData, port_scan_data: Dict[int, PortScanData]
):
def _process_scan_results(self, ip: str, scan_results: IPScanResults):
victim_host = VictimHost(ip)
has_open_port = False
Propagator._process_ping_scan_results(victim_host, scan_results.ping_scan_data)
has_open_port = Propagator._process_tcp_scan_results(
victim_host, scan_results.port_scan_data
)
Propagator._process_fingerprinter_results(victim_host, scan_results.fingerprint_data)
if has_open_port:
self._hosts_to_exploit.put(victim_host)
self._telemetry_messenger.send_telemetry(ScanTelem(victim_host))
@staticmethod
def _process_ping_scan_results(victim_host: VictimHost, ping_scan_data: PingScanData):
victim_host.icmp = ping_scan_data.response_received
if ping_scan_data.os is not None:
victim_host.os["type"] = ping_scan_data.os
@staticmethod
def _process_tcp_scan_results(victim_host: VictimHost, port_scan_data: PortScanData) -> bool:
has_open_port = False
for psd in port_scan_data.values():
if psd.status == PortStatus.OPEN:
has_open_port = True
@ -71,10 +85,23 @@ class Propagator:
if psd.banner is not None:
victim_host.services[psd.service]["banner"] = psd.banner
if has_open_port:
self._hosts_to_exploit.put(victim_host)
return has_open_port
self._telemetry_messenger.send_telemetry(ScanTelem(victim_host))
@staticmethod
def _process_fingerprinter_results(victim_host: VictimHost, fingerprint_data: FingerprintData):
for fd in fingerprint_data.values():
# TODO: This logic preserves the existing behavior prior to introducing IMaster and
# IPuppet, but it is possibly flawed. Different fingerprinters may detect
# different os types or versions, and this logic isn't sufficient to handle those
# conflicts. Reevaluate this logic when we overhaul our scanners/fingerprinters.
if fd.os_type is not None:
victim_host.os["type"] = fd.os_type
if ("version" not in victim_host.os) and (fd.os_version is not None):
victim_host.os["version"] = fd.os_version
for service, details in fd.services.items():
victim_host.services.setdefault(service, {}).update(details)
def _exploit_targets(self, scan_thread: Thread, stop: Event):
pass

View File

@ -181,8 +181,7 @@ class SMBFinger(HostFinger):
host.services[SMB_SERVICE]["name"] = service_client
if "version" not in host.os:
host.os["version"] = os_version
else:
host.services[SMB_SERVICE]["os-version"] = os_version
return True
except Exception as exc:
logger.debug("Error getting smb fingerprint: %s", exc)

View File

@ -28,8 +28,7 @@ class SSHFinger(HostFinger):
os_version = banner.split(" ").pop().strip()
if "version" not in host.os:
host.os["version"] = os_version
else:
host.services[service]["os-version"] = os_version
break
def get_host_fingerprint(self, host):

View File

@ -4,6 +4,7 @@ from typing import Dict, Tuple
from infection_monkey.i_puppet import (
ExploiterResultData,
FingerprintData,
IPuppet,
PingScanData,
PortScanData,
@ -193,29 +194,43 @@ class MockPuppet(IPuppet):
return _get_empty_results(port)
def fingerprint(self, name: str, host: str) -> Dict:
def fingerprint(
self,
name: str,
host: str,
ping_scan_data: PingScanData,
port_scan_data: Dict[int, PortScanData],
) -> FingerprintData:
logger.debug(f"fingerprint({name}, {host})")
empty_fingerprint_data = FingerprintData(None, None, {})
dot_1_results = {
"SMBFinger": {
"os": {"type": "windows", "version": "vista"},
"services": {"tcp-445": {"name": "SSH", "os": "linux"}},
}
"SMBFinger": FingerprintData(
"windows", "vista", {"tcp-445": {"name": "smb_service_name"}}
)
}
dot_3_results = {
"SSHFinger": {"os": "linux", "services": {"tcp-22": {"name": "SSH"}}},
"HTTPFinger": {
"services": {"tcp-https": {"name": "http", "data": ("SERVER_HEADERS", DOT_3)}}
"SSHFinger": FingerprintData(
"linux", "ubuntu", {"tcp-22": {"name": "SSH", "banner": "SSH BANNER"}}
),
"HTTPFinger": FingerprintData(
None,
None,
{
"tcp-80": {"name": "http", "data": ("SERVER_HEADERS", False)},
"tcp-443": {"name": "http", "data": ("SERVER_HEADERS_2", True)},
},
),
}
if host == DOT_1:
return dot_1_results.get(name, {})
return dot_1_results.get(name, empty_fingerprint_data)
if host == DOT_3:
return dot_3_results.get(name, {})
return dot_3_results.get(name, empty_fingerprint_data)
return {}
return empty_fingerprint_data
def exploit_host(
self, name: str, host: str, options: Dict, interrupt: threading.Event

View File

@ -488,6 +488,9 @@ class ConfigService:
formatted_network_scan_config["icmp"] = ConfigService._format_icmp_scan_from_flat_config(
config
)
formatted_network_scan_config[
"fingerprinters"
] = ConfigService._format_fingerprinters_from_flat_config(config)
return formatted_network_scan_config
@ -529,6 +532,15 @@ class ConfigService:
return formatted_icmp_scan_config
@staticmethod
def _format_fingerprinters_from_flat_config(config: Dict):
flat_fingerprinter_classes_field = "finger_classes"
formatted_fingerprinters = config[flat_fingerprinter_classes_field]
config.pop(flat_fingerprinter_classes_field)
return formatted_fingerprinters
@staticmethod
def _format_targets_from_flat_config(config: Dict):
flat_blocked_ips_field = "blocked_ips"

View File

@ -66,7 +66,6 @@
"SMBFinger",
"SSHFinger",
"HTTPFinger",
"MySQLFinger",
"MSSQLFinger",
"ElasticFinger"
],

View File

@ -4,7 +4,7 @@ from unittest.mock import MagicMock
import pytest
from infection_monkey.i_puppet import PortScanData, PortStatus
from infection_monkey.i_puppet import FingerprintData, PortScanData, PortStatus
from infection_monkey.master import IPScanner
from infection_monkey.puppet.mock_puppet import MockPuppet
@ -29,6 +29,7 @@ def scan_config():
"icmp": {
"timeout_ms": 1000,
},
"fingerprinters": {"HTTPFinger", "SMBFinger", "SSHFinger"},
}
@ -50,9 +51,20 @@ def assert_port_status(port_scan_data, expected_open_ports: Set[int]):
assert psd.status == PortStatus.CLOSED
def assert_scan_results_no_1(ip, ping_scan_data, port_scan_data):
assert ip == "10.0.0.1"
def assert_scan_results(ip, scan_results):
ping_scan_data = scan_results.ping_scan_data
port_scan_data = scan_results.port_scan_data
fingerprint_data = scan_results.fingerprint_data
if ip == "10.0.0.1":
assert_scan_results_no_1(ping_scan_data, port_scan_data, fingerprint_data)
elif ip == "10.0.0.3":
assert_scan_results_no_3(ping_scan_data, port_scan_data, fingerprint_data)
else:
assert_scan_results_host_down(ip, ping_scan_data, port_scan_data, fingerprint_data)
def assert_scan_results_no_1(ping_scan_data, port_scan_data, fingerprint_data):
assert ping_scan_data.response_received is True
assert ping_scan_data.os == WINDOWS_OS
@ -70,11 +82,22 @@ def assert_scan_results_no_1(ip, ping_scan_data, port_scan_data):
assert psd_3389.service == "tcp-3389"
assert_port_status(port_scan_data, {445, 3389})
assert_fingerprint_results_no_1(fingerprint_data)
def assert_scan_results_no_3(ip, ping_scan_data, port_scan_data):
assert ip == "10.0.0.3"
def assert_fingerprint_results_no_1(fingerprint_data):
assert len(fingerprint_data.keys()) == 3
assert fingerprint_data["SSHFinger"].services == {}
assert fingerprint_data["HTTPFinger"].services == {}
assert fingerprint_data["SMBFinger"].os_type == WINDOWS_OS
assert fingerprint_data["SMBFinger"].os_version == "vista"
assert len(fingerprint_data["SMBFinger"].services.keys()) == 1
assert fingerprint_data["SMBFinger"].services["tcp-445"]["name"] == "smb_service_name"
def assert_scan_results_no_3(ping_scan_data, port_scan_data, fingerprint_data):
assert ping_scan_data.response_received is True
assert ping_scan_data.os == LINUX_OS
assert len(port_scan_data.keys()) == 6
@ -91,15 +114,36 @@ def assert_scan_results_no_3(ip, ping_scan_data, port_scan_data):
assert psd_22.service == "tcp-22"
assert_port_status(port_scan_data, {22, 443})
assert_fingerprint_results_no_3(fingerprint_data)
def assert_scan_results_host_down(ip, ping_scan_data, port_scan_data):
def assert_fingerprint_results_no_3(fingerprint_data):
assert len(fingerprint_data.keys()) == 3
assert fingerprint_data["SMBFinger"].services == {}
assert fingerprint_data["SSHFinger"].os_type == LINUX_OS
assert fingerprint_data["SSHFinger"].os_version == "ubuntu"
assert len(fingerprint_data["SSHFinger"].services.keys()) == 1
assert fingerprint_data["SSHFinger"].services["tcp-22"]["name"] == "SSH"
assert fingerprint_data["SSHFinger"].services["tcp-22"]["banner"] == "SSH BANNER"
assert len(fingerprint_data["HTTPFinger"].services.keys()) == 2
assert fingerprint_data["HTTPFinger"].services["tcp-80"]["name"] == "http"
assert fingerprint_data["HTTPFinger"].services["tcp-80"]["data"] == ("SERVER_HEADERS", False)
assert fingerprint_data["HTTPFinger"].services["tcp-443"]["name"] == "http"
assert fingerprint_data["HTTPFinger"].services["tcp-443"]["data"] == ("SERVER_HEADERS_2", True)
def assert_scan_results_host_down(ip, ping_scan_data, port_scan_data, fingerprint_data):
assert ip not in {"10.0.0.1", "10.0.0.3"}
assert ping_scan_data.response_received is False
assert len(port_scan_data.keys()) == 6
assert_port_status(port_scan_data, set())
assert fingerprint_data == {}
def test_scan_single_ip(callback, scan_config, stop):
ips = ["10.0.0.1"]
@ -109,8 +153,8 @@ def test_scan_single_ip(callback, scan_config, stop):
callback.assert_called_once()
(ip, ping_scan_data, port_scan_data) = callback.call_args_list[0][0]
assert_scan_results_no_1(ip, ping_scan_data, port_scan_data)
(ip, scan_results) = callback.call_args_list[0][0]
assert_scan_results(ip, scan_results)
def test_scan_multiple_ips(callback, scan_config, stop):
@ -121,17 +165,17 @@ def test_scan_multiple_ips(callback, scan_config, stop):
assert callback.call_count == 4
(ip, ping_scan_data, port_scan_data) = callback.call_args_list[0][0]
assert_scan_results_no_1(ip, ping_scan_data, port_scan_data)
(ip, scan_results) = callback.call_args_list[0][0]
assert_scan_results(ip, scan_results)
(ip, ping_scan_data, port_scan_data) = callback.call_args_list[1][0]
assert_scan_results_host_down(ip, ping_scan_data, port_scan_data)
(ip, scan_results) = callback.call_args_list[1][0]
assert_scan_results(ip, scan_results)
(ip, ping_scan_data, port_scan_data) = callback.call_args_list[2][0]
assert_scan_results_no_3(ip, ping_scan_data, port_scan_data)
(ip, scan_results) = callback.call_args_list[2][0]
assert_scan_results(ip, scan_results)
(ip, ping_scan_data, port_scan_data) = callback.call_args_list[3][0]
assert_scan_results_host_down(ip, ping_scan_data, port_scan_data)
(ip, scan_results) = callback.call_args_list[3][0]
assert_scan_results(ip, scan_results)
def test_scan_lots_of_ips(callback, scan_config, stop):
@ -182,3 +226,25 @@ def test_interrupt_port_scanning(callback, scan_config, stop):
ns.scan(ips, scan_config, callback, stop)
assert puppet.scan_tcp_port.call_count == 2
def test_interrupt_fingerprinting(callback, scan_config, stop):
def stopable_fingerprint(*_):
# Block all threads here until 2 threads reach this barrier, then set stop
# and test that neither thread scans any more ports
stopable_fingerprint.barrier.wait()
stop.set()
return FingerprintData(None, None, {})
stopable_fingerprint.barrier = Barrier(2)
puppet = MockPuppet()
puppet.fingerprint = MagicMock(side_effect=stopable_fingerprint)
ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"]
ns = IPScanner(puppet, num_workers=2)
ns.scan(ips, scan_config, callback, stop)
assert puppet.fingerprint.call_count == 2

View File

@ -1,61 +1,98 @@
from threading import Event
from infection_monkey.i_puppet import PingScanData, PortScanData, PortStatus
from infection_monkey.master import Propagator
from infection_monkey.i_puppet import FingerprintData, PingScanData, PortScanData, PortStatus
from infection_monkey.master import IPScanResults, Propagator
dot_1_results = (
empty_fingerprint_data = FingerprintData(None, None, {})
dot_1_results = IPScanResults(
PingScanData(True, "windows"),
{
22: PortScanData(22, PortStatus.CLOSED, None, None),
445: PortScanData(445, PortStatus.OPEN, "SMB BANNER", "tcp-445"),
3389: PortScanData(3389, PortStatus.OPEN, "", "tcp-3389"),
},
{
"SMBFinger": FingerprintData("windows", "vista", {"tcp-445": {"name": "smb_service_name"}}),
"SSHFinger": empty_fingerprint_data,
"HTTPFinger": empty_fingerprint_data,
},
)
dot_3_results = (
dot_3_results = IPScanResults(
PingScanData(True, "linux"),
{
22: PortScanData(22, PortStatus.OPEN, "SSH BANNER", "tcp-22"),
443: PortScanData(443, PortStatus.OPEN, "HTTPS BANNER", "tcp-443"),
3389: PortScanData(3389, PortStatus.CLOSED, "", None),
},
{
"SSHFinger": FingerprintData(
"linux", "ubuntu", {"tcp-22": {"name": "SSH", "banner": "SSH BANNER"}}
),
"HTTPFinger": FingerprintData(
None,
None,
{
"tcp-80": {"name": "http", "data": ("SERVER_HEADERS", False)},
"tcp-443": {"name": "http", "data": ("SERVER_HEADERS_2", True)},
},
),
"SMBFinger": empty_fingerprint_data,
},
)
dead_host_results = (
dead_host_results = IPScanResults(
PingScanData(False, None),
{
22: PortScanData(22, PortStatus.CLOSED, None, None),
443: PortScanData(443, PortStatus.CLOSED, None, None),
3389: PortScanData(3389, PortStatus.CLOSED, "", None),
},
{},
)
dot_1_services = {
"tcp-445": {"display_name": "unknown(TCP)", "port": 445, "banner": "SMB BANNER"},
"tcp-445": {
"name": "smb_service_name",
"display_name": "unknown(TCP)",
"port": 445,
"banner": "SMB BANNER",
},
"tcp-3389": {"display_name": "unknown(TCP)", "port": 3389, "banner": ""},
}
dot_3_services = {
"tcp-22": {"display_name": "unknown(TCP)", "port": 22, "banner": "SSH BANNER"},
"tcp-443": {"display_name": "unknown(TCP)", "port": 443, "banner": "HTTPS BANNER"},
"tcp-22": {"name": "SSH", "display_name": "unknown(TCP)", "port": 22, "banner": "SSH BANNER"},
"tcp-80": {"name": "http", "data": ("SERVER_HEADERS", False)},
"tcp-443": {
"name": "http",
"display_name": "unknown(TCP)",
"port": 443,
"banner": "HTTPS BANNER",
"data": ("SERVER_HEADERS_2", True),
},
}
class MockIPScanner:
def scan(self, ips_to_scan, options, results_callback, stop):
def scan(self, ips_to_scan, _, results_callback, stop):
for ip in ips_to_scan:
if ip.endswith(".1"):
results_callback(ip, *dot_1_results)
results_callback(ip, dot_1_results)
elif ip.endswith(".3"):
results_callback(ip, *dot_3_results)
results_callback(ip, dot_3_results)
else:
results_callback(ip, *dead_host_results)
results_callback(ip, dead_host_results)
def test_scan_result_processing(telemetry_messenger_spy):
p = Propagator(telemetry_messenger_spy, MockIPScanner())
p.propagate(
{"targets": {"subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"]}, "network_scan": {}},
{
"targets": {"subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"]},
"network_scan": {},
},
Event(),
)
@ -68,11 +105,13 @@ def test_scan_result_processing(telemetry_messenger_spy):
if ip.endswith(".1"):
assert data["service_count"] == 2
assert data["machine"]["os"]["type"] == "windows"
assert data["machine"]["os"]["version"] == "vista"
assert data["machine"]["services"] == dot_1_services
assert data["machine"]["icmp"] is True
elif ip.endswith(".3"):
assert data["service_count"] == 2
assert data["service_count"] == 3
assert data["machine"]["os"]["type"] == "linux"
assert data["machine"]["os"]["version"] == "ubuntu"
assert data["machine"]["services"] == dot_3_services
assert data["machine"]["icmp"] is True
else:

View File

@ -143,6 +143,13 @@ def test_format_config_for_agent__network_scan(flat_monkey_config):
"icmp": {
"timeout_ms": 1000,
},
"fingerprinters": [
"SMBFinger",
"SSHFinger",
"HTTPFinger",
"MSSQLFinger",
"ElasticFinger",
],
}
ConfigService.format_flat_config_for_agent(flat_monkey_config)
@ -153,3 +160,4 @@ def test_format_config_for_agent__network_scan(flat_monkey_config):
assert "tcp_scan_timeout" not in flat_monkey_config
assert "tcp_target_ports" not in flat_monkey_config
assert "ping_scan_timeout" not in flat_monkey_config
assert "finger_classes" not in flat_monkey_config