Merge branch '1605-resolve-circular-dependency' into agent-refactor

Issue #1605

monkey/infection_monkey/exploit/elasticgroovy.py

@@ -22,7 +22,7 @@ from infection_monkey.model import (
     ID_STRING,
     WGET_HTTP_UPLOAD,
 )
-from infection_monkey.network.elasticfinger import ES_PORT
+from infection_monkey.network_scanning.elasticfinger import ES_PORT
 from infection_monkey.telemetry.attack.t1197_telem import T1197Telem
 
 logger = logging.getLogger(__name__)

monkey/infection_monkey/exploit/smbexec.py

@@ -8,8 +8,8 @@ from infection_monkey.exploit.HostExploiter import HostExploiter
 from infection_monkey.exploit.tools.helpers import get_monkey_depth, get_target_monkey
 from infection_monkey.exploit.tools.smb_tools import SmbTools
 from infection_monkey.model import DROPPER_CMDLINE_DETACHED_WINDOWS, MONKEY_CMDLINE_DETACHED_WINDOWS
-from infection_monkey.network.smbfinger import SMBFinger
 from infection_monkey.network.tools import check_tcp_port
+from infection_monkey.network_scanning.smbfinger import SMBFinger
 from infection_monkey.telemetry.attack.t1035_telem import T1035Telem
 from infection_monkey.utils.commands import build_monkey_commandline
 

monkey/infection_monkey/i_puppet/i_puppet.py

@@ -5,6 +5,8 @@ from dataclasses import dataclass
 from enum import Enum
 from typing import Dict, Iterable, List, Mapping, Sequence
 
+from infection_monkey.model import VictimHost
+
 from . import PluginType
 from .credential_collection import Credentials
 
@@ -110,15 +112,14 @@ class IPuppet(metaclass=abc.ABCMeta):
         :rtype: FingerprintData
         """
 
-    # TODO: host should be VictimHost, at the moment it can't because of circular dependency
     @abc.abstractmethod
     def exploit_host(
-        self, name: str, host: object, options: Dict, interrupt: threading.Event
+        self, name: str, host: VictimHost, options: Dict, interrupt: threading.Event
     ) -> ExploiterResultData:
         """
         Runs an exploiter against a remote host
         :param str name: The name of the exploiter to run
-        :param object host: The domain name or IP address of a host
+        :param VictimHost host: A VictimHost object representing the target to exploit
         :param Dict options: A dictionary containing options that modify the behavior of the
                              exploiter
         :param threading.Event interrupt: A threading.Event object that signals the exploit to stop

monkey/infection_monkey/master/propagator.py

@@ -12,7 +12,7 @@ from infection_monkey.i_puppet import (
 )
 from infection_monkey.model import VictimHost, VictimHostFactory
 from infection_monkey.network import NetworkAddress, NetworkInterface
-from infection_monkey.network.scan_target_generator import compile_scan_target_list
+from infection_monkey.network_scanning.scan_target_generator import compile_scan_target_list
 from infection_monkey.telemetry.exploit_telem import ExploitTelem
 from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger
 from infection_monkey.telemetry.scan_telem import ScanTelem

monkey/infection_monkey/monkey.py

@@ -22,13 +22,13 @@ from infection_monkey.master import AutomatedMaster
 from infection_monkey.master.control_channel import ControlChannel
 from infection_monkey.model import DELAY_DELETE_CMD, VictimHostFactory
 from infection_monkey.network import NetworkInterface
-from infection_monkey.network.elasticsearch_fingerprinter import ElasticSearchFingerprinter
 from infection_monkey.network.firewall import app as firewall
-from infection_monkey.network.http_fingerprinter import HTTPFingerprinter
 from infection_monkey.network.info import get_local_network_interfaces
-from infection_monkey.network.mssql_fingerprinter import MSSQLFingerprinter
-from infection_monkey.network.smb_fingerprinter import SMBFingerprinter
-from infection_monkey.network.ssh_fingerprinter import SSHFingerprinter
+from infection_monkey.network_scanning.elasticsearch_fingerprinter import ElasticSearchFingerprinter
+from infection_monkey.network_scanning.http_fingerprinter import HTTPFingerprinter
+from infection_monkey.network_scanning.mssql_fingerprinter import MSSQLFingerprinter
+from infection_monkey.network_scanning.smb_fingerprinter import SMBFingerprinter
+from infection_monkey.network_scanning.ssh_fingerprinter import SSHFingerprinter
 from infection_monkey.payload.ransomware.ransomware_payload import RansomwarePayload
 from infection_monkey.puppet.puppet import Puppet
 from infection_monkey.system_singleton import SystemSingleton

monkey/infection_monkey/network/__init__.py

@@ -1,3 +1 @@
-from .scan_target_generator import NetworkAddress, NetworkInterface
-from .ping_scanner import ping
-from .tcp_scanner import scan_tcp_ports
+from .info import NetworkAddress, NetworkInterface

monkey/infection_monkey/network/info.py

@@ -1,6 +1,7 @@
 import itertools
 import socket
 import struct
+from collections import namedtuple
 from ipaddress import IPv4Network
 from random import randint  # noqa: DUO102
 from typing import List
@@ -11,8 +12,6 @@ import psutil
 from common.network.network_range import CidrRange
 from infection_monkey.utils.environment import is_windows_os
 
-from . import NetworkInterface
-
 # Timeout for monkey connections
 TIMEOUT = 15
 LOOPBACK_NAME = b"lo"
@@ -21,6 +20,9 @@ SIOCGIFNETMASK = 0x891B  # get network PA mask
 RTF_UP = 0x0001  # Route usable
 RTF_REJECT = 0x0200
 
+NetworkInterface = namedtuple("NetworkInterface", ("address", "netmask"))
+NetworkAddress = namedtuple("NetworkAddress", ("ip", "domain"))
+
 
 def get_local_network_interfaces() -> List[NetworkInterface]:
     network_interfaces = []

monkey/infection_monkey/network/network_scanner.py

@@ -1,136 +0,0 @@
-import logging
-from multiprocessing.dummy import Pool
-
-from common.network.network_range import NetworkRange
-from infection_monkey.config import WormConfiguration
-from infection_monkey.model.victim_host_generator import VictimHostGenerator
-from infection_monkey.network.info import get_interfaces_ranges, local_ips
-from infection_monkey.network.ping_scanner import PingScanner
-from infection_monkey.network.tcp_scanner import TcpScanner
-
-logger = logging.getLogger(__name__)
-
-ITERATION_BLOCK_SIZE = 5
-
-
-class NetworkScanner(object):
-    def __init__(self):
-        self._ip_addresses = None
-        self._ranges = None
-        self.scanners = [TcpScanner(), PingScanner()]
-
-    def initialize(self):
-        """
-        Set up scanning.
-        based on configuration: scans local network and/or scans fixed list of IPs/subnets.
-        """
-        # get local ip addresses
-        self._ip_addresses = local_ips()
-
-        if not self._ip_addresses:
-            raise Exception("Cannot find local IP address for the machine")
-
-        logger.info("Found local IP addresses of the machine: %r", self._ip_addresses)
-        # for fixed range, only scan once.
-        self._ranges = [
-            NetworkRange.get_range_obj(address_str=x) for x in WormConfiguration.subnet_scan_list
-        ]
-        if WormConfiguration.local_network_scan:
-            self._ranges += get_interfaces_ranges()
-        self._ranges += self._get_inaccessible_subnets_ips()
-        logger.info("Base local networks to scan are: %r", self._ranges)
-
-    # TODO remove afret agent refactoring,
-    #  it's already handled in network.scan_target_generator._get_inaccessible_subnets_ips
-    def _get_inaccessible_subnets_ips(self):
-        """
-        For each of the machine's IPs, checks if it's in one of the subnets specified in the
-        'inaccessible_subnets' config value. If so, all other subnets in the config value
-        shouldn't be accessible.
-        All these subnets are returned.
-        :return: A list of subnets that shouldn't be accessible from the machine the monkey is
-        running on.
-        """
-        subnets_to_scan = []
-        if len(WormConfiguration.inaccessible_subnets) > 1:
-            for subnet_str in WormConfiguration.inaccessible_subnets:
-                if NetworkScanner._is_any_ip_in_subnet(
-                    [str(x) for x in self._ip_addresses], subnet_str
-                ):
-                    # If machine has IPs from 2 different subnets in the same group, there's no
-                    # point checking the other
-                    # subnet.
-                    for other_subnet_str in WormConfiguration.inaccessible_subnets:
-                        if other_subnet_str == subnet_str:
-                            continue
-                        if not NetworkScanner._is_any_ip_in_subnet(
-                            [str(x) for x in self._ip_addresses], other_subnet_str
-                        ):
-                            subnets_to_scan.append(NetworkRange.get_range_obj(other_subnet_str))
-                    break
-
-        return subnets_to_scan
-
-    def get_victim_machines(self, max_find=5, stop_callback=None):
-        """
-        Finds machines according to the ranges specified in the object
-        :param max_find: Max number of victims to find regardless of ranges
-        :param stop_callback: A callback to check at any point if we should stop scanning
-        :return: yields a sequence of VictimHost instances
-        """
-        # We currently use the ITERATION_BLOCK_SIZE as the pool size, however, this may not be
-        # the best decision
-        # However, the decision what ITERATION_BLOCK_SIZE also requires balancing network usage (
-        # pps and bw)
-        # Because we are using this to spread out IO heavy tasks, we can probably go a lot higher
-        # than CPU core size
-        # But again, balance
-        pool = Pool(ITERATION_BLOCK_SIZE)
-        victim_generator = VictimHostGenerator(
-            self._ranges, WormConfiguration.blocked_ips, local_ips()
-        )
-
-        victims_count = 0
-        for victim_chunk in victim_generator.generate_victims(ITERATION_BLOCK_SIZE):
-            logger.debug("Scanning for potential victims in chunk %r", victim_chunk)
-
-            # check before running scans
-            if stop_callback and stop_callback():
-                logger.debug("Got stop signal")
-                return
-
-            results = pool.map(self.scan_machine, victim_chunk)
-            resulting_victims = [x for x in results if x is not None]
-            for victim in resulting_victims:
-                logger.debug("Found potential victim: %r", victim)
-                victims_count += 1
-                yield victim
-
-                if victims_count >= max_find:
-                    logger.debug("Found max needed victims (%d), stopping scan", max_find)
-                    return
-
-    @staticmethod
-    # TODO remove afret agent refactoring,
-    #  it's already handled in network.scan_target_generator._is_any_ip_in_subnet
-    def _is_any_ip_in_subnet(ip_addresses, subnet_str):
-        for ip_address in ip_addresses:
-            if NetworkRange.get_range_obj(subnet_str).is_in_range(ip_address):
-                return True
-        return False
-
-    def scan_machine(self, victim):
-        """
-        Scans specific machine using instance scanners
-        :param victim: VictimHost machine
-        :return: Victim or None if victim isn't alive
-        """
-        logger.debug("Scanning target address: %r", victim)
-        if any(scanner.is_host_alive(victim) for scanner in self.scanners):
-            logger.debug("Found potential target_ip: %r", victim)
-            return victim
-        else:
-            return None
-
-    def on_island(self, server):
-        return bool([x for x in self._ip_addresses if x in server])

monkey/infection_monkey/network_scanning/__init__.py

@@ -0,0 +1,2 @@
+from .ping_scanner import ping
+from .tcp_scanner import scan_tcp_ports

monkey/infection_monkey/network_scanning/scan_target_generator.py

@@ -1,13 +1,10 @@
 import itertools
 import logging
 import socket
-from collections import namedtuple
 from typing import List
 
 from common.network.network_range import InvalidNetworkRangeError, NetworkRange
-
-NetworkInterface = namedtuple("NetworkInterface", ("address", "netmask"))
-NetworkAddress = namedtuple("NetworkAddress", ("ip", "domain"))
+from infection_monkey.network import NetworkAddress, NetworkInterface
 
 logger = logging.getLogger(__name__)
 

monkey/infection_monkey/puppet/mock_puppet.py

@@ -14,6 +14,7 @@ from infection_monkey.i_puppet import (
     PortStatus,
     PostBreachData,
 )
+from infection_monkey.model import VictimHost
 
 DOT_1 = "10.0.0.1"
 DOT_2 = "10.0.0.2"
@@ -136,7 +137,7 @@ class MockPuppet(IPuppet):
 
     # TODO: host should be VictimHost, at the moment it can't because of circular dependency
     def exploit_host(
-        self, name: str, host: object, options: Dict, interrupt: threading.Event
+        self, name: str, host: VictimHost, options: Dict, interrupt: threading.Event
     ) -> ExploiterResultData:
         logger.debug(f"exploit_hosts({name}, {host}, {options})")
         attempts = [

monkey/infection_monkey/puppet/puppet.py

@@ -2,7 +2,7 @@ import logging
 import threading
 from typing import Dict, List, Sequence
 
-from infection_monkey import network
+from infection_monkey import network_scanning
 from infection_monkey.i_puppet import (
     Credentials,
     ExploiterResultData,
@@ -13,6 +13,7 @@ from infection_monkey.i_puppet import (
     PortScanData,
     PostBreachData,
 )
+from infection_monkey.model import VictimHost
 
 from ..telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger
 from .mock_puppet import MockPuppet
@@ -40,12 +41,12 @@ class Puppet(IPuppet):
         return self._mock_puppet.run_pba(name, options)
 
     def ping(self, host: str, timeout: float = 1) -> PingScanData:
-        return network.ping(host, timeout)
+        return network_scanning.ping(host, timeout)
 
     def scan_tcp_ports(
         self, host: str, ports: List[int], timeout: float = 3
     ) -> Dict[int, PortScanData]:
-        return network.scan_tcp_ports(host, ports, timeout)
+        return network_scanning.scan_tcp_ports(host, ports, timeout)
 
     def fingerprint(
         self,
@@ -58,9 +59,8 @@ class Puppet(IPuppet):
         fingerprinter = self._plugin_registry.get_plugin(name, PluginType.FINGERPRINTER)
         return fingerprinter.get_host_fingerprint(host, ping_scan_data, port_scan_data, options)
 
-    # TODO: host should be VictimHost, at the moment it can't because of circular dependency
     def exploit_host(
-        self, name: str, host: object, options: Dict, interrupt: threading.Event
+        self, name: str, host: VictimHost, options: Dict, interrupt: threading.Event
     ) -> ExploiterResultData:
         exploiter = self._plugin_registry.get_plugin(name, PluginType.EXPLOITER)
         return exploiter.exploit_host(host, self._telemetry_messenger, options)

monkey/tests/unit_tests/infection_monkey/model/test_victim_host_factory.py

@@ -3,7 +3,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from infection_monkey.model import VictimHostFactory
-from infection_monkey.network.scan_target_generator import NetworkAddress
+from infection_monkey.network import NetworkAddress
 
 
 @pytest.fixture

monkey/tests/unit_tests/infection_monkey/network_scanning/test_elasticsearch_fingerprinter.py

@@ -4,7 +4,10 @@ import pytest
 
 from common.common_consts.network_consts import ES_SERVICE
 from infection_monkey.i_puppet import PortScanData, PortStatus
-from infection_monkey.network.elasticsearch_fingerprinter import ES_PORT, ElasticSearchFingerprinter
+from infection_monkey.network_scanning.elasticsearch_fingerprinter import (
+    ES_PORT,
+    ElasticSearchFingerprinter,
+)
 
 PORT_SCAN_DATA_OPEN = {ES_PORT: PortScanData(ES_PORT, PortStatus.OPEN, "", f"tcp-{ES_PORT}")}
 PORT_SCAN_DATA_CLOSED = {ES_PORT: PortScanData(ES_PORT, PortStatus.CLOSED, "", f"tcp-{ES_PORT}")}
@@ -26,7 +29,7 @@ def test_successful(monkeypatch, fingerprinter):
         "version": {"number": "1.0.0"},
     }
     monkeypatch.setattr(
-        "infection_monkey.network.elasticsearch_fingerprinter._query_elasticsearch",
+        "infection_monkey.network_scanning.elasticsearch_fingerprinter._query_elasticsearch",
         lambda _: successful_server_response,
     )
 
@@ -49,7 +52,7 @@ def test_successful(monkeypatch, fingerprinter):
 def test_fingerprinting_skipped_if_port_closed(monkeypatch, fingerprinter, port_scan_data):
     mock_query_elasticsearch = MagicMock()
     monkeypatch.setattr(
-        "infection_monkey.network.elasticsearch_fingerprinter._query_elasticsearch",
+        "infection_monkey.network_scanning.elasticsearch_fingerprinter._query_elasticsearch",
         mock_query_elasticsearch,
     )
 
@@ -70,7 +73,7 @@ def test_fingerprinting_skipped_if_port_closed(monkeypatch, fingerprinter, port_
 )
 def test_no_response_from_server(monkeypatch, fingerprinter, mock_query_function):
     monkeypatch.setattr(
-        "infection_monkey.network.elasticsearch_fingerprinter._query_elasticsearch",
+        "infection_monkey.network_scanning.elasticsearch_fingerprinter._query_elasticsearch",
         mock_query_function,
     )
 

monkey/tests/unit_tests/infection_monkey/network_scanning/test_http_fingerprinter.py

@@ -3,7 +3,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from infection_monkey.i_puppet import PortScanData, PortStatus
-from infection_monkey.network.http_fingerprinter import HTTPFingerprinter
+from infection_monkey.network_scanning.http_fingerprinter import HTTPFingerprinter
 
 OPTIONS = {"http_ports": [80, 443, 8080, 9200]}
 
@@ -24,7 +24,7 @@ def mock_get_server_from_headers():
 @pytest.fixture(autouse=True)
 def patch_get_server_from_headers(monkeypatch, mock_get_server_from_headers):
     monkeypatch.setattr(
-        "infection_monkey.network.http_fingerprinter._get_server_from_headers",
+        "infection_monkey.network_scanning.http_fingerprinter._get_server_from_headers",
         mock_get_server_from_headers,
     )
 

monkey/tests/unit_tests/infection_monkey/network_scanning/test_mssql_fingerprinter.py

@@ -4,7 +4,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from infection_monkey.i_puppet import PortScanData, PortStatus
-from infection_monkey.network.mssql_fingerprinter import (
+from infection_monkey.network_scanning.mssql_fingerprinter import (
     MSSQL_SERVICE,
     SQL_BROWSER_DEFAULT_PORT,
     MSSQLFingerprinter,
@@ -36,7 +36,7 @@ def test_mssql_fingerprint_successful(monkeypatch, fingerprinter):
         b"IsClustered;No;Version;11.1.1111.111;tcp;1433;np;blah_blah;;"
     )
     monkeypatch.setattr(
-        "infection_monkey.network.mssql_fingerprinter._query_mssql_for_instance_data",
+        "infection_monkey.network_scanning.mssql_fingerprinter._query_mssql_for_instance_data",
         lambda _: successful_server_response,
     )
 
@@ -69,7 +69,7 @@ def test_mssql_fingerprint_successful(monkeypatch, fingerprinter):
 )
 def test_mssql_no_response_from_server(monkeypatch, fingerprinter, mock_query_function):
     monkeypatch.setattr(
-        "infection_monkey.network.mssql_fingerprinter._query_mssql_for_instance_data",
+        "infection_monkey.network_scanning.mssql_fingerprinter._query_mssql_for_instance_data",
         mock_query_function,
     )
 
@@ -89,7 +89,7 @@ def test_mssql_wrong_response_from_server(monkeypatch, fingerprinter):
         b"Pellentesque ultrices ornare libero, ;;"
     )
     monkeypatch.setattr(
-        "infection_monkey.network.mssql_fingerprinter._query_mssql_for_instance_data",
+        "infection_monkey.network_scanning.mssql_fingerprinter._query_mssql_for_instance_data",
         lambda _: mangled_server_response,
     )
 

monkey/tests/unit_tests/infection_monkey/network_scanning/test_ping.py

@@ -3,7 +3,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from infection_monkey.network import ping
+from infection_monkey.network_scanning import ping
 
 LINUX_SUCCESS_OUTPUT = """
 PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.

monkey/tests/unit_tests/infection_monkey/network_scanning/test_scan_target_generator.py

@@ -3,11 +3,8 @@ from itertools import chain
 import pytest
 
 from common.network.network_range import InvalidNetworkRangeError
-from infection_monkey.network.scan_target_generator import (
-    NetworkAddress,
-    NetworkInterface,
-    compile_scan_target_list,
-)
+from infection_monkey.network import NetworkAddress, NetworkInterface
+from infection_monkey.network_scanning.scan_target_generator import compile_scan_target_list
 
 
 def compile_ranges_only(ranges):

monkey/tests/unit_tests/infection_monkey/network_scanning/test_ssh_fingerprinter.py

@@ -1,7 +1,7 @@
 import pytest
 
 from infection_monkey.i_puppet import FingerprintData, PortScanData, PortStatus
-from infection_monkey.network.ssh_fingerprinter import SSHFingerprinter
+from infection_monkey.network_scanning.ssh_fingerprinter import SSHFingerprinter
 
 
 @pytest.fixture

monkey/tests/unit_tests/infection_monkey/network_scanning/test_tcp_scanning.py

@@ -1,7 +1,7 @@
 import pytest
 
 from infection_monkey.i_puppet import PortStatus
-from infection_monkey.network import scan_tcp_ports
+from infection_monkey.network_scanning import scan_tcp_ports
 
 PORTS_TO_SCAN = [22, 80, 8080, 143, 445, 2222]
 
@@ -11,7 +11,7 @@ OPEN_PORTS_DATA = {22: "SSH-banner", 80: "", 2222: "SSH2-banner"}
 @pytest.fixture
 def patch_check_tcp_ports(monkeypatch, open_ports_data):
     monkeypatch.setattr(
-        "infection_monkey.network.tcp_scanner._check_tcp_ports",
+        "infection_monkey.network_scanning.tcp_scanner._check_tcp_ports",
         lambda *_: open_ports_data,
     )