From 80a095b6574b325fe091070f8d5c9413331543ba Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 3 Oct 2022 14:45:29 +0300 Subject: [PATCH 1/8] Agent: Use NetworkPort instead of Port --- monkey/infection_monkey/master/ip_scan_results.py | 5 ++--- monkey/infection_monkey/master/propagator.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/monkey/infection_monkey/master/ip_scan_results.py b/monkey/infection_monkey/master/ip_scan_results.py index 06bf7cd2f..8abce95e6 100644 --- a/monkey/infection_monkey/master/ip_scan_results.py +++ b/monkey/infection_monkey/master/ip_scan_results.py @@ -1,15 +1,14 @@ from dataclasses import dataclass from typing import Dict -from common.types import PingScanData +from common.types import NetworkPort, PingScanData from infection_monkey.i_puppet import FingerprintData, PortScanData -Port = int FingerprinterName = str @dataclass class IPScanResults: ping_scan_data: PingScanData - port_scan_data: Dict[Port, PortScanData] + port_scan_data: Dict[NetworkPort, PortScanData] fingerprint_data: Dict[FingerprinterName, FingerprintData] diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index 21a4708fb..f39d69d50 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -10,7 +10,7 @@ from common.agent_configuration import ( PropagationConfiguration, ScanTargetConfiguration, ) -from common.types import PingScanData, PortStatus +from common.types import NetworkPort, PingScanData, PortStatus from infection_monkey.i_puppet import ExploiterResultData, FingerprintData, PortScanData from infection_monkey.model import VictimHost, VictimHostFactory from infection_monkey.network import NetworkAddress @@ -21,7 +21,7 @@ from infection_monkey.telemetry.scan_telem import ScanTelem from infection_monkey.utils.threading import create_daemon_thread from . import Exploiter, IPScanner, IPScanResults -from .ip_scan_results import FingerprinterName, Port +from .ip_scan_results import FingerprinterName logger = logging.getLogger() @@ -146,7 +146,7 @@ class Propagator: @staticmethod def _process_tcp_scan_results( - victim_host: VictimHost, port_scan_data: Mapping[Port, PortScanData] + victim_host: VictimHost, port_scan_data: Mapping[NetworkPort, PortScanData] ): for psd in filter( lambda scan_data: scan_data.status == PortStatus.OPEN, port_scan_data.values() From a390c97b709941fb79a88753c3e119a57af0ae7b Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 3 Oct 2022 17:59:17 +0300 Subject: [PATCH 2/8] Island: Add tcp_connections to node --- monkey/monkey_island/cc/models/node.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monkey/monkey_island/cc/models/node.py b/monkey/monkey_island/cc/models/node.py index 715e52bb3..d992d1836 100644 --- a/monkey/monkey_island/cc/models/node.py +++ b/monkey/monkey_island/cc/models/node.py @@ -4,6 +4,7 @@ from pydantic import Field from typing_extensions import TypeAlias from common.base_models import MutableInfectionMonkeyBaseModel +from common.types import SocketAddress from . import CommunicationType, MachineID @@ -24,3 +25,6 @@ class Node(MutableInfectionMonkeyBaseModel): connections: NodeConnections """All outbound connections from this node to other machines""" + + tcp_connections: Mapping[MachineID, FrozenSet[SocketAddress]] = {} + """All successfull outbound TCP connections""" From 8bf1d1f46fb8d928640114b757c12f44a15b0e13 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 3 Oct 2022 18:01:49 +0300 Subject: [PATCH 3/8] Island, Common: Add services to machine.py --- monkey/common/types.py | 6 +++++- monkey/monkey_island/cc/models/machine.py | 7 +++++-- vulture_allowlist.py | 10 +++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/monkey/common/types.py b/monkey/common/types.py index 499df9339..049c7b7de 100644 --- a/monkey/common/types.py +++ b/monkey/common/types.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import Enum +from enum import Enum, auto from ipaddress import IPv4Address from typing import Dict, List, Optional, Union from uuid import UUID @@ -28,6 +28,10 @@ JSONSerializable = Union[ # type: ignore[misc] ] +class NetworkServiceNameEnum(Enum): + UNKNOWN = auto() + + class NetworkPort(ConstrainedInt): """ Define network port as constrainer integer. diff --git a/monkey/monkey_island/cc/models/machine.py b/monkey/monkey_island/cc/models/machine.py index ece877b9e..9175b49a2 100644 --- a/monkey/monkey_island/cc/models/machine.py +++ b/monkey/monkey_island/cc/models/machine.py @@ -1,12 +1,12 @@ from ipaddress import IPv4Interface -from typing import Optional, Sequence +from typing import Mapping, Optional, Sequence from pydantic import Field, validator from common import OperatingSystem from common.base_models import MutableInfectionMonkeyBaseModel from common.transforms import make_immutable_sequence -from common.types import HardwareID +from common.types import HardwareID, NetworkServiceNameEnum, SocketAddress from . import MachineID @@ -35,6 +35,9 @@ class Machine(MutableInfectionMonkeyBaseModel): hostname: str = "" """The hostname of the machine""" + network_services: Mapping[SocketAddress, NetworkServiceNameEnum] + """All network services found running on the machine""" + _make_immutable_sequence = validator("network_interfaces", pre=True, allow_reuse=True)( make_immutable_sequence ) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 0ff6b710d..62deedcd9 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -9,14 +9,13 @@ from common.agent_configuration.agent_sub_configurations import ( ) from common.agent_events import ExploitationEvent, PingScanEvent, PropagationEvent, TCPScanEvent from common.credentials import Credentials, LMHash, NTHash -from common.types import NetworkPort from infection_monkey.exploit.HostExploiter.HostExploiter import ( _publish_exploitation_event, _publish_propagation_event, ) +from common.types import NetworkPort, NetworkServiceNameEnum from infection_monkey.exploit.log4shell_utils.ldap_server import LDAPServerFactory -from monkey_island.cc.event_queue import IslandEventTopic, PyPubSubIslandEventQueue -from monkey_island.cc.models import Report +from monkey_island.cc.models import Machine, Node, Report from monkey_island.cc.models.networkmap import Arc, NetworkMap from monkey_island.cc.repository import MongoAgentRepository, MongoMachineRepository from monkey_island.cc.repository.attack.IMitigationsRepository import IMitigationsRepository @@ -340,3 +339,8 @@ SCANNED EXPLOITED CC CC_TUNNEL + +# TODO remove when 2267 is done +NetworkServiceNameEnum.UNKNOWN +Machine.network_services +Node.tcp_connections From f6ed8a997c3a84ed8540294043f2ddf4d6d7c608 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 4 Oct 2022 12:10:47 -0400 Subject: [PATCH 4/8] Common: Rename NetworkServiceNameEnum -> NetworkService "Name" and "Enum" are redundant in this case --- monkey/common/types.py | 2 +- monkey/monkey_island/cc/models/machine.py | 4 ++-- vulture_allowlist.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/common/types.py b/monkey/common/types.py index 049c7b7de..00b77ca10 100644 --- a/monkey/common/types.py +++ b/monkey/common/types.py @@ -28,7 +28,7 @@ JSONSerializable = Union[ # type: ignore[misc] ] -class NetworkServiceNameEnum(Enum): +class NetworkService(Enum): UNKNOWN = auto() diff --git a/monkey/monkey_island/cc/models/machine.py b/monkey/monkey_island/cc/models/machine.py index 9175b49a2..96fc35e80 100644 --- a/monkey/monkey_island/cc/models/machine.py +++ b/monkey/monkey_island/cc/models/machine.py @@ -6,7 +6,7 @@ from pydantic import Field, validator from common import OperatingSystem from common.base_models import MutableInfectionMonkeyBaseModel from common.transforms import make_immutable_sequence -from common.types import HardwareID, NetworkServiceNameEnum, SocketAddress +from common.types import HardwareID, NetworkService, SocketAddress from . import MachineID @@ -35,7 +35,7 @@ class Machine(MutableInfectionMonkeyBaseModel): hostname: str = "" """The hostname of the machine""" - network_services: Mapping[SocketAddress, NetworkServiceNameEnum] + network_services: Mapping[SocketAddress, NetworkService] """All network services found running on the machine""" _make_immutable_sequence = validator("network_interfaces", pre=True, allow_reuse=True)( diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 62deedcd9..0dbb29f50 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -13,7 +13,7 @@ from infection_monkey.exploit.HostExploiter.HostExploiter import ( _publish_exploitation_event, _publish_propagation_event, ) -from common.types import NetworkPort, NetworkServiceNameEnum +from common.types import NetworkPort, NetworkService from infection_monkey.exploit.log4shell_utils.ldap_server import LDAPServerFactory from monkey_island.cc.models import Machine, Node, Report from monkey_island.cc.models.networkmap import Arc, NetworkMap From eb3daf84f1b099b8521b172974de95ac017f00a6 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 4 Oct 2022 12:13:04 -0400 Subject: [PATCH 5/8] Common: Use strings for NetworkService Enum values --- monkey/common/types.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/monkey/common/types.py b/monkey/common/types.py index 00b77ca10..55ebb4620 100644 --- a/monkey/common/types.py +++ b/monkey/common/types.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import Enum, auto +from enum import Enum from ipaddress import IPv4Address from typing import Dict, List, Optional, Union from uuid import UUID @@ -29,7 +29,14 @@ JSONSerializable = Union[ # type: ignore[misc] class NetworkService(Enum): - UNKNOWN = auto() + """ + An Enum representing network services + + This Enum represents all network services that Infection Monkey supports. The value of each + member is the member's name in all lower-case characters. + """ + + UNKNOWN = "unknown" class NetworkPort(ConstrainedInt): From d8cf5d33ddda3448dec00dfafe6b333536866f76 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 4 Oct 2022 12:29:56 -0400 Subject: [PATCH 6/8] Common: Extract MutableInfectionMonkeyModelConfig --- monkey/common/base_models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/monkey/common/base_models.py b/monkey/common/base_models.py index c2df68c5f..94e744d3a 100644 --- a/monkey/common/base_models.py +++ b/monkey/common/base_models.py @@ -10,6 +10,11 @@ class InfectionMonkeyModelConfig: extra = Extra.forbid +class MutableInfectionMonkeyModelConfig(InfectionMonkeyModelConfig): + allow_mutation = True + validate_assignment = True + + class InfectionMonkeyBaseModel(BaseModel): class Config(InfectionMonkeyModelConfig): pass @@ -47,6 +52,5 @@ class InfectionMonkeyBaseModel(BaseModel): class MutableInfectionMonkeyBaseModel(InfectionMonkeyBaseModel): - class Config(InfectionMonkeyModelConfig): - allow_mutation = True - validate_assignment = True + class Config(MutableInfectionMonkeyModelConfig): + pass From 8799a60f47e7d4957fd67aa56778ffcdf2ed8549 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 4 Oct 2022 12:41:42 -0400 Subject: [PATCH 7/8] Island: Fix serialization/deserialization of Machine.network_services --- monkey/monkey_island/cc/models/machine.py | 32 +++++++++++++++-- .../monkey_island/cc/models/test_machine.py | 36 ++++++++++++++++++- vulture_allowlist.py | 2 ++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/models/machine.py b/monkey/monkey_island/cc/models/machine.py index 96fc35e80..a4dfbc982 100644 --- a/monkey/monkey_island/cc/models/machine.py +++ b/monkey/monkey_island/cc/models/machine.py @@ -1,19 +1,45 @@ +import json from ipaddress import IPv4Interface -from typing import Mapping, Optional, Sequence +from typing import Any, Dict, Mapping, Optional, Sequence from pydantic import Field, validator from common import OperatingSystem -from common.base_models import MutableInfectionMonkeyBaseModel +from common.base_models import MutableInfectionMonkeyBaseModel, MutableInfectionMonkeyModelConfig from common.transforms import make_immutable_sequence from common.types import HardwareID, NetworkService, SocketAddress from . import MachineID +def _serialize_network_services(machine_dict: Dict, *, default): + machine_dict["network_services"] = { + str(addr): val for addr, val in machine_dict["network_services"].items() + } + return json.dumps(machine_dict, default=default) + + class Machine(MutableInfectionMonkeyBaseModel): """Represents machines, VMs, or other network nodes discovered by Infection Monkey""" + class Config(MutableInfectionMonkeyModelConfig): + json_dumps = _serialize_network_services + + @validator("network_services", pre=True) + def _socketaddress_from_string(cls, v: Any) -> Any: + if not isinstance(v, Mapping): + # Let pydantic's type validation handle this + return v + + new_network_services = {} + for addr, service in v.items(): + if isinstance(addr, SocketAddress): + new_network_services[addr] = service + else: + new_network_services[SocketAddress.from_string(addr)] = service + + return new_network_services + id: MachineID = Field(..., allow_mutation=False) """Uniquely identifies the machine within the island""" @@ -35,7 +61,7 @@ class Machine(MutableInfectionMonkeyBaseModel): hostname: str = "" """The hostname of the machine""" - network_services: Mapping[SocketAddress, NetworkService] + network_services: Mapping[SocketAddress, NetworkService] = Field(default_factory=dict) """All network services found running on the machine""" _make_immutable_sequence = validator("network_interfaces", pre=True, allow_reuse=True)( diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/test_machine.py b/monkey/tests/unit_tests/monkey_island/cc/models/test_machine.py index b63006d35..cecfd6fd5 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/test_machine.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/test_machine.py @@ -6,8 +6,12 @@ from typing import MutableSequence import pytest from common import OperatingSystem +from common.types import NetworkService, SocketAddress from monkey_island.cc.models import Machine +SOCKET_ADDR_1 = "192.168.1.10:5000" +SOCKET_ADDR_2 = "192.168.1.10:8080" + MACHINE_OBJECT_DICT = MappingProxyType( { "id": 1, @@ -17,6 +21,10 @@ MACHINE_OBJECT_DICT = MappingProxyType( "operating_system": OperatingSystem.WINDOWS, "operating_system_version": "eXtra Problems", "hostname": "my.host", + "network_services": { + SocketAddress.from_string(SOCKET_ADDR_1): NetworkService.UNKNOWN, + SocketAddress.from_string(SOCKET_ADDR_2): NetworkService.UNKNOWN, + }, } ) @@ -26,9 +34,13 @@ MACHINE_SIMPLE_DICT = MappingProxyType( "hardware_id": uuid.getnode(), "island": True, "network_interfaces": ["10.0.0.1/24", "192.168.5.32/16"], - "operating_system": "windows", + "operating_system": OperatingSystem.WINDOWS.value, "operating_system_version": "eXtra Problems", "hostname": "my.host", + "network_services": { + SOCKET_ADDR_1: NetworkService.UNKNOWN.value, + SOCKET_ADDR_2: NetworkService.UNKNOWN.value, + }, } ) @@ -60,6 +72,11 @@ def test_to_dict(): ("operating_system", "bsd"), ("operating_system_version", {}), ("hostname", []), + ("network_services", 42), + ("network_services", [SOCKET_ADDR_1]), + ("network_services", None), + ("network_services", {SOCKET_ADDR_1: "Hello"}), + ("network_services", {SocketAddress.from_string(SOCKET_ADDR_1): "Hello"}), ], ) def test_construct_invalid_field__type_error(key, value): @@ -77,6 +94,7 @@ def test_construct_invalid_field__type_error(key, value): ("hardware_id", 0), ("network_interfaces", [1, "stuff", 3]), ("network_interfaces", ["10.0.0.1/16", 2, []]), + ("network_services", {"192.168.": NetworkService.UNKNOWN.value}), ], ) def test_construct_invalid_field__value_error(key, value): @@ -230,3 +248,19 @@ def test_hostname_default_value(): m = Machine(**missing_hostname_dict) assert m.hostname == "" + + +def test_set_network_services_validates(): + m = Machine(**MACHINE_OBJECT_DICT) + + with pytest.raises(ValueError): + m.network_services = {"not-an-ip": NetworkService.UNKNOWN.value} + + +def test_set_network_services_default_value(): + missing_network_services = MACHINE_OBJECT_DICT.copy() + del missing_network_services["network_services"] + + m = Machine(**missing_network_services) + + assert m.network_services == {} diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 0dbb29f50..fc890fd8e 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -343,4 +343,6 @@ CC_TUNNEL # TODO remove when 2267 is done NetworkServiceNameEnum.UNKNOWN Machine.network_services +Machine.config.json_dumps +Machine._socketaddress_from_string Node.tcp_connections From 10e3c97489b9514a49318e2164f35dfd5f390b7d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 4 Oct 2022 15:09:02 -0400 Subject: [PATCH 8/8] Island: Use Tuple[SocketAddress] for tcp_connections There are serialization issues when using FrozenSet because pydantic converts the SocketAddress to a dict, which is not hashable. There are probably ways to work around this, but it's not worth the effort at thsi time. If performance becomes an issue (doubtful) we can revisit using a frozenset instead. --- monkey/monkey_island/cc/models/node.py | 4 ++-- .../monkey_island/cc/models/test_node.py | 20 ++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/models/node.py b/monkey/monkey_island/cc/models/node.py index d992d1836..ada8aac19 100644 --- a/monkey/monkey_island/cc/models/node.py +++ b/monkey/monkey_island/cc/models/node.py @@ -1,4 +1,4 @@ -from typing import FrozenSet, Mapping +from typing import FrozenSet, Mapping, Tuple from pydantic import Field from typing_extensions import TypeAlias @@ -26,5 +26,5 @@ class Node(MutableInfectionMonkeyBaseModel): connections: NodeConnections """All outbound connections from this node to other machines""" - tcp_connections: Mapping[MachineID, FrozenSet[SocketAddress]] = {} + tcp_connections: Mapping[MachineID, Tuple[SocketAddress, ...]] = {} """All successfull outbound TCP connections""" diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/test_node.py b/monkey/tests/unit_tests/monkey_island/cc/models/test_node.py index 74a83860c..e50c493ef 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/test_node.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/test_node.py @@ -2,6 +2,7 @@ from typing import MutableSequence import pytest +from common.types import SocketAddress from monkey_island.cc.models import CommunicationType, Node @@ -11,13 +12,21 @@ def test_constructor(): 6: frozenset((CommunicationType.SCANNED,)), 7: frozenset((CommunicationType.SCANNED, CommunicationType.EXPLOITED)), } + tcp_connections = { + 6: tuple( + (SocketAddress(ip="192.168.1.1", port=80), SocketAddress(ip="192.168.1.1", port=443)) + ), + 7: tuple((SocketAddress(ip="192.168.1.2", port=22),)), + } n = Node( - machine_id=1, + machine_id=machine_id, connections=connections, + tcp_connections=tcp_connections, ) assert n.machine_id == machine_id assert n.connections == connections + assert n.tcp_connections == tcp_connections def test_serialization(): @@ -27,9 +36,12 @@ def test_serialization(): "6": [CommunicationType.CC.value, CommunicationType.SCANNED.value], "7": [CommunicationType.EXPLOITED.value, CommunicationType.CC.value], }, + "tcp_connections": { + "6": [{"ip": "192.168.1.1", "port": 80}, {"ip": "192.168.1.1", "port": 443}], + "7": [{"ip": "192.168.1.2", "port": 22}], + }, } - # "6": frozenset((CommunicationType.CC, CommunicationType.SCANNED)), - # "7": frozenset((CommunicationType.EXPLOITED, CommunicationType.CC)), + n = Node(**node_dict) serialized_node = n.dict(simplify=True) @@ -44,6 +56,8 @@ def test_serialization(): for key, value in serialized_node["connections"].items(): assert set(value) == set(node_dict["connections"][key]) + assert serialized_node["tcp_connections"] == node_dict["tcp_connections"] + def test_machine_id_immutable(): n = Node(machine_id=1, connections={})