From 8537f1fcb75429378693dcd0a161bc5710ddc5b6 Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 16:20:46 +0000 Subject: [PATCH 01/21] Island: Add get_machines to IMachineRepository --- .../cc/repository/i_machine_repository.py | 9 +++++ .../cc/repository/mongo_machine_repository.py | 8 +++++ .../test_mongo_machine_repository.py | 35 +++++++++++++++---- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/repository/i_machine_repository.py b/monkey/monkey_island/cc/repository/i_machine_repository.py index 85e8e71f3..7cea0bb02 100644 --- a/monkey/monkey_island/cc/repository/i_machine_repository.py +++ b/monkey/monkey_island/cc/repository/i_machine_repository.py @@ -53,6 +53,15 @@ class IMachineRepository(ABC): :raises RetrievalError: If an error occurs while attempting to retrieve the `Machine` """ + @abstractmethod + def get_machines(self) -> Sequence[Machine]: + """ + Get all machines in the repository + + :return: A sequence of all stored `Machine`s + :raises RetrievalError: If an error occurs while attempting to retrieve the `Machine`s + """ + @abstractmethod def get_machines_by_ip(self, ip: IPv4Address) -> Sequence[Machine]: """ diff --git a/monkey/monkey_island/cc/repository/mongo_machine_repository.py b/monkey/monkey_island/cc/repository/mongo_machine_repository.py index fab038694..75cbf2d46 100644 --- a/monkey/monkey_island/cc/repository/mongo_machine_repository.py +++ b/monkey/monkey_island/cc/repository/mongo_machine_repository.py @@ -69,6 +69,14 @@ class MongoMachineRepository(IMachineRepository): return Machine(**machine_dict) + def get_machines(self) -> Sequence[Machine]: + try: + cursor = self._machines_collection.find({}, {MONGO_OBJECT_ID_KEY: False}) + except Exception as err: + raise RetrievalError(f"Error retrieving machines: {err}") + + return list(map(lambda m: Machine(**m), cursor)) + def get_machines_by_ip(self, ip: IPv4Address) -> Sequence[Machine]: ip_regex = "^" + str(ip).replace(".", "\\.") + "\\/.*$" query = {"network_interfaces": {"$elemMatch": {"$regex": ip_regex}}} diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_mongo_machine_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_mongo_machine_repository.py index 90d0af1f2..bbd35283b 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_mongo_machine_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_mongo_machine_repository.py @@ -93,6 +93,11 @@ def machine_repository(mongo_client) -> IMachineRepository: return MongoMachineRepository(mongo_client) +@pytest.fixture +def empty_machine_repository() -> IMachineRepository: + return MongoMachineRepository(mongomock.MongoClient()) + + def test_get_new_id__unique_id(machine_repository): new_machine_id = machine_repository.get_new_id() @@ -107,8 +112,7 @@ def test_get_new_id__multiple_unique_ids(machine_repository): assert id_1 != id_2 -def test_get_new_id__new_id_for_empty_repo(machine_repository): - empty_machine_repository = MongoMachineRepository(mongomock.MongoClient()) +def test_get_new_id__new_id_for_empty_repo(empty_machine_repository): id_1 = empty_machine_repository.get_new_id() id_2 = empty_machine_repository.get_new_id() @@ -202,7 +206,7 @@ def test_get_machine_by_hardware_id__retrieval_error(error_raising_machine_repos error_raising_machine_repository.get_machine_by_hardware_id(1) -def test_get_machine_by_ip(machine_repository): +def test_get_machines_by_ip(machine_repository): expected_machine = MACHINES[0] expected_machine_ip = expected_machine.network_interfaces[0].ip @@ -212,7 +216,7 @@ def test_get_machine_by_ip(machine_repository): assert retrieved_machines[0] == expected_machine -def test_get_machine_by_ip__multiple_results(machine_repository): +def test_get_machines_by_ip__multiple_results(machine_repository): search_ip = MACHINES[3].network_interfaces[0].ip retrieved_machines = machine_repository.get_machines_by_ip(search_ip) @@ -222,16 +226,35 @@ def test_get_machine_by_ip__multiple_results(machine_repository): assert MACHINES[3] in retrieved_machines -def test_get_machine_by_ip__not_found(machine_repository): +def test_get_machines_by_ip__not_found(machine_repository): with pytest.raises(UnknownRecordError): machine_repository.get_machines_by_ip("1.1.1.1") -def test_get_machine_by_ip__retrieval_error(error_raising_machine_repository): +def test_get_machines_by_ip__retrieval_error(error_raising_machine_repository): with pytest.raises(RetrievalError): error_raising_machine_repository.get_machines_by_ip("1.1.1.1") +def test_get_machines(machine_repository): + retrieved_machines = machine_repository.get_machines() + + assert len(retrieved_machines) == len(MACHINES) + for machine in MACHINES: + assert machine in retrieved_machines + + +def test_get_machines__empty_repository(empty_machine_repository): + retrieved_machines = empty_machine_repository.get_machines() + + assert len(retrieved_machines) == 0 + + +def test_get_machines__retrieval_error(error_raising_machine_repository): + with pytest.raises(RetrievalError): + error_raising_machine_repository.get_machines() + + def test_reset(machine_repository): # Ensure the repository is not empty preexisting_machine = machine_repository.get_machine_by_id(MACHINES[0].id) From 85a5cb32091644cb0d00f0e12ab3a7db54280bae Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 17:11:29 +0000 Subject: [PATCH 02/21] Island: Add ReportService.get_all_machines --- .../cc/services/reporting/report.py | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 52ed04df9..729967aba 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -1,16 +1,20 @@ import functools import ipaddress import logging -from itertools import chain, product -from typing import List +from itertools import chain, filterfalse, product, tee +from typing import Iterable, List, Optional from common.network.network_range import NetworkRange from common.network.network_utils import get_my_ip_addresses_legacy, get_network_interfaces from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst from monkey_island.cc.database import mongo -from monkey_island.cc.models import Monkey +from monkey_island.cc.models import Machine, Monkey from monkey_island.cc.models.report import get_report, save_report -from monkey_island.cc.repository import IAgentConfigurationRepository, ICredentialsRepository +from monkey_island.cc.repository import ( + IAgentConfigurationRepository, + ICredentialsRepository, + IMachineRepository, +) from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.reporting.exploitations.manual_exploitation import get_manual_monkeys from monkey_island.cc.services.reporting.exploitations.monkey_exploitation import ( @@ -31,9 +35,10 @@ logger = logging.getLogger(__name__) class ReportService: - _aws_service = None - _agent_configuration_repository = None - _credentials_repository = None + _aws_service: Optional[AWSService] = None + _agent_configuration_repository: Optional[IAgentConfigurationRepository] = None + _credentials_repository: Optional[ICredentialsRepository] = None + _machine_repository: Optional[IMachineRepository] = None class DerivedIssueEnum: ZEROLOGON_PASS_RESTORE_FAILED = "zerologon_pass_restore_failed" @@ -44,10 +49,12 @@ class ReportService: aws_service: AWSService, agent_configuration_repository: IAgentConfigurationRepository, credentials_repository: ICredentialsRepository, + machine_repository: IMachineRepository, ): cls._aws_service = aws_service cls._agent_configuration_repository = agent_configuration_repository cls._credentials_repository = credentials_repository + cls._machine_repository = machine_repository # This should pull from Simulation entity @staticmethod @@ -123,6 +130,18 @@ class ReportService: return formatted_nodes + @classmethod + def get_all_machines(cls) -> Iterable[Machine]: + if cls._machine_repository is None: + return iter(()) + machines = cls._machine_repository.get_machines() + t1, t2 = tee(machines) + + def is_island(machine: Machine): + return machine.island + + return chain(filter(is_island, t1), *filterfalse(is_island, t2)) + @staticmethod def get_all_displayed_nodes(): nodes_without_monkeys = [ From 49c6839c10ca3dedc85c13d42216d7e426b8594c Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 17:13:57 +0000 Subject: [PATCH 03/21] Island: Add ReportService.get_accessible_machines --- .../cc/services/reporting/report.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 729967aba..353f19b19 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -8,12 +8,13 @@ from common.network.network_range import NetworkRange from common.network.network_utils import get_my_ip_addresses_legacy, get_network_interfaces from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst from monkey_island.cc.database import mongo -from monkey_island.cc.models import Machine, Monkey +from monkey_island.cc.models import CommunicationType, Machine, Monkey from monkey_island.cc.models.report import get_report, save_report from monkey_island.cc.repository import ( IAgentConfigurationRepository, ICredentialsRepository, IMachineRepository, + INodeRepository, ) from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.reporting.exploitations.manual_exploitation import get_manual_monkeys @@ -39,6 +40,7 @@ class ReportService: _agent_configuration_repository: Optional[IAgentConfigurationRepository] = None _credentials_repository: Optional[ICredentialsRepository] = None _machine_repository: Optional[IMachineRepository] = None + _node_repository: Optional[INodeRepository] = None class DerivedIssueEnum: ZEROLOGON_PASS_RESTORE_FAILED = "zerologon_pass_restore_failed" @@ -50,11 +52,13 @@ class ReportService: agent_configuration_repository: IAgentConfigurationRepository, credentials_repository: ICredentialsRepository, machine_repository: IMachineRepository, + node_repository: INodeRepository, ): cls._aws_service = aws_service cls._agent_configuration_repository = agent_configuration_repository cls._credentials_repository = credentials_repository cls._machine_repository = machine_repository + cls._node_repository = node_repository # This should pull from Simulation entity @staticmethod @@ -130,6 +134,21 @@ class ReportService: return formatted_nodes + @classmethod + def get_accessible_machines(cls, machine: Machine): + if cls._node_repository is None or cls._machine_repository is None: + return [] + + nodes = cls._node_repository.get_nodes() + machine_iter = (node for node in nodes if node.machine_id == machine.id) + accessible_machines = set() + for source in machine_iter: + for dest, conn in source.connections.items(): + if CommunicationType.SCANNED in conn: + accessible_machines.add(dest) + + return [cls._machine_repository.get_machine_by_id(id) for id in accessible_machines] + @classmethod def get_all_machines(cls) -> Iterable[Machine]: if cls._machine_repository is None: From 8acf2d9e91c2be3e30b49683a04ba1602c6c85bf Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 17:15:31 +0000 Subject: [PATCH 04/21] Island: Refactor ReportService.get_scanned Update ReportService.get_scanned to use repositories instead of services --- .../cc/services/reporting/report.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 353f19b19..256f681aa 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -113,20 +113,23 @@ class ReportService: def get_scanned(): formatted_nodes = [] - nodes = ReportService.get_all_displayed_nodes() + machines = ReportService.get_all_machines() - for node in nodes: + for machine in machines: # This information should be evident from the map, not sure a table/list is a good way # to display it anyways - nodes_that_can_access_current_node = node["accessible_from_nodes_hostnames"] + addresses = [str(iface.ip) for iface in machine.network_interfaces] + accessible_machines = [ + m.hostname for m in ReportService.get_accessible_machines(machine) + ] formatted_nodes.append( { - "label": node["label"], - "ip_addresses": node["ip_addresses"], - "accessible_from_nodes": nodes_that_can_access_current_node, - "services": node["services"], - "domain_name": node["domain_name"], - "pba_results": node["pba_results"] if "pba_results" in node else "None", + "label": machine.hostname, + "ip_addresses": addresses, + "accessible_from_nodes": accessible_machines, + "services": [], + "domain_name": "", + "pba_results": "None", } ) From 520b212c69f40fdcb717b697d0444c47736c66b2 Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 17:17:28 +0000 Subject: [PATCH 05/21] Island: Remove ReportService.get_all_displayed_nodes --- .../monkey_island/cc/services/reporting/report.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 256f681aa..23460634e 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -164,19 +164,6 @@ class ReportService: return chain(filter(is_island, t1), *filterfalse(is_island, t2)) - @staticmethod - def get_all_displayed_nodes(): - nodes_without_monkeys = [ - NodeService.get_displayed_node_by_id(node["_id"], True) - for node in mongo.db.node.find({}, {"_id": 1}) - ] - nodes_with_monkeys = [ - NodeService.get_displayed_node_by_id(monkey["_id"], True) - for monkey in mongo.db.monkey.find({}, {"_id": 1}) - ] - nodes = nodes_without_monkeys + nodes_with_monkeys - return nodes - @staticmethod def process_exploit(exploit) -> ExploiterReportInfo: exploiter_type = exploit["data"]["exploiter"] From 274010062162ffa3f7794e68a95ec173325cfcf5 Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 19:41:33 +0000 Subject: [PATCH 06/21] Island: Connect new ReportService dependencies --- monkey/monkey_island/cc/services/initialize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monkey/monkey_island/cc/services/initialize.py b/monkey/monkey_island/cc/services/initialize.py index 40839c388..9e205e820 100644 --- a/monkey/monkey_island/cc/services/initialize.py +++ b/monkey/monkey_island/cc/services/initialize.py @@ -94,6 +94,8 @@ def initialize_services(container: DIContainer, data_dir: Path): container.resolve(AWSService), container.resolve(IAgentConfigurationRepository), container.resolve(ICredentialsRepository), + container.resolve(IMachineRepository), + container.resolve(INodeRepository), ) From bbd606501ece769fadea68abbc629f158eeba429 Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Wed, 28 Sep 2022 20:34:18 +0000 Subject: [PATCH 07/21] Island: Fix bug in reporting --- monkey/monkey_island/cc/services/reporting/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 23460634e..d0d41cbb1 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -162,7 +162,7 @@ class ReportService: def is_island(machine: Machine): return machine.island - return chain(filter(is_island, t1), *filterfalse(is_island, t2)) + return chain(filter(is_island, t1), filterfalse(is_island, t2)) @staticmethod def process_exploit(exploit) -> ExploiterReportInfo: From dd0c5047438eac5756daac06c0c964cb0de98cf9 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Thu, 29 Sep 2022 11:52:57 +0300 Subject: [PATCH 08/21] UT: Add machines and node to report tests --- .../cc/services/reporting/test_report.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py new file mode 100644 index 000000000..89930871c --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -0,0 +1,34 @@ +from ipaddress import IPv4Interface + +from monkey_island.cc.models import CommunicationType, Machine, Node + +ISLAND_MACHINE = Machine( + id=0, + island=True, + hardware_id=5, + network_interfaces=[IPv4Interface("10.10.10.99/24")], +) + +MACHINE_A = Machine( + id=1, + hardware_id=9, + network_interfaces=[IPv4Interface("10.10.10.1/24")], +) + +MACHINE_B = Machine( + id=2, + hardware_id=9, + network_interfaces=[IPv4Interface("10.10.10.2/24")], +) + +MACHINE_C = Machine( + id=3, + hardware_id=9, + network_interfaces=[IPv4Interface("10.10.10.3/24")], +) + +NODES = [ + Node(id=1, connections={"2", CommunicationType.EXPLOITED}), + Node(id=0, connections={"1", CommunicationType.SCANNED}), + Node(id=3, connections={"0", CommunicationType.CC}), +] From f7a30e460893c3fc5865770c6fcb0448fd2a10d0 Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Thu, 29 Sep 2022 20:03:11 +0000 Subject: [PATCH 09/21] Island: Raise errors if repositories do not exist --- monkey/monkey_island/cc/services/reporting/report.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index d0d41cbb1..aa70b3895 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -139,8 +139,10 @@ class ReportService: @classmethod def get_accessible_machines(cls, machine: Machine): - if cls._node_repository is None or cls._machine_repository is None: - return [] + if cls._node_repository is None: + raise RuntimeError("Node repository does not exist") + elif cls._machine_repository is None: + raise RuntimeError("Machine repository does not exist") nodes = cls._node_repository.get_nodes() machine_iter = (node for node in nodes if node.machine_id == machine.id) @@ -155,7 +157,7 @@ class ReportService: @classmethod def get_all_machines(cls) -> Iterable[Machine]: if cls._machine_repository is None: - return iter(()) + raise RuntimeError("Machine repository does not exist") machines = cls._machine_repository.get_machines() t1, t2 = tee(machines) From f0f858eba562bf0f4491bde99c9b13148bf6f5c8 Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Thu, 29 Sep 2022 20:06:46 +0000 Subject: [PATCH 10/21] UT: Fix bad values in test_report --- .../monkey_island/cc/services/reporting/test_report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index 89930871c..1afad4d95 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -3,7 +3,7 @@ from ipaddress import IPv4Interface from monkey_island.cc.models import CommunicationType, Machine, Node ISLAND_MACHINE = Machine( - id=0, + id=99, island=True, hardware_id=5, network_interfaces=[IPv4Interface("10.10.10.99/24")], @@ -28,7 +28,7 @@ MACHINE_C = Machine( ) NODES = [ - Node(id=1, connections={"2", CommunicationType.EXPLOITED}), - Node(id=0, connections={"1", CommunicationType.SCANNED}), - Node(id=3, connections={"0", CommunicationType.CC}), + Node(machine_id=1, connections={"2": {CommunicationType.EXPLOITED}}), + Node(machine_id=99, connections={"1": {CommunicationType.SCANNED}}), + Node(machine_id=3, connections={"99": {CommunicationType.CC}}), ] From 9a82e467997cbfaac40366b89497da2d41f78786 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 30 Sep 2022 11:39:52 +0300 Subject: [PATCH 11/21] Island: Make machine model hashable Machines can be hashed based on their ID. This allows us to add them to set for example and make sure no duplicate entries are there --- monkey/monkey_island/cc/models/machine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monkey/monkey_island/cc/models/machine.py b/monkey/monkey_island/cc/models/machine.py index 99d4ae5eb..ece877b9e 100644 --- a/monkey/monkey_island/cc/models/machine.py +++ b/monkey/monkey_island/cc/models/machine.py @@ -38,3 +38,6 @@ class Machine(MutableInfectionMonkeyBaseModel): _make_immutable_sequence = validator("network_interfaces", pre=True, allow_reuse=True)( make_immutable_sequence ) + + def __hash__(self): + return self.id From 3d27e42ff377131a0d03329ea5b912dfe185d950 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 30 Sep 2022 11:46:05 +0300 Subject: [PATCH 12/21] Island: Simplify the report of scanned machines --- .../cc/services/reporting/report.py | 71 +++++++------------ .../cc/services/reporting/test_report.py | 55 ++++++++++++-- 2 files changed, 76 insertions(+), 50 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index aa70b3895..20991d532 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -1,8 +1,8 @@ import functools import ipaddress import logging -from itertools import chain, filterfalse, product, tee -from typing import Iterable, List, Optional +from itertools import chain, product +from typing import List, Optional from common.network.network_range import NetworkRange from common.network.network_utils import get_my_ip_addresses_legacy, get_network_interfaces @@ -35,7 +35,6 @@ logger = logging.getLogger(__name__) class ReportService: - _aws_service: Optional[AWSService] = None _agent_configuration_repository: Optional[IAgentConfigurationRepository] = None _credentials_repository: Optional[ICredentialsRepository] = None @@ -113,58 +112,42 @@ class ReportService: def get_scanned(): formatted_nodes = [] - machines = ReportService.get_all_machines() + machines = ReportService._machine_repository.get_machines() for machine in machines: - # This information should be evident from the map, not sure a table/list is a good way - # to display it anyways - addresses = [str(iface.ip) for iface in machine.network_interfaces] - accessible_machines = [ - m.hostname for m in ReportService.get_accessible_machines(machine) - ] - formatted_nodes.append( - { - "label": machine.hostname, - "ip_addresses": addresses, - "accessible_from_nodes": accessible_machines, - "services": [], - "domain_name": "", - "pba_results": "None", - } - ) - - logger.info("Scanned nodes generated for reporting") + accessible_from = ReportService.get_scanners_of_machine(machine) + if accessible_from: + formatted_nodes.append( + { + "hostname": machine.hostname, + "ip_addresses": machine.network_interfaces, + "accessible_from_nodes": accessible_from, + "domain_name": "", + # TODO add services + "services": [], + } + ) return formatted_nodes @classmethod - def get_accessible_machines(cls, machine: Machine): - if cls._node_repository is None: + def get_scanners_of_machine(cls, machine: Machine) -> List[Machine]: + if not cls._node_repository: raise RuntimeError("Node repository does not exist") - elif cls._machine_repository is None: + if not cls._machine_repository: raise RuntimeError("Machine repository does not exist") nodes = cls._node_repository.get_nodes() - machine_iter = (node for node in nodes if node.machine_id == machine.id) - accessible_machines = set() - for source in machine_iter: - for dest, conn in source.connections.items(): - if CommunicationType.SCANNED in conn: - accessible_machines.add(dest) + scanner_machines = set() + for node in nodes: + for dest, conn in node.connections.items(): + if CommunicationType.SCANNED in conn and dest == machine.id: + scanner_machine = ReportService._machine_repository.get_machine_by_id( + node.machine_id + ) + scanner_machines.add(scanner_machine) - return [cls._machine_repository.get_machine_by_id(id) for id in accessible_machines] - - @classmethod - def get_all_machines(cls) -> Iterable[Machine]: - if cls._machine_repository is None: - raise RuntimeError("Machine repository does not exist") - machines = cls._machine_repository.get_machines() - t1, t2 = tee(machines) - - def is_island(machine: Machine): - return machine.island - - return chain(filter(is_island, t1), filterfalse(is_island, t2)) + return list(scanner_machines) @staticmethod def process_exploit(exploit) -> ExploiterReportInfo: diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index 1afad4d95..c445bbed7 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -1,34 +1,77 @@ from ipaddress import IPv4Interface +from unittest.mock import MagicMock from monkey_island.cc.models import CommunicationType, Machine, Node +from monkey_island.cc.services.reporting.report import ReportService ISLAND_MACHINE = Machine( id=99, island=True, + hostname="Island", hardware_id=5, network_interfaces=[IPv4Interface("10.10.10.99/24")], ) -MACHINE_A = Machine( +MACHINE_1 = Machine( id=1, hardware_id=9, + hostname="machine_1", network_interfaces=[IPv4Interface("10.10.10.1/24")], ) -MACHINE_B = Machine( +MACHINE_2 = Machine( id=2, hardware_id=9, network_interfaces=[IPv4Interface("10.10.10.2/24")], ) -MACHINE_C = Machine( +MACHINE_3 = Machine( id=3, hardware_id=9, network_interfaces=[IPv4Interface("10.10.10.3/24")], ) NODES = [ - Node(machine_id=1, connections={"2": {CommunicationType.EXPLOITED}}), - Node(machine_id=99, connections={"1": {CommunicationType.SCANNED}}), - Node(machine_id=3, connections={"99": {CommunicationType.CC}}), + Node( + machine_id=1, + connections={"2": frozenset([CommunicationType.EXPLOITED, CommunicationType.SCANNED])}, + ), + Node(machine_id=99, connections={"1": frozenset([CommunicationType.SCANNED])}), + Node( + machine_id=3, + connections={"99": frozenset([CommunicationType.CC, CommunicationType.EXPLOITED])}, + ), ] + +MACHINES = [MACHINE_1, MACHINE_2, MACHINE_3, ISLAND_MACHINE] + +EXPECTED_SCANNED_MACHINES = [ + { + "hostname": MACHINE_1.hostname, + "ip_addresses": MACHINE_1.network_interfaces, + "accessible_from_nodes": [ISLAND_MACHINE], + "services": [], + "domain_name": "", + }, + { + "hostname": MACHINE_2.hostname, + "ip_addresses": MACHINE_2.network_interfaces, + "accessible_from_nodes": [MACHINE_1], + "services": [], + "domain_name": "", + }, +] + + +def get_machine_by_id(machine_id): + return [machine for machine in MACHINES if machine_id == machine.id][0] + + +def test_get_scanned(): + ReportService._node_repository = MagicMock() + ReportService._node_repository.get_nodes.return_value = NODES + ReportService._machine_repository = MagicMock() + ReportService._machine_repository.get_machines.return_value = MACHINES + ReportService._machine_repository.get_machine_by_id = get_machine_by_id + scanned = ReportService.get_scanned() + assert scanned == EXPECTED_SCANNED_MACHINES From 550c375abc6539ba373564a3e808042fec811a47 Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Fri, 30 Sep 2022 09:54:12 +0000 Subject: [PATCH 13/21] Agent: Fix exception handling for failed delivery of agent events If there's an exception while sending the event to the island it should be logged as exception, not warning (events are not being stored, that's a big no-no). Also, the log should contain proper information of stack trace, not event type --- monkey/infection_monkey/agent_event_forwarder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/agent_event_forwarder.py b/monkey/infection_monkey/agent_event_forwarder.py index f253255d3..cf585a4bb 100644 --- a/monkey/infection_monkey/agent_event_forwarder.py +++ b/monkey/infection_monkey/agent_event_forwarder.py @@ -79,8 +79,8 @@ class BatchingAgentEventForwarder: try: logger.debug(f"Sending Agent events to Island: {events}") self._island_api_client.send_events(events) - except Exception as err: - logger.warning(f"Exception caught when connecting to the Island: {err}") + except Exception: + logger.exception("Exception caught when connecting to the Island") def _send_remaining_events(self): self._send_events_to_island() From d974b03ab01f9285259d4789621522e2e760ca2b Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Fri, 30 Sep 2022 10:33:59 +0000 Subject: [PATCH 14/21] Common: Register PingScanEvent to event serializer --- monkey/common/agent_event_serializers/register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monkey/common/agent_event_serializers/register.py b/monkey/common/agent_event_serializers/register.py index fbe353db9..0a820f714 100644 --- a/monkey/common/agent_event_serializers/register.py +++ b/monkey/common/agent_event_serializers/register.py @@ -1,4 +1,4 @@ -from common.agent_events import CredentialsStolenEvent +from common.agent_events import CredentialsStolenEvent, PingScanEvent from . import AgentEventSerializerRegistry, PydanticAgentEventSerializer @@ -9,3 +9,4 @@ def register_common_agent_event_serializers( event_serializer_registry[CredentialsStolenEvent] = PydanticAgentEventSerializer( CredentialsStolenEvent ) + event_serializer_registry[PingScanEvent] = PydanticAgentEventSerializer(PingScanEvent) From 9048f72030ac83db6636fb83ec407b1f4d776def Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Fri, 30 Sep 2022 12:41:09 +0000 Subject: [PATCH 15/21] Island: Convert scanned machines to json serializable data --- monkey/monkey_island/cc/services/reporting/report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 20991d532..e0d49339a 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -120,8 +120,10 @@ class ReportService: formatted_nodes.append( { "hostname": machine.hostname, - "ip_addresses": machine.network_interfaces, - "accessible_from_nodes": accessible_from, + "ip_addresses": list(map(str, machine.network_interfaces)), + "accessible_from_nodes": list( + map(lambda m: m.dict(simplify=True), accessible_from) + ), "domain_name": "", # TODO add services "services": [], From d0d08f7649961dc9a7819096b30bd9f8ac51c2cf Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Fri, 30 Sep 2022 12:42:28 +0000 Subject: [PATCH 16/21] UI: Fix the rendering of scanned machines in UI --- .../ui/src/components/report-components/SecurityReport.js | 2 ++ .../components/report-components/common/RenderArrays.js | 4 ++++ .../report-components/security/ScannedServers.js | 8 ++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index fd1accfb8..3b314ad9b 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -538,9 +538,11 @@ class ReportPageComponent extends AuthComponent { + {/* Post breach data should be separated from scanned machines
+ */}
; }; +export let renderMachineArray = function(array) { + return <>{array.map(x =>
{x.network_interfaces[0]}
)}; +} + export let renderLimitedArray = function (array, limit, className='', diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js index 229bbfed2..b2a58d85f 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js @@ -1,19 +1,20 @@ import React from 'react'; import ReactTable from 'react-table'; import Pluralize from 'pluralize'; -import {renderArray, renderIpAddresses} from '../common/RenderArrays'; +import {renderArray, renderIpAddresses, renderMachineArray} from '../common/RenderArrays'; const columns = [ { Header: 'Scanned Servers', columns: [ - {Header: 'Machine', accessor: 'label'}, + {Header: 'Machine', id: 'machine', accessor: x => x.ip_addresses[0]}, { Header: 'IP Addresses', id: 'ip_addresses', accessor: x => renderIpAddresses(x) }, - {Header: 'Accessible From', id: 'accessible_from_nodes', accessor: x => renderArray(x.accessible_from_nodes)}, + {Header: 'Accessible From', id: 'accessible_from_nodes', + accessor: x => renderMachineArray(x.accessible_from_nodes)}, {Header: 'Services', id: 'services', accessor: x => renderArray(x.services)} ] } @@ -27,7 +28,6 @@ class ScannedServersComponent extends React.Component { } render() { - let defaultPageSize = this.props.data.length > pageSize ? pageSize : this.props.data.length; let showPagination = this.props.data.length > pageSize; From e8ed30660e354f2a47041c09aca2b324b725bdec Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 30 Sep 2022 15:54:47 +0300 Subject: [PATCH 17/21] UT: Fix report test --- .../monkey_island/cc/services/reporting/test_report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index c445bbed7..75d269701 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -48,15 +48,15 @@ MACHINES = [MACHINE_1, MACHINE_2, MACHINE_3, ISLAND_MACHINE] EXPECTED_SCANNED_MACHINES = [ { "hostname": MACHINE_1.hostname, - "ip_addresses": MACHINE_1.network_interfaces, - "accessible_from_nodes": [ISLAND_MACHINE], + "ip_addresses": list(map(str, MACHINE_1.network_interfaces)), + "accessible_from_nodes": [ISLAND_MACHINE.dict(simplify=True)], "services": [], "domain_name": "", }, { "hostname": MACHINE_2.hostname, - "ip_addresses": MACHINE_2.network_interfaces, - "accessible_from_nodes": [MACHINE_1], + "ip_addresses": list(map(str, MACHINE_2.network_interfaces)), + "accessible_from_nodes": [MACHINE_1.dict(simplify=True)], "services": [], "domain_name": "", }, From 1bf4407b20776ef3efe90d9f8d6a886dbe3f63b1 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 30 Sep 2022 15:56:23 +0300 Subject: [PATCH 18/21] UI: Remove temporarily unused import in SecurityReport.js --- .../src/components/report-components/SecurityReport.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index 3b314ad9b..ba3f53232 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -2,7 +2,6 @@ import React, {Fragment} from 'react'; import Pluralize from 'pluralize'; import BreachedServers from 'components/report-components/security/BreachedServers'; import ScannedServers from 'components/report-components/security/ScannedServers'; -import PostBreach from 'components/report-components/security/PostBreach'; import {ReactiveGraph} from 'components/reactive-graph/ReactiveGraph'; import {edgeGroupToColor, getOptions} from 'components/map/MapOptions'; import StolenPasswords from 'components/report-components/security/StolenPasswords'; @@ -23,7 +22,7 @@ import {smbPasswordReport, smbPthReport} from './security/issues/SmbIssue'; import {hadoopIssueOverview, hadoopIssueReport} from './security/issues/HadoopIssue'; import {mssqlIssueOverview, mssqlIssueReport} from './security/issues/MssqlIssue'; import {wmiPasswordIssueReport, wmiPthIssueReport} from './security/issues/WmiIssue'; -import {sshKeysReport, shhIssueReport, sshIssueOverview} from './security/issues/SshIssue'; +import {shhIssueReport, sshIssueOverview, sshKeysReport} from './security/issues/SshIssue'; import {log4shellIssueOverview, log4shellIssueReport} from './security/issues/Log4ShellIssue'; import { crossSegmentIssueOverview, @@ -31,11 +30,13 @@ import { islandCrossSegmentIssueReport } from './security/issues/CrossSegmentIssue'; import { - sharedCredsDomainIssueReport, sharedCredsIssueReport, sharedLocalAdminsIssueReport, sharedAdminsDomainIssueOverview, + sharedCredsDomainIssueReport, + sharedCredsIssueReport, + sharedLocalAdminsIssueReport, sharedPasswordsIssueOverview } from './security/issues/SharedPasswordsIssue'; -import {tunnelIssueReport, tunnelIssueOverview} from './security/issues/TunnelIssue'; +import {tunnelIssueOverview, tunnelIssueReport} from './security/issues/TunnelIssue'; import {stolenCredsIssueOverview} from './security/issues/StolenCredsIssue'; import {strongUsersOnCritIssueReport} from './security/issues/StrongUsersOnCritIssue'; import { From b7566a805b456b0a6821b42cdc01b645a2c890b8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 30 Sep 2022 14:22:02 -0400 Subject: [PATCH 19/21] Island: Use list comprehension instead of map() --- .../monkey_island/cc/repository/mongo_machine_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/repository/mongo_machine_repository.py b/monkey/monkey_island/cc/repository/mongo_machine_repository.py index 75cbf2d46..8e1934cc7 100644 --- a/monkey/monkey_island/cc/repository/mongo_machine_repository.py +++ b/monkey/monkey_island/cc/repository/mongo_machine_repository.py @@ -75,7 +75,7 @@ class MongoMachineRepository(IMachineRepository): except Exception as err: raise RetrievalError(f"Error retrieving machines: {err}") - return list(map(lambda m: Machine(**m), cursor)) + return [Machine(**m) for m in cursor] def get_machines_by_ip(self, ip: IPv4Address) -> Sequence[Machine]: ip_regex = "^" + str(ip).replace(".", "\\.") + "\\/.*$" @@ -86,7 +86,7 @@ class MongoMachineRepository(IMachineRepository): except Exception as err: raise RetrievalError(f'Error retrieving machines with ip "{ip}": {err}') - machines = list(map(lambda m: Machine(**m), cursor)) + machines = [Machine(**m) for m in cursor] if len(machines) == 0: raise UnknownRecordError(f'No machines found with IP "{ip}"') From 519f48b6d80ccebb0fa25ad04de6aed30ae6727e Mon Sep 17 00:00:00 2001 From: Kekoa Kaaikala Date: Fri, 30 Sep 2022 18:20:04 +0000 Subject: [PATCH 20/21] Island: Use ip address in scan report --- monkey/monkey_island/cc/services/reporting/report.py | 2 +- .../monkey_island/cc/services/reporting/test_report.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index e0d49339a..31827a403 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -120,7 +120,7 @@ class ReportService: formatted_nodes.append( { "hostname": machine.hostname, - "ip_addresses": list(map(str, machine.network_interfaces)), + "ip_addresses": [str(iface.ip) for iface in machine.network_interfaces], "accessible_from_nodes": list( map(lambda m: m.dict(simplify=True), accessible_from) ), diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index 75d269701..cd38d6a6b 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -48,14 +48,14 @@ MACHINES = [MACHINE_1, MACHINE_2, MACHINE_3, ISLAND_MACHINE] EXPECTED_SCANNED_MACHINES = [ { "hostname": MACHINE_1.hostname, - "ip_addresses": list(map(str, MACHINE_1.network_interfaces)), + "ip_addresses": [str(iface.ip) for iface in MACHINE_1.network_interfaces], "accessible_from_nodes": [ISLAND_MACHINE.dict(simplify=True)], "services": [], "domain_name": "", }, { "hostname": MACHINE_2.hostname, - "ip_addresses": list(map(str, MACHINE_2.network_interfaces)), + "ip_addresses": [str(iface.ip) for iface in MACHINE_2.network_interfaces], "accessible_from_nodes": [MACHINE_1.dict(simplify=True)], "services": [], "domain_name": "", From 807193ece562ea7862c90bbcaafe58823eb3e42d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 30 Sep 2022 14:29:31 -0400 Subject: [PATCH 21/21] Island: Use list comprehension instead of map() --- monkey/monkey_island/cc/services/reporting/report.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 31827a403..b3673dd90 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -121,9 +121,7 @@ class ReportService: { "hostname": machine.hostname, "ip_addresses": [str(iface.ip) for iface in machine.network_interfaces], - "accessible_from_nodes": list( - map(lambda m: m.dict(simplify=True), accessible_from) - ), + "accessible_from_nodes": [m.dict(simplify=True) for m in accessible_from], "domain_name": "", # TODO add services "services": [],