Merge pull request #2134 from guardicore/2092-bb-tests

2092 bb tests

envs/monkey_zoo/blackbox/analyzers/zerologon_analyzer.py

@@ -1,6 +1,7 @@
 from pprint import pformat
 from typing import List
 
+from common.credentials import CredentialComponentType, Credentials
 from envs.monkey_zoo.blackbox.analyzers.analyzer import Analyzer
 from envs.monkey_zoo.blackbox.analyzers.analyzer_log import AnalyzerLog
 from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
@@ -26,19 +27,23 @@ class ZerologonAnalyzer(Analyzer):
         return is_creds_gathered and is_creds_restored
 
     def _analyze_credential_gathering(self) -> bool:
-        config = self.island_client.get_config()
+        propagation_credentials = self.island_client.get_propagation_credentials()
-        credentials_on_island = ZerologonAnalyzer._get_relevant_credentials(config)
+        credentials_on_island = ZerologonAnalyzer._get_relevant_credentials(propagation_credentials)
         return self._is_all_credentials_in_list(credentials_on_island)
 
     @staticmethod
-    def _get_relevant_credentials(config: dict):
+    def _get_relevant_credentials(propagation_credentials: Credentials) -> List[str]:
-        credentials_on_island = []
+        credentials_on_island = set()
-        # TODO: Pull configured credentials and put usernames, nt and lm hashes into
+
-        # credentials_island
+        for credentials in propagation_credentials:
-        # credentials_on_island.extend(dpath.util.get(config["configuration"], USER_LIST_PATH))
+            if credentials.identity.credential_type is CredentialComponentType.USERNAME:
-        # credentials_on_island.extend(dpath.util.get(config["configuration"], NTLM_HASH_LIST_PATH))
+                credentials_on_island.update([credentials.identity.username])
-        # credentials_on_island.extend(dpath.util.get(config["configuration"], LM_HASH_LIST_PATH))
+            if credentials.secret.credential_type is CredentialComponentType.NT_HASH:
-        return credentials_on_island
+                credentials_on_island.update([credentials.secret.nt_hash])
+            if credentials.secret.credential_type is CredentialComponentType.LM_HASH:
+                credentials_on_island.update([credentials.secret.lm_hash])
+
+        return list(credentials_on_island)
 
     def _is_all_credentials_in_list(self, all_creds: List[str]) -> bool:
         credentials_missing = [cred for cred in self.expected_credentials if cred not in all_creds]

envs/monkey_zoo/blackbox/conftest.py

@@ -32,6 +32,11 @@ def no_gcp(request):
     return request.config.getoption("--no-gcp")
 
 
+@pytest.fixture(scope="session")
+def machines_to_start(request):
+    return request.config.getoption("-k")
+
+
 def pytest_runtest_setup(item):
     if "skip_powershell_reuse" in item.keywords and item.config.getoption(
         "--skip-powershell-reuse"

envs/monkey_zoo/blackbox/gcp_test_machine_list.py

@@ -26,3 +26,70 @@ GCP_TEST_MACHINE_LIST = {
         "log4j-tomcat-52",
     ],
 }
+
+DEPTH_2_A = {
+    "europe-west3-a": [
+        "sshkeys-11",
+        "sshkeys-12",
+    ]
+}
+
+
+DEPTH_1_A = {
+    "europe-west3-a": ["hadoop-2", "hadoop-3", "mssql-16", "mimikatz-14", "mimikatz-15"],
+    "europe-west1-b": [
+        "log4j-logstash-55",
+        "log4j-logstash-56",
+        "log4j-solr-49",
+        "log4j-solr-50",
+        "log4j-tomcat-51",
+        "log4j-tomcat-52",
+    ],
+}
+
+DEPTH_3_A = {
+    "europe-west3-a": [
+        "tunneling-9",
+        "tunneling-10",
+        "tunneling-11",
+        "tunneling-12",
+        "mimikatz-15",
+    ],
+    "europe-west1-b": [
+        "powershell-3-45",
+        "powershell-3-46",
+        "powershell-3-47",
+        "powershell-3-48",
+    ],
+}
+
+POWERSHELL_EXPLOITER_REUSE = {
+    "europe-west1-b": [
+        "powershell-3-46",
+    ]
+}
+
+ZEROLOGON = {
+    "europe-west3-a": [
+        "zerologon-25",
+    ],
+}
+
+WMI_AND_MIMIKATZ = {
+    "europe-west3-a": [
+        "mimikatz-14",
+        "mimikatz-15",
+    ]
+}
+
+SMB_PTH = {"europe-west3-a": ["mimikatz-15"]}
+
+GCP_SINGLE_TEST_LIST = {
+    "test_depth_2_a": DEPTH_2_A,
+    "test_depth_1_a": DEPTH_1_A,
+    "test_depth_3_a": DEPTH_3_A,
+    "test_powershell_exploiter_credentials_reuse": POWERSHELL_EXPLOITER_REUSE,
+    "test_zerologon_exploiter": ZEROLOGON,
+    "test_wmi_and_mimikatz_exploiters": WMI_AND_MIMIKATZ,
+    "test_smb_pth": SMB_PTH,
+}

envs/monkey_zoo/blackbox/island_client/monkey_island_client.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import time
-from typing import Union
+from typing import Sequence, Union
 
 from bson import json_util
 
@@ -29,23 +29,41 @@ class MonkeyIslandClient(object):
     def get_api_status(self):
         return self.requests.get("api")
 
-    def get_config(self):
+    def get_propagation_credentials(self) -> Sequence[Credentials]:
-        return json.loads(self.requests.get("api/agent-configuration").content)
+        response = self.requests.get("api/propagation-credentials")
+        return [Credentials.from_mapping(credentials) for credentials in response.json()]
 
     @avoid_race_condition
     def import_config(self, test_configuration: TestConfiguration):
-        self.requests.post_json(
+        self._import_config(test_configuration)
+        self._import_credentials(test_configuration.propagation_credentials)
+
+    @avoid_race_condition
+    def _import_config(self, test_configuration: TestConfiguration):
+        response = self.requests.post_json(
             "api/agent-configuration",
             json=AgentConfiguration.to_mapping(test_configuration.agent_configuration),
         )
+        if response.ok:
+            LOGGER.info("Configuration is imported.")
+        else:
+            LOGGER.error(f"Failed to import config: {response}")
+            assert False
+
+    @avoid_race_condition
+    def _import_credentials(self, propagation_credentials: Credentials):
         serialized_propagation_credentials = [
-            Credentials.to_mapping(credentials)
+            Credentials.to_mapping(credentials) for credentials in propagation_credentials
-            for credentials in test_configuration.propagation_credentials
         ]
-        self.requests.post_json(
+        response = self.requests.post_json(
             "/api/propagation-credentials/configured-credentials",
             json=serialized_propagation_credentials,
         )
+        if response.ok:
+            LOGGER.info("Credentials are imported.")
+        else:
+            LOGGER.error(f"Failed to import credentials: {response}")
+            assert False
 
     @avoid_race_condition
     def run_monkey_local(self):

envs/monkey_zoo/blackbox/test_blackbox.py

@@ -6,7 +6,10 @@ import pytest
 
 from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer
 from envs.monkey_zoo.blackbox.analyzers.zerologon_analyzer import ZerologonAnalyzer
-from envs.monkey_zoo.blackbox.gcp_test_machine_list import GCP_TEST_MACHINE_LIST
+from envs.monkey_zoo.blackbox.gcp_test_machine_list import (
+    GCP_SINGLE_TEST_LIST,
+    GCP_TEST_MACHINE_LIST,
+)
 from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
 from envs.monkey_zoo.blackbox.island_client.test_configuration_parser import get_target_ips
 from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler
@@ -35,18 +38,21 @@ LOGGER = logging.getLogger(__name__)
 
 
 @pytest.fixture(autouse=True, scope="session")
-def GCPHandler(request, no_gcp):
+def GCPHandler(request, no_gcp, machines_to_start):
     if not no_gcp:
+        list_machines = GCP_TEST_MACHINE_LIST
+        if machines_to_start:
+            list_machines = GCP_SINGLE_TEST_LIST[machines_to_start]
         try:
             initialize_gcp_client()
-            start_machines(GCP_TEST_MACHINE_LIST)
+            start_machines(list_machines)
         except Exception as e:
             LOGGER.error("GCP Handler failed to initialize: %s." % e)
             pytest.exit("Encountered an error while starting GCP machines. Stopping the tests.")
         wait_machine_bootup()
 
         def fin():
-            stop_machines(GCP_TEST_MACHINE_LIST)
+            stop_machines(list_machines)
 
         request.addfinalizer(fin)
 

envs/monkey_zoo/blackbox/test_configurations/depth_1_a.py

@@ -5,6 +5,7 @@ from .noop import noop_test_configuration
 from .utils import (
     add_credential_collectors,
     add_exploiters,
+    add_fingerprinters,
     add_http_ports,
     add_subnets,
     add_tcp_ports,
@@ -34,6 +35,12 @@ def _add_exploiters(agent_configuration: AgentConfiguration) -> AgentConfigurati
     return add_exploiters(agent_configuration, brute_force=brute_force, vulnerability=vulnerability)
 
 
+def _add_fingerprinters(agent_configuration: AgentConfiguration) -> AgentConfiguration:
+    fingerprinters = [PluginConfiguration(name="http", options={})]
+
+    return add_fingerprinters(agent_configuration, fingerprinters)
+
+
 def _add_subnets(agent_configuration: AgentConfiguration) -> AgentConfiguration:
     subnets = [
         "10.2.2.2",
@@ -69,18 +76,17 @@ def _add_http_ports(agent_configuration: AgentConfiguration) -> AgentConfigurati
     return add_http_ports(agent_configuration, HTTP_PORTS)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_fingerprinters(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
-agent_configuration = _add_credential_collectors(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
-agent_configuration = _add_http_ports(agent_configuration)
+test_configuration = _add_credential_collectors(test_configuration)
+test_configuration = _add_http_ports(test_configuration)
 
 depth_1_a_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )
-
-
 CREDENTIALS = (
     Credentials(Username("m0nk3y"), None),
     Credentials(None, Password("Ivrrw5zEzs")),

envs/monkey_zoo/blackbox/test_configurations/depth_2_a.py

@@ -34,13 +34,13 @@ def _add_tcp_ports(agent_configuration: AgentConfiguration) -> AgentConfiguratio
     return add_tcp_ports(agent_configuration, ports)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 2)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 2)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
 
 depth_2_a_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )
 
 

envs/monkey_zoo/blackbox/test_configurations/depth_3_a.py

@@ -48,14 +48,14 @@ def _add_tcp_ports(agent_configuration: AgentConfiguration) -> AgentConfiguratio
     return add_tcp_ports(agent_configuration, ports)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 3)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 3)
-agent_configuration = set_keep_tunnel_open_time(noop_test_configuration.agent_configuration, 20)
+test_configuration = set_keep_tunnel_open_time(test_configuration, 20)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
 
 depth_3_a_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )
 
 

envs/monkey_zoo/blackbox/test_configurations/powershell_credentials_reuse.py

@@ -30,11 +30,11 @@ def _add_tcp_ports(agent_configuration: AgentConfiguration) -> AgentConfiguratio
     return add_tcp_ports(agent_configuration, ports)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
 
 powershell_credentials_reuse_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )

envs/monkey_zoo/blackbox/test_configurations/smb_pth.py

@@ -33,14 +33,14 @@ def _add_tcp_ports(agent_configuration: AgentConfiguration) -> AgentConfiguratio
     return add_tcp_ports(agent_configuration, ports)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 3)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 3)
-agent_configuration = set_keep_tunnel_open_time(agent_configuration, 20)
+test_configuration = set_keep_tunnel_open_time(test_configuration, 20)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
 
 smb_pth_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )
 
 

envs/monkey_zoo/blackbox/test_configurations/utils.py

@@ -28,6 +28,16 @@ def add_exploiters(
     return replace_exploitation_configuration(agent_configuration, exploitation_configuration)
 
 
+def add_fingerprinters(
+    agent_configuration: AgentConfiguration, fingerprinters: Sequence[PluginConfiguration]
+) -> AgentConfiguration:
+    network_scan_configuration = replace(
+        agent_configuration.propagation.network_scan, fingerprinters=fingerprinters
+    )
+
+    return replace_network_scan_configuration(agent_configuration, network_scan_configuration)
+
+
 def add_tcp_ports(
     agent_configuration: AgentConfiguration, tcp_ports: Sequence[int]
 ) -> AgentConfiguration:

envs/monkey_zoo/blackbox/test_configurations/wmi_mimikatz.py

@@ -40,15 +40,15 @@ def _add_tcp_ports(agent_configuration: AgentConfiguration) -> AgentConfiguratio
     return add_tcp_ports(agent_configuration, ports)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
-agent_configuration = _add_credential_collectors(agent_configuration)
+test_configuration = _add_credential_collectors(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
-agent_configuration = _add_credential_collectors(agent_configuration)
+test_configuration = _add_credential_collectors(test_configuration)
 
 wmi_mimikatz_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )
 
 

envs/monkey_zoo/blackbox/test_configurations/zerologon.py

@@ -27,11 +27,11 @@ def _add_subnets(agent_configuration: AgentConfiguration) -> AgentConfiguration:
     return add_subnets(agent_configuration, subnets)
 
 
-agent_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
+test_configuration = set_maximum_depth(noop_test_configuration.agent_configuration, 1)
-agent_configuration = _add_exploiters(agent_configuration)
+test_configuration = _add_exploiters(test_configuration)
-agent_configuration = _add_tcp_ports(agent_configuration)
+test_configuration = _add_tcp_ports(test_configuration)
-agent_configuration = _add_subnets(agent_configuration)
+test_configuration = _add_subnets(test_configuration)
 
 zerologon_test_configuration = replace_agent_configuration(
-    noop_test_configuration, agent_configuration
+    noop_test_configuration, test_configuration
 )

monkey/infection_monkey/exploit/zerologon.py

@@ -285,7 +285,12 @@ class ZerologonExploiter(HostExploiter):
         self, user: str, lmhash: str, nthash: str
     ) -> None:
         self.telemetry_messenger.send_telemetry(
-            CredentialsTelem([Credentials([Username(user)], [LMHash(lmhash), NTHash(nthash)])])
+            CredentialsTelem(
+                [
+                    Credentials(Username(user), LMHash(lmhash)),
+                    Credentials(Username(user), NTHash(nthash)),
+                ]
+            )
         )
 
     def get_original_pwd_nthash(self, username: str, user_pwd_hashes: List[str]) -> str:

monkey/infection_monkey/master/propagator.py

@@ -1,7 +1,8 @@
 import logging
+from dataclasses import replace
 from queue import Queue
 from threading import Event
-from typing import List
+from typing import List, Sequence
 
 from common.agent_configuration import (
     ExploitationConfiguration,
@@ -53,10 +54,14 @@ class Propagator:
         network_scan_completed = Event()
         self._hosts_to_exploit = Queue()
 
+        network_scan = self._add_http_ports_to_fingerprinters(
+            propagation_config.network_scan, propagation_config.exploitation.options.http_ports
+        )
+
         scan_thread = create_daemon_thread(
             target=self._scan_network,
             name="PropagatorScanThread",
-            args=(propagation_config.network_scan, stop),
+            args=(network_scan, stop),
         )
         exploit_thread = create_daemon_thread(
             target=self._exploit_hosts,
@@ -74,6 +79,23 @@ class Propagator:
 
         logger.info("Finished attempting to propagate")
 
+    @staticmethod
+    def _add_http_ports_to_fingerprinters(
+        network_scan: NetworkScanConfiguration, http_ports: Sequence[int]
+    ) -> NetworkScanConfiguration:
+        # This is a hack to add http_ports to the options of fingerprinters
+        # It will be reworked. See https://github.com/guardicore/monkey/issues/2136
+        modified_fingerprinters = [*network_scan.fingerprinters]
+        for i, fingerprinter in enumerate(modified_fingerprinters):
+            if fingerprinter.name != "http":
+                continue
+
+            modified_options = fingerprinter.options.copy()
+            modified_options["http_ports"] = list(http_ports)
+            modified_fingerprinters[i] = replace(fingerprinter, options=modified_options)
+
+        return replace(network_scan, fingerprinters=modified_fingerprinters)
+
     def _scan_network(self, scan_config: NetworkScanConfiguration, stop: Event):
         logger.info("Starting network scan")
 

monkey/monkey_island/cc/services/attack/technique_reports/technique_report_tools.py

@@ -31,7 +31,7 @@ def censor_password(password, plain_chars=3, secret_chars=5):
     """
     if not password:
         return ""
-    password = get_datastore_encryptor().decrypt(password)
+    password = get_datastore_encryptor().decrypt(password.encode()).decode()
     return password[0:plain_chars] + "*" * secret_chars
 
 
@@ -45,5 +45,5 @@ def censor_hash(str_hash, plain_chars=5):
     """
     if not str_hash:
         return ""
-    str_hash = get_datastore_encryptor().decrypt(str_hash)
+    str_hash = get_datastore_encryptor().decrypt(str_hash.encode()).decode()
     return str_hash[0:plain_chars] + " ..."

monkey/monkey_island/cc/services/representations.py

@@ -22,6 +22,8 @@ class APIEncoder(JSONEncoder):
             return value.name
         if issubclass(type(value), IJSONSerializable):
             return loads(value.__class__.to_json(value))
+        if issubclass(type(value), set):
+            return list(value)
         try:
             return JSONEncoder.default(self, value)
         except TypeError:

monkey/monkey_island/cc/services/telemetry/processing/exploit.py

@@ -52,4 +52,6 @@ def encrypt_exploit_creds(telemetry_json):
             credential = attempts[i][field]
             if credential:  # PowerShell exploiter's telem may have `None` here
                 if len(credential) > 0:
-                    attempts[i][field] = get_datastore_encryptor().encrypt(credential)
+                    attempts[i][field] = (
+                        get_datastore_encryptor().encrypt(credential.encode()).decode()
+                    )