Merge pull request #2042 from guardicore/2001-use-new-configuration-endpoint

2001 use new configuration endpoint

monkey/infection_monkey/master/control_channel.py

@@ -52,17 +52,16 @@ class ControlChannel(IControlChannel):
     def get_config(self) -> AgentConfiguration:
         try:
             response = requests.get(  # noqa: DUO123
-                f"https://{self._control_channel_server}/api/agent",
+                f"https://{self._control_channel_server}/api/agent-configuration",
                 verify=False,
                 proxies=self._proxies,
                 timeout=SHORT_REQUEST_TIMEOUT,
             )
             response.raise_for_status()
 
-            config_dict = json.loads(response.text)["config"]
             logger.debug(f"Received configuration:\n{pformat(json.loads(response.text))}")
 
-            return AgentConfiguration.from_mapping(config_dict)
+            return AgentConfiguration.from_json(response.text)
         except (
             json.JSONDecodeError,
             requests.exceptions.ConnectionError,

monkey/monkey_island/cc/resources/agent_configuration.py

@@ -15,7 +15,7 @@ class AgentConfiguration(AbstractResource):
     def __init__(self, agent_configuration_repository: IAgentConfigurationRepository):
         self._agent_configuration_repository = agent_configuration_repository
 
-    @jwt_required
+    # Used by the agent. Can't secure
     def get(self):
         configuration = self._agent_configuration_repository.get_configuration()
         configuration_json = AgentConfigurationObject.to_json(configuration)
@@ -23,7 +23,6 @@ class AgentConfiguration(AbstractResource):
 
     @jwt_required
     def post(self):
-
         try:
             configuration_object = AgentConfigurationObject.from_json(request.data)
             self._agent_configuration_repository.store_configuration(configuration_object)

monkey/monkey_island/cc/resources/monkey.py

@@ -8,7 +8,6 @@ from monkey_island.cc.models.monkey_ttl import create_monkey_ttl_document
 from monkey_island.cc.resources.AbstractResource import AbstractResource
 from monkey_island.cc.resources.utils.semaphores import agent_killing_mutex
 from monkey_island.cc.server_utils.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS
-from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.edge.edge import EdgeService
 from monkey_island.cc.services.node import NodeService
 
@@ -22,10 +21,6 @@ class Monkey(AbstractResource):
         "/api/agent/<string:guid>",
     ]
 
-    # Used by monkey. can't secure.
-    def get(self):
-        return {"config": ConfigService.format_flat_config_for_agent()}
-
     # Used by monkey. can't secure.
     # Called on monkey wakeup to initialize local configuration
     def post(self, **kw):

monkey/monkey_island/cc/services/config.py

@@ -2,8 +2,7 @@ import collections
 import copy
 import functools
 import logging
-import re
+from typing import Dict, List
-from typing import Any, Dict, List
 
 from jsonschema import Draft4Validator, validators
 
@@ -44,8 +43,6 @@ SENSITIVE_SSH_KEY_FIELDS = [
     SensitiveField(path="public_key", field_encryptor=StringEncryptor),
 ]
 
-SMB_DOWNLOAD_TIMEOUT = 30
-
 
 class ConfigService:
     default_config = None
@@ -349,246 +346,3 @@ class ConfigService:
             "exploit_ntlm_hash_list": config.get("exploit_ntlm_hash_list", []),
             "exploit_ssh_keys": config.get("exploit_ssh_keys", []),
         }
-
-    @staticmethod
-    def format_flat_config_for_agent():
-        config = ConfigService.get_flat_config()
-        ConfigService._remove_credentials_from_flat_config(config)
-        ConfigService._format_payloads_from_flat_config(config)
-        ConfigService._format_pbas_from_flat_config(config)
-        ConfigService._format_propagation_from_flat_config(config)
-        ConfigService._format_credential_collectors(config)
-
-        # Ok, I'll admit this is just sort of jammed in here. But this code is going away very soon.
-        del config["HTTP_PORTS"]
-
-        return config
-
-    @staticmethod
-    def _remove_credentials_from_flat_config(config: Dict):
-        fields_to_remove = {
-            "exploit_lm_hash_list",
-            "exploit_ntlm_hash_list",
-            "exploit_password_list",
-            "exploit_ssh_keys",
-            "exploit_user_list",
-        }
-
-        for field in fields_to_remove:
-            config.pop(field, None)
-
-    @staticmethod
-    def _format_credential_collectors(config: Dict):
-        collectors = [
-            {"name": collector, "options": {}} for collector in config["credential_collectors"]
-        ]
-        config["credential_collectors"] = collectors
-
-    @staticmethod
-    def _format_payloads_from_flat_config(config: Dict):
-        config.setdefault("payloads", []).append(
-            {"name": "ransomware", "options": config["ransomware"]}
-        )
-        config.pop("ransomware", None)
-
-    @staticmethod
-    def _format_pbas_from_flat_config(config: Dict):
-        flat_linux_command_field = "custom_PBA_linux_cmd"
-        flat_linux_filename_field = "PBA_linux_filename"
-        flat_windows_command_field = "custom_PBA_windows_cmd"
-        flat_windows_filename_field = "PBA_windows_filename"
-
-        formatted_pbas_config = [
-            {"name": pba, "options": {}} for pba in config.get("post_breach_actions", [])
-        ]
-
-        config["custom_pbas"] = {
-            "linux_command": config.get(flat_linux_command_field, ""),
-            "linux_filename": config.get(flat_linux_filename_field, ""),
-            "windows_command": config.get(flat_windows_command_field, ""),
-            "windows_filename": config.get(flat_windows_filename_field, ""),
-        }
-
-        config["post_breach_actions"] = formatted_pbas_config
-
-        config.pop(flat_linux_command_field, None)
-        config.pop(flat_linux_filename_field, None)
-        config.pop(flat_windows_command_field, None)
-        config.pop(flat_windows_filename_field, None)
-
-    @staticmethod
-    def _format_propagation_from_flat_config(config: Dict):
-        formatted_propagation_config = {"network_scan": {}, "maximum_depth": {}, "exploitation": {}}
-
-        formatted_propagation_config[
-            "network_scan"
-        ] = ConfigService._format_network_scan_from_flat_config(config)
-
-        formatted_propagation_config[
-            "exploitation"
-        ] = ConfigService._format_exploiters_from_flat_config(config)
-
-        formatted_propagation_config["maximum_depth"] = config["depth"]
-        del config["depth"]
-
-        config["propagation"] = formatted_propagation_config
-
-    @staticmethod
-    def _format_network_scan_from_flat_config(config: Dict) -> Dict[str, Any]:
-        formatted_network_scan_config = {"tcp": {}, "icmp": {}, "fingerprinters": [], "targets": {}}
-
-        formatted_network_scan_config["tcp"] = ConfigService._format_tcp_scan_from_flat_config(
-            config
-        )
-        formatted_network_scan_config["icmp"] = ConfigService._format_icmp_scan_from_flat_config(
-            config
-        )
-        formatted_network_scan_config[
-            "fingerprinters"
-        ] = ConfigService._format_fingerprinters_from_flat_config(config)
-
-        formatted_network_scan_config["targets"] = ConfigService._format_targets_from_flat_config(
-            config
-        )
-
-        return formatted_network_scan_config
-
-    @staticmethod
-    def _format_tcp_scan_from_flat_config(config: Dict) -> Dict[str, Any]:
-        flat_http_ports_field = "HTTP_PORTS"
-        flat_tcp_timeout_field = "tcp_scan_timeout"
-        flat_tcp_ports_field = "tcp_target_ports"
-
-        formatted_tcp_scan_config = {}
-
-        formatted_tcp_scan_config["timeout"] = config[flat_tcp_timeout_field] / 1000
-
-        ports = ConfigService._union_tcp_and_http_ports(
-            config[flat_tcp_ports_field], config[flat_http_ports_field]
-        )
-        formatted_tcp_scan_config["ports"] = ports
-
-        # Do not remove HTTP_PORTS field. Other components besides scanning need it.
-        config.pop(flat_tcp_timeout_field, None)
-        config.pop(flat_tcp_ports_field, None)
-
-        return formatted_tcp_scan_config
-
-    @staticmethod
-    def _union_tcp_and_http_ports(tcp_ports: List[int], http_ports: List[int]) -> List[int]:
-        combined_ports = list(set(tcp_ports) | set(http_ports))
-
-        return sorted(combined_ports)
-
-    @staticmethod
-    def _format_icmp_scan_from_flat_config(config: Dict) -> Dict[str, Any]:
-        flat_ping_timeout_field = "ping_scan_timeout"
-
-        formatted_icmp_scan_config = {}
-        formatted_icmp_scan_config["timeout"] = config[flat_ping_timeout_field] / 1000
-
-        config.pop(flat_ping_timeout_field, None)
-
-        return formatted_icmp_scan_config
-
-    @staticmethod
-    def _format_fingerprinters_from_flat_config(config: Dict) -> List[Dict[str, Any]]:
-        flat_fingerprinter_classes_field = "finger_classes"
-        flat_http_ports_field = "HTTP_PORTS"
-
-        formatted_fingerprinters = [
-            {"name": f, "options": {}} for f in sorted(config[flat_fingerprinter_classes_field])
-        ]
-
-        for fp in formatted_fingerprinters:
-            if fp["name"] == "HTTPFinger":
-                fp["options"] = {"http_ports": sorted(config[flat_http_ports_field])}
-
-            fp["name"] = ConfigService._translate_fingerprinter_name(fp["name"])
-
-        config.pop(flat_fingerprinter_classes_field)
-        return formatted_fingerprinters
-
-    @staticmethod
-    def _translate_fingerprinter_name(name: str) -> str:
-        # This translates names like "HTTPFinger" to "http". "HTTPFinger" is an old classname on the
-        # agent-side and is therefore unnecessarily couples the island to the fingerprinter's
-        # implementation within the agent. For the time being, fingerprinters will have names like
-        # "http", "ssh", "elastic", etc. This will be revisited when fingerprinters become plugins.
-        return re.sub(r"Finger", "", name).lower()
-
-    @staticmethod
-    def _format_targets_from_flat_config(config: Dict) -> Dict[str, Any]:
-        flat_blocked_ips_field = "blocked_ips"
-        flat_inaccessible_subnets_field = "inaccessible_subnets"
-        flat_local_network_scan_field = "local_network_scan"
-        flat_subnet_scan_list_field = "subnet_scan_list"
-
-        formatted_scan_targets_config = {}
-
-        formatted_scan_targets_config[flat_blocked_ips_field] = config[flat_blocked_ips_field]
-        formatted_scan_targets_config[flat_inaccessible_subnets_field] = config[
-            flat_inaccessible_subnets_field
-        ]
-        formatted_scan_targets_config[flat_local_network_scan_field] = config[
-            flat_local_network_scan_field
-        ]
-        formatted_scan_targets_config["subnets"] = config[flat_subnet_scan_list_field]
-
-        config.pop(flat_blocked_ips_field, None)
-        config.pop(flat_inaccessible_subnets_field, None)
-        config.pop(flat_local_network_scan_field, None)
-        config.pop(flat_subnet_scan_list_field, None)
-
-        return formatted_scan_targets_config
-
-    @staticmethod
-    def _format_exploiters_from_flat_config(config: Dict) -> Dict[str, List[Dict[str, Any]]]:
-        flat_config_exploiter_classes_field = "exploiter_classes"
-        brute_force_category = "brute_force"
-        vulnerability_category = "vulnerability"
-        brute_force_exploiters = {
-            "MSSQLExploiter",
-            "PowerShellExploiter",
-            "SSHExploiter",
-            "SmbExploiter",
-            "WmiExploiter",
-        }
-
-        exploit_options = {}
-
-        exploit_options["http_ports"] = sorted(config["HTTP_PORTS"])
-
-        formatted_exploiters_config = {
-            "options": exploit_options,
-            "brute_force": [],
-            "vulnerability": [],
-        }
-
-        for exploiter in sorted(config[flat_config_exploiter_classes_field]):
-            category = (
-                brute_force_category
-                if exploiter in brute_force_exploiters
-                else vulnerability_category
-            )
-
-            formatted_exploiters_config[category].append({"name": exploiter, "options": {}})
-
-        config.pop(flat_config_exploiter_classes_field, None)
-
-        formatted_exploiters_config = ConfigService._add_smb_download_timeout_to_exploiters(
-            formatted_exploiters_config
-        )
-        return formatted_exploiters_config
-
-    @staticmethod
-    def _add_smb_download_timeout_to_exploiters(
-        formatted_config: Dict,
-    ) -> Dict[str, List[Dict[str, Any]]]:
-        new_config = copy.deepcopy(formatted_config)
-        uses_smb_timeout = {"SmbExploiter", "WmiExploiter"}
-
-        for exploiter in filter(lambda e: e["name"] in uses_smb_timeout, new_config["brute_force"]):
-            exploiter["options"]["smb_download_timeout"] = SMB_DOWNLOAD_TIMEOUT
-
-        return new_config

monkey/tests/unit_tests/monkey_island/cc/services/test_config.py

@@ -1,80 +1,9 @@
-import pytest
-
 from monkey_island.cc.services.config import ConfigService
 
 # If tests fail because config path is changed, sync with
 # monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage/RunOptions.js
 
 
-@pytest.fixture(autouse=True)
-def mock_flat_config(monkeypatch, flat_monkey_config):
-    monkeypatch.setattr(
-        "monkey_island.cc.services.config.ConfigService.get_flat_config", lambda: flat_monkey_config
-    )
-
-
-def test_format_config_for_agent__credentials_removed():
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert "exploit_lm_hash_list" not in flat_monkey_config
-    assert "exploit_ntlm_hash_list" not in flat_monkey_config
-    assert "exploit_password_list" not in flat_monkey_config
-    assert "exploit_ssh_keys" not in flat_monkey_config
-    assert "exploit_user_list" not in flat_monkey_config
-
-
-def test_format_config_for_agent__ransomware_payload():
-    expected_ransomware_options = {
-        "encryption": {
-            "enabled": True,
-            "directories": {
-                "linux_target_dir": "/tmp/ransomware-target",
-                "windows_target_dir": "C:\\windows\\temp\\ransomware-target",
-            },
-        },
-        "other_behaviors": {"readme": True},
-    }
-
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert "payloads" in flat_monkey_config
-    assert flat_monkey_config["payloads"][0]["name"] == "ransomware"
-    assert flat_monkey_config["payloads"][0]["options"] == expected_ransomware_options
-
-    assert "ransomware" not in flat_monkey_config
-
-
-def test_format_config_for_agent__pbas():
-    expected_pbas_config = [
-        {"name": "CommunicateAsBackdoorUser", "options": {}},
-        {"name": "ModifyShellStartupFiles", "options": {}},
-        {"name": "ScheduleJobs", "options": {}},
-        {"name": "Timestomping", "options": {}},
-        {"name": "AccountDiscovery", "options": {}},
-    ]
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert "post_breach_actions" in flat_monkey_config
-    assert flat_monkey_config["post_breach_actions"] == expected_pbas_config
-
-    assert "custom_PBA_linux_cmd" not in flat_monkey_config
-    assert "PBA_linux_filename" not in flat_monkey_config
-    assert "custom_PBA_windows_cmd" not in flat_monkey_config
-    assert "PBA_windows_filename" not in flat_monkey_config
-
-
-def test_format_config_for_custom_pbas():
-    custom_config = {
-        "linux_command": "bash test.sh",
-        "windows_command": "powershell test.ps1",
-        "linux_filename": "test.sh",
-        "windows_filename": "test.ps1",
-    }
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert flat_monkey_config["custom_pbas"] == custom_config
-
-
 def test_get_config_propagation_credentials_from_flat_config(flat_monkey_config):
     expected_creds = {
         "exploit_lm_hash_list": ["lm_hash_1", "lm_hash_2"],
@@ -86,125 +15,3 @@ def test_get_config_propagation_credentials_from_flat_config(flat_monkey_config)
 
     creds = ConfigService.get_config_propagation_credentials_from_flat_config(flat_monkey_config)
     assert creds == expected_creds
-
-
-def test_format_config_for_agent__propagation():
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert "propagation" in flat_monkey_config
-    assert "network_scan" in flat_monkey_config["propagation"]
-    assert "exploitation" in flat_monkey_config["propagation"]
-
-
-def test_format_config_for_agent__network_scan():
-    expected_network_scan_config = {
-        "tcp": {
-            "timeout": 3.0,
-            "ports": [
-                22,
-                80,
-                135,
-                443,
-                445,
-                2222,
-                3306,
-                3389,
-                7001,
-                8008,
-                8080,
-                8088,
-                9200,
-            ],
-        },
-        "icmp": {
-            "timeout": 1.0,
-        },
-        "targets": {
-            "blocked_ips": ["192.168.1.1", "192.168.1.100"],
-            "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"],
-            "local_network_scan": True,
-            "subnets": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"],
-        },
-        "fingerprinters": [
-            {"name": "elastic", "options": {}},
-            {
-                "name": "http",
-                "options": {"http_ports": [80, 443, 7001, 8008, 8080, 9200]},
-            },
-            {"name": "mssql", "options": {}},
-            {"name": "smb", "options": {}},
-            {"name": "ssh", "options": {}},
-        ],
-    }
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert "propagation" in flat_monkey_config
-    assert "network_scan" in flat_monkey_config["propagation"]
-    assert flat_monkey_config["propagation"]["network_scan"] == expected_network_scan_config
-
-    assert "tcp_scan_timeout" not in flat_monkey_config
-    assert "tcp_target_ports" not in flat_monkey_config
-    assert "ping_scan_timeout" not in flat_monkey_config
-    assert "finger_classes" not in flat_monkey_config
-
-
-def test_format_config_for_agent__propagation_network_scan_targets():
-    expected_targets = {
-        "blocked_ips": ["192.168.1.1", "192.168.1.100"],
-        "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"],
-        "local_network_scan": True,
-        "subnets": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"],
-    }
-
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert flat_monkey_config["propagation"]["network_scan"]["targets"] == expected_targets
-    assert "blocked_ips" not in flat_monkey_config
-    assert "inaccessible_subnets" not in flat_monkey_config
-    assert "local_network_scan" not in flat_monkey_config
-    assert "subnet_scan_list" not in flat_monkey_config
-
-
-def test_format_config_for_agent__exploiters():
-    expected_exploiters_config = {
-        "options": {
-            "http_ports": [80, 443, 7001, 8008, 8080, 9200],
-        },
-        "brute_force": [
-            {"name": "MSSQLExploiter", "options": {}},
-            {
-                "name": "PowerShellExploiter",
-                "options": {},
-            },
-            {"name": "SSHExploiter", "options": {}},
-            {
-                "name": "SmbExploiter",
-                "options": {"smb_download_timeout": 30},
-            },
-            {
-                "name": "WmiExploiter",
-                "options": {"smb_download_timeout": 30},
-            },
-        ],
-        "vulnerability": [
-            {
-                "name": "HadoopExploiter",
-                "options": {},
-            },
-            {
-                "name": "Log4ShellExploiter",
-                "options": {},
-            },
-            {
-                "name": "ZerologonExploiter",
-                "options": {},
-            },
-        ],
-    }
-    flat_monkey_config = ConfigService.format_flat_config_for_agent()
-
-    assert "propagation" in flat_monkey_config
-    assert "exploitation" in flat_monkey_config["propagation"]
-
-    assert flat_monkey_config["propagation"]["exploitation"] == expected_exploiters_config
-    assert "exploiter_classes" not in flat_monkey_config