Island: refactor report generation to take credentials from model

Reporting used to fetch credentials from telemetries, but they are no longer stored. Instead, credentials are being fetched from stolen_credentials collection

monkey/monkey_island/cc/services/attack/technique_reports/T1003.py

@@ -1,7 +1,7 @@
 from common.utils.attack_utils import ScanStatus
-from monkey_island.cc.database import mongo
+from monkey_island.cc.models import StolenCredentials
 from monkey_island.cc.services.attack.technique_reports import AttackTechnique
-from monkey_island.cc.services.reporting.report import ReportService
+from monkey_island.cc.services.reporting.stolen_credentials import get_stolen_creds
 
 
 class T1003(AttackTechnique):
@@ -14,29 +14,10 @@ class T1003(AttackTechnique):
     scanned_msg = ""
     used_msg = "Monkey successfully obtained some credentials from systems on the network."
 
-    query = {
-        "$or": [
-            {
-                "telem_category": "system_info",
-                "$and": [
-                    {"data.credentials": {"$exists": True}},
-                    {"data.credentials": {"$gt": {}}},
-                ],
-            },  # $gt: {} checks if field is not an empty object
-            {
-                "telem_category": "exploit",
-                "$and": [
-                    {"data.info.credentials": {"$exists": True}},
-                    {"data.info.credentials": {"$gt": {}}},
-                ],
-            },
-        ]
-    }
-
     @staticmethod
     def get_report_data():
         def get_technique_status_and_data():
-            if mongo.db.telemetry.count_documents(T1003.query):
+            if list(StolenCredentials.objects()):
                 status = ScanStatus.USED.value
             else:
                 status = ScanStatus.UNSCANNED.value
@@ -47,6 +28,5 @@ class T1003(AttackTechnique):
 
         data.update(T1003.get_message_and_status(status))
         data.update(T1003.get_mitigation_by_status(status))
-        data["stolen_creds"] = ReportService.get_stolen_creds()
+        data["stolen_creds"] = get_stolen_creds()
-        data["stolen_creds"].extend(ReportService.get_ssh_keys())
         return data

monkey/monkey_island/cc/services/reporting/report.py

@@ -16,7 +16,6 @@ from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst
 from monkey_island.cc.database import mongo
 from monkey_island.cc.models import Monkey
 from monkey_island.cc.models.report import get_report, save_report
-from monkey_island.cc.models.telemetries import get_telemetry_by_query
 from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.configuration.utils import (
     get_config_network_segments_as_subnet_groups,
@@ -26,22 +25,21 @@ from monkey_island.cc.services.reporting.exploitations.manual_exploitation impor
 from monkey_island.cc.services.reporting.exploitations.monkey_exploitation import (
     get_monkey_exploited,
 )
-from monkey_island.cc.services.reporting.issue_processing.exploit_processing.exploiter_descriptor_enum import (  # noqa: E501
-    ExploiterDescriptorEnum,
-)
-from monkey_island.cc.services.reporting.issue_processing.exploit_processing.processors.cred_exploit import (  # noqa: E501
-    CredentialType,
-)
-from monkey_island.cc.services.reporting.issue_processing.exploit_processing.processors.exploit import (  # noqa: E501
-    ExploiterReportInfo,
-)
 from monkey_island.cc.services.reporting.pth_report import PTHReportService
 from monkey_island.cc.services.reporting.report_exporter_manager import ReportExporterManager
 from monkey_island.cc.services.reporting.report_generation_synchronisation import (
     safe_generate_regular_report,
 )
+from monkey_island.cc.services.reporting.stolen_credentials import (
+    extract_ssh_keys,
+    get_stolen_creds,
+)
 from monkey_island.cc.services.utils.network_utils import get_subnets, local_ip_addresses
 
+from .issue_processing.exploit_processing.exploiter_descriptor_enum import ExploiterDescriptorEnum
+from .issue_processing.exploit_processing.processors.cred_exploit import CredentialType
+from .issue_processing.exploit_processing.processors.exploit import ExploiterReportInfo
+
 logger = logging.getLogger(__name__)
 
 
@@ -133,104 +131,6 @@ class ReportService:
         nodes = nodes_without_monkeys + nodes_with_monkeys
         return nodes
 
-    @staticmethod
-    def get_stolen_creds():
-        creds = []
-
-        stolen_system_info_creds = ReportService._get_credentials_from_system_info_telems()
-        creds.extend(stolen_system_info_creds)
-
-        stolen_exploit_creds = ReportService._get_credentials_from_exploit_telems()
-        creds.extend(stolen_exploit_creds)
-
-        logger.info("Stolen creds generated for reporting")
-        return creds
-
-    @staticmethod
-    def _get_credentials_from_system_info_telems():
-        formatted_creds = []
-        for telem in get_telemetry_by_query(
-            {"telem_category": "system_info", "data.credentials": {"$exists": True}},
-            {"data.credentials": 1, "monkey_guid": 1},
-        ):
-            creds = telem["data"]["credentials"]
-            origin = NodeService.get_monkey_by_guid(telem["monkey_guid"])["hostname"]
-            formatted_creds.extend(ReportService._format_creds_for_reporting(telem, creds, origin))
-        return formatted_creds
-
-    @staticmethod
-    def _get_credentials_from_exploit_telems():
-        formatted_creds = []
-        for telem in mongo.db.telemetry.find(
-            {"telem_category": "exploit", "data.info.credentials": {"$exists": True}},
-            {"data.info.credentials": 1, "data.machine": 1, "monkey_guid": 1},
-        ):
-            creds = telem["data"]["info"]["credentials"]
-            domain_name = telem["data"]["machine"]["domain_name"]
-            ip = telem["data"]["machine"]["ip_addr"]
-            origin = domain_name if domain_name else ip
-            formatted_creds.extend(ReportService._format_creds_for_reporting(telem, creds, origin))
-        return formatted_creds
-
-    @staticmethod
-    def _format_creds_for_reporting(telem, monkey_creds, origin):
-        creds = []
-        CRED_TYPE_DICT = {
-            "password": "Clear Password",
-            "lm_hash": "LM hash",
-            "ntlm_hash": "NTLM hash",
-        }
-        if len(monkey_creds) == 0:
-            return []
-
-        for user in monkey_creds:
-            for cred_type in CRED_TYPE_DICT:
-                if cred_type not in monkey_creds[user] or not monkey_creds[user][cred_type]:
-                    continue
-                username = (
-                    monkey_creds[user]["username"] if "username" in monkey_creds[user] else user
-                )
-                cred_row = {
-                    "username": username,
-                    "type": CRED_TYPE_DICT[cred_type],
-                    "origin": origin,
-                }
-                if cred_row not in creds:
-                    creds.append(cred_row)
-        return creds
-
-    @staticmethod
-    def get_ssh_keys():
-        """
-        Return private ssh keys found as credentials
-        :return: List of credentials
-        """
-        creds = []
-        for telem in mongo.db.telemetry.find(
-            {"telem_category": "system_info", "data.ssh_info": {"$exists": True}},
-            {"data.ssh_info": 1, "monkey_guid": 1},
-        ):
-            origin = NodeService.get_monkey_by_guid(telem["monkey_guid"])["hostname"]
-            if telem["data"]["ssh_info"]:
-                # Pick out all ssh keys not yet included in creds
-                ssh_keys = [
-                    {
-                        "username": key_pair["name"],
-                        "type": "Clear SSH private key",
-                        "origin": origin,
-                    }
-                    for key_pair in telem["data"]["ssh_info"]
-                    if key_pair["private_key"]
-                    and {
-                        "username": key_pair["name"],
-                        "type": "Clear SSH private key",
-                        "origin": origin,
-                    }
-                    not in creds
-                ]
-                creds.extend(ssh_keys)
-        return creds
-
     @staticmethod
     def process_exploit(exploit) -> ExploiterReportInfo:
         exploiter_type = exploit["data"]["exploiter"]
@@ -564,6 +464,7 @@ class ReportService:
         issue_set = ReportService.get_issue_set(issues, config_users, config_passwords)
         cross_segment_issues = ReportService.get_cross_segment_issues()
         monkey_latest_modify_time = Monkey.get_latest_modifytime()
+        stolen_creds = get_stolen_creds()
 
         scanned_nodes = ReportService.get_scanned()
         exploited_cnt = len(get_monkey_exploited())
@@ -585,8 +486,8 @@ class ReportService:
             "glance": {
                 "scanned": scanned_nodes,
                 "exploited_cnt": exploited_cnt,
-                "stolen_creds": ReportService.get_stolen_creds(),
+                "stolen_creds": stolen_creds,
-                "ssh_keys": ReportService.get_ssh_keys(),
+                "ssh_keys": extract_ssh_keys(stolen_creds),
                 "strong_users": PTHReportService.get_strong_users_on_crit_details(),
             },
             "recommendations": {"issues": issues, "domain_issues": domain_issues},

monkey/monkey_island/cc/services/reporting/stolen_credentials.py

@@ -0,0 +1,59 @@
+import logging
+from typing import Mapping, Sequence
+
+from common.common_consts.credential_component_type import CredentialComponentType
+from monkey_island.cc.models import StolenCredentials
+
+logger = logging.getLogger(__name__)
+
+
+def get_stolen_creds() -> Sequence[Mapping]:
+    stolen_creds = _fetch_from_db()
+    stolen_creds = _format_creds_for_reporting(stolen_creds)
+
+    logger.info("Stolen creds generated for reporting")
+    return stolen_creds
+
+
+def extract_ssh_keys(credentials: Sequence[Mapping]) -> Sequence[Mapping]:
+    ssh_keys = []
+    for credential in credentials:
+        if credential["_type"] == CredentialComponentType.SSH_KEYPAIR.name:
+            ssh_keys.append(credential)
+    return ssh_keys
+
+
+def _fetch_from_db() -> Sequence[StolenCredentials]:
+    return list(StolenCredentials.objects())
+
+
+def _format_creds_for_reporting(credentials: Sequence[StolenCredentials]):
+    formatted_creds = []
+    cred_type_dict = {
+        CredentialComponentType.PASSWORD.name: "Clear Password",
+        CredentialComponentType.LM_HASH.name: "LM hash",
+        CredentialComponentType.NT_HASH.name: "NTLM hash",
+        CredentialComponentType.SSH_KEYPAIR.name: "Clear SSH private key",
+    }
+
+    for cred in credentials:
+        for secret_type in cred.secrets:
+            if secret_type not in cred_type_dict:
+                continue
+            username = _get_username(cred)
+            cred_row = {
+                "username": username,
+                "_type": secret_type,
+                "type": cred_type_dict[secret_type],
+                "origin": cred.monkey.hostname,
+            }
+            if cred_row not in formatted_creds:
+                formatted_creds.append(cred_row)
+    return formatted_creds
+
+
+def _get_username(credentials: StolenCredentials) -> str:
+    if credentials.identities:
+        return credentials.identities[0]["username"]
+    else:
+        return ""

monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js

@@ -556,7 +556,7 @@ class ReportPageComponent extends AuthComponent {
         </div>
 
         <div style={{marginBottom: '20px'}}>
-          <StolenPasswords data={this.state.report.glance.stolen_creds.concat(this.state.report.glance.ssh_keys)}/>
+          <StolenPasswords data={this.state.report.glance.stolen_creds}/>
         </div>
         <div>
           <StrongUsers data={this.state.report.glance.strong_users}/>

monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_stolen_credentials.py

@@ -0,0 +1,97 @@
+import pytest
+
+from common.common_consts.credential_component_type import CredentialComponentType
+from monkey_island.cc.models import Monkey, StolenCredentials
+from monkey_island.cc.services.reporting.stolen_credentials import (
+    extract_ssh_keys,
+    get_stolen_creds,
+)
+
+monkey_hostname = "fake_hostname"
+fake_monkey_guid = "abc"
+
+fake_username = "m0nk3y_user"
+fake_nt_hash = "c1c58f96cdf212b50837bc11a00be47c"
+fake_lm_hash = "299BD128C1101FD6"
+fake_password = "trytostealthis"
+fake_ssh_key = "RSA_fake_key"
+fake_credentials = {
+    "identities": [{"username": fake_username, "credential_type": "USERNAME"}],
+    "secrets": [
+        CredentialComponentType.NT_HASH.name,
+        CredentialComponentType.LM_HASH.name,
+        CredentialComponentType.PASSWORD.name,
+        CredentialComponentType.SSH_KEYPAIR.name,
+    ],
+}
+
+
+@pytest.fixture
+def fake_monkey():
+    monkey = Monkey()
+    monkey.guid = fake_monkey_guid
+    monkey.hostname = monkey_hostname
+    monkey.save()
+    return monkey.id
+
+
+@pytest.mark.usefixture("uses_database")
+def test_get_credentials(fake_monkey):
+    StolenCredentials(
+        identities=fake_credentials["identities"],
+        secrets=fake_credentials["secrets"],
+        monkey=fake_monkey,
+    ).save()
+
+    credentials = get_stolen_creds()
+
+    result1 = {
+        "origin": monkey_hostname,
+        "_type": CredentialComponentType.NT_HASH.name,
+        "type": "NTLM hash",
+        "username": fake_username,
+    }
+    result2 = {
+        "origin": monkey_hostname,
+        "_type": CredentialComponentType.LM_HASH.name,
+        "type": "LM hash",
+        "username": fake_username,
+    }
+    result3 = {
+        "origin": monkey_hostname,
+        "_type": CredentialComponentType.PASSWORD.name,
+        "type": "Clear Password",
+        "username": fake_username,
+    }
+    result4 = {
+        "origin": monkey_hostname,
+        "_type": CredentialComponentType.SSH_KEYPAIR.name,
+        "type": "Clear SSH private key",
+        "username": fake_username,
+    }
+    assert result1 in credentials
+    assert result2 in credentials
+    assert result3 in credentials
+    assert result4 in credentials
+
+
+@pytest.mark.usefixtures("uses_database")
+def test_extract_ssh_keys(fake_monkey):
+    StolenCredentials(
+        identities=fake_credentials["identities"],
+        secrets=fake_credentials["secrets"],
+        monkey=fake_monkey,
+    ).save()
+
+    credentials = get_stolen_creds()
+    keys = extract_ssh_keys(credentials)
+
+    assert len(keys) == 1
+
+    result = {
+        "origin": monkey_hostname,
+        "_type": CredentialComponentType.SSH_KEYPAIR.name,
+        "type": "Clear SSH private key",
+        "username": fake_username,
+    }
+    assert result in keys