Island: don't store credential telemetries

Credential telemetries are not stored on the database to prevent the need to encrypt credentials and query database directly. Instead, credentials are parsed into a document that doesn't contain secrets and is easily queryable

monkey/monkey_island/cc/models/__init__.py

@@ -7,3 +7,4 @@ from .monkey import Monkey
 from .monkey_ttl import MonkeyTtl
 from .pba_results import PbaResults
 from monkey_island.cc.models.report.report import Report
+from .stolen_credentials import StolenCredentials

monkey/monkey_island/cc/models/stolen_credentials.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from mongoengine import Document, ListField, ReferenceField
+
+from monkey_island.cc.models import Monkey
+from monkey_island.cc.services.telemetry.processing.credentials import Credentials
+
+
+class StolenCredentials(Document):
+    """
+    This class has 2 main section:
+        *   The schema section defines the DB fields in the document. This is the data of the
+        object.
+        *   The logic section defines complex questions we can ask about a single document which
+        are asked multiple
+            times, somewhat like an API.
+    """
+
+    # SCHEMA
+    monkey = ReferenceField(Monkey)
+    identities = ListField()
+    secrets = ListField()
+
+    @staticmethod
+    def from_credentials(credentials: Credentials) -> StolenCredentials:
+        stolen_creds = StolenCredentials()
+
+        stolen_creds.secrets = [secret["credential_type"] for secret in credentials.secrets]
+        stolen_creds.identities = credentials.identities
+        stolen_creds.monkey = Monkey.get_single_monkey_by_guid(credentials.monkey_guid).id
+        return stolen_creds

monkey/monkey_island/cc/models/telemetries/telemetry_dal.py

@@ -5,23 +5,9 @@ from typing import List
 from monkey_island.cc.database import mongo
 from monkey_island.cc.models import CommandControlChannel
 from monkey_island.cc.models.telemetries.telemetry import Telemetry
-from monkey_island.cc.server_utils.encryption import (
-    FieldNotFoundError,
-    MimikatzResultsEncryptor,
-    SensitiveField,
-    decrypt_dict,
-    encrypt_dict,
-)
-
-sensitive_fields = [SensitiveField("data.credentials", MimikatzResultsEncryptor)]
 
 
 def save_telemetry(telemetry_dict: dict):
-    try:
-        telemetry_dict = encrypt_dict(sensitive_fields, telemetry_dict)
-    except FieldNotFoundError:
-        pass  # Not all telemetries require encryption
-
     cc_channel = CommandControlChannel(
         src=telemetry_dict["command_control_channel"]["src"],
         dst=telemetry_dict["command_control_channel"]["dst"],
@@ -35,14 +21,5 @@ def save_telemetry(telemetry_dict: dict):
     ).save()
 
 
-# A lot of codebase is using queries for telemetry collection and document field encryption is
-# not yet implemented in mongoengine. To avoid big time investment, queries are used for now.
 def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]:
-    telemetries = mongo.db.telemetry.find(query, output_fields)
-    decrypted_list = []
-    for telemetry in telemetries:
-        try:
-            decrypted_list.append(decrypt_dict(sensitive_fields, telemetry))
-        except FieldNotFoundError:
-            decrypted_list.append(telemetry)
-    return decrypted_list
+    return mongo.db.telemetry.find(query, output_fields)

monkey/monkey_island/cc/resources/telemetry.py

@@ -6,10 +6,9 @@ import dateutil
 import flask_restful
 from flask import request
 
-from common.common_consts.telem_categories import TelemCategoryEnum
 from monkey_island.cc.database import mongo
 from monkey_island.cc.models.monkey import Monkey
-from monkey_island.cc.models.telemetries import get_telemetry_by_query, save_telemetry
+from monkey_island.cc.models.telemetries import get_telemetry_by_query
 from monkey_island.cc.resources.auth.auth import jwt_required
 from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore
 from monkey_island.cc.services.node import NodeService
@@ -61,8 +60,6 @@ class Telemetry(flask_restful.Resource):
 
         process_telemetry(telemetry_json)
 
-        save_telemetry(telemetry_json)
-
         return {}, 201
 
     @staticmethod
@@ -80,10 +77,5 @@ class Telemetry(flask_restful.Resource):
                 monkey_label = telem_monkey_guid
             x["monkey"] = monkey_label
             objects.append(x)
-            if x["telem_category"] == TelemCategoryEnum.SYSTEM_INFO and "credentials" in x["data"]:
-                for user in x["data"]["credentials"]:
-                    if -1 != user.find(","):
-                        new_user = user.replace(",", ".")
-                        x["data"]["credentials"][new_user] = x["data"]["credentials"].pop(user)
 
         return objects

monkey/monkey_island/cc/server_utils/encryption/__init__.py

@@ -23,5 +23,4 @@ from .dict_encryptor import (
     FieldNotFoundError,
 )
 from .field_encryptors.i_field_encryptor import IFieldEncryptor
-from .field_encryptors.mimikatz_results_encryptor import MimikatzResultsEncryptor
 from .field_encryptors.string_list_encryptor import StringListEncryptor

monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py

@@ -1,3 +1,2 @@
 from .i_field_encryptor import IFieldEncryptor
-from .mimikatz_results_encryptor import MimikatzResultsEncryptor
 from .string_list_encryptor import StringListEncryptor

monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py

@@ -1,29 +0,0 @@
-import logging
-
-from ..data_store_encryptor import get_datastore_encryptor
-from . import IFieldEncryptor
-
-logger = logging.getLogger(__name__)
-
-
-class MimikatzResultsEncryptor(IFieldEncryptor):
-
-    secret_types = ["password", "ntlm_hash", "lm_hash"]
-
-    @staticmethod
-    def encrypt(results: dict) -> dict:
-        for _, credentials in results.items():
-            for secret_type in MimikatzResultsEncryptor.secret_types:
-                credentials[secret_type] = get_datastore_encryptor().encrypt(
-                    credentials[secret_type]
-                )
-        return results
-
-    @staticmethod
-    def decrypt(results: dict) -> dict:
-        for _, credentials in results.items():
-            for secret_type in MimikatzResultsEncryptor.secret_types:
-                credentials[secret_type] = get_datastore_encryptor().decrypt(
-                    credentials[secret_type]
-                )
-        return results

monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py

@@ -3,6 +3,7 @@ from itertools import chain
 from typing import Mapping
 
 from common.common_consts.credential_component_type import CredentialComponentType
+from monkey_island.cc.models import StolenCredentials
 
 from .credentials import Credentials
 from .identities.username_processor import process_username
@@ -29,6 +30,12 @@ def parse_credentials(telemetry_dict: Mapping):
     ]
 
     for credential in credentials:
+        _store_in_db(credential)
         for cred_comp in chain(credential.identities, credential.secrets):
             credential_type = CredentialComponentType[cred_comp["credential_type"]]
             CREDENTIAL_COMPONENT_PROCESSORS[credential_type](cred_comp, credential)
+
+
+def _store_in_db(credentials: Credentials):
+    stolen_cred_doc = StolenCredentials.from_credentials(credentials)
+    stolen_cred_doc.save()

monkey/monkey_island/cc/services/telemetry/processing/processing.py

@@ -1,9 +1,11 @@
 import logging
 
 from common.common_consts.telem_categories import TelemCategoryEnum
+from monkey_island.cc.models.telemetries import save_telemetry
 from monkey_island.cc.services.telemetry.processing.aws_info import process_aws_telemetry
-from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import\
-    parse_credentials
+from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
+    parse_credentials,
+)
 from monkey_island.cc.services.telemetry.processing.exploit import process_exploit_telemetry
 from monkey_island.cc.services.telemetry.processing.post_breach import process_post_breach_telemetry
 from monkey_island.cc.services.telemetry.processing.scan import process_scan_telemetry
@@ -25,6 +27,10 @@ TELEMETRY_CATEGORY_TO_PROCESSING_FUNC = {
     TelemCategoryEnum.TUNNEL: process_tunnel_telemetry,
 }
 
+# Don't save credential telemetries in telemetries collection.
+# Credentials are stored in StolenCredentials documents
+UNSAVED_TELEMETRIES = [TelemCategoryEnum.CREDENTIALS]
+
 
 def process_telemetry(telemetry_json):
     try:
@@ -33,6 +39,10 @@ def process_telemetry(telemetry_json):
             TELEMETRY_CATEGORY_TO_PROCESSING_FUNC[telem_category](telemetry_json)
         else:
             logger.info("Got unknown type of telemetry: %s" % telem_category)
+
+        if telem_category not in UNSAVED_TELEMETRIES:
+            save_telemetry(telemetry_json)
+
     except Exception as ex:
         logger.error(
             "Exception caught while processing telemetry. Info: {}".format(ex), exc_info=True

monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py

@@ -3,17 +3,27 @@ from datetime import datetime
 import mongoengine
 import pytest
 
+from monkey_island.cc.models import Monkey
 from monkey_island.cc.services.config import ConfigService
 
+fake_monkey_guid = "272405690278083"
+fake_ip_address = "192.168.56.1"
+
 
 @pytest.fixture
-def fake_mongo(monkeypatch):
+def fake_mongo(monkeypatch, uses_encryptor):
     mongo = mongoengine.connection.get_connection()
     monkeypatch.setattr("monkey_island.cc.services.config.mongo", mongo)
     config = ConfigService.get_default_config()
     ConfigService.update_config(config, should_encrypt=True)
 
 
+@pytest.fixture
+def insert_fake_monkey():
+    monkey = Monkey(guid=fake_monkey_guid, ip_addresses=[fake_ip_address])
+    monkey.save()
+
+
 CREDENTIAL_TELEM_TEMPLATE = {
     "monkey_guid": "272405690278083",
     "telem_category": "credentials",

monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py

@@ -6,12 +6,14 @@ from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials
     CREDENTIAL_TELEM_TEMPLATE,
 )
 
+from common.common_consts.credential_component_type import CredentialComponentType
 from common.config_value_paths import (
     LM_HASH_LIST_PATH,
     NTLM_HASH_LIST_PATH,
     PASSWORD_LIST_PATH,
     USER_LIST_PATH,
 )
+from monkey_island.cc.models import StolenCredentials
 from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
     parse_credentials,
@@ -51,21 +53,21 @@ cred_empty_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
 cred_empty_telem["data"] = [{"identities": [], "secrets": []}]
 
 
-@pytest.mark.usefixtures("uses_database", "fake_mongo")
+@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
 def test_cred_username_parsing():
     parse_credentials(cred_telem_usernames)
     config = ConfigService.get_config(should_decrypt=True)
     assert fake_username in dpath.util.get(config, USER_LIST_PATH)
 
 
-@pytest.mark.usefixtures("uses_database", "fake_mongo")
+@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
 def test_cred_special_username_parsing():
     parse_credentials(cred_telem_special_usernames)
     config = ConfigService.get_config(should_decrypt=True)
     assert fake_special_username in dpath.util.get(config, USER_LIST_PATH)
 
 
-@pytest.mark.usefixtures("uses_database", "fake_mongo")
+@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
 def test_cred_telemetry_parsing():
     parse_credentials(cred_telem)
     config = ConfigService.get_config(should_decrypt=True)
@@ -75,7 +77,20 @@ def test_cred_telemetry_parsing():
     assert fake_password in dpath.util.get(config, PASSWORD_LIST_PATH)
 
 
-@pytest.mark.usefixtures("uses_database", "fake_mongo")
+@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
+def test_cred_storage_in_db():
+    parse_credentials(cred_telem)
+    cred_docs = list(StolenCredentials.objects())
+    assert len(cred_docs) == 1
+
+    stolen_creds = cred_docs[0]
+    assert fake_username == stolen_creds.identities[0]["username"]
+    assert CredentialComponentType.PASSWORD.name in stolen_creds.secrets
+    assert CredentialComponentType.LM_HASH.name in stolen_creds.secrets
+    assert CredentialComponentType.NT_HASH.name in stolen_creds.secrets
+
+
+@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
 def test_empty_cred_telemetry_parsing():
     default_config = deepcopy(ConfigService.get_config(should_decrypt=True))
     default_usernames = dpath.util.get(default_config, USER_LIST_PATH)

monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py

@@ -4,18 +4,15 @@ import dpath.util
 import pytest
 from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials.conftest import (
     CREDENTIAL_TELEM_TEMPLATE,
+    fake_ip_address,
 )
 
 from common.config_value_paths import SSH_KEYS_PATH, USER_LIST_PATH
-from monkey_island.cc.models import Monkey
 from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
     parse_credentials,
 )
 
-fake_monkey_guid = "272405690278083"
-fake_ip_address = "192.168.56.1"
-
 fake_private_key = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFlczI1N\n"
 fake_partial_secret = {"private_key": fake_private_key, "credential_type": "SSH_KEYPAIR"}
 
@@ -35,12 +32,6 @@ ssh_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
 ssh_telem["data"] = [{"identities": [fake_identity], "secrets": [fake_secret_full]}]
 
 
-@pytest.fixture
-def insert_fake_monkey():
-    monkey = Monkey(guid=fake_monkey_guid, ip_addresses=[fake_ip_address])
-    monkey.save()
-
-
 @pytest.mark.usefixtures("uses_encryptor", "uses_database", "fake_mongo", "insert_fake_monkey")
 def test_ssh_credential_parsing():
     parse_credentials(ssh_telem)