diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 212a20396..c293ae2e7 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -7,3 +7,4 @@ from .monkey import Monkey from .monkey_ttl import MonkeyTtl from .pba_results import PbaResults from monkey_island.cc.models.report.report import Report +from .stolen_credentials import StolenCredentials diff --git a/monkey/monkey_island/cc/models/stolen_credentials.py b/monkey/monkey_island/cc/models/stolen_credentials.py new file mode 100644 index 000000000..fea6068bd --- /dev/null +++ b/monkey/monkey_island/cc/models/stolen_credentials.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from mongoengine import Document, ListField, ReferenceField + +from monkey_island.cc.models import Monkey +from monkey_island.cc.services.telemetry.processing.credentials import Credentials + + +class StolenCredentials(Document): + """ + This class has 2 main section: + * The schema section defines the DB fields in the document. This is the data of the + object. + * The logic section defines complex questions we can ask about a single document which + are asked multiple + times, somewhat like an API. + """ + + # SCHEMA + monkey = ReferenceField(Monkey) + identities = ListField() + secrets = ListField() + + @staticmethod + def from_credentials(credentials: Credentials) -> StolenCredentials: + stolen_creds = StolenCredentials() + + stolen_creds.secrets = [secret["credential_type"] for secret in credentials.secrets] + stolen_creds.identities = credentials.identities + stolen_creds.monkey = Monkey.get_single_monkey_by_guid(credentials.monkey_guid).id + return stolen_creds diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py index d6425238f..a46242419 100644 --- a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py +++ b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py @@ -5,23 +5,9 @@ from typing import List from monkey_island.cc.database import mongo from monkey_island.cc.models import CommandControlChannel from monkey_island.cc.models.telemetries.telemetry import Telemetry -from monkey_island.cc.server_utils.encryption import ( - FieldNotFoundError, - MimikatzResultsEncryptor, - SensitiveField, - decrypt_dict, - encrypt_dict, -) - -sensitive_fields = [SensitiveField("data.credentials", MimikatzResultsEncryptor)] def save_telemetry(telemetry_dict: dict): - try: - telemetry_dict = encrypt_dict(sensitive_fields, telemetry_dict) - except FieldNotFoundError: - pass # Not all telemetries require encryption - cc_channel = CommandControlChannel( src=telemetry_dict["command_control_channel"]["src"], dst=telemetry_dict["command_control_channel"]["dst"], @@ -35,14 +21,5 @@ def save_telemetry(telemetry_dict: dict): ).save() -# A lot of codebase is using queries for telemetry collection and document field encryption is -# not yet implemented in mongoengine. To avoid big time investment, queries are used for now. def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]: - telemetries = mongo.db.telemetry.find(query, output_fields) - decrypted_list = [] - for telemetry in telemetries: - try: - decrypted_list.append(decrypt_dict(sensitive_fields, telemetry)) - except FieldNotFoundError: - decrypted_list.append(telemetry) - return decrypted_list + return mongo.db.telemetry.find(query, output_fields) diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index 1158e82f0..3358788f3 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -6,10 +6,9 @@ import dateutil import flask_restful from flask import request -from common.common_consts.telem_categories import TelemCategoryEnum from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey import Monkey -from monkey_island.cc.models.telemetries import get_telemetry_by_query, save_telemetry +from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.resources.auth.auth import jwt_required from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService @@ -61,8 +60,6 @@ class Telemetry(flask_restful.Resource): process_telemetry(telemetry_json) - save_telemetry(telemetry_json) - return {}, 201 @staticmethod @@ -80,10 +77,5 @@ class Telemetry(flask_restful.Resource): monkey_label = telem_monkey_guid x["monkey"] = monkey_label objects.append(x) - if x["telem_category"] == TelemCategoryEnum.SYSTEM_INFO and "credentials" in x["data"]: - for user in x["data"]["credentials"]: - if -1 != user.find(","): - new_user = user.replace(",", ".") - x["data"]["credentials"][new_user] = x["data"]["credentials"].pop(user) return objects diff --git a/monkey/monkey_island/cc/server_utils/encryption/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/__init__.py index 16ac78cbe..4cfe67fe2 100644 --- a/monkey/monkey_island/cc/server_utils/encryption/__init__.py +++ b/monkey/monkey_island/cc/server_utils/encryption/__init__.py @@ -23,5 +23,4 @@ from .dict_encryptor import ( FieldNotFoundError, ) from .field_encryptors.i_field_encryptor import IFieldEncryptor -from .field_encryptors.mimikatz_results_encryptor import MimikatzResultsEncryptor from .field_encryptors.string_list_encryptor import StringListEncryptor diff --git a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py index 7c938d25b..1ceedf768 100644 --- a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py +++ b/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py @@ -1,3 +1,2 @@ from .i_field_encryptor import IFieldEncryptor -from .mimikatz_results_encryptor import MimikatzResultsEncryptor from .string_list_encryptor import StringListEncryptor diff --git a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py b/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py deleted file mode 100644 index 31f597e60..000000000 --- a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging - -from ..data_store_encryptor import get_datastore_encryptor -from . import IFieldEncryptor - -logger = logging.getLogger(__name__) - - -class MimikatzResultsEncryptor(IFieldEncryptor): - - secret_types = ["password", "ntlm_hash", "lm_hash"] - - @staticmethod - def encrypt(results: dict) -> dict: - for _, credentials in results.items(): - for secret_type in MimikatzResultsEncryptor.secret_types: - credentials[secret_type] = get_datastore_encryptor().encrypt( - credentials[secret_type] - ) - return results - - @staticmethod - def decrypt(results: dict) -> dict: - for _, credentials in results.items(): - for secret_type in MimikatzResultsEncryptor.secret_types: - credentials[secret_type] = get_datastore_encryptor().decrypt( - credentials[secret_type] - ) - return results diff --git a/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py b/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py index 9df47f91d..5c6d15631 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py @@ -3,6 +3,7 @@ from itertools import chain from typing import Mapping from common.common_consts.credential_component_type import CredentialComponentType +from monkey_island.cc.models import StolenCredentials from .credentials import Credentials from .identities.username_processor import process_username @@ -29,6 +30,12 @@ def parse_credentials(telemetry_dict: Mapping): ] for credential in credentials: + _store_in_db(credential) for cred_comp in chain(credential.identities, credential.secrets): credential_type = CredentialComponentType[cred_comp["credential_type"]] CREDENTIAL_COMPONENT_PROCESSORS[credential_type](cred_comp, credential) + + +def _store_in_db(credentials: Credentials): + stolen_cred_doc = StolenCredentials.from_credentials(credentials) + stolen_cred_doc.save() diff --git a/monkey/monkey_island/cc/services/telemetry/processing/processing.py b/monkey/monkey_island/cc/services/telemetry/processing/processing.py index abea5dc38..709097ee0 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/processing.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/processing.py @@ -1,9 +1,11 @@ import logging from common.common_consts.telem_categories import TelemCategoryEnum +from monkey_island.cc.models.telemetries import save_telemetry from monkey_island.cc.services.telemetry.processing.aws_info import process_aws_telemetry -from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import\ - parse_credentials +from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import ( + parse_credentials, +) from monkey_island.cc.services.telemetry.processing.exploit import process_exploit_telemetry from monkey_island.cc.services.telemetry.processing.post_breach import process_post_breach_telemetry from monkey_island.cc.services.telemetry.processing.scan import process_scan_telemetry @@ -25,6 +27,10 @@ TELEMETRY_CATEGORY_TO_PROCESSING_FUNC = { TelemCategoryEnum.TUNNEL: process_tunnel_telemetry, } +# Don't save credential telemetries in telemetries collection. +# Credentials are stored in StolenCredentials documents +UNSAVED_TELEMETRIES = [TelemCategoryEnum.CREDENTIALS] + def process_telemetry(telemetry_json): try: @@ -33,6 +39,10 @@ def process_telemetry(telemetry_json): TELEMETRY_CATEGORY_TO_PROCESSING_FUNC[telem_category](telemetry_json) else: logger.info("Got unknown type of telemetry: %s" % telem_category) + + if telem_category not in UNSAVED_TELEMETRIES: + save_telemetry(telemetry_json) + except Exception as ex: logger.error( "Exception caught while processing telemetry. Info: {}".format(ex), exc_info=True diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py index d2891678e..0088995f3 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py @@ -3,17 +3,27 @@ from datetime import datetime import mongoengine import pytest +from monkey_island.cc.models import Monkey from monkey_island.cc.services.config import ConfigService +fake_monkey_guid = "272405690278083" +fake_ip_address = "192.168.56.1" + @pytest.fixture -def fake_mongo(monkeypatch): +def fake_mongo(monkeypatch, uses_encryptor): mongo = mongoengine.connection.get_connection() monkeypatch.setattr("monkey_island.cc.services.config.mongo", mongo) config = ConfigService.get_default_config() ConfigService.update_config(config, should_encrypt=True) +@pytest.fixture +def insert_fake_monkey(): + monkey = Monkey(guid=fake_monkey_guid, ip_addresses=[fake_ip_address]) + monkey.save() + + CREDENTIAL_TELEM_TEMPLATE = { "monkey_guid": "272405690278083", "telem_category": "credentials", diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py index 2ad17431d..5cef3e387 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py @@ -6,12 +6,14 @@ from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials CREDENTIAL_TELEM_TEMPLATE, ) +from common.common_consts.credential_component_type import CredentialComponentType from common.config_value_paths import ( LM_HASH_LIST_PATH, NTLM_HASH_LIST_PATH, PASSWORD_LIST_PATH, USER_LIST_PATH, ) +from monkey_island.cc.models import StolenCredentials from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import ( parse_credentials, @@ -51,21 +53,21 @@ cred_empty_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE) cred_empty_telem["data"] = [{"identities": [], "secrets": []}] -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_username_parsing(): parse_credentials(cred_telem_usernames) config = ConfigService.get_config(should_decrypt=True) assert fake_username in dpath.util.get(config, USER_LIST_PATH) -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_special_username_parsing(): parse_credentials(cred_telem_special_usernames) config = ConfigService.get_config(should_decrypt=True) assert fake_special_username in dpath.util.get(config, USER_LIST_PATH) -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_telemetry_parsing(): parse_credentials(cred_telem) config = ConfigService.get_config(should_decrypt=True) @@ -75,7 +77,20 @@ def test_cred_telemetry_parsing(): assert fake_password in dpath.util.get(config, PASSWORD_LIST_PATH) -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") +def test_cred_storage_in_db(): + parse_credentials(cred_telem) + cred_docs = list(StolenCredentials.objects()) + assert len(cred_docs) == 1 + + stolen_creds = cred_docs[0] + assert fake_username == stolen_creds.identities[0]["username"] + assert CredentialComponentType.PASSWORD.name in stolen_creds.secrets + assert CredentialComponentType.LM_HASH.name in stolen_creds.secrets + assert CredentialComponentType.NT_HASH.name in stolen_creds.secrets + + +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_empty_cred_telemetry_parsing(): default_config = deepcopy(ConfigService.get_config(should_decrypt=True)) default_usernames = dpath.util.get(default_config, USER_LIST_PATH) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py index 2d012c4ee..52abf5705 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py @@ -4,18 +4,15 @@ import dpath.util import pytest from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials.conftest import ( CREDENTIAL_TELEM_TEMPLATE, + fake_ip_address, ) from common.config_value_paths import SSH_KEYS_PATH, USER_LIST_PATH -from monkey_island.cc.models import Monkey from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import ( parse_credentials, ) -fake_monkey_guid = "272405690278083" -fake_ip_address = "192.168.56.1" - fake_private_key = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFlczI1N\n" fake_partial_secret = {"private_key": fake_private_key, "credential_type": "SSH_KEYPAIR"} @@ -35,12 +32,6 @@ ssh_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE) ssh_telem["data"] = [{"identities": [fake_identity], "secrets": [fake_secret_full]}] -@pytest.fixture -def insert_fake_monkey(): - monkey = Monkey(guid=fake_monkey_guid, ip_addresses=[fake_ip_address]) - monkey.save() - - @pytest.mark.usefixtures("uses_encryptor", "uses_database", "fake_mongo", "insert_fake_monkey") def test_ssh_credential_parsing(): parse_credentials(ssh_telem)