From afc98667c448e07646b31b6ad9be667daca5feca Mon Sep 17 00:00:00 2001 From: vakarisz Date: Wed, 23 Feb 2022 14:50:16 +0200 Subject: [PATCH 1/8] Island: remove unused "creds" properties from monkey model --- monkey/monkey_island/cc/models/__init__.py | 1 - monkey/monkey_island/cc/models/creds.py | 10 ---------- monkey/monkey_island/cc/models/monkey.py | 1 - vulture_allowlist.py | 2 -- 4 files changed, 14 deletions(-) delete mode 100644 monkey/monkey_island/cc/models/creds.py diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index cab95ae18..212a20396 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -3,7 +3,6 @@ from .command_control_channel import CommandControlChannel # Order of importing matters here, for registering the embedded and referenced documents before # using them. from .config import Config -from .creds import Creds from .monkey import Monkey from .monkey_ttl import MonkeyTtl from .pba_results import PbaResults diff --git a/monkey/monkey_island/cc/models/creds.py b/monkey/monkey_island/cc/models/creds.py deleted file mode 100644 index d0861846d..000000000 --- a/monkey/monkey_island/cc/models/creds.py +++ /dev/null @@ -1,10 +0,0 @@ -from mongoengine import EmbeddedDocument - - -class Creds(EmbeddedDocument): - """ - TODO get an example of this data, and make it strict - """ - - meta = {"strict": False} - pass diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index af17e45a2..c7fe734b6 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -38,7 +38,6 @@ class Monkey(Document): # SCHEMA guid = StringField(required=True) config = EmbeddedDocumentField("Config") - creds = ListField(EmbeddedDocumentField("Creds")) dead = BooleanField() description = StringField() hostname = StringField() diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 67399ff55..8e5a99516 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -74,10 +74,8 @@ meta # unused variable (monkey/monkey_island/cc/models/zero_trust/finding.py:37 meta # unused variable (monkey/monkey_island/cc/models/monkey_ttl.py:34) expire_at # unused variable (monkey/monkey_island/cc/models/monkey_ttl.py:36) meta # unused variable (monkey/monkey_island/cc/models/config.py:11) -meta # unused variable (monkey/monkey_island/cc/models/creds.py:9) meta # unused variable (monkey/monkey_island/cc/models/edge.py:5) Config # unused class (monkey/monkey_island/cc/models/config.py:4) -Creds # unused class (monkey/monkey_island/cc/models/creds.py:4) _.do_CONNECT # unused method (monkey/infection_monkey/transport/http.py:151) _.do_POST # unused method (monkey/infection_monkey/transport/http.py:122) _.do_HEAD # unused method (monkey/infection_monkey/transport/http.py:61) From 0ecfbff1e4c9730209ca099e60fe2ae0a4353572 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Thu, 24 Feb 2022 17:41:01 +0200 Subject: [PATCH 2/8] Island: don't store credential telemetries Credential telemetries are not stored on the database to prevent the need to encrypt credentials and query database directly. Instead, credentials are parsed into a document that doesn't contain secrets and is easily queryable --- monkey/monkey_island/cc/models/__init__.py | 1 + .../cc/models/stolen_credentials.py | 31 +++++++++++++++++++ .../cc/models/telemetries/telemetry_dal.py | 25 +-------------- .../monkey_island/cc/resources/telemetry.py | 10 +----- .../cc/server_utils/encryption/__init__.py | 1 - .../encryption/field_encryptors/__init__.py | 1 - .../mimikatz_results_encryptor.py | 29 ----------------- .../credentials/credentials_parser.py | 7 +++++ .../telemetry/processing/processing.py | 14 +++++++-- .../processing/credentials/conftest.py | 12 ++++++- .../credentials/test_credential_processing.py | 23 +++++++++++--- .../credentials/test_ssh_key_processing.py | 11 +------ 12 files changed, 84 insertions(+), 81 deletions(-) create mode 100644 monkey/monkey_island/cc/models/stolen_credentials.py delete mode 100644 monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 212a20396..c293ae2e7 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -7,3 +7,4 @@ from .monkey import Monkey from .monkey_ttl import MonkeyTtl from .pba_results import PbaResults from monkey_island.cc.models.report.report import Report +from .stolen_credentials import StolenCredentials diff --git a/monkey/monkey_island/cc/models/stolen_credentials.py b/monkey/monkey_island/cc/models/stolen_credentials.py new file mode 100644 index 000000000..fea6068bd --- /dev/null +++ b/monkey/monkey_island/cc/models/stolen_credentials.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from mongoengine import Document, ListField, ReferenceField + +from monkey_island.cc.models import Monkey +from monkey_island.cc.services.telemetry.processing.credentials import Credentials + + +class StolenCredentials(Document): + """ + This class has 2 main section: + * The schema section defines the DB fields in the document. This is the data of the + object. + * The logic section defines complex questions we can ask about a single document which + are asked multiple + times, somewhat like an API. + """ + + # SCHEMA + monkey = ReferenceField(Monkey) + identities = ListField() + secrets = ListField() + + @staticmethod + def from_credentials(credentials: Credentials) -> StolenCredentials: + stolen_creds = StolenCredentials() + + stolen_creds.secrets = [secret["credential_type"] for secret in credentials.secrets] + stolen_creds.identities = credentials.identities + stolen_creds.monkey = Monkey.get_single_monkey_by_guid(credentials.monkey_guid).id + return stolen_creds diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py index d6425238f..a46242419 100644 --- a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py +++ b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py @@ -5,23 +5,9 @@ from typing import List from monkey_island.cc.database import mongo from monkey_island.cc.models import CommandControlChannel from monkey_island.cc.models.telemetries.telemetry import Telemetry -from monkey_island.cc.server_utils.encryption import ( - FieldNotFoundError, - MimikatzResultsEncryptor, - SensitiveField, - decrypt_dict, - encrypt_dict, -) - -sensitive_fields = [SensitiveField("data.credentials", MimikatzResultsEncryptor)] def save_telemetry(telemetry_dict: dict): - try: - telemetry_dict = encrypt_dict(sensitive_fields, telemetry_dict) - except FieldNotFoundError: - pass # Not all telemetries require encryption - cc_channel = CommandControlChannel( src=telemetry_dict["command_control_channel"]["src"], dst=telemetry_dict["command_control_channel"]["dst"], @@ -35,14 +21,5 @@ def save_telemetry(telemetry_dict: dict): ).save() -# A lot of codebase is using queries for telemetry collection and document field encryption is -# not yet implemented in mongoengine. To avoid big time investment, queries are used for now. def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]: - telemetries = mongo.db.telemetry.find(query, output_fields) - decrypted_list = [] - for telemetry in telemetries: - try: - decrypted_list.append(decrypt_dict(sensitive_fields, telemetry)) - except FieldNotFoundError: - decrypted_list.append(telemetry) - return decrypted_list + return mongo.db.telemetry.find(query, output_fields) diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index 1158e82f0..3358788f3 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -6,10 +6,9 @@ import dateutil import flask_restful from flask import request -from common.common_consts.telem_categories import TelemCategoryEnum from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey import Monkey -from monkey_island.cc.models.telemetries import get_telemetry_by_query, save_telemetry +from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.resources.auth.auth import jwt_required from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService @@ -61,8 +60,6 @@ class Telemetry(flask_restful.Resource): process_telemetry(telemetry_json) - save_telemetry(telemetry_json) - return {}, 201 @staticmethod @@ -80,10 +77,5 @@ class Telemetry(flask_restful.Resource): monkey_label = telem_monkey_guid x["monkey"] = monkey_label objects.append(x) - if x["telem_category"] == TelemCategoryEnum.SYSTEM_INFO and "credentials" in x["data"]: - for user in x["data"]["credentials"]: - if -1 != user.find(","): - new_user = user.replace(",", ".") - x["data"]["credentials"][new_user] = x["data"]["credentials"].pop(user) return objects diff --git a/monkey/monkey_island/cc/server_utils/encryption/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/__init__.py index 16ac78cbe..4cfe67fe2 100644 --- a/monkey/monkey_island/cc/server_utils/encryption/__init__.py +++ b/monkey/monkey_island/cc/server_utils/encryption/__init__.py @@ -23,5 +23,4 @@ from .dict_encryptor import ( FieldNotFoundError, ) from .field_encryptors.i_field_encryptor import IFieldEncryptor -from .field_encryptors.mimikatz_results_encryptor import MimikatzResultsEncryptor from .field_encryptors.string_list_encryptor import StringListEncryptor diff --git a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py index 7c938d25b..1ceedf768 100644 --- a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py +++ b/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/__init__.py @@ -1,3 +1,2 @@ from .i_field_encryptor import IFieldEncryptor -from .mimikatz_results_encryptor import MimikatzResultsEncryptor from .string_list_encryptor import StringListEncryptor diff --git a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py b/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py deleted file mode 100644 index 31f597e60..000000000 --- a/monkey/monkey_island/cc/server_utils/encryption/field_encryptors/mimikatz_results_encryptor.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging - -from ..data_store_encryptor import get_datastore_encryptor -from . import IFieldEncryptor - -logger = logging.getLogger(__name__) - - -class MimikatzResultsEncryptor(IFieldEncryptor): - - secret_types = ["password", "ntlm_hash", "lm_hash"] - - @staticmethod - def encrypt(results: dict) -> dict: - for _, credentials in results.items(): - for secret_type in MimikatzResultsEncryptor.secret_types: - credentials[secret_type] = get_datastore_encryptor().encrypt( - credentials[secret_type] - ) - return results - - @staticmethod - def decrypt(results: dict) -> dict: - for _, credentials in results.items(): - for secret_type in MimikatzResultsEncryptor.secret_types: - credentials[secret_type] = get_datastore_encryptor().decrypt( - credentials[secret_type] - ) - return results diff --git a/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py b/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py index 9df47f91d..5c6d15631 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/credentials/credentials_parser.py @@ -3,6 +3,7 @@ from itertools import chain from typing import Mapping from common.common_consts.credential_component_type import CredentialComponentType +from monkey_island.cc.models import StolenCredentials from .credentials import Credentials from .identities.username_processor import process_username @@ -29,6 +30,12 @@ def parse_credentials(telemetry_dict: Mapping): ] for credential in credentials: + _store_in_db(credential) for cred_comp in chain(credential.identities, credential.secrets): credential_type = CredentialComponentType[cred_comp["credential_type"]] CREDENTIAL_COMPONENT_PROCESSORS[credential_type](cred_comp, credential) + + +def _store_in_db(credentials: Credentials): + stolen_cred_doc = StolenCredentials.from_credentials(credentials) + stolen_cred_doc.save() diff --git a/monkey/monkey_island/cc/services/telemetry/processing/processing.py b/monkey/monkey_island/cc/services/telemetry/processing/processing.py index abea5dc38..709097ee0 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/processing.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/processing.py @@ -1,9 +1,11 @@ import logging from common.common_consts.telem_categories import TelemCategoryEnum +from monkey_island.cc.models.telemetries import save_telemetry from monkey_island.cc.services.telemetry.processing.aws_info import process_aws_telemetry -from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import\ - parse_credentials +from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import ( + parse_credentials, +) from monkey_island.cc.services.telemetry.processing.exploit import process_exploit_telemetry from monkey_island.cc.services.telemetry.processing.post_breach import process_post_breach_telemetry from monkey_island.cc.services.telemetry.processing.scan import process_scan_telemetry @@ -25,6 +27,10 @@ TELEMETRY_CATEGORY_TO_PROCESSING_FUNC = { TelemCategoryEnum.TUNNEL: process_tunnel_telemetry, } +# Don't save credential telemetries in telemetries collection. +# Credentials are stored in StolenCredentials documents +UNSAVED_TELEMETRIES = [TelemCategoryEnum.CREDENTIALS] + def process_telemetry(telemetry_json): try: @@ -33,6 +39,10 @@ def process_telemetry(telemetry_json): TELEMETRY_CATEGORY_TO_PROCESSING_FUNC[telem_category](telemetry_json) else: logger.info("Got unknown type of telemetry: %s" % telem_category) + + if telem_category not in UNSAVED_TELEMETRIES: + save_telemetry(telemetry_json) + except Exception as ex: logger.error( "Exception caught while processing telemetry. Info: {}".format(ex), exc_info=True diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py index d2891678e..0088995f3 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/conftest.py @@ -3,17 +3,27 @@ from datetime import datetime import mongoengine import pytest +from monkey_island.cc.models import Monkey from monkey_island.cc.services.config import ConfigService +fake_monkey_guid = "272405690278083" +fake_ip_address = "192.168.56.1" + @pytest.fixture -def fake_mongo(monkeypatch): +def fake_mongo(monkeypatch, uses_encryptor): mongo = mongoengine.connection.get_connection() monkeypatch.setattr("monkey_island.cc.services.config.mongo", mongo) config = ConfigService.get_default_config() ConfigService.update_config(config, should_encrypt=True) +@pytest.fixture +def insert_fake_monkey(): + monkey = Monkey(guid=fake_monkey_guid, ip_addresses=[fake_ip_address]) + monkey.save() + + CREDENTIAL_TELEM_TEMPLATE = { "monkey_guid": "272405690278083", "telem_category": "credentials", diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py index 2ad17431d..5cef3e387 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py @@ -6,12 +6,14 @@ from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials CREDENTIAL_TELEM_TEMPLATE, ) +from common.common_consts.credential_component_type import CredentialComponentType from common.config_value_paths import ( LM_HASH_LIST_PATH, NTLM_HASH_LIST_PATH, PASSWORD_LIST_PATH, USER_LIST_PATH, ) +from monkey_island.cc.models import StolenCredentials from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import ( parse_credentials, @@ -51,21 +53,21 @@ cred_empty_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE) cred_empty_telem["data"] = [{"identities": [], "secrets": []}] -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_username_parsing(): parse_credentials(cred_telem_usernames) config = ConfigService.get_config(should_decrypt=True) assert fake_username in dpath.util.get(config, USER_LIST_PATH) -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_special_username_parsing(): parse_credentials(cred_telem_special_usernames) config = ConfigService.get_config(should_decrypt=True) assert fake_special_username in dpath.util.get(config, USER_LIST_PATH) -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_telemetry_parsing(): parse_credentials(cred_telem) config = ConfigService.get_config(should_decrypt=True) @@ -75,7 +77,20 @@ def test_cred_telemetry_parsing(): assert fake_password in dpath.util.get(config, PASSWORD_LIST_PATH) -@pytest.mark.usefixtures("uses_database", "fake_mongo") +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") +def test_cred_storage_in_db(): + parse_credentials(cred_telem) + cred_docs = list(StolenCredentials.objects()) + assert len(cred_docs) == 1 + + stolen_creds = cred_docs[0] + assert fake_username == stolen_creds.identities[0]["username"] + assert CredentialComponentType.PASSWORD.name in stolen_creds.secrets + assert CredentialComponentType.LM_HASH.name in stolen_creds.secrets + assert CredentialComponentType.NT_HASH.name in stolen_creds.secrets + + +@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_empty_cred_telemetry_parsing(): default_config = deepcopy(ConfigService.get_config(should_decrypt=True)) default_usernames = dpath.util.get(default_config, USER_LIST_PATH) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py index 2d012c4ee..52abf5705 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py @@ -4,18 +4,15 @@ import dpath.util import pytest from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials.conftest import ( CREDENTIAL_TELEM_TEMPLATE, + fake_ip_address, ) from common.config_value_paths import SSH_KEYS_PATH, USER_LIST_PATH -from monkey_island.cc.models import Monkey from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import ( parse_credentials, ) -fake_monkey_guid = "272405690278083" -fake_ip_address = "192.168.56.1" - fake_private_key = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFlczI1N\n" fake_partial_secret = {"private_key": fake_private_key, "credential_type": "SSH_KEYPAIR"} @@ -35,12 +32,6 @@ ssh_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE) ssh_telem["data"] = [{"identities": [fake_identity], "secrets": [fake_secret_full]}] -@pytest.fixture -def insert_fake_monkey(): - monkey = Monkey(guid=fake_monkey_guid, ip_addresses=[fake_ip_address]) - monkey.save() - - @pytest.mark.usefixtures("uses_encryptor", "uses_database", "fake_mongo", "insert_fake_monkey") def test_ssh_credential_parsing(): parse_credentials(ssh_telem) From cf56fcbef24399c312cff969267e9e0c8aa0e40c Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 25 Feb 2022 10:17:18 +0200 Subject: [PATCH 3/8] UT: removed telemetry encryption test --- .../models/telemetries/test_telemetry_dal.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py index 77003ec8f..d3582bf79 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py @@ -51,28 +51,6 @@ def fake_mongo(monkeypatch): monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry_dal.mongo", mongo) -@pytest.mark.slow -@pytest.mark.usefixtures("uses_database", "uses_encryptor") -def test_telemetry_encryption(): - secret_keys = ["password", "lm_hash", "ntlm_hash"] - - save_telemetry(MOCK_TELEMETRY) - - encrypted_telemetry = Telemetry.objects.first() - for user in MOCK_CREDENTIALS.keys(): - assert encrypted_telemetry["data"]["credentials"][user]["username"] == user - - for s in secret_keys: - assert encrypted_telemetry["data"]["credentials"][user][s] != MOCK_CREDENTIALS[user][s] - - decrypted_telemetry = get_telemetry_by_query({})[0] - for user in MOCK_CREDENTIALS.keys(): - assert decrypted_telemetry["data"]["credentials"][user]["username"] == user - - for s in secret_keys: - assert decrypted_telemetry["data"]["credentials"][user][s] == MOCK_CREDENTIALS[user][s] - - @pytest.mark.slow @pytest.mark.usefixtures("uses_database", "uses_encryptor") def test_no_encryption_needed(): From 02d81771a94196c18788195018335e117c3ab7b4 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 25 Feb 2022 17:13:19 +0200 Subject: [PATCH 4/8] Island: remove remaining references to "creds" property of monkey --- monkey/monkey_island/cc/resources/monkey.py | 3 --- monkey/monkey_island/cc/services/node.py | 9 --------- 2 files changed, 12 deletions(-) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index ae8493398..b32ecbb83 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -69,7 +69,6 @@ class Monkey(flask_restful.Resource): def post(self, **kw): with agent_killing_mutex: monkey_json = json.loads(request.data) - monkey_json["creds"] = [] monkey_json["dead"] = False if "keepalive" in monkey_json: monkey_json["keepalive"] = dateutil.parser.parse(monkey_json["keepalive"]) @@ -163,8 +162,6 @@ class Monkey(flask_restful.Resource): EdgeService.update_all_dst_nodes( old_dst_node_id=node_id, new_dst_node_id=new_monkey_id ) - for creds in existing_node["creds"]: - NodeService.add_credentials_to_monkey(new_monkey_id, creds) mongo.db.node.remove({"_id": node_id}) return {"id": new_monkey_id} diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index 79c3408bf..74fb1b091 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -202,7 +202,6 @@ class NodeService: "ip_addresses": [ip_address], "domain_name": domain_name, "exploited": False, - "creds": [], "os": {"type": "unknown", "version": "unknown"}, } ) @@ -318,14 +317,6 @@ class NodeService: def is_monkey_finished_running(): return NodeService.is_any_monkey_exists() and not NodeService.is_any_monkey_alive() - @staticmethod - def add_credentials_to_monkey(monkey_id, creds): - mongo.db.monkey.update({"_id": monkey_id}, {"$push": {"creds": creds}}) - - @staticmethod - def add_credentials_to_node(node_id, creds): - mongo.db.node.update({"_id": node_id}, {"$push": {"creds": creds}}) - @staticmethod def get_node_or_monkey_by_ip(ip_address): node = NodeService.get_node_by_ip(ip_address) From 40820a5ba501d9b9affc7680ceaa882ec212fa4e Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 25 Feb 2022 17:28:07 +0200 Subject: [PATCH 5/8] Island: refactor report generation to take credentials from model Reporting used to fetch credentials from telemetries, but they are no longer stored. Instead, credentials are being fetched from stolen_credentials collection --- .../attack/technique_reports/T1003.py | 28 +--- .../cc/services/reporting/report.py | 121 ++---------------- .../services/reporting/stolen_credentials.py | 59 +++++++++ .../report-components/SecurityReport.js | 2 +- .../reporting/test_stolen_credentials.py | 97 ++++++++++++++ 5 files changed, 172 insertions(+), 135 deletions(-) create mode 100644 monkey/monkey_island/cc/services/reporting/stolen_credentials.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_stolen_credentials.py diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/T1003.py b/monkey/monkey_island/cc/services/attack/technique_reports/T1003.py index c842436dd..81cd7ad69 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/T1003.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/T1003.py @@ -1,7 +1,7 @@ from common.utils.attack_utils import ScanStatus -from monkey_island.cc.database import mongo +from monkey_island.cc.models import StolenCredentials from monkey_island.cc.services.attack.technique_reports import AttackTechnique -from monkey_island.cc.services.reporting.report import ReportService +from monkey_island.cc.services.reporting.stolen_credentials import get_stolen_creds class T1003(AttackTechnique): @@ -14,29 +14,10 @@ class T1003(AttackTechnique): scanned_msg = "" used_msg = "Monkey successfully obtained some credentials from systems on the network." - query = { - "$or": [ - { - "telem_category": "system_info", - "$and": [ - {"data.credentials": {"$exists": True}}, - {"data.credentials": {"$gt": {}}}, - ], - }, # $gt: {} checks if field is not an empty object - { - "telem_category": "exploit", - "$and": [ - {"data.info.credentials": {"$exists": True}}, - {"data.info.credentials": {"$gt": {}}}, - ], - }, - ] - } - @staticmethod def get_report_data(): def get_technique_status_and_data(): - if mongo.db.telemetry.count_documents(T1003.query): + if list(StolenCredentials.objects()): status = ScanStatus.USED.value else: status = ScanStatus.UNSCANNED.value @@ -47,6 +28,5 @@ class T1003(AttackTechnique): data.update(T1003.get_message_and_status(status)) data.update(T1003.get_mitigation_by_status(status)) - data["stolen_creds"] = ReportService.get_stolen_creds() - data["stolen_creds"].extend(ReportService.get_ssh_keys()) + data["stolen_creds"] = get_stolen_creds() return data diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 8d93d8062..3ac0c0364 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -16,7 +16,6 @@ from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey from monkey_island.cc.models.report import get_report, save_report -from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.configuration.utils import ( get_config_network_segments_as_subnet_groups, @@ -26,22 +25,21 @@ from monkey_island.cc.services.reporting.exploitations.manual_exploitation impor from monkey_island.cc.services.reporting.exploitations.monkey_exploitation import ( get_monkey_exploited, ) -from monkey_island.cc.services.reporting.issue_processing.exploit_processing.exploiter_descriptor_enum import ( # noqa: E501 - ExploiterDescriptorEnum, -) -from monkey_island.cc.services.reporting.issue_processing.exploit_processing.processors.cred_exploit import ( # noqa: E501 - CredentialType, -) -from monkey_island.cc.services.reporting.issue_processing.exploit_processing.processors.exploit import ( # noqa: E501 - ExploiterReportInfo, -) from monkey_island.cc.services.reporting.pth_report import PTHReportService from monkey_island.cc.services.reporting.report_exporter_manager import ReportExporterManager from monkey_island.cc.services.reporting.report_generation_synchronisation import ( safe_generate_regular_report, ) +from monkey_island.cc.services.reporting.stolen_credentials import ( + extract_ssh_keys, + get_stolen_creds, +) from monkey_island.cc.services.utils.network_utils import get_subnets, local_ip_addresses +from .issue_processing.exploit_processing.exploiter_descriptor_enum import ExploiterDescriptorEnum +from .issue_processing.exploit_processing.processors.cred_exploit import CredentialType +from .issue_processing.exploit_processing.processors.exploit import ExploiterReportInfo + logger = logging.getLogger(__name__) @@ -133,104 +131,6 @@ class ReportService: nodes = nodes_without_monkeys + nodes_with_monkeys return nodes - @staticmethod - def get_stolen_creds(): - creds = [] - - stolen_system_info_creds = ReportService._get_credentials_from_system_info_telems() - creds.extend(stolen_system_info_creds) - - stolen_exploit_creds = ReportService._get_credentials_from_exploit_telems() - creds.extend(stolen_exploit_creds) - - logger.info("Stolen creds generated for reporting") - return creds - - @staticmethod - def _get_credentials_from_system_info_telems(): - formatted_creds = [] - for telem in get_telemetry_by_query( - {"telem_category": "system_info", "data.credentials": {"$exists": True}}, - {"data.credentials": 1, "monkey_guid": 1}, - ): - creds = telem["data"]["credentials"] - origin = NodeService.get_monkey_by_guid(telem["monkey_guid"])["hostname"] - formatted_creds.extend(ReportService._format_creds_for_reporting(telem, creds, origin)) - return formatted_creds - - @staticmethod - def _get_credentials_from_exploit_telems(): - formatted_creds = [] - for telem in mongo.db.telemetry.find( - {"telem_category": "exploit", "data.info.credentials": {"$exists": True}}, - {"data.info.credentials": 1, "data.machine": 1, "monkey_guid": 1}, - ): - creds = telem["data"]["info"]["credentials"] - domain_name = telem["data"]["machine"]["domain_name"] - ip = telem["data"]["machine"]["ip_addr"] - origin = domain_name if domain_name else ip - formatted_creds.extend(ReportService._format_creds_for_reporting(telem, creds, origin)) - return formatted_creds - - @staticmethod - def _format_creds_for_reporting(telem, monkey_creds, origin): - creds = [] - CRED_TYPE_DICT = { - "password": "Clear Password", - "lm_hash": "LM hash", - "ntlm_hash": "NTLM hash", - } - if len(monkey_creds) == 0: - return [] - - for user in monkey_creds: - for cred_type in CRED_TYPE_DICT: - if cred_type not in monkey_creds[user] or not monkey_creds[user][cred_type]: - continue - username = ( - monkey_creds[user]["username"] if "username" in monkey_creds[user] else user - ) - cred_row = { - "username": username, - "type": CRED_TYPE_DICT[cred_type], - "origin": origin, - } - if cred_row not in creds: - creds.append(cred_row) - return creds - - @staticmethod - def get_ssh_keys(): - """ - Return private ssh keys found as credentials - :return: List of credentials - """ - creds = [] - for telem in mongo.db.telemetry.find( - {"telem_category": "system_info", "data.ssh_info": {"$exists": True}}, - {"data.ssh_info": 1, "monkey_guid": 1}, - ): - origin = NodeService.get_monkey_by_guid(telem["monkey_guid"])["hostname"] - if telem["data"]["ssh_info"]: - # Pick out all ssh keys not yet included in creds - ssh_keys = [ - { - "username": key_pair["name"], - "type": "Clear SSH private key", - "origin": origin, - } - for key_pair in telem["data"]["ssh_info"] - if key_pair["private_key"] - and { - "username": key_pair["name"], - "type": "Clear SSH private key", - "origin": origin, - } - not in creds - ] - creds.extend(ssh_keys) - return creds - @staticmethod def process_exploit(exploit) -> ExploiterReportInfo: exploiter_type = exploit["data"]["exploiter"] @@ -564,6 +464,7 @@ class ReportService: issue_set = ReportService.get_issue_set(issues, config_users, config_passwords) cross_segment_issues = ReportService.get_cross_segment_issues() monkey_latest_modify_time = Monkey.get_latest_modifytime() + stolen_creds = get_stolen_creds() scanned_nodes = ReportService.get_scanned() exploited_cnt = len(get_monkey_exploited()) @@ -585,8 +486,8 @@ class ReportService: "glance": { "scanned": scanned_nodes, "exploited_cnt": exploited_cnt, - "stolen_creds": ReportService.get_stolen_creds(), - "ssh_keys": ReportService.get_ssh_keys(), + "stolen_creds": stolen_creds, + "ssh_keys": extract_ssh_keys(stolen_creds), "strong_users": PTHReportService.get_strong_users_on_crit_details(), }, "recommendations": {"issues": issues, "domain_issues": domain_issues}, diff --git a/monkey/monkey_island/cc/services/reporting/stolen_credentials.py b/monkey/monkey_island/cc/services/reporting/stolen_credentials.py new file mode 100644 index 000000000..f65b26bb3 --- /dev/null +++ b/monkey/monkey_island/cc/services/reporting/stolen_credentials.py @@ -0,0 +1,59 @@ +import logging +from typing import Mapping, Sequence + +from common.common_consts.credential_component_type import CredentialComponentType +from monkey_island.cc.models import StolenCredentials + +logger = logging.getLogger(__name__) + + +def get_stolen_creds() -> Sequence[Mapping]: + stolen_creds = _fetch_from_db() + stolen_creds = _format_creds_for_reporting(stolen_creds) + + logger.info("Stolen creds generated for reporting") + return stolen_creds + + +def extract_ssh_keys(credentials: Sequence[Mapping]) -> Sequence[Mapping]: + ssh_keys = [] + for credential in credentials: + if credential["_type"] == CredentialComponentType.SSH_KEYPAIR.name: + ssh_keys.append(credential) + return ssh_keys + + +def _fetch_from_db() -> Sequence[StolenCredentials]: + return list(StolenCredentials.objects()) + + +def _format_creds_for_reporting(credentials: Sequence[StolenCredentials]): + formatted_creds = [] + cred_type_dict = { + CredentialComponentType.PASSWORD.name: "Clear Password", + CredentialComponentType.LM_HASH.name: "LM hash", + CredentialComponentType.NT_HASH.name: "NTLM hash", + CredentialComponentType.SSH_KEYPAIR.name: "Clear SSH private key", + } + + for cred in credentials: + for secret_type in cred.secrets: + if secret_type not in cred_type_dict: + continue + username = _get_username(cred) + cred_row = { + "username": username, + "_type": secret_type, + "type": cred_type_dict[secret_type], + "origin": cred.monkey.hostname, + } + if cred_row not in formatted_creds: + formatted_creds.append(cred_row) + return formatted_creds + + +def _get_username(credentials: StolenCredentials) -> str: + if credentials.identities: + return credentials.identities[0]["username"] + else: + return "" diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index 932879fea..f058a3069 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -556,7 +556,7 @@ class ReportPageComponent extends AuthComponent {
- +
diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_stolen_credentials.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_stolen_credentials.py new file mode 100644 index 000000000..e3a7f6570 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_stolen_credentials.py @@ -0,0 +1,97 @@ +import pytest + +from common.common_consts.credential_component_type import CredentialComponentType +from monkey_island.cc.models import Monkey, StolenCredentials +from monkey_island.cc.services.reporting.stolen_credentials import ( + extract_ssh_keys, + get_stolen_creds, +) + +monkey_hostname = "fake_hostname" +fake_monkey_guid = "abc" + +fake_username = "m0nk3y_user" +fake_nt_hash = "c1c58f96cdf212b50837bc11a00be47c" +fake_lm_hash = "299BD128C1101FD6" +fake_password = "trytostealthis" +fake_ssh_key = "RSA_fake_key" +fake_credentials = { + "identities": [{"username": fake_username, "credential_type": "USERNAME"}], + "secrets": [ + CredentialComponentType.NT_HASH.name, + CredentialComponentType.LM_HASH.name, + CredentialComponentType.PASSWORD.name, + CredentialComponentType.SSH_KEYPAIR.name, + ], +} + + +@pytest.fixture +def fake_monkey(): + monkey = Monkey() + monkey.guid = fake_monkey_guid + monkey.hostname = monkey_hostname + monkey.save() + return monkey.id + + +@pytest.mark.usefixture("uses_database") +def test_get_credentials(fake_monkey): + StolenCredentials( + identities=fake_credentials["identities"], + secrets=fake_credentials["secrets"], + monkey=fake_monkey, + ).save() + + credentials = get_stolen_creds() + + result1 = { + "origin": monkey_hostname, + "_type": CredentialComponentType.NT_HASH.name, + "type": "NTLM hash", + "username": fake_username, + } + result2 = { + "origin": monkey_hostname, + "_type": CredentialComponentType.LM_HASH.name, + "type": "LM hash", + "username": fake_username, + } + result3 = { + "origin": monkey_hostname, + "_type": CredentialComponentType.PASSWORD.name, + "type": "Clear Password", + "username": fake_username, + } + result4 = { + "origin": monkey_hostname, + "_type": CredentialComponentType.SSH_KEYPAIR.name, + "type": "Clear SSH private key", + "username": fake_username, + } + assert result1 in credentials + assert result2 in credentials + assert result3 in credentials + assert result4 in credentials + + +@pytest.mark.usefixtures("uses_database") +def test_extract_ssh_keys(fake_monkey): + StolenCredentials( + identities=fake_credentials["identities"], + secrets=fake_credentials["secrets"], + monkey=fake_monkey, + ).save() + + credentials = get_stolen_creds() + keys = extract_ssh_keys(credentials) + + assert len(keys) == 1 + + result = { + "origin": monkey_hostname, + "_type": CredentialComponentType.SSH_KEYPAIR.name, + "type": "Clear SSH private key", + "username": fake_username, + } + assert result in keys From 748178a00cece8cccbc5756772c00570e92480d9 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 28 Feb 2022 16:45:36 +0200 Subject: [PATCH 6/8] Island: small style improvements in stolen_credentials.py --- .../cc/services/reporting/stolen_credentials.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/stolen_credentials.py b/monkey/monkey_island/cc/services/reporting/stolen_credentials.py index f65b26bb3..a1f596ad5 100644 --- a/monkey/monkey_island/cc/services/reporting/stolen_credentials.py +++ b/monkey/monkey_island/cc/services/reporting/stolen_credentials.py @@ -16,11 +16,7 @@ def get_stolen_creds() -> Sequence[Mapping]: def extract_ssh_keys(credentials: Sequence[Mapping]) -> Sequence[Mapping]: - ssh_keys = [] - for credential in credentials: - if credential["_type"] == CredentialComponentType.SSH_KEYPAIR.name: - ssh_keys.append(credential) - return ssh_keys + return [c for c in credentials if c["_type"] == CredentialComponentType.SSH_KEYPAIR.name] def _fetch_from_db() -> Sequence[StolenCredentials]: @@ -53,7 +49,4 @@ def _format_creds_for_reporting(credentials: Sequence[StolenCredentials]): def _get_username(credentials: StolenCredentials) -> str: - if credentials.identities: - return credentials.identities[0]["username"] - else: - return "" + return credentials.identities[0]["username"] if credentials.identities else "" From 4f58a69c54b461618c5b32420deeb8258dd70612 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 28 Feb 2022 16:59:15 +0200 Subject: [PATCH 7/8] UT: added slow marks and changed some names, related to credential tests --- .../monkey_island/cc/services/reporting/test_report.py | 6 +++--- .../processing/credentials/test_credential_processing.py | 5 +++++ .../processing/credentials/test_ssh_key_processing.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index c33f0087b..bfe19ea83 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -158,18 +158,18 @@ def test_get_stolen_creds_exploit(fake_mongo): @pytest.mark.slow @pytest.mark.usefixtures("uses_database", "uses_encryptor") -def test_get_stolen_creds_system_info(fake_mongo): +def test_get_stolen_creds_from_db(fake_mongo): fake_mongo.db.monkey.insert_one(MONKEY_TELEM) save_telemetry(SYSTEM_INFO_TELEMETRY_TELEM) stolen_creds_system_info = ReportService.get_stolen_creds() - expected_stolen_creds_system_info = [ + expected_stolen_creds_from_db = [ {"origin": HOSTNAME, "type": "Clear Password", "username": USER}, {"origin": HOSTNAME, "type": "LM hash", "username": USER}, {"origin": HOSTNAME, "type": "NTLM hash", "username": USER}, ] - assert expected_stolen_creds_system_info == stolen_creds_system_info + assert expected_stolen_creds_from_db == stolen_creds_system_info @pytest.mark.usefixtures("uses_database") diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py index 5cef3e387..5fcccc7a0 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_credential_processing.py @@ -53,6 +53,7 @@ cred_empty_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE) cred_empty_telem["data"] = [{"identities": [], "secrets": []}] +@pytest.mark.slow @pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_username_parsing(): parse_credentials(cred_telem_usernames) @@ -60,6 +61,7 @@ def test_cred_username_parsing(): assert fake_username in dpath.util.get(config, USER_LIST_PATH) +@pytest.mark.slow @pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_special_username_parsing(): parse_credentials(cred_telem_special_usernames) @@ -67,6 +69,7 @@ def test_cred_special_username_parsing(): assert fake_special_username in dpath.util.get(config, USER_LIST_PATH) +@pytest.mark.slow @pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_telemetry_parsing(): parse_credentials(cred_telem) @@ -77,6 +80,7 @@ def test_cred_telemetry_parsing(): assert fake_password in dpath.util.get(config, PASSWORD_LIST_PATH) +@pytest.mark.slow @pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_cred_storage_in_db(): parse_credentials(cred_telem) @@ -90,6 +94,7 @@ def test_cred_storage_in_db(): assert CredentialComponentType.NT_HASH.name in stolen_creds.secrets +@pytest.mark.slow @pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey") def test_empty_cred_telemetry_parsing(): default_config = deepcopy(ConfigService.get_config(should_decrypt=True)) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py index 52abf5705..900166847 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/credentials/test_ssh_key_processing.py @@ -32,6 +32,7 @@ ssh_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE) ssh_telem["data"] = [{"identities": [fake_identity], "secrets": [fake_secret_full]}] +@pytest.mark.slow @pytest.mark.usefixtures("uses_encryptor", "uses_database", "fake_mongo", "insert_fake_monkey") def test_ssh_credential_parsing(): parse_credentials(ssh_telem) From 52c041379765d41ef3767c0c3bea9bf12b28972d Mon Sep 17 00:00:00 2001 From: vakarisz Date: Tue, 1 Mar 2022 11:31:47 +0200 Subject: [PATCH 8/8] Island, UT: remove credential processing from exploit telemetry Credentials should be sent via credential telemetry, not exploit telemetry. This will remove the need to maintain duplicate code of credential extraction --- monkey/infection_monkey/exploit/zerologon.py | 2 + .../services/telemetry/processing/exploit.py | 15 -- .../exploitations/test_monkey_exploitation.py | 134 ++++++++++++- .../cc/services/reporting/test_report.py | 183 ------------------ 4 files changed, 131 insertions(+), 203 deletions(-) delete mode 100644 monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py diff --git a/monkey/infection_monkey/exploit/zerologon.py b/monkey/infection_monkey/exploit/zerologon.py index a882b17de..f05983d92 100644 --- a/monkey/infection_monkey/exploit/zerologon.py +++ b/monkey/infection_monkey/exploit/zerologon.py @@ -276,6 +276,8 @@ class ZerologonExploiter(HostExploiter): ) def add_extracted_creds_to_exploit_info(self, user: str, lmhash: str, nthash: str) -> None: + # TODO exploit_info["credentials"] is discontinued, + # refactor to send a credential telemetry self.exploit_info["credentials"].update( { user: { diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index a867267d0..d035dedd3 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -4,7 +4,6 @@ import dateutil from monkey_island.cc.models import Monkey from monkey_island.cc.server_utils.encryption import get_datastore_encryptor -from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.edge.displayed_edge import EdgeService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.utils import ( @@ -20,7 +19,6 @@ def process_exploit_telemetry(telemetry_json): edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) update_network_with_exploit(edge, telemetry_json) update_node_credentials_from_successful_attempts(edge, telemetry_json) - add_exploit_extracted_creds_to_config(telemetry_json) check_machine_exploited( current_monkey=Monkey.get_single_monkey_by_guid(telemetry_json["monkey_guid"]), @@ -31,19 +29,6 @@ def process_exploit_telemetry(telemetry_json): ) -def add_exploit_extracted_creds_to_config(telemetry_json): - if "credentials" in telemetry_json["data"]["info"]: - creds = telemetry_json["data"]["info"]["credentials"] - for user in creds: - ConfigService.creds_add_username(creds[user]["username"]) - if "password" in creds[user] and creds[user]["password"]: - ConfigService.creds_add_password(creds[user]["password"]) - if "lm_hash" in creds[user] and creds[user]["lm_hash"]: - ConfigService.creds_add_lm_hash(creds[user]["lm_hash"]) - if "ntlm_hash" in creds[user] and creds[user]["ntlm_hash"]: - ConfigService.creds_add_ntlm_hash(creds[user]["ntlm_hash"]) - - def update_node_credentials_from_successful_attempts(edge: EdgeService, telemetry_json): for attempt in telemetry_json["data"]["attempts"]: if attempt["result"]: diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/exploitations/test_monkey_exploitation.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/exploitations/test_monkey_exploitation.py index 1c0377807..9995ba795 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/exploitations/test_monkey_exploitation.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/exploitations/test_monkey_exploitation.py @@ -1,13 +1,137 @@ -from tests.unit_tests.monkey_island.cc.services.reporting.test_report import ( - NODE_DICT, - NODE_DICT_DUPLICATE_EXPLOITS, - NODE_DICT_FAILED_EXPLOITS, -) +import datetime +from copy import deepcopy + +from bson import ObjectId from monkey_island.cc.services.reporting.exploitations.monkey_exploitation import ( get_exploits_used_on_node, ) +TELEM_ID = { + "exploit_creds": ObjectId(b"123456789000"), + "system_info_creds": ObjectId(b"987654321000"), + "no_creds": ObjectId(b"112233445566"), + "monkey": ObjectId(b"665544332211"), +} +MONKEY_GUID = "67890" +USER = "user-name" +PWD = "password123" +LM_HASH = "e52cac67419a9a22664345140a852f61" +NT_HASH = "a9fdfa038c4b75ebc76dc855dd74f0da" +VICTIM_IP = "0.0.0.0" +VICTIM_DOMAIN_NAME = "domain-name" +HOSTNAME = "name-of-host" + +# Below telem constants only contain fields relevant to current tests + +EXPLOIT_TELEMETRY_TELEM = { + "_id": TELEM_ID["exploit_creds"], + "monkey_guid": MONKEY_GUID, + "telem_category": "exploit", + "data": { + "machine": { + "ip_addr": VICTIM_IP, + "domain_name": VICTIM_DOMAIN_NAME, + }, + "info": { + "credentials": { + USER: { + "username": USER, + "lm_hash": LM_HASH, + "ntlm_hash": NT_HASH, + } + } + }, + }, +} + +SYSTEM_INFO_TELEMETRY_TELEM = { + "_id": TELEM_ID["system_info_creds"], + "monkey_guid": MONKEY_GUID, + "telem_category": "system_info", + "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), + "command_control_channel": { + "src": "192.168.56.1", + "dst": "192.168.56.2", + }, + "data": { + "credentials": { + USER: { + "password": PWD, + "lm_hash": LM_HASH, + "ntlm_hash": NT_HASH, + } + } + }, +} + +NO_CREDS_TELEMETRY_TELEM = { + "_id": TELEM_ID["no_creds"], + "monkey_guid": MONKEY_GUID, + "telem_category": "exploit", + "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), + "command_control_channel": { + "src": "192.168.56.1", + "dst": "192.168.56.2", + }, + "data": { + "machine": { + "ip_addr": VICTIM_IP, + "domain_name": VICTIM_DOMAIN_NAME, + }, + "info": {"credentials": {}}, + }, +} + +MONKEY_TELEM = {"_id": TELEM_ID["monkey"], "guid": MONKEY_GUID, "hostname": HOSTNAME} + +NODE_DICT = { + "id": "602f62118e30cf35830ff8e4", + "label": "WinDev2010Eval.mshome.net", + "group": "monkey_windows", + "os": "windows", + "dead": True, + "exploits": [ + { + "exploitation_result": True, + "exploiter": "DrupalExploiter", + "info": { + "display_name": "Drupal Server", + "started": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000), + "finished": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000), + "vulnerable_urls": [], + "vulnerable_ports": [], + "executed_cmds": [], + }, + "attempts": [], + "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), + "origin": "MonkeyIsland : 192.168.56.1", + }, + { + "exploitation_result": True, + "exploiter": "ZerologonExploiter", + "info": { + "display_name": "Zerologon", + "started": datetime.datetime(2021, 2, 19, 9, 0, 15, 16000), + "finished": datetime.datetime(2021, 2, 19, 9, 0, 15, 17000), + "vulnerable_urls": [], + "vulnerable_ports": [], + "executed_cmds": [], + }, + "attempts": [], + "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 15, 60000), + "origin": "MonkeyIsland : 192.168.56.1", + }, + ], +} + +NODE_DICT_DUPLICATE_EXPLOITS = deepcopy(NODE_DICT) +NODE_DICT_DUPLICATE_EXPLOITS["exploits"][1] = NODE_DICT_DUPLICATE_EXPLOITS["exploits"][0] + +NODE_DICT_FAILED_EXPLOITS = deepcopy(NODE_DICT) +NODE_DICT_FAILED_EXPLOITS["exploits"][0]["exploitation_result"] = False +NODE_DICT_FAILED_EXPLOITS["exploits"][1]["exploitation_result"] = False + def test_get_exploits_used_on_node__2_exploits(): exploits = get_exploits_used_on_node(NODE_DICT) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py deleted file mode 100644 index bfe19ea83..000000000 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ /dev/null @@ -1,183 +0,0 @@ -import datetime -from copy import deepcopy - -import mongoengine -import pytest -from bson import ObjectId - -from monkey_island.cc.models.telemetries import save_telemetry -from monkey_island.cc.services.reporting.report import ReportService - -TELEM_ID = { - "exploit_creds": ObjectId(b"123456789000"), - "system_info_creds": ObjectId(b"987654321000"), - "no_creds": ObjectId(b"112233445566"), - "monkey": ObjectId(b"665544332211"), -} -MONKEY_GUID = "67890" -USER = "user-name" -PWD = "password123" -LM_HASH = "e52cac67419a9a22664345140a852f61" -NT_HASH = "a9fdfa038c4b75ebc76dc855dd74f0da" -VICTIM_IP = "0.0.0.0" -VICTIM_DOMAIN_NAME = "domain-name" -HOSTNAME = "name-of-host" - -# Below telem constants only contain fields relevant to current tests - -EXPLOIT_TELEMETRY_TELEM = { - "_id": TELEM_ID["exploit_creds"], - "monkey_guid": MONKEY_GUID, - "telem_category": "exploit", - "data": { - "machine": { - "ip_addr": VICTIM_IP, - "domain_name": VICTIM_DOMAIN_NAME, - }, - "info": { - "credentials": { - USER: { - "username": USER, - "lm_hash": LM_HASH, - "ntlm_hash": NT_HASH, - } - } - }, - }, -} - -SYSTEM_INFO_TELEMETRY_TELEM = { - "_id": TELEM_ID["system_info_creds"], - "monkey_guid": MONKEY_GUID, - "telem_category": "system_info", - "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), - "command_control_channel": { - "src": "192.168.56.1", - "dst": "192.168.56.2", - }, - "data": { - "credentials": { - USER: { - "password": PWD, - "lm_hash": LM_HASH, - "ntlm_hash": NT_HASH, - } - } - }, -} - -NO_CREDS_TELEMETRY_TELEM = { - "_id": TELEM_ID["no_creds"], - "monkey_guid": MONKEY_GUID, - "telem_category": "exploit", - "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), - "command_control_channel": { - "src": "192.168.56.1", - "dst": "192.168.56.2", - }, - "data": { - "machine": { - "ip_addr": VICTIM_IP, - "domain_name": VICTIM_DOMAIN_NAME, - }, - "info": {"credentials": {}}, - }, -} - -MONKEY_TELEM = {"_id": TELEM_ID["monkey"], "guid": MONKEY_GUID, "hostname": HOSTNAME} - -NODE_DICT = { - "id": "602f62118e30cf35830ff8e4", - "label": "WinDev2010Eval.mshome.net", - "group": "monkey_windows", - "os": "windows", - "dead": True, - "exploits": [ - { - "exploitation_result": True, - "exploiter": "DrupalExploiter", - "info": { - "display_name": "Drupal Server", - "started": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000), - "finished": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000), - "vulnerable_urls": [], - "vulnerable_ports": [], - "executed_cmds": [], - }, - "attempts": [], - "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), - "origin": "MonkeyIsland : 192.168.56.1", - }, - { - "exploitation_result": True, - "exploiter": "ZerologonExploiter", - "info": { - "display_name": "Zerologon", - "started": datetime.datetime(2021, 2, 19, 9, 0, 15, 16000), - "finished": datetime.datetime(2021, 2, 19, 9, 0, 15, 17000), - "vulnerable_urls": [], - "vulnerable_ports": [], - "executed_cmds": [], - }, - "attempts": [], - "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 15, 60000), - "origin": "MonkeyIsland : 192.168.56.1", - }, - ], -} - -NODE_DICT_DUPLICATE_EXPLOITS = deepcopy(NODE_DICT) -NODE_DICT_DUPLICATE_EXPLOITS["exploits"][1] = NODE_DICT_DUPLICATE_EXPLOITS["exploits"][0] - -NODE_DICT_FAILED_EXPLOITS = deepcopy(NODE_DICT) -NODE_DICT_FAILED_EXPLOITS["exploits"][0]["exploitation_result"] = False -NODE_DICT_FAILED_EXPLOITS["exploits"][1]["exploitation_result"] = False - - -@pytest.fixture -def fake_mongo(monkeypatch): - mongo = mongoengine.connection.get_connection() - monkeypatch.setattr("monkey_island.cc.services.reporting.report.mongo", mongo) - monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry_dal.mongo", mongo) - monkeypatch.setattr("monkey_island.cc.services.node.mongo", mongo) - return mongo - - -@pytest.mark.usefixtures("uses_database") -def test_get_stolen_creds_exploit(fake_mongo): - fake_mongo.db.telemetry.insert_one(EXPLOIT_TELEMETRY_TELEM) - - stolen_creds_exploit = ReportService.get_stolen_creds() - expected_stolen_creds_exploit = [ - {"origin": VICTIM_DOMAIN_NAME, "type": "LM hash", "username": USER}, - {"origin": VICTIM_DOMAIN_NAME, "type": "NTLM hash", "username": USER}, - ] - - assert expected_stolen_creds_exploit == stolen_creds_exploit - - -@pytest.mark.slow -@pytest.mark.usefixtures("uses_database", "uses_encryptor") -def test_get_stolen_creds_from_db(fake_mongo): - fake_mongo.db.monkey.insert_one(MONKEY_TELEM) - save_telemetry(SYSTEM_INFO_TELEMETRY_TELEM) - - stolen_creds_system_info = ReportService.get_stolen_creds() - expected_stolen_creds_from_db = [ - {"origin": HOSTNAME, "type": "Clear Password", "username": USER}, - {"origin": HOSTNAME, "type": "LM hash", "username": USER}, - {"origin": HOSTNAME, "type": "NTLM hash", "username": USER}, - ] - - assert expected_stolen_creds_from_db == stolen_creds_system_info - - -@pytest.mark.usefixtures("uses_database") -def test_get_stolen_creds_no_creds(fake_mongo): - fake_mongo.db.monkey.insert_one(MONKEY_TELEM) - save_telemetry(NO_CREDS_TELEMETRY_TELEM) - - stolen_creds_no_creds = ReportService.get_stolen_creds() - expected_stolen_creds_no_creds = [] - - assert expected_stolen_creds_no_creds == stolen_creds_no_creds