From f1c7cf40477686e32b83f0c3bc33789a46e47420 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 21 Sep 2021 15:32:05 +0300 Subject: [PATCH 01/16] Generalize report_encryptor.py into document_encryptor.py and extract the sensitive fields to report_encryptor.py --- .../cc/models/utils/document_encryptor.py | 54 +++++++++++++++++++ .../cc/models/utils/report_encryptor.py | 50 ----------------- .../monkey_island/cc/models/__init__.py | 0 3 files changed, 54 insertions(+), 50 deletions(-) create mode 100644 monkey/monkey_island/cc/models/utils/document_encryptor.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/models/__init__.py diff --git a/monkey/monkey_island/cc/models/utils/document_encryptor.py b/monkey/monkey_island/cc/models/utils/document_encryptor.py new file mode 100644 index 000000000..0a0c2a62e --- /dev/null +++ b/monkey/monkey_island/cc/models/utils/document_encryptor.py @@ -0,0 +1,54 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable, List, Type + +import dpath.util + +from monkey_island.cc.models.utils.field_types.field_type_abc import FieldTypeABC + + +@dataclass +class SensitiveField: + path: str + path_separator = "." + field_type: Type[FieldTypeABC] + + +class DocumentEncryptor(ABC): + @property + @abstractmethod + def sensitive_fields(self) -> List[SensitiveField]: + pass + + @classmethod + def encrypt(cls, document_dict: dict) -> dict: + for sensitive_field in cls.sensitive_fields: + DocumentEncryptor._apply_operation_to_document_field( + document_dict, sensitive_field, sensitive_field.field_type.encrypt + ) + + return document_dict + + @classmethod + def decrypt(cls, document_dict: dict) -> dict: + for sensitive_field in cls.sensitive_fields: + DocumentEncryptor._apply_operation_to_document_field( + document_dict, sensitive_field, sensitive_field.field_type.decrypt + ) + return document_dict + + @staticmethod + def _apply_operation_to_document_field( + report: dict, sensitive_field: SensitiveField, operation: Callable + ): + field_value = dpath.util.get( + report, sensitive_field.path, sensitive_field.path_separator, None + ) + if field_value is None: + raise Exception( + f"Can't encrypt object because the path {sensitive_field.path} doesn't exist." + ) + + modified_value = operation(field_value) + + dpath.util.set(report, sensitive_field.path, modified_value, sensitive_field.path_separator) diff --git a/monkey/monkey_island/cc/models/utils/report_encryptor.py b/monkey/monkey_island/cc/models/utils/report_encryptor.py index d5e31f4d8..e69de29bb 100644 --- a/monkey/monkey_island/cc/models/utils/report_encryptor.py +++ b/monkey/monkey_island/cc/models/utils/report_encryptor.py @@ -1,50 +0,0 @@ -from dataclasses import dataclass -from typing import Callable, Type - -import dpath.util - -from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor -from monkey_island.cc.models.utils.field_encryptors.string_list_encryptor import StringListEncryptor - - -@dataclass -class SensitiveField: - path: str - path_separator = "." - field_type: Type[IFieldEncryptor] - - -sensitive_fields = [ - SensitiveField(path="overview.config_passwords", field_type=StringListEncryptor) -] - - -def encrypt(report: dict) -> dict: - for sensitive_field in sensitive_fields: - _apply_operation_to_report_field( - report, sensitive_field, sensitive_field.field_type.encrypt - ) - - return report - - -def decrypt(report: dict) -> dict: - for sensitive_field in sensitive_fields: - _apply_operation_to_report_field( - report, sensitive_field, sensitive_field.field_type.decrypt - ) - return report - - -def _apply_operation_to_report_field( - report: dict, sensitive_field: SensitiveField, operation: Callable -): - field_value = dpath.util.get(report, sensitive_field.path, sensitive_field.path_separator, None) - if field_value is None: - raise Exception( - f"Can't encrypt object because the path {sensitive_field.path} doesn't exist." - ) - - modified_value = operation(field_value) - - dpath.util.set(report, sensitive_field.path, modified_value, sensitive_field.path_separator) diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/models/__init__.py new file mode 100644 index 000000000..e69de29bb From f3865d022bb25365ebb9749beb3e1a8e86ba6040 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 22 Sep 2021 13:41:40 +0300 Subject: [PATCH 02/16] Change mongomock_fixtures.py to drop the whole database instead of specified collections. This makes it easier to add new database related tests, because we no longer need to modify the mongomock_fixtures.py to also drop a particular collection we are testing. --- .../monkey_island/cc/mongomock_fixtures.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py b/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py index 26a41685a..aa00aa69a 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py +++ b/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py @@ -1,32 +1,20 @@ import mongoengine import pytest -from monkey_island.cc.models import Monkey -from monkey_island.cc.models.edge import Edge -from monkey_island.cc.models.zero_trust.finding import Finding +MOCK_DB_NAME = "mongoenginetest" @pytest.fixture(scope="module", autouse=True) def change_to_mongo_mock(): # Make sure tests are working with mongomock mongoengine.disconnect() - mongoengine.connect("mongoenginetest", host="mongomock://localhost") + mongoengine.connect(MOCK_DB_NAME, host="mongomock://localhost") @pytest.fixture(scope="function") def uses_database(): - _clean_edge_db() - _clean_monkey_db() - _clean_finding_db() + _drop_database() -def _clean_monkey_db(): - Monkey.objects().delete() - - -def _clean_edge_db(): - Edge.objects().delete() - - -def _clean_finding_db(): - Finding.objects().delete() +def _drop_database(): + mongoengine.connection.get_connection().drop_database(MOCK_DB_NAME) From 854ce4e1e1a0bde6698d266418aec954151bf7f5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 22 Sep 2021 15:36:51 +0300 Subject: [PATCH 03/16] Refactor DocumentEncryptor class into a series of methods. DocumentEncryptor class serves no purpose because it holds no state, sensitive_fields can be passed as a parameter to methods --- monkey/monkey_island/cc/models/report.py | 14 +++- .../__init__.py} | 0 .../monkey_island/cc/models/utils/__init__.py | 0 .../cc/models/utils/document_encryptor.py | 70 +++++++++---------- .../models/utils/field_encryptors/__init__.py | 0 .../cc/models/test_report_model.py | 20 +++--- 6 files changed, 52 insertions(+), 52 deletions(-) rename monkey/monkey_island/cc/models/{utils/report_encryptor.py => telemetries/__init__.py} (100%) create mode 100644 monkey/monkey_island/cc/models/utils/__init__.py create mode 100644 monkey/monkey_island/cc/models/utils/field_encryptors/__init__.py diff --git a/monkey/monkey_island/cc/models/report.py b/monkey/monkey_island/cc/models/report.py index 4158a5244..fb1ab3cea 100644 --- a/monkey/monkey_island/cc/models/report.py +++ b/monkey/monkey_island/cc/models/report.py @@ -3,7 +3,13 @@ from __future__ import annotations from bson import json_util from mongoengine import DictField, Document -from monkey_island.cc.models.utils import report_encryptor +from monkey_island.cc.models.utils import document_encryptor +from monkey_island.cc.models.utils.document_encryptor import SensitiveField +from monkey_island.cc.models.utils.field_encryptors.string_list_encryptor import StringListEncryptor + +sensitive_fields = [ + SensitiveField(path="overview.config_passwords", field_encryptor=StringListEncryptor) +] class Report(Document): @@ -18,7 +24,7 @@ class Report(Document): @staticmethod def save_report(report_dict: dict): report_dict = _encode_dot_char_before_mongo_insert(report_dict) - report_dict = report_encryptor.encrypt(report_dict) + report_dict = document_encryptor.encrypt(sensitive_fields, report_dict) Report.objects.delete() Report( overview=report_dict["overview"], @@ -30,7 +36,9 @@ class Report(Document): @staticmethod def get_report() -> dict: report_dict = Report.objects.first().to_mongo() - return _decode_dot_char_before_mongo_insert(report_encryptor.decrypt(report_dict)) + return _decode_dot_char_before_mongo_insert( + document_encryptor.decrypt(sensitive_fields, report_dict) + ) def _encode_dot_char_before_mongo_insert(report_dict): diff --git a/monkey/monkey_island/cc/models/utils/report_encryptor.py b/monkey/monkey_island/cc/models/telemetries/__init__.py similarity index 100% rename from monkey/monkey_island/cc/models/utils/report_encryptor.py rename to monkey/monkey_island/cc/models/telemetries/__init__.py diff --git a/monkey/monkey_island/cc/models/utils/__init__.py b/monkey/monkey_island/cc/models/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/cc/models/utils/document_encryptor.py b/monkey/monkey_island/cc/models/utils/document_encryptor.py index 0a0c2a62e..afec95afa 100644 --- a/monkey/monkey_island/cc/models/utils/document_encryptor.py +++ b/monkey/monkey_island/cc/models/utils/document_encryptor.py @@ -1,54 +1,48 @@ -from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Callable, List, Type import dpath.util -from monkey_island.cc.models.utils.field_types.field_type_abc import FieldTypeABC +from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor + + +class FieldNotFoundError(Exception): + pass @dataclass class SensitiveField: path: str path_separator = "." - field_type: Type[FieldTypeABC] + field_encryptor: Type[IFieldEncryptor] -class DocumentEncryptor(ABC): - @property - @abstractmethod - def sensitive_fields(self) -> List[SensitiveField]: - pass - - @classmethod - def encrypt(cls, document_dict: dict) -> dict: - for sensitive_field in cls.sensitive_fields: - DocumentEncryptor._apply_operation_to_document_field( - document_dict, sensitive_field, sensitive_field.field_type.encrypt - ) - - return document_dict - - @classmethod - def decrypt(cls, document_dict: dict) -> dict: - for sensitive_field in cls.sensitive_fields: - DocumentEncryptor._apply_operation_to_document_field( - document_dict, sensitive_field, sensitive_field.field_type.decrypt - ) - return document_dict - - @staticmethod - def _apply_operation_to_document_field( - report: dict, sensitive_field: SensitiveField, operation: Callable - ): - field_value = dpath.util.get( - report, sensitive_field.path, sensitive_field.path_separator, None +def encrypt(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict: + for sensitive_field in sensitive_fields: + _apply_operation_to_document_field( + document_dict, sensitive_field, sensitive_field.field_encryptor.encrypt ) - if field_value is None: - raise Exception( - f"Can't encrypt object because the path {sensitive_field.path} doesn't exist." - ) - modified_value = operation(field_value) + return document_dict - dpath.util.set(report, sensitive_field.path, modified_value, sensitive_field.path_separator) + +def decrypt(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict: + for sensitive_field in sensitive_fields: + _apply_operation_to_document_field( + document_dict, sensitive_field, sensitive_field.field_encryptor.decrypt + ) + return document_dict + + +def _apply_operation_to_document_field( + report: dict, sensitive_field: SensitiveField, operation: Callable +): + field_value = dpath.util.get(report, sensitive_field.path, sensitive_field.path_separator, None) + if field_value is None: + raise FieldNotFoundError( + f"Can't encrypt object because the path {sensitive_field.path} doesn't exist." + ) + + modified_value = operation(field_value) + + dpath.util.set(report, sensitive_field.path, modified_value, sensitive_field.path_separator) diff --git a/monkey/monkey_island/cc/models/utils/field_encryptors/__init__.py b/monkey/monkey_island/cc/models/utils/field_encryptors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py b/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py index 6ad1a78fb..b88b7d0b0 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py @@ -4,8 +4,8 @@ from typing import List import pytest from monkey_island.cc.models import Report +from monkey_island.cc.models.utils.document_encryptor import SensitiveField from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor -from monkey_island.cc.models.utils.report_encryptor import SensitiveField MOCK_SENSITIVE_FIELD_CONTENTS = ["the_string", "the_string2"] MOCK_REPORT_DICT = { @@ -19,32 +19,30 @@ MOCK_REPORT_DICT = { } -class MockFieldEncryptor(IFieldEncryptor): +class MockStringListEncryptor(IFieldEncryptor): plaintext = [] @staticmethod def encrypt(value: List[str]) -> List[str]: - return [MockFieldEncryptor._encrypt(v) for v in value] + return [MockStringListEncryptor._encrypt(v) for v in value] @staticmethod def _encrypt(value: str) -> str: - MockFieldEncryptor.plaintext.append(value) - return f"ENCRYPTED_{str(len(MockFieldEncryptor.plaintext) - 1)}" + MockStringListEncryptor.plaintext.append(value) + return f"ENCRYPTED_{str(len(MockStringListEncryptor.plaintext) - 1)}" @staticmethod def decrypt(value: List[str]) -> List[str]: - return MockFieldEncryptor.plaintext + return MockStringListEncryptor.plaintext @pytest.fixture(autouse=True) def patch_sensitive_fields(monkeypatch): mock_sensitive_fields = [ - SensitiveField("overview.foo.the_key", MockFieldEncryptor), - SensitiveField("overview.bar.the_key", MockFieldEncryptor), + SensitiveField("overview.foo.the_key", MockStringListEncryptor), + SensitiveField("overview.bar.the_key", MockStringListEncryptor), ] - monkeypatch.setattr( - "monkey_island.cc.models.utils.report_encryptor.sensitive_fields", mock_sensitive_fields - ) + monkeypatch.setattr("monkey_island.cc.models.report.sensitive_fields", mock_sensitive_fields) @pytest.mark.usefixtures("uses_database") From b2db5e77c4bb65faee70037ba3b8490b79670c3a Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 22 Sep 2021 15:38:03 +0300 Subject: [PATCH 04/16] Change test_string_list_encryptor.py to re-use fixture "uses_encryptor" rather than implementing the same fixture locally --- monkey/tests/unit_tests/monkey_island/cc/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monkey/tests/unit_tests/monkey_island/cc/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/conftest.py index 438ee3fef..f7d28b9de 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/conftest.py @@ -10,6 +10,8 @@ from tests.unit_tests.monkey_island.cc.server_utils.encryption.test_password_bas STANDARD_PLAINTEXT_MONKEY_CONFIG_FILENAME, ) +from monkey_island.cc.server_utils.encryptor import initialize_encryptor + @pytest.fixture def monkey_config(data_for_tests_dir): @@ -23,3 +25,8 @@ def monkey_config(data_for_tests_dir): @pytest.fixture def monkey_config_json(monkey_config): return json.dumps(monkey_config) + + +@pytest.fixture +def uses_encryptor(data_for_tests_dir): + initialize_encryptor(data_for_tests_dir) From 989d0ffd846d764cff62d2de66d684a73c5f28c6 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 22 Sep 2021 16:10:32 +0300 Subject: [PATCH 05/16] Add unit tests for telemetry model --- .../telemetries/test_telemetry_model.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py new file mode 100644 index 000000000..578aff235 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py @@ -0,0 +1,89 @@ +from copy import deepcopy +from datetime import datetime + +import pytest + +from monkey_island.cc.models.telemetries.telemetry import Telemetry +from monkey_island.cc.models.utils.document_encryptor import SensitiveField +from monkey_island.cc.models.utils.field_encryptors.mimikatz_results_encryptor import ( + MimikatzResultsEncryptor, +) + +MOCK_CREDENTIALS = { + "Vakaris": { + "username": "M0nk3y", + "password": "", + "ntlm_hash": "e87f2f73e353f1d95e42ce618601b61f", + "lm_hash": "", + }, + "user": {"username": "user", "password": "test", "ntlm_hash": "", "lm_hash": ""}, +} + +MOCK_DATA_DICT = { + "network_info": {}, + "credentials": deepcopy(MOCK_CREDENTIALS), + "mimikatz": deepcopy(MOCK_CREDENTIALS), +} + +MOCK_TELEMETRY = { + "timestamp": datetime.now(), + "command_control_channel": { + "src": "192.168.56.1", + "dst": "192.168.56.2", + }, + "monkey_guid": "211375648895908", + "telem_category": "system_info", + "data": MOCK_DATA_DICT, +} + +MOCK_NO_ENCRYPTION_NEEDED_TELEMETRY = { + "timestamp": datetime.now(), + "command_control_channel": { + "src": "192.168.56.1", + "dst": "192.168.56.2", + }, + "monkey_guid": "211375648895908", + "telem_category": "state", + "data": {"done": False}, +} + +MOCK_SENSITIVE_FIELDS = [ + SensitiveField("data.credentials", MimikatzResultsEncryptor), + SensitiveField("data.mimikatz", MimikatzResultsEncryptor), +] + + +@pytest.fixture(autouse=True) +def patch_sensitive_fields(monkeypatch): + monkeypatch.setattr( + "monkey_island.cc.models.telemetries.telemetry.sensitive_fields", + MOCK_SENSITIVE_FIELDS, + ) + + +@pytest.mark.usefixtures("uses_database", "uses_encryptor") +def test_telemetry_encryption(monkeypatch): + + Telemetry.save_telemetry(MOCK_TELEMETRY) + assert ( + not Telemetry.objects.first()["data"]["credentials"]["user"]["password"] + == MOCK_CREDENTIALS["user"]["password"] + ) + assert ( + not Telemetry.objects.first()["data"]["mimikatz"]["Vakaris"]["ntlm_hash"] + == MOCK_CREDENTIALS["Vakaris"]["ntlm_hash"] + ) + assert ( + Telemetry.get_telemetry()["data"]["credentials"]["user"]["password"] + == MOCK_CREDENTIALS["user"]["password"] + ) + assert ( + Telemetry.get_telemetry()["data"]["mimikatz"]["Vakaris"]["ntlm_hash"] + == MOCK_CREDENTIALS["Vakaris"]["ntlm_hash"] + ) + + +@pytest.mark.usefixtures("uses_database", "uses_encryptor") +def test_no_encryption_needed(monkeypatch, data_for_tests_dir): + # Make sure telemetry save doesn't break when telemetry doesn't need encryption + Telemetry.save_telemetry(MOCK_NO_ENCRYPTION_NEEDED_TELEMETRY) From 1ab0fe7b138947400a29408ffed2bbb96ef39199 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 22 Sep 2021 16:12:50 +0300 Subject: [PATCH 06/16] Add Telemetry model --- .../cc/models/telemetries/telemetry.py | 50 +++++++++++++++++++ .../mimikatz_results_encryptor.py | 21 ++++++++ .../cc/models/telemetries/__init__.py | 0 3 files changed, 71 insertions(+) create mode 100644 monkey/monkey_island/cc/models/telemetries/telemetry.py create mode 100644 monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/models/telemetries/__init__.py diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry.py b/monkey/monkey_island/cc/models/telemetries/telemetry.py new file mode 100644 index 000000000..bbefeb92f --- /dev/null +++ b/monkey/monkey_island/cc/models/telemetries/telemetry.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from mongoengine import DateTimeField, DictField, Document, EmbeddedDocumentField, StringField + +from monkey_island.cc.models import CommandControlChannel +from monkey_island.cc.models.utils import document_encryptor +from monkey_island.cc.models.utils.document_encryptor import FieldNotFoundError, SensitiveField +from monkey_island.cc.models.utils.field_encryptors.mimikatz_results_encryptor import ( + MimikatzResultsEncryptor, +) + +sensitive_fields = [ + SensitiveField("data.credentials", MimikatzResultsEncryptor), + SensitiveField("data.mimikatz", MimikatzResultsEncryptor), +] + + +class Telemetry(Document): + + data = DictField(required=True) + timestamp = DateTimeField(required=True) + monkey_guid = StringField(required=True) + telem_category = StringField(required=True) + command_control_channel = EmbeddedDocumentField(CommandControlChannel) + + meta = {"strict": False} + + @staticmethod + def save_telemetry(telemetry_dict: dict): + try: + telemetry_dict = document_encryptor.encrypt(sensitive_fields, telemetry_dict) + except FieldNotFoundError: + pass # Not all telemetries require encryption + + cc_channel = CommandControlChannel( + src=telemetry_dict["command_control_channel"]["src"], + dst=telemetry_dict["command_control_channel"]["dst"], + ) + Telemetry( + data=telemetry_dict["data"], + timestamp=telemetry_dict["timestamp"], + monkey_guid=telemetry_dict["monkey_guid"], + telem_category=telemetry_dict["telem_category"], + command_control_channel=cc_channel, + ).save() + + @staticmethod + def get_telemetry() -> dict: + telemetry_dict = Telemetry.objects.first().to_mongo() + return document_encryptor.decrypt(sensitive_fields, telemetry_dict) diff --git a/monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py b/monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py new file mode 100644 index 000000000..c924a64e9 --- /dev/null +++ b/monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py @@ -0,0 +1,21 @@ +from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor +from monkey_island.cc.server_utils.encryptor import get_encryptor + + +class MimikatzResultsEncryptor(IFieldEncryptor): + + secret_types = ["password", "ntlm_hash", "lm_hash"] + + @staticmethod + def encrypt(results: dict) -> dict: + for _, credentials in results.items(): + for secret_type in MimikatzResultsEncryptor.secret_types: + credentials[secret_type] = get_encryptor().enc(credentials[secret_type]) + return results + + @staticmethod + def decrypt(results: dict) -> dict: + for _, credentials in results.items(): + for secret_type in MimikatzResultsEncryptor.secret_types: + credentials[secret_type] = get_encryptor().dec(credentials[secret_type]) + return results diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/__init__.py new file mode 100644 index 000000000..e69de29bb From 3781095f25b4e1e9130068c7bd1a72510c3f0105 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 24 Sep 2021 12:17:39 +0300 Subject: [PATCH 07/16] Change the mock database name to "db", because all of the codebase is using this database. This change enables us to write unit tests without the need to patch the the database name in all of the mongo queries that look like "mongo.db.collection" --- .../tests/unit_tests/monkey_island/cc/mongomock_fixtures.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py b/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py index aa00aa69a..4fac539c1 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py +++ b/monkey/tests/unit_tests/monkey_island/cc/mongomock_fixtures.py @@ -1,7 +1,10 @@ import mongoengine import pytest -MOCK_DB_NAME = "mongoenginetest" +# Database name has to match the db used in the codebase, +# else the name needs to be mocked during tests. +# Currently its used like so: "mongo.db.telemetry.find()". +MOCK_DB_NAME = "db" @pytest.fixture(scope="module", autouse=True) From e6ad125be97d93aa8c09a0cd84596240c55ffb6b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 24 Sep 2021 12:42:31 +0300 Subject: [PATCH 08/16] Change the telemetry model to have a method for fetching the telemetries based on queries. Telemetry code mainly uses queries and mongoengine has no good way of field encryption, that's why this method prefers to handle queries rather than Telemetry models --- .../cc/models/telemetries/__init__.py | 1 + .../cc/models/telemetries/telemetry.py | 19 ++++++++++++++----- .../telemetries/test_telemetry_model.py | 13 ++++++++++--- vulture_allowlist.py | 3 +++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/models/telemetries/__init__.py b/monkey/monkey_island/cc/models/telemetries/__init__.py index e69de29bb..b3421bbd4 100644 --- a/monkey/monkey_island/cc/models/telemetries/__init__.py +++ b/monkey/monkey_island/cc/models/telemetries/__init__.py @@ -0,0 +1 @@ +from .telemetry import Telemetry # noqa: F401 diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry.py b/monkey/monkey_island/cc/models/telemetries/telemetry.py index bbefeb92f..371484b85 100644 --- a/monkey/monkey_island/cc/models/telemetries/telemetry.py +++ b/monkey/monkey_island/cc/models/telemetries/telemetry.py @@ -1,7 +1,10 @@ from __future__ import annotations -from mongoengine import DateTimeField, DictField, Document, EmbeddedDocumentField, StringField +from typing import List +from mongoengine import DateTimeField, Document, DynamicField, EmbeddedDocumentField, StringField + +from monkey_island.cc.database import mongo from monkey_island.cc.models import CommandControlChannel from monkey_island.cc.models.utils import document_encryptor from monkey_island.cc.models.utils.document_encryptor import FieldNotFoundError, SensitiveField @@ -17,7 +20,7 @@ sensitive_fields = [ class Telemetry(Document): - data = DictField(required=True) + data = DynamicField(required=True) timestamp = DateTimeField(required=True) monkey_guid = StringField(required=True) telem_category = StringField(required=True) @@ -45,6 +48,12 @@ class Telemetry(Document): ).save() @staticmethod - def get_telemetry() -> dict: - telemetry_dict = Telemetry.objects.first().to_mongo() - return document_encryptor.decrypt(sensitive_fields, telemetry_dict) + def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]: + telemetries = mongo.db.telemetry.find(query, output_fields) + decrypted_list = [] + for telemetry in telemetries: + try: + decrypted_list.append(document_encryptor.decrypt(sensitive_fields, telemetry)) + except FieldNotFoundError: + decrypted_list.append(telemetry) + return decrypted_list diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py index 578aff235..860ae05fb 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py @@ -1,9 +1,10 @@ from copy import deepcopy from datetime import datetime +import mongoengine import pytest -from monkey_island.cc.models.telemetries.telemetry import Telemetry +from monkey_island.cc.models.telemetries import Telemetry from monkey_island.cc.models.utils.document_encryptor import SensitiveField from monkey_island.cc.models.utils.field_encryptors.mimikatz_results_encryptor import ( MimikatzResultsEncryptor, @@ -61,6 +62,12 @@ def patch_sensitive_fields(monkeypatch): ) +@pytest.fixture(autouse=True) +def fake_mongo(monkeypatch): + mongo = mongoengine.connection.get_connection() + monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry.mongo", mongo) + + @pytest.mark.usefixtures("uses_database", "uses_encryptor") def test_telemetry_encryption(monkeypatch): @@ -74,11 +81,11 @@ def test_telemetry_encryption(monkeypatch): == MOCK_CREDENTIALS["Vakaris"]["ntlm_hash"] ) assert ( - Telemetry.get_telemetry()["data"]["credentials"]["user"]["password"] + Telemetry.get_telemetry_by_query({})[0]["data"]["credentials"]["user"]["password"] == MOCK_CREDENTIALS["user"]["password"] ) assert ( - Telemetry.get_telemetry()["data"]["mimikatz"]["Vakaris"]["ntlm_hash"] + Telemetry.get_telemetry_by_query({})[0]["data"]["mimikatz"]["Vakaris"]["ntlm_hash"] == MOCK_CREDENTIALS["Vakaris"]["ntlm_hash"] ) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 905cc74ad..d58d4ea9b 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -4,6 +4,7 @@ dead or is kept deliberately. Referencing these in a file like this makes sure t Vulture doesn't mark these as dead again. """ from monkey_island.cc.models import Report +from monkey_island.cc.models.telemetries import Telemetry fake_monkey_dir_path # unused variable (monkey/tests/infection_monkey/post_breach/actions/test_users_custom_pba.py:37) set_os_linux # unused variable (monkey/tests/infection_monkey/post_breach/actions/test_users_custom_pba.py:37) @@ -182,6 +183,8 @@ Report.glance Report.meta_info Report.meta Report.save_report +Telemetry.save_telemetry +Telemetry.get_telemetry_by_query # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23) From ace60052da10595e820c7f1568c1e48519477797 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 24 Sep 2021 13:12:43 +0300 Subject: [PATCH 09/16] Alter usages of telemetry collection in report to store/fetch system info telemetry using the Telemetry model This is required to automatically encrypt/decrypt the telemetries and it's a good practice to have a DAL for telemetries --- .../blackbox/telemetry_blackbox_endpoint.py | 4 +-- .../monkey_island/cc/resources/telemetry.py | 8 +++--- .../cc/services/reporting/report.py | 3 ++- .../cc/services/reporting/test_report.py | 27 ++++++++++++++----- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py b/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py index 5573e5152..f784bc323 100644 --- a/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py +++ b/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py @@ -2,7 +2,7 @@ import flask_restful from bson import json_util from flask import request -from monkey_island.cc.database import mongo +from monkey_island.cc.models.telemetries import Telemetry from monkey_island.cc.resources.auth.auth import jwt_required @@ -10,4 +10,4 @@ class TelemetryBlackboxEndpoint(flask_restful.Resource): @jwt_required def get(self, **kw): find_query = json_util.loads(request.args.get("find_query")) - return {"results": list(mongo.db.telemetry.find(find_query))} + return {"results": list(Telemetry.get_telemetry_by_query(find_query))} diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index 525197f0f..3ab2c1242 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -9,6 +9,7 @@ from flask import request from common.common_consts.telem_categories import TelemCategoryEnum from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey import Monkey +from monkey_island.cc.models.telemetries.telemetry import Telemetry as TelemetryModel from monkey_island.cc.resources.auth.auth import jwt_required from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService @@ -37,7 +38,7 @@ class Telemetry(flask_restful.Resource): find_filter["timestamp"] = {"$gt": dateutil.parser.parse(timestamp)} result["objects"] = self.telemetry_to_displayed_telemetry( - mongo.db.telemetry.find(find_filter) + TelemetryModel.get_telemetry_by_query(query=find_filter) ) return result @@ -60,8 +61,9 @@ class Telemetry(flask_restful.Resource): process_telemetry(telemetry_json) - telem_id = mongo.db.telemetry.insert(telemetry_json) - return mongo.db.telemetry.find_one_or_404({"_id": telem_id}) + TelemetryModel.save_telemetry(telemetry_json) + + return {}, 201 @staticmethod def telemetry_to_displayed_telemetry(telemetry): diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 7d14d4f4a..32675f1d5 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -15,6 +15,7 @@ from common.network.network_range import NetworkRange from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey, Report +from monkey_island.cc.models.telemetries import Telemetry from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.configuration.utils import ( get_config_network_segments_as_subnet_groups, @@ -165,7 +166,7 @@ class ReportService: @staticmethod def _get_credentials_from_system_info_telems(): formatted_creds = [] - for telem in mongo.db.telemetry.find( + for telem in Telemetry.get_telemetry_by_query( {"telem_category": "system_info", "data.credentials": {"$exists": True}}, {"data.credentials": 1, "monkey_guid": 1}, ): diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index 0093e4235..9f845b263 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -1,10 +1,11 @@ import datetime from copy import deepcopy -import mongomock +import mongoengine import pytest from bson import ObjectId +from monkey_island.cc.models.telemetries import Telemetry from monkey_island.cc.services.reporting.report import ReportService TELEM_ID = { @@ -49,6 +50,11 @@ SYSTEM_INFO_TELEMETRY_TELEM = { "_id": TELEM_ID["system_info_creds"], "monkey_guid": MONKEY_GUID, "telem_category": "system_info", + "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), + "command_control_channel": { + "src": "192.168.56.1", + "dst": "192.168.56.2", + }, "data": { "credentials": { USER: { @@ -64,6 +70,11 @@ NO_CREDS_TELEMETRY_TELEM = { "_id": TELEM_ID["no_creds"], "monkey_guid": MONKEY_GUID, "telem_category": "exploit", + "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000), + "command_control_channel": { + "src": "192.168.56.1", + "dst": "192.168.56.2", + }, "data": { "machine": { "ip_addr": VICTIM_IP, @@ -125,13 +136,14 @@ NODE_DICT_FAILED_EXPLOITS["exploits"][1]["result"] = False @pytest.fixture def fake_mongo(monkeypatch): - mongo = mongomock.MongoClient() + mongo = mongoengine.connection.get_connection() monkeypatch.setattr("monkey_island.cc.services.reporting.report.mongo", mongo) + monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry.mongo", mongo) monkeypatch.setattr("monkey_island.cc.services.node.mongo", mongo) return mongo -def test_get_stolen_creds_exploit(fake_mongo): +def test_get_stolen_creds_exploit(fake_mongo, uses_database): fake_mongo.db.telemetry.insert_one(EXPLOIT_TELEMETRY_TELEM) stolen_creds_exploit = ReportService.get_stolen_creds() @@ -143,9 +155,9 @@ def test_get_stolen_creds_exploit(fake_mongo): assert expected_stolen_creds_exploit == stolen_creds_exploit -def test_get_stolen_creds_system_info(fake_mongo): +def test_get_stolen_creds_system_info(fake_mongo, uses_database): fake_mongo.db.monkey.insert_one(MONKEY_TELEM) - fake_mongo.db.telemetry.insert_one(SYSTEM_INFO_TELEMETRY_TELEM) + Telemetry.save_telemetry(SYSTEM_INFO_TELEMETRY_TELEM) stolen_creds_system_info = ReportService.get_stolen_creds() expected_stolen_creds_system_info = [ @@ -157,8 +169,9 @@ def test_get_stolen_creds_system_info(fake_mongo): assert expected_stolen_creds_system_info == stolen_creds_system_info -def test_get_stolen_creds_no_creds(fake_mongo): - fake_mongo.db.telemetry.insert_one(NO_CREDS_TELEMETRY_TELEM) +def test_get_stolen_creds_no_creds(fake_mongo, uses_database): + fake_mongo.db.monkey.insert_one(MONKEY_TELEM) + Telemetry.save_telemetry(NO_CREDS_TELEMETRY_TELEM) stolen_creds_no_creds = ReportService.get_stolen_creds() expected_stolen_creds_no_creds = [] From 51f6fbe3569272a311ae266f62ab08ec0f9b34a7 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 27 Sep 2021 16:29:41 +0300 Subject: [PATCH 10/16] Adjust island conftest.py to also rename the encryptor to datastore_encryptor --- monkey/tests/unit_tests/monkey_island/cc/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/conftest.py index f7d28b9de..9cca0caab 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/conftest.py @@ -10,7 +10,7 @@ from tests.unit_tests.monkey_island.cc.server_utils.encryption.test_password_bas STANDARD_PLAINTEXT_MONKEY_CONFIG_FILENAME, ) -from monkey_island.cc.server_utils.encryptor import initialize_encryptor +from monkey_island.cc.server_utils.encryption import initialize_datastore_encryptor @pytest.fixture @@ -29,4 +29,4 @@ def monkey_config_json(monkey_config): @pytest.fixture def uses_encryptor(data_for_tests_dir): - initialize_encryptor(data_for_tests_dir) + initialize_datastore_encryptor(data_for_tests_dir) From 46f263be5f58e605e5afa813ebff659fb350b097 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 27 Sep 2021 16:51:59 +0300 Subject: [PATCH 11/16] Separate the telemetry document from telemetry_dal, also extracted external interface into __init__.py files --- monkey/monkey_island/cc/models/report.py | 9 ++-- .../cc/models/telemetries/__init__.py | 2 +- .../cc/models/telemetries/telemetry.py | 45 ------------------ .../cc/models/telemetries/telemetry_dal.py | 46 +++++++++++++++++++ .../monkey_island/cc/models/utils/__init__.py | 0 .../models/utils/field_encryptors/__init__.py | 0 .../mimikatz_results_encryptor.py | 21 --------- .../blackbox/telemetry_blackbox_endpoint.py | 4 +- .../monkey_island/cc/resources/telemetry.py | 6 +-- .../cc/services/reporting/report.py | 4 +- monkey/monkey_island/cc/utils/__init__.py | 1 + .../dict_encryptor.py} | 2 +- .../cc/utils/field_encryptors/__init__.py | 3 ++ .../field_encryptors/i_field_encryptor.py | 0 .../mimikatz_results_encryptor.py | 34 ++++++++++++++ .../field_encryptors/string_list_encryptor.py | 2 +- ...lemetry_model.py => test_telemetry_dal.py} | 25 +++++----- .../cc/models/test_report_model.py | 4 +- .../test_string_list_encryptor.py | 2 +- .../cc/services/reporting/test_report.py | 17 ++++--- 20 files changed, 123 insertions(+), 104 deletions(-) create mode 100644 monkey/monkey_island/cc/models/telemetries/telemetry_dal.py delete mode 100644 monkey/monkey_island/cc/models/utils/__init__.py delete mode 100644 monkey/monkey_island/cc/models/utils/field_encryptors/__init__.py delete mode 100644 monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py create mode 100644 monkey/monkey_island/cc/utils/__init__.py rename monkey/monkey_island/cc/{models/utils/document_encryptor.py => utils/dict_encryptor.py} (93%) create mode 100644 monkey/monkey_island/cc/utils/field_encryptors/__init__.py rename monkey/monkey_island/cc/{models => }/utils/field_encryptors/i_field_encryptor.py (100%) create mode 100644 monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py rename monkey/monkey_island/cc/{models => }/utils/field_encryptors/string_list_encryptor.py (81%) rename monkey/tests/unit_tests/monkey_island/cc/models/telemetries/{test_telemetry_model.py => test_telemetry_dal.py} (74%) diff --git a/monkey/monkey_island/cc/models/report.py b/monkey/monkey_island/cc/models/report.py index fb1ab3cea..cbdde0dcb 100644 --- a/monkey/monkey_island/cc/models/report.py +++ b/monkey/monkey_island/cc/models/report.py @@ -3,9 +3,8 @@ from __future__ import annotations from bson import json_util from mongoengine import DictField, Document -from monkey_island.cc.models.utils import document_encryptor -from monkey_island.cc.models.utils.document_encryptor import SensitiveField -from monkey_island.cc.models.utils.field_encryptors.string_list_encryptor import StringListEncryptor +from monkey_island.cc.utils import SensitiveField, dict_encryptor +from monkey_island.cc.utils.field_encryptors import StringListEncryptor sensitive_fields = [ SensitiveField(path="overview.config_passwords", field_encryptor=StringListEncryptor) @@ -24,7 +23,7 @@ class Report(Document): @staticmethod def save_report(report_dict: dict): report_dict = _encode_dot_char_before_mongo_insert(report_dict) - report_dict = document_encryptor.encrypt(sensitive_fields, report_dict) + report_dict = dict_encryptor.encrypt(sensitive_fields, report_dict) Report.objects.delete() Report( overview=report_dict["overview"], @@ -37,7 +36,7 @@ class Report(Document): def get_report() -> dict: report_dict = Report.objects.first().to_mongo() return _decode_dot_char_before_mongo_insert( - document_encryptor.decrypt(sensitive_fields, report_dict) + dict_encryptor.decrypt(sensitive_fields, report_dict) ) diff --git a/monkey/monkey_island/cc/models/telemetries/__init__.py b/monkey/monkey_island/cc/models/telemetries/__init__.py index b3421bbd4..6c00785f7 100644 --- a/monkey/monkey_island/cc/models/telemetries/__init__.py +++ b/monkey/monkey_island/cc/models/telemetries/__init__.py @@ -1 +1 @@ -from .telemetry import Telemetry # noqa: F401 +from .telemetry_dal import save_telemetry, get_telemetry_by_query diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry.py b/monkey/monkey_island/cc/models/telemetries/telemetry.py index 371484b85..3da3d2b10 100644 --- a/monkey/monkey_island/cc/models/telemetries/telemetry.py +++ b/monkey/monkey_island/cc/models/telemetries/telemetry.py @@ -1,21 +1,6 @@ -from __future__ import annotations - -from typing import List - from mongoengine import DateTimeField, Document, DynamicField, EmbeddedDocumentField, StringField -from monkey_island.cc.database import mongo from monkey_island.cc.models import CommandControlChannel -from monkey_island.cc.models.utils import document_encryptor -from monkey_island.cc.models.utils.document_encryptor import FieldNotFoundError, SensitiveField -from monkey_island.cc.models.utils.field_encryptors.mimikatz_results_encryptor import ( - MimikatzResultsEncryptor, -) - -sensitive_fields = [ - SensitiveField("data.credentials", MimikatzResultsEncryptor), - SensitiveField("data.mimikatz", MimikatzResultsEncryptor), -] class Telemetry(Document): @@ -27,33 +12,3 @@ class Telemetry(Document): command_control_channel = EmbeddedDocumentField(CommandControlChannel) meta = {"strict": False} - - @staticmethod - def save_telemetry(telemetry_dict: dict): - try: - telemetry_dict = document_encryptor.encrypt(sensitive_fields, telemetry_dict) - except FieldNotFoundError: - pass # Not all telemetries require encryption - - cc_channel = CommandControlChannel( - src=telemetry_dict["command_control_channel"]["src"], - dst=telemetry_dict["command_control_channel"]["dst"], - ) - Telemetry( - data=telemetry_dict["data"], - timestamp=telemetry_dict["timestamp"], - monkey_guid=telemetry_dict["monkey_guid"], - telem_category=telemetry_dict["telem_category"], - command_control_channel=cc_channel, - ).save() - - @staticmethod - def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]: - telemetries = mongo.db.telemetry.find(query, output_fields) - decrypted_list = [] - for telemetry in telemetries: - try: - decrypted_list.append(document_encryptor.decrypt(sensitive_fields, telemetry)) - except FieldNotFoundError: - decrypted_list.append(telemetry) - return decrypted_list diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py new file mode 100644 index 000000000..88e617725 --- /dev/null +++ b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import List + +from monkey_island.cc.database import mongo +from monkey_island.cc.models import CommandControlChannel +from monkey_island.cc.models.telemetries.telemetry import Telemetry +from monkey_island.cc.utils import FieldNotFoundError, SensitiveField, dict_encryptor +from monkey_island.cc.utils.field_encryptors import MimikatzResultsEncryptor + +sensitive_fields = [ + SensitiveField("data.credentials", MimikatzResultsEncryptor), + SensitiveField("data.mimikatz", MimikatzResultsEncryptor), +] + + +def save_telemetry(telemetry_dict: dict): + try: + telemetry_dict = dict_encryptor.encrypt(sensitive_fields, telemetry_dict) + except FieldNotFoundError: + pass # Not all telemetries require encryption + + cc_channel = CommandControlChannel( + src=telemetry_dict["command_control_channel"]["src"], + dst=telemetry_dict["command_control_channel"]["dst"], + ) + Telemetry( + data=telemetry_dict["data"], + timestamp=telemetry_dict["timestamp"], + monkey_guid=telemetry_dict["monkey_guid"], + telem_category=telemetry_dict["telem_category"], + command_control_channel=cc_channel, + ).save() + + +# A lot of codebase is using queries for telemetry collection and document field encryption is +# not yet implemented in mongoengine. To avoid big time investment, queries are used for now. +def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]: + telemetries = mongo.db.telemetry.find(query, output_fields) + decrypted_list = [] + for telemetry in telemetries: + try: + decrypted_list.append(dict_encryptor.decrypt(sensitive_fields, telemetry)) + except FieldNotFoundError: + decrypted_list.append(telemetry) + return decrypted_list diff --git a/monkey/monkey_island/cc/models/utils/__init__.py b/monkey/monkey_island/cc/models/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/monkey/monkey_island/cc/models/utils/field_encryptors/__init__.py b/monkey/monkey_island/cc/models/utils/field_encryptors/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py b/monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py deleted file mode 100644 index c924a64e9..000000000 --- a/monkey/monkey_island/cc/models/utils/field_encryptors/mimikatz_results_encryptor.py +++ /dev/null @@ -1,21 +0,0 @@ -from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor -from monkey_island.cc.server_utils.encryptor import get_encryptor - - -class MimikatzResultsEncryptor(IFieldEncryptor): - - secret_types = ["password", "ntlm_hash", "lm_hash"] - - @staticmethod - def encrypt(results: dict) -> dict: - for _, credentials in results.items(): - for secret_type in MimikatzResultsEncryptor.secret_types: - credentials[secret_type] = get_encryptor().enc(credentials[secret_type]) - return results - - @staticmethod - def decrypt(results: dict) -> dict: - for _, credentials in results.items(): - for secret_type in MimikatzResultsEncryptor.secret_types: - credentials[secret_type] = get_encryptor().dec(credentials[secret_type]) - return results diff --git a/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py b/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py index f784bc323..f1e958e3e 100644 --- a/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py +++ b/monkey/monkey_island/cc/resources/blackbox/telemetry_blackbox_endpoint.py @@ -2,7 +2,7 @@ import flask_restful from bson import json_util from flask import request -from monkey_island.cc.models.telemetries import Telemetry +from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.resources.auth.auth import jwt_required @@ -10,4 +10,4 @@ class TelemetryBlackboxEndpoint(flask_restful.Resource): @jwt_required def get(self, **kw): find_query = json_util.loads(request.args.get("find_query")) - return {"results": list(Telemetry.get_telemetry_by_query(find_query))} + return {"results": list(get_telemetry_by_query(find_query))} diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index 3ab2c1242..1158e82f0 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -9,7 +9,7 @@ from flask import request from common.common_consts.telem_categories import TelemCategoryEnum from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey import Monkey -from monkey_island.cc.models.telemetries.telemetry import Telemetry as TelemetryModel +from monkey_island.cc.models.telemetries import get_telemetry_by_query, save_telemetry from monkey_island.cc.resources.auth.auth import jwt_required from monkey_island.cc.resources.blackbox.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService @@ -38,7 +38,7 @@ class Telemetry(flask_restful.Resource): find_filter["timestamp"] = {"$gt": dateutil.parser.parse(timestamp)} result["objects"] = self.telemetry_to_displayed_telemetry( - TelemetryModel.get_telemetry_by_query(query=find_filter) + get_telemetry_by_query(query=find_filter) ) return result @@ -61,7 +61,7 @@ class Telemetry(flask_restful.Resource): process_telemetry(telemetry_json) - TelemetryModel.save_telemetry(telemetry_json) + save_telemetry(telemetry_json) return {}, 201 diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 32675f1d5..17e6ce037 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -15,7 +15,7 @@ from common.network.network_range import NetworkRange from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey, Report -from monkey_island.cc.models.telemetries import Telemetry +from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.configuration.utils import ( get_config_network_segments_as_subnet_groups, @@ -166,7 +166,7 @@ class ReportService: @staticmethod def _get_credentials_from_system_info_telems(): formatted_creds = [] - for telem in Telemetry.get_telemetry_by_query( + for telem in get_telemetry_by_query( {"telem_category": "system_info", "data.credentials": {"$exists": True}}, {"data.credentials": 1, "monkey_guid": 1}, ): diff --git a/monkey/monkey_island/cc/utils/__init__.py b/monkey/monkey_island/cc/utils/__init__.py new file mode 100644 index 000000000..abe040645 --- /dev/null +++ b/monkey/monkey_island/cc/utils/__init__.py @@ -0,0 +1 @@ +from .dict_encryptor import FieldNotFoundError, SensitiveField diff --git a/monkey/monkey_island/cc/models/utils/document_encryptor.py b/monkey/monkey_island/cc/utils/dict_encryptor.py similarity index 93% rename from monkey/monkey_island/cc/models/utils/document_encryptor.py rename to monkey/monkey_island/cc/utils/dict_encryptor.py index afec95afa..9a6d1d3d0 100644 --- a/monkey/monkey_island/cc/models/utils/document_encryptor.py +++ b/monkey/monkey_island/cc/utils/dict_encryptor.py @@ -3,7 +3,7 @@ from typing import Callable, List, Type import dpath.util -from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor +from monkey_island.cc.utils.field_encryptors import IFieldEncryptor class FieldNotFoundError(Exception): diff --git a/monkey/monkey_island/cc/utils/field_encryptors/__init__.py b/monkey/monkey_island/cc/utils/field_encryptors/__init__.py new file mode 100644 index 000000000..7c938d25b --- /dev/null +++ b/monkey/monkey_island/cc/utils/field_encryptors/__init__.py @@ -0,0 +1,3 @@ +from .i_field_encryptor import IFieldEncryptor +from .mimikatz_results_encryptor import MimikatzResultsEncryptor +from .string_list_encryptor import StringListEncryptor diff --git a/monkey/monkey_island/cc/models/utils/field_encryptors/i_field_encryptor.py b/monkey/monkey_island/cc/utils/field_encryptors/i_field_encryptor.py similarity index 100% rename from monkey/monkey_island/cc/models/utils/field_encryptors/i_field_encryptor.py rename to monkey/monkey_island/cc/utils/field_encryptors/i_field_encryptor.py diff --git a/monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py b/monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py new file mode 100644 index 000000000..6708ec40c --- /dev/null +++ b/monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py @@ -0,0 +1,34 @@ +import logging + +from monkey_island.cc.server_utils.encryption import get_datastore_encryptor +from monkey_island.cc.utils.field_encryptors import IFieldEncryptor + +logger = logging.getLogger(__name__) + + +class MimikatzResultsEncryptor(IFieldEncryptor): + + secret_types = ["password", "ntlm_hash", "lm_hash"] + + @staticmethod + def encrypt(results: dict) -> dict: + for _, credentials in results.items(): + for secret_type in MimikatzResultsEncryptor.secret_types: + try: + credentials[secret_type] = get_datastore_encryptor().enc( + credentials[secret_type] + ) + except ValueError as e: + logger.error( + f"Failed encrypting sensitive field for " + f"user {credentials['username']}! Error: {e}" + ) + credentials[secret_type] = get_datastore_encryptor().enc("") + return results + + @staticmethod + def decrypt(results: dict) -> dict: + for _, credentials in results.items(): + for secret_type in MimikatzResultsEncryptor.secret_types: + credentials[secret_type] = get_datastore_encryptor().dec(credentials[secret_type]) + return results diff --git a/monkey/monkey_island/cc/models/utils/field_encryptors/string_list_encryptor.py b/monkey/monkey_island/cc/utils/field_encryptors/string_list_encryptor.py similarity index 81% rename from monkey/monkey_island/cc/models/utils/field_encryptors/string_list_encryptor.py rename to monkey/monkey_island/cc/utils/field_encryptors/string_list_encryptor.py index 089155289..f939c0e22 100644 --- a/monkey/monkey_island/cc/models/utils/field_encryptors/string_list_encryptor.py +++ b/monkey/monkey_island/cc/utils/field_encryptors/string_list_encryptor.py @@ -1,7 +1,7 @@ from typing import List -from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor from monkey_island.cc.server_utils.encryption import get_datastore_encryptor +from monkey_island.cc.utils.field_encryptors import IFieldEncryptor class StringListEncryptor(IFieldEncryptor): diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py similarity index 74% rename from monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py rename to monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py index 860ae05fb..f25e8ffb3 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_model.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py @@ -4,11 +4,10 @@ from datetime import datetime import mongoengine import pytest -from monkey_island.cc.models.telemetries import Telemetry -from monkey_island.cc.models.utils.document_encryptor import SensitiveField -from monkey_island.cc.models.utils.field_encryptors.mimikatz_results_encryptor import ( - MimikatzResultsEncryptor, -) +from monkey_island.cc.models.telemetries import get_telemetry_by_query, save_telemetry +from monkey_island.cc.models.telemetries.telemetry import Telemetry +from monkey_island.cc.utils import SensitiveField +from monkey_island.cc.utils.field_encryptors import MimikatzResultsEncryptor MOCK_CREDENTIALS = { "Vakaris": { @@ -57,7 +56,7 @@ MOCK_SENSITIVE_FIELDS = [ @pytest.fixture(autouse=True) def patch_sensitive_fields(monkeypatch): monkeypatch.setattr( - "monkey_island.cc.models.telemetries.telemetry.sensitive_fields", + "monkey_island.cc.models.telemetries.telemetry_dal.sensitive_fields", MOCK_SENSITIVE_FIELDS, ) @@ -65,13 +64,13 @@ def patch_sensitive_fields(monkeypatch): @pytest.fixture(autouse=True) def fake_mongo(monkeypatch): mongo = mongoengine.connection.get_connection() - monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry.mongo", mongo) + monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry_dal.mongo", mongo) @pytest.mark.usefixtures("uses_database", "uses_encryptor") -def test_telemetry_encryption(monkeypatch): +def test_telemetry_encryption(): - Telemetry.save_telemetry(MOCK_TELEMETRY) + save_telemetry(MOCK_TELEMETRY) assert ( not Telemetry.objects.first()["data"]["credentials"]["user"]["password"] == MOCK_CREDENTIALS["user"]["password"] @@ -81,16 +80,16 @@ def test_telemetry_encryption(monkeypatch): == MOCK_CREDENTIALS["Vakaris"]["ntlm_hash"] ) assert ( - Telemetry.get_telemetry_by_query({})[0]["data"]["credentials"]["user"]["password"] + get_telemetry_by_query({})[0]["data"]["credentials"]["user"]["password"] == MOCK_CREDENTIALS["user"]["password"] ) assert ( - Telemetry.get_telemetry_by_query({})[0]["data"]["mimikatz"]["Vakaris"]["ntlm_hash"] + get_telemetry_by_query({})[0]["data"]["mimikatz"]["Vakaris"]["ntlm_hash"] == MOCK_CREDENTIALS["Vakaris"]["ntlm_hash"] ) @pytest.mark.usefixtures("uses_database", "uses_encryptor") -def test_no_encryption_needed(monkeypatch, data_for_tests_dir): +def test_no_encryption_needed(): # Make sure telemetry save doesn't break when telemetry doesn't need encryption - Telemetry.save_telemetry(MOCK_NO_ENCRYPTION_NEEDED_TELEMETRY) + save_telemetry(MOCK_NO_ENCRYPTION_NEEDED_TELEMETRY) diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py b/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py index b88b7d0b0..0c8fd90de 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py @@ -4,8 +4,8 @@ from typing import List import pytest from monkey_island.cc.models import Report -from monkey_island.cc.models.utils.document_encryptor import SensitiveField -from monkey_island.cc.models.utils.field_encryptors.i_field_encryptor import IFieldEncryptor +from monkey_island.cc.utils import SensitiveField +from monkey_island.cc.utils.field_encryptors import IFieldEncryptor MOCK_SENSITIVE_FIELD_CONTENTS = ["the_string", "the_string2"] MOCK_REPORT_DICT = { diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py b/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py index a93397392..ac46898c0 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py @@ -1,7 +1,7 @@ import pytest -from monkey_island.cc.models.utils.field_encryptors.string_list_encryptor import StringListEncryptor from monkey_island.cc.server_utils.encryption import initialize_datastore_encryptor +from monkey_island.cc.utils.field_encryptors import StringListEncryptor MOCK_STRING_LIST = ["test_1", "test_2"] EMPTY_LIST = [] diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py index 9f845b263..6829965f2 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/reporting/test_report.py @@ -5,7 +5,7 @@ import mongoengine import pytest from bson import ObjectId -from monkey_island.cc.models.telemetries import Telemetry +from monkey_island.cc.models.telemetries import save_telemetry from monkey_island.cc.services.reporting.report import ReportService TELEM_ID = { @@ -138,12 +138,13 @@ NODE_DICT_FAILED_EXPLOITS["exploits"][1]["result"] = False def fake_mongo(monkeypatch): mongo = mongoengine.connection.get_connection() monkeypatch.setattr("monkey_island.cc.services.reporting.report.mongo", mongo) - monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry.mongo", mongo) + monkeypatch.setattr("monkey_island.cc.models.telemetries.telemetry_dal.mongo", mongo) monkeypatch.setattr("monkey_island.cc.services.node.mongo", mongo) return mongo -def test_get_stolen_creds_exploit(fake_mongo, uses_database): +@pytest.mark.usefixtures("uses_database") +def test_get_stolen_creds_exploit(fake_mongo): fake_mongo.db.telemetry.insert_one(EXPLOIT_TELEMETRY_TELEM) stolen_creds_exploit = ReportService.get_stolen_creds() @@ -155,9 +156,10 @@ def test_get_stolen_creds_exploit(fake_mongo, uses_database): assert expected_stolen_creds_exploit == stolen_creds_exploit -def test_get_stolen_creds_system_info(fake_mongo, uses_database): +@pytest.mark.usefixtures("uses_database") +def test_get_stolen_creds_system_info(fake_mongo): fake_mongo.db.monkey.insert_one(MONKEY_TELEM) - Telemetry.save_telemetry(SYSTEM_INFO_TELEMETRY_TELEM) + save_telemetry(SYSTEM_INFO_TELEMETRY_TELEM) stolen_creds_system_info = ReportService.get_stolen_creds() expected_stolen_creds_system_info = [ @@ -169,9 +171,10 @@ def test_get_stolen_creds_system_info(fake_mongo, uses_database): assert expected_stolen_creds_system_info == stolen_creds_system_info -def test_get_stolen_creds_no_creds(fake_mongo, uses_database): +@pytest.mark.usefixtures("uses_database") +def test_get_stolen_creds_no_creds(fake_mongo): fake_mongo.db.monkey.insert_one(MONKEY_TELEM) - Telemetry.save_telemetry(NO_CREDS_TELEMETRY_TELEM) + save_telemetry(NO_CREDS_TELEMETRY_TELEM) stolen_creds_no_creds = ReportService.get_stolen_creds() expected_stolen_creds_no_creds = [] From 8b9973238e88dd187c167f3c4d7b00ab14e15d97 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 27 Sep 2021 16:59:11 +0300 Subject: [PATCH 12/16] Add CHANGELOG.md entry about fixed plaintext credentials in mongodb --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e603c37f..003d425c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ Changelog](https://keepachangelog.com/en/1.0.0/). - Overlapping Guardicore logo in the landing page. #1441 - PBA table collapse in security report on data change. #1423 - Unsigned Windows agent binaries in Linux packages are now signed. #1444 +- Some of the gathered credentials no longer appear in database plaintext. #1454 ### Security - Generate a random password when creating a new user for CommunicateAsNewUser From 1160ac6af000cb8f793c0c43ebb09949d17792c1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 28 Sep 2021 10:52:54 +0300 Subject: [PATCH 13/16] Refactor dictionary and sensitive mongo field encryption by moving it to server_utils/encryption --- monkey/monkey_island/cc/models/__init__.py | 2 +- .../cc/models/report/__init__.py | 1 + .../cc/models/telemetries/telemetry_dal.py | 13 +++++++++---- .../cc/server_utils/encryption/__init__.py | 8 ++++++++ .../encryption/dict_encryption/__init__.py | 0 .../dict_encryption}/dict_encryptor.py | 8 +++++--- .../field_encryptors/__init__.py | 0 .../field_encryptors/i_field_encryptor.py | 0 .../mimikatz_results_encryptor.py | 4 +++- .../field_encryptors/string_list_encryptor.py | 4 +++- monkey/monkey_island/cc/utils/__init__.py | 1 - .../models/telemetries/test_telemetry_dal.py | 6 ++++-- ...est_report_model.py => test_report_dal.py} | 19 ++++++++++++------- .../test_string_list_encryptor.py | 4 +++- 14 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 monkey/monkey_island/cc/models/report/__init__.py create mode 100644 monkey/monkey_island/cc/server_utils/encryption/dict_encryption/__init__.py rename monkey/monkey_island/cc/{utils => server_utils/encryption/dict_encryption}/dict_encryptor.py (80%) rename monkey/monkey_island/cc/{utils => server_utils/encryption/dict_encryption}/field_encryptors/__init__.py (100%) rename monkey/monkey_island/cc/{utils => server_utils/encryption/dict_encryption}/field_encryptors/i_field_encryptor.py (100%) rename monkey/monkey_island/cc/{utils => server_utils/encryption/dict_encryption}/field_encryptors/mimikatz_results_encryptor.py (91%) rename monkey/monkey_island/cc/{utils => server_utils/encryption/dict_encryption}/field_encryptors/string_list_encryptor.py (78%) delete mode 100644 monkey/monkey_island/cc/utils/__init__.py rename monkey/tests/unit_tests/monkey_island/cc/models/{test_report_model.py => test_report_dal.py} (76%) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 3464154b5..cab95ae18 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -7,4 +7,4 @@ from .creds import Creds from .monkey import Monkey from .monkey_ttl import MonkeyTtl from .pba_results import PbaResults -from .report import Report +from monkey_island.cc.models.report.report import Report diff --git a/monkey/monkey_island/cc/models/report/__init__.py b/monkey/monkey_island/cc/models/report/__init__.py new file mode 100644 index 000000000..ba3d7d9e0 --- /dev/null +++ b/monkey/monkey_island/cc/models/report/__init__.py @@ -0,0 +1 @@ +from .report_dal import save_report, get_report diff --git a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py index 88e617725..c036c5776 100644 --- a/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py +++ b/monkey/monkey_island/cc/models/telemetries/telemetry_dal.py @@ -5,8 +5,13 @@ from typing import List from monkey_island.cc.database import mongo from monkey_island.cc.models import CommandControlChannel from monkey_island.cc.models.telemetries.telemetry import Telemetry -from monkey_island.cc.utils import FieldNotFoundError, SensitiveField, dict_encryptor -from monkey_island.cc.utils.field_encryptors import MimikatzResultsEncryptor +from monkey_island.cc.server_utils.encryption import ( + FieldNotFoundError, + MimikatzResultsEncryptor, + SensitiveField, + decrypt_dict, + encrypt_dict, +) sensitive_fields = [ SensitiveField("data.credentials", MimikatzResultsEncryptor), @@ -16,7 +21,7 @@ sensitive_fields = [ def save_telemetry(telemetry_dict: dict): try: - telemetry_dict = dict_encryptor.encrypt(sensitive_fields, telemetry_dict) + telemetry_dict = encrypt_dict(sensitive_fields, telemetry_dict) except FieldNotFoundError: pass # Not all telemetries require encryption @@ -40,7 +45,7 @@ def get_telemetry_by_query(query: dict, output_fields=None) -> List[dict]: decrypted_list = [] for telemetry in telemetries: try: - decrypted_list.append(dict_encryptor.decrypt(sensitive_fields, telemetry)) + decrypted_list.append(decrypt_dict(sensitive_fields, telemetry)) except FieldNotFoundError: decrypted_list.append(telemetry) return decrypted_list diff --git a/monkey/monkey_island/cc/server_utils/encryption/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/__init__.py index a41240be1..7d806139c 100644 --- a/monkey/monkey_island/cc/server_utils/encryption/__init__.py +++ b/monkey/monkey_island/cc/server_utils/encryption/__init__.py @@ -11,3 +11,11 @@ from monkey_island.cc.server_utils.encryption.data_store_encryptor import ( get_datastore_encryptor, initialize_datastore_encryptor, ) +from .dict_encryption.dict_encryptor import ( + SensitiveField, + encrypt_dict, + decrypt_dict, + FieldNotFoundError, +) +from .dict_encryption.field_encryptors.mimikatz_results_encryptor import MimikatzResultsEncryptor +from .dict_encryption.field_encryptors.string_list_encryptor import StringListEncryptor diff --git a/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/cc/utils/dict_encryptor.py b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/dict_encryptor.py similarity index 80% rename from monkey/monkey_island/cc/utils/dict_encryptor.py rename to monkey/monkey_island/cc/server_utils/encryption/dict_encryption/dict_encryptor.py index 9a6d1d3d0..a95a761e0 100644 --- a/monkey/monkey_island/cc/utils/dict_encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/dict_encryptor.py @@ -3,7 +3,9 @@ from typing import Callable, List, Type import dpath.util -from monkey_island.cc.utils.field_encryptors import IFieldEncryptor +from monkey_island.cc.server_utils.encryption.dict_encryption.field_encryptors import ( + IFieldEncryptor, +) class FieldNotFoundError(Exception): @@ -17,7 +19,7 @@ class SensitiveField: field_encryptor: Type[IFieldEncryptor] -def encrypt(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict: +def encrypt_dict(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict: for sensitive_field in sensitive_fields: _apply_operation_to_document_field( document_dict, sensitive_field, sensitive_field.field_encryptor.encrypt @@ -26,7 +28,7 @@ def encrypt(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict return document_dict -def decrypt(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict: +def decrypt_dict(sensitive_fields: List[SensitiveField], document_dict: dict) -> dict: for sensitive_field in sensitive_fields: _apply_operation_to_document_field( document_dict, sensitive_field, sensitive_field.field_encryptor.decrypt diff --git a/monkey/monkey_island/cc/utils/field_encryptors/__init__.py b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/__init__.py similarity index 100% rename from monkey/monkey_island/cc/utils/field_encryptors/__init__.py rename to monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/__init__.py diff --git a/monkey/monkey_island/cc/utils/field_encryptors/i_field_encryptor.py b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/i_field_encryptor.py similarity index 100% rename from monkey/monkey_island/cc/utils/field_encryptors/i_field_encryptor.py rename to monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/i_field_encryptor.py diff --git a/monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/mimikatz_results_encryptor.py similarity index 91% rename from monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py rename to monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/mimikatz_results_encryptor.py index 6708ec40c..6261f5147 100644 --- a/monkey/monkey_island/cc/utils/field_encryptors/mimikatz_results_encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/mimikatz_results_encryptor.py @@ -1,7 +1,9 @@ import logging from monkey_island.cc.server_utils.encryption import get_datastore_encryptor -from monkey_island.cc.utils.field_encryptors import IFieldEncryptor +from monkey_island.cc.server_utils.encryption.dict_encryption.field_encryptors import ( + IFieldEncryptor, +) logger = logging.getLogger(__name__) diff --git a/monkey/monkey_island/cc/utils/field_encryptors/string_list_encryptor.py b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/string_list_encryptor.py similarity index 78% rename from monkey/monkey_island/cc/utils/field_encryptors/string_list_encryptor.py rename to monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/string_list_encryptor.py index f939c0e22..46eef09cb 100644 --- a/monkey/monkey_island/cc/utils/field_encryptors/string_list_encryptor.py +++ b/monkey/monkey_island/cc/server_utils/encryption/dict_encryption/field_encryptors/string_list_encryptor.py @@ -1,7 +1,9 @@ from typing import List from monkey_island.cc.server_utils.encryption import get_datastore_encryptor -from monkey_island.cc.utils.field_encryptors import IFieldEncryptor +from monkey_island.cc.server_utils.encryption.dict_encryption.field_encryptors import ( + IFieldEncryptor, +) class StringListEncryptor(IFieldEncryptor): diff --git a/monkey/monkey_island/cc/utils/__init__.py b/monkey/monkey_island/cc/utils/__init__.py deleted file mode 100644 index abe040645..000000000 --- a/monkey/monkey_island/cc/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .dict_encryptor import FieldNotFoundError, SensitiveField diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py index f25e8ffb3..d6a35760a 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/telemetries/test_telemetry_dal.py @@ -6,8 +6,10 @@ import pytest from monkey_island.cc.models.telemetries import get_telemetry_by_query, save_telemetry from monkey_island.cc.models.telemetries.telemetry import Telemetry -from monkey_island.cc.utils import SensitiveField -from monkey_island.cc.utils.field_encryptors import MimikatzResultsEncryptor +from monkey_island.cc.server_utils.encryption import SensitiveField +from monkey_island.cc.server_utils.encryption.dict_encryption.field_encryptors import ( + MimikatzResultsEncryptor, +) MOCK_CREDENTIALS = { "Vakaris": { diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py b/monkey/tests/unit_tests/monkey_island/cc/models/test_report_dal.py similarity index 76% rename from monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py rename to monkey/tests/unit_tests/monkey_island/cc/models/test_report_dal.py index 0c8fd90de..5d3d5a49a 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/test_report_model.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/test_report_dal.py @@ -4,8 +4,11 @@ from typing import List import pytest from monkey_island.cc.models import Report -from monkey_island.cc.utils import SensitiveField -from monkey_island.cc.utils.field_encryptors import IFieldEncryptor +from monkey_island.cc.models.report import get_report, save_report +from monkey_island.cc.server_utils.encryption import SensitiveField +from monkey_island.cc.server_utils.encryption.dict_encryption.field_encryptors import ( + IFieldEncryptor, +) MOCK_SENSITIVE_FIELD_CONTENTS = ["the_string", "the_string2"] MOCK_REPORT_DICT = { @@ -42,26 +45,28 @@ def patch_sensitive_fields(monkeypatch): SensitiveField("overview.foo.the_key", MockStringListEncryptor), SensitiveField("overview.bar.the_key", MockStringListEncryptor), ] - monkeypatch.setattr("monkey_island.cc.models.report.sensitive_fields", mock_sensitive_fields) + monkeypatch.setattr( + "monkey_island.cc.models.report.report_dal.sensitive_fields", mock_sensitive_fields + ) @pytest.mark.usefixtures("uses_database") def test_report_encryption(): - Report.save_report(MOCK_REPORT_DICT) + save_report(MOCK_REPORT_DICT) assert Report.objects.first()["overview"]["foo"]["the_key"] == ["ENCRYPTED_0", "ENCRYPTED_1"] assert Report.objects.first()["overview"]["bar"]["the_key"] == [] - assert Report.get_report()["overview"]["foo"]["the_key"] == MOCK_SENSITIVE_FIELD_CONTENTS + assert get_report()["overview"]["foo"]["the_key"] == MOCK_SENSITIVE_FIELD_CONTENTS @pytest.mark.usefixtures("uses_database") def test_report_dot_encoding(): mrd = copy.deepcopy(MOCK_REPORT_DICT) mrd["meta_info"] = {"foo.bar": "baz"} - Report.save_report(mrd) + save_report(mrd) assert "foo.bar" not in Report.objects.first()["meta_info"] assert "foo,,,bar" in Report.objects.first()["meta_info"] - report = Report.get_report() + report = get_report() assert "foo.bar" in report["meta_info"] diff --git a/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py b/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py index ac46898c0..d02ad5bbb 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py +++ b/monkey/tests/unit_tests/monkey_island/cc/models/utils/field_encryptors/test_string_list_encryptor.py @@ -1,7 +1,9 @@ import pytest from monkey_island.cc.server_utils.encryption import initialize_datastore_encryptor -from monkey_island.cc.utils.field_encryptors import StringListEncryptor +from monkey_island.cc.server_utils.encryption.dict_encryption.field_encryptors import ( + StringListEncryptor, +) MOCK_STRING_LIST = ["test_1", "test_2"] EMPTY_LIST = [] From a24eb841c132aa189d769fe7c97f2ce3bf3ef1d6 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 28 Sep 2021 10:58:23 +0300 Subject: [PATCH 14/16] Extract DAL interface for report model into a separate report_dal.py file --- monkey/monkey_island/cc/models/report.py | 60 ------------------- .../monkey_island/cc/models/report/report.py | 13 ++++ .../cc/models/report/report_dal.py | 52 ++++++++++++++++ .../cc/services/reporting/report.py | 7 ++- 4 files changed, 69 insertions(+), 63 deletions(-) delete mode 100644 monkey/monkey_island/cc/models/report.py create mode 100644 monkey/monkey_island/cc/models/report/report.py create mode 100644 monkey/monkey_island/cc/models/report/report_dal.py diff --git a/monkey/monkey_island/cc/models/report.py b/monkey/monkey_island/cc/models/report.py deleted file mode 100644 index cbdde0dcb..000000000 --- a/monkey/monkey_island/cc/models/report.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from bson import json_util -from mongoengine import DictField, Document - -from monkey_island.cc.utils import SensitiveField, dict_encryptor -from monkey_island.cc.utils.field_encryptors import StringListEncryptor - -sensitive_fields = [ - SensitiveField(path="overview.config_passwords", field_encryptor=StringListEncryptor) -] - - -class Report(Document): - - overview = DictField(required=True) - glance = DictField(required=True) - recommendations = DictField(required=True) - meta_info = DictField(required=True) - - meta = {"strict": False} - - @staticmethod - def save_report(report_dict: dict): - report_dict = _encode_dot_char_before_mongo_insert(report_dict) - report_dict = dict_encryptor.encrypt(sensitive_fields, report_dict) - Report.objects.delete() - Report( - overview=report_dict["overview"], - glance=report_dict["glance"], - recommendations=report_dict["recommendations"], - meta_info=report_dict["meta_info"], - ).save() - - @staticmethod - def get_report() -> dict: - report_dict = Report.objects.first().to_mongo() - return _decode_dot_char_before_mongo_insert( - dict_encryptor.decrypt(sensitive_fields, report_dict) - ) - - -def _encode_dot_char_before_mongo_insert(report_dict): - """ - mongodb doesn't allow for '.' and '$' in a key's name, this function replaces the '.' - char with the unicode - ,,, combo instead. - :return: dict with formatted keys with no dots. - """ - report_as_json = json_util.dumps(report_dict).replace(".", ",,,") - return json_util.loads(report_as_json) - - -def _decode_dot_char_before_mongo_insert(report_dict): - """ - this function replaces the ',,,' combo with the '.' char instead. - :return: report dict with formatted keys (',,,' -> '.') - """ - report_as_json = json_util.dumps(report_dict).replace(",,,", ".") - return json_util.loads(report_as_json) diff --git a/monkey/monkey_island/cc/models/report/report.py b/monkey/monkey_island/cc/models/report/report.py new file mode 100644 index 000000000..8de3124e6 --- /dev/null +++ b/monkey/monkey_island/cc/models/report/report.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from mongoengine import DictField, Document + + +class Report(Document): + + overview = DictField(required=True) + glance = DictField(required=True) + recommendations = DictField(required=True) + meta_info = DictField(required=True) + + meta = {"strict": False} diff --git a/monkey/monkey_island/cc/models/report/report_dal.py b/monkey/monkey_island/cc/models/report/report_dal.py new file mode 100644 index 000000000..be7bade9e --- /dev/null +++ b/monkey/monkey_island/cc/models/report/report_dal.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from bson import json_util + +from monkey_island.cc.models.report.report import Report +from monkey_island.cc.server_utils.encryption import ( + SensitiveField, + StringListEncryptor, + decrypt_dict, + encrypt_dict, +) + +sensitive_fields = [ + SensitiveField(path="overview.config_passwords", field_encryptor=StringListEncryptor) +] + + +def save_report(report_dict: dict): + report_dict = _encode_dot_char_before_mongo_insert(report_dict) + report_dict = encrypt_dict(sensitive_fields, report_dict) + Report.objects.delete() + Report( + overview=report_dict["overview"], + glance=report_dict["glance"], + recommendations=report_dict["recommendations"], + meta_info=report_dict["meta_info"], + ).save() + + +def get_report() -> dict: + report_dict = Report.objects.first().to_mongo() + return _decode_dot_char_before_mongo_insert(decrypt_dict(sensitive_fields, report_dict)) + + +def _encode_dot_char_before_mongo_insert(report_dict): + """ + mongodb doesn't allow for '.' and '$' in a key's name, this function replaces the '.' + char with the unicode + ,,, combo instead. + :return: dict with formatted keys with no dots. + """ + report_as_json = json_util.dumps(report_dict).replace(".", ",,,") + return json_util.loads(report_as_json) + + +def _decode_dot_char_before_mongo_insert(report_dict): + """ + this function replaces the ',,,' combo with the '.' char instead. + :return: report dict with formatted keys (',,,' -> '.') + """ + report_as_json = json_util.dumps(report_dict).replace(",,,", ".") + return json_util.loads(report_as_json) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 17e6ce037..d0ac2939f 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -14,7 +14,8 @@ from common.config_value_paths import ( from common.network.network_range import NetworkRange from common.network.segmentation_utils import get_ip_in_src_and_not_in_dst from monkey_island.cc.database import mongo -from monkey_island.cc.models import Monkey, Report +from monkey_island.cc.models import Monkey +from monkey_island.cc.models.report import get_report, save_report from monkey_island.cc.models.telemetries import get_telemetry_by_query from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.configuration.utils import ( @@ -635,7 +636,7 @@ class ReportService: "meta_info": {"latest_monkey_modifytime": monkey_latest_modify_time}, } ReportExporterManager().export(report) - Report.save_report(report) + save_report(report) return report @staticmethod @@ -697,4 +698,4 @@ class ReportService: if not ReportService.is_latest_report_exists(): return safe_generate_regular_report() - return Report.get_report() + return get_report() From d79892427b4d2f732d50545b160c9ffdfbb49aa1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 28 Sep 2021 10:59:04 +0300 Subject: [PATCH 15/16] Moved credential encryption in mongo CHANGELOG.md entry from Fixes to Security --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 003d425c8..a151cbf17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,8 @@ Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - Generate a random password when creating a new user for CommunicateAsNewUser PBA. #1434 +- Credentials gathered from victim machines are no longer stored plaintext in the database. #1454 + ## [1.11.0] - 2021-08-13 ### Added From 8b9ddb0c4bfc9929520850a563543667a116d819 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 28 Sep 2021 11:00:14 +0300 Subject: [PATCH 16/16] Removed unnecessary vulture ignores from whitelist --- vulture_allowlist.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index d58d4ea9b..bab90e90d 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -4,7 +4,6 @@ dead or is kept deliberately. Referencing these in a file like this makes sure t Vulture doesn't mark these as dead again. """ from monkey_island.cc.models import Report -from monkey_island.cc.models.telemetries import Telemetry fake_monkey_dir_path # unused variable (monkey/tests/infection_monkey/post_breach/actions/test_users_custom_pba.py:37) set_os_linux # unused variable (monkey/tests/infection_monkey/post_breach/actions/test_users_custom_pba.py:37) @@ -182,9 +181,6 @@ Report.recommendations Report.glance Report.meta_info Report.meta -Report.save_report -Telemetry.save_telemetry -Telemetry.get_telemetry_by_query # these are not needed for it to work, but may be useful extra information to understand what's going on WINDOWS_PBA_TYPE # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)