Merge pull request #2075 from guardicore/1965-mongo-credentials-repository

1965 mongo credentials repository

monkey/common/credentials/credentials.py

@@ -200,3 +200,13 @@ class Credentials:
         :return: A JSON string representing an array of Credentials objects
         """
         return "[" + ",".join([Credentials.to_json(c) for c in credentials]) + "]"
+
+    @staticmethod
+    def to_mapping(credentials: Credentials) -> Mapping:
+        """
+        Serialize a Credentials object to a Mapping
+
+        :param credentials: A Credentials object
+        :return: A mapping representing a Credentials object
+        """
+        return CredentialsSchema().dump(credentials)

monkey/monkey_island/Pipfile

@@ -31,6 +31,7 @@ pefile = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requirement
 marshmallow = "*"
 marshmallow-enum = "*"
 readerwriterlock = "*"
+pymongo = "*"
 
 [dev-packages]
 virtualenv = ">=20.0.26"

monkey/monkey_island/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "91b8cfcf1408b3709300f47d420c550fe355df76ad396e455049fef1cceca3ad"
+            "sha256": "4dae5f6c39b19f146ce303e0e1b6c4a52f45128e44b07ceefacf9d8c493d00a2"
         },
         "pipfile-spec": 6,
         "requires": {

monkey/monkey_island/cc/repository/__init__.py

@@ -1,13 +1,19 @@
 from .errors import RemovalError, RetrievalError, StorageError
+
+
 from .i_file_repository import FileNotFoundError, IFileRepository
+from .i_agent_binary_repository import IAgentBinaryRepository
+from .i_agent_configuration_repository import IAgentConfigurationRepository
+from .i_simulation_repository import ISimulationRepository
+from .i_credentials_repository import ICredentialsRepository
+
+
 from .local_storage_file_repository import LocalStorageFileRepository
 from .file_repository_caching_decorator import FileRepositoryCachingDecorator
 from .file_repository_locking_decorator import FileRepositoryLockingDecorator
 from .file_repository_logging_decorator import FileRepositoryLoggingDecorator
-from .i_agent_binary_repository import IAgentBinaryRepository
+
 from .agent_binary_repository import AgentBinaryRepository
-from .i_agent_configuration_repository import IAgentConfigurationRepository
 from .file_agent_configuration_repository import FileAgentConfigurationRepository
-from .i_simulation_repository import ISimulationRepository
 from .file_simulation_repository import FileSimulationRepository
-from .i_credentials_repository import ICredentialsRepository
+from .mongo_credentials_repository import MongoCredentialsRepository

monkey/monkey_island/cc/repository/i_credentials_repository.py

@@ -43,7 +43,7 @@ class ICredentialsRepository(ABC):
         """
         pass
 
-    def save_configured_credentials(self, credentials: Credentials):
+    def save_configured_credentials(self, credentials: Sequence[Credentials]):
         """
         Save credentials that were configured.
 
@@ -52,7 +52,7 @@ class ICredentialsRepository(ABC):
         """
         pass
 
-    def save_stolen_credentials(self, credentials: Credentials):
+    def save_stolen_credentials(self, credentials: Sequence[Credentials]):
         """
         Save credentials that were stolen during a simulation.
 

monkey/monkey_island/cc/repository/mongo_credentials_repository.py

@@ -0,0 +1,85 @@
+from typing import Sequence
+
+from pymongo import MongoClient
+
+from common.credentials import Credentials
+from monkey_island.cc.repository import RemovalError, RetrievalError, StorageError
+from monkey_island.cc.repository.i_credentials_repository import ICredentialsRepository
+
+
+class MongoCredentialsRepository(ICredentialsRepository):
+    """
+    Store credentials in a mongo database that can be used to propagate around the network.
+    """
+
+    def __init__(self, mongo: MongoClient):
+        self._mongo = mongo
+
+    def get_configured_credentials(self) -> Sequence[Credentials]:
+        return MongoCredentialsRepository._get_credentials_from_collection(
+            self._mongo.db.configured_credentials
+        )
+
+    def get_stolen_credentials(self) -> Sequence[Credentials]:
+        return MongoCredentialsRepository._get_credentials_from_collection(
+            self._mongo.db.stolen_credentials
+        )
+
+    def get_all_credentials(self) -> Sequence[Credentials]:
+        configured_credentials = self.get_configured_credentials()
+        stolen_credentials = self.get_stolen_credentials()
+
+        return [*configured_credentials, *stolen_credentials]
+
+    def save_configured_credentials(self, credentials: Sequence[Credentials]):
+        # TODO: Fix deduplication of Credentials in mongo
+        MongoCredentialsRepository._save_credentials_to_collection(
+            credentials, self._mongo.db.configured_credentials
+        )
+
+    def save_stolen_credentials(self, credentials: Sequence[Credentials]):
+        MongoCredentialsRepository._save_credentials_to_collection(
+            credentials, self._mongo.db.stolen_credentials
+        )
+
+    def remove_configured_credentials(self):
+        MongoCredentialsRepository._remove_credentials_fom_collection(
+            self._mongo.db.configured_credentials
+        )
+
+    def remove_stolen_credentials(self):
+        MongoCredentialsRepository._remove_credentials_fom_collection(
+            self._mongo.db.stolen_credentials
+        )
+
+    def remove_all_credentials(self):
+        self.remove_configured_credentials()
+        self.remove_stolen_credentials()
+
+    @staticmethod
+    def _get_credentials_from_collection(collection) -> Sequence[Credentials]:
+        try:
+            collection_result = []
+            list_collection_result = list(collection.find({}))
+            for c in list_collection_result:
+                del c["_id"]
+                collection_result.append(Credentials.from_mapping(c))
+
+            return collection_result
+        except Exception as err:
+            raise RetrievalError(err)
+
+    @staticmethod
+    def _save_credentials_to_collection(credentials: Sequence[Credentials], collection):
+        try:
+            for c in credentials:
+                collection.insert_one(Credentials.to_mapping(c))
+        except Exception as err:
+            raise StorageError(err)
+
+    @staticmethod
+    def _remove_credentials_fom_collection(collection):
+        try:
+            collection.delete_many({})
+        except RemovalError as err:
+            raise err

monkey/monkey_island/cc/resources/propagation_credentials.py

@@ -1,5 +1,6 @@
-from flask import jsonify
+from flask import make_response
 
+from common.credentials import Credentials
 from monkey_island.cc.repository import ICredentialsRepository
 from monkey_island.cc.resources.AbstractResource import AbstractResource
 
@@ -13,4 +14,4 @@ class PropagationCredentials(AbstractResource):
     def get(self):
         propagation_credentials = self._credentials_repository.get_all_credentials()
 
-        return jsonify(propagation_credentials)
+        return make_response(Credentials.to_json_array(propagation_credentials), 200)

monkey/monkey_island/cc/services/initialize.py

@@ -1,6 +1,8 @@
 import logging
 from pathlib import Path
 
+from pymongo import MongoClient
+
 from common import DIContainer
 from common.aws import AWSInstance
 from common.configuration import (
@@ -18,15 +20,18 @@ from monkey_island.cc.repository import (
     FileSimulationRepository,
     IAgentBinaryRepository,
     IAgentConfigurationRepository,
+    ICredentialsRepository,
     IFileRepository,
     ISimulationRepository,
     LocalStorageFileRepository,
+    MongoCredentialsRepository,
     RetrievalError,
 )
 from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
 from monkey_island.cc.services import AWSService, IslandModeService
 from monkey_island.cc.services.post_breach_files import PostBreachFilesService
 from monkey_island.cc.services.run_local_monkey import LocalMonkeyRunService
+from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL
 
 from . import AuthenticationService, JsonFileUserDatastore
 from .reporting.report import ReportService
@@ -39,7 +44,10 @@ AGENT_BINARIES_PATH = Path(MONKEY_ISLAND_ABS_PATH) / "cc" / "binaries"
 def initialize_services(data_dir: Path) -> DIContainer:
     container = DIContainer()
     _register_conventions(container, data_dir)
+
     container.register_instance(AWSInstance, AWSInstance())
+    container.register_instance(MongoClient, MongoClient(MONGO_URL, serverSelectionTimeoutMS=100))
+
     _register_repositories(container, data_dir)
     _register_services(container)
 
@@ -73,6 +81,9 @@ def _register_repositories(container: DIContainer, data_dir: Path):
         IAgentConfigurationRepository, container.resolve(FileAgentConfigurationRepository)
     )
     container.register_instance(ISimulationRepository, container.resolve(FileSimulationRepository))
+    container.register_instance(
+        ICredentialsRepository, container.resolve(MongoCredentialsRepository)
+    )
 
 
 def _decorate_file_repository(file_repository: IFileRepository) -> IFileRepository:

monkey/tests/monkey_island/stub_propagation_credentials_repository.py

@@ -1,38 +1,25 @@
 from typing import Sequence
 
+from common.credentials import Credentials, LMHash, NTHash, Password, Username
 from monkey_island.cc.repository import ICredentialsRepository
-from monkey_island.cc.services.telemetry.processing.credentials import Credentials
 
 fake_username = "m0nk3y_user"
 fake_special_username = "m0nk3y.user"
-fake_nt_hash = "c1c58f96cdf212b50837bc11a00be47c"
-fake_lm_hash = "299BD128C1101FD6"
+fake_nt_hash = "C1C58F96CDF212B50837BC11A00BE47C"
+fake_lm_hash = "299BD128C1101FD6299BD128C1101FD6"
 fake_password_1 = "trytostealthis"
 fake_password_2 = "password"
 fake_password_3 = "12345678"
-PROPAGATION_CREDENTIALS_1 = {
-    "identities": [{"username": fake_username, "credential_type": "USERNAME"}],
-    "secrets": [
-        {"nt_hash": fake_nt_hash, "credential_type": "NT_HASH"},
-        {"lm_hash": fake_lm_hash, "credential_type": "LM_HASH"},
-        {"password": fake_password_1, "credential_type": "PASSWORD"},
-    ],
-}
-
-PROPAGATION_CREDENTIALS_2 = {
-    "identities": [
-        {"username": fake_username, "credential_type": "USERNAME"},
-        {"username": fake_special_username, "credential_type": "USERNAME"},
-    ],
-    "secrets": [
-        {"password": fake_password_1, "credential_type": "PASSWORD"},
-        {"password": fake_password_2, "credential_type": "PASSWORD"},
-        {"password": fake_password_3, "credential_type": "PASSWORD"},
-    ],
-}
+PROPAGATION_CREDENTIALS_1 = Credentials(
+    identities=(Username(fake_username),),
+    secrets=(NTHash(fake_nt_hash), LMHash(fake_lm_hash), Password(fake_password_1)),
+)
+PROPAGATION_CREDENTIALS_2 = Credentials(
+    identities=(Username(fake_username), Username(fake_special_username)),
+    secrets=(Password(fake_password_1), Password(fake_password_2), Password(fake_password_3)),
+)
 
 
-# TODO: Use Credentials from common.credentials when serialization is implemented
 class StubPropagationCredentialsRepository(ICredentialsRepository):
     def get_configured_credentials(self) -> Sequence[Credentials]:
         pass
@@ -42,15 +29,12 @@ class StubPropagationCredentialsRepository(ICredentialsRepository):
 
     def get_all_credentials(self) -> Sequence[Credentials]:
 
-        return [
-            Credentials.from_mapping(PROPAGATION_CREDENTIALS_1, monkey_guid="some_guid"),
-            Credentials.from_mapping(PROPAGATION_CREDENTIALS_2, monkey_guid="second_guid"),
-        ]
+        return [PROPAGATION_CREDENTIALS_1, PROPAGATION_CREDENTIALS_2]
 
-    def save_configured_credentials(self, credentials: Credentials):
+    def save_configured_credentials(self, credentials: Sequence[Credentials]):
         pass
 
-    def save_stolen_credentials(self, credentials: Credentials):
+    def save_stolen_credentials(self, credentials: Sequence[Credentials]):
         pass
 
     def remove_configured_credentials(self):

monkey/tests/unit_tests/common/credentials/test_credentials.py

@@ -56,6 +56,12 @@ def test_credentials_serialization_json():
     assert json.loads(serialized_credentials) == CREDENTIALS_DICT
 
 
+def test_credentials_serialization_mapping():
+    serialized_credentials = Credentials.to_mapping(CREDENTIALS_OBJECT)
+
+    assert serialized_credentials == CREDENTIALS_DICT
+
+
 def test_credentials_deserialization__from_mapping():
     deserialized_credentials = Credentials.from_mapping(CREDENTIALS_DICT)
 

monkey/tests/unit_tests/monkey_island/cc/repository/test_mongo_credentials_repository.py

@@ -0,0 +1,115 @@
+import mongomock
+import pytest
+
+from common.credentials import Credentials, LMHash, NTHash, Password, SSHKeypair, Username
+from monkey_island.cc.repository import MongoCredentialsRepository
+
+USER1 = "test_user_1"
+USER2 = "test_user_2"
+USER3 = "test_user_3"
+PASSWORD = "12435"
+PASSWORD2 = "password"
+PASSWORD3 = "lozinka"
+LM_HASH = "AEBD4DE384C7EC43AAD3B435B51404EE"
+NT_HASH = "7A21990FCD3D759941E45C490F143D5F"
+PUBLIC_KEY = "MY_PUBLIC_KEY"
+PRIVATE_KEY = "MY_PRIVATE_KEY"
+
+IDENTITIES_1 = (Username(USER1), Username(USER2))
+SECRETS_1 = (
+    Password(PASSWORD),
+    LMHash(LM_HASH),
+    NTHash(NT_HASH),
+    SSHKeypair(PRIVATE_KEY, PUBLIC_KEY),
+)
+CREDENTIALS_OBJECT_1 = Credentials(IDENTITIES_1, SECRETS_1)
+
+IDENTITIES_2 = (Username(USER3),)
+SECRETS_2 = (Password(PASSWORD2), Password(PASSWORD3))
+CREDENTIALS_OBJECT_2 = Credentials(IDENTITIES_2, SECRETS_2)
+
+
+CONFIGURED_CREDENTIALS = [CREDENTIALS_OBJECT_1]
+
+STOLEN_CREDENTIALS = [CREDENTIALS_OBJECT_2]
+
+CREDENTIALS_LIST = [CREDENTIALS_OBJECT_1, CREDENTIALS_OBJECT_2]
+
+
+@pytest.fixture
+def mongo_repository():
+    mongo = mongomock.MongoClient()
+
+    return MongoCredentialsRepository(mongo)
+
+
+def test_mongo_repository_get_configured(mongo_repository):
+
+    actual_configured_credentials = mongo_repository.get_configured_credentials()
+
+    assert actual_configured_credentials == []
+
+
+def test_mongo_repository_get_stolen(mongo_repository):
+
+    actual_stolen_credentials = mongo_repository.get_stolen_credentials()
+
+    assert actual_stolen_credentials == []
+
+
+def test_mongo_repository_get_all(mongo_repository):
+
+    actual_credentials = mongo_repository.get_all_credentials()
+
+    assert actual_credentials == []
+
+
+def test_mongo_repository_configured(mongo_repository):
+
+    mongo_repository.save_configured_credentials(CREDENTIALS_LIST)
+
+    actual_configured_credentials = mongo_repository.get_configured_credentials()
+
+    assert actual_configured_credentials == CREDENTIALS_LIST
+
+    mongo_repository.remove_configured_credentials()
+
+    actual_configured_credentials = mongo_repository.get_configured_credentials()
+
+    assert actual_configured_credentials == []
+
+
+def test_mongo_repository_stolen(mongo_repository):
+
+    mongo_repository.save_configured_credentials(CONFIGURED_CREDENTIALS)
+    mongo_repository.save_stolen_credentials(STOLEN_CREDENTIALS)
+
+    actual_stolen_credentials = mongo_repository.get_stolen_credentials()
+
+    assert actual_stolen_credentials == STOLEN_CREDENTIALS
+
+    mongo_repository.remove_stolen_credentials()
+
+    actual_stolen_credentials = mongo_repository.get_stolen_credentials()
+
+    assert actual_stolen_credentials == []
+
+
+def test_mongo_repository_all(mongo_repository):
+
+    mongo_repository.save_configured_credentials(CONFIGURED_CREDENTIALS)
+    mongo_repository.save_stolen_credentials(STOLEN_CREDENTIALS)
+
+    actual_credentials = mongo_repository.get_all_credentials()
+
+    assert actual_credentials == CREDENTIALS_LIST
+
+    mongo_repository.remove_all_credentials()
+
+    actual_credentials = mongo_repository.get_all_credentials()
+    actual_stolen_credentials = mongo_repository.get_stolen_credentials()
+    actual_configured_credentials = mongo_repository.get_configured_credentials()
+
+    assert actual_credentials == []
+    assert actual_stolen_credentials == []
+    assert actual_configured_credentials == []

monkey/tests/unit_tests/monkey_island/cc/resources/test_propagation_credentials.py

@@ -1,5 +1,3 @@
-import json
-
 import pytest
 from tests.common import StubDIContainer
 from tests.monkey_island import (
@@ -9,6 +7,7 @@ from tests.monkey_island import (
 )
 from tests.unit_tests.monkey_island.conftest import get_url_for_resource
 
+from common.credentials import Credentials
 from monkey_island.cc.repository import ICredentialsRepository
 from monkey_island.cc.resources.propagation_credentials import PropagationCredentials
 
@@ -27,13 +26,9 @@ def test_propagation_credentials_endpoint_get(flask_client):
     propagation_credentials_url = get_url_for_resource(PropagationCredentials)
 
     resp = flask_client.get(propagation_credentials_url)
+    actual_propagation_credentials = Credentials.from_json_array(resp.text)
 
     assert resp.status_code == 200
-    actual_propagation_credentials = json.loads(resp.data)
     assert len(actual_propagation_credentials) == 2
-
-    # TODO: delete the removal of monkey_guid key when the serialization of credentials
-    del actual_propagation_credentials[0]["monkey_guid"]
     assert actual_propagation_credentials[0] == PROPAGATION_CREDENTIALS_1
-    del actual_propagation_credentials[1]["monkey_guid"]
     assert actual_propagation_credentials[1] == PROPAGATION_CREDENTIALS_2