Merge pull request #2077 from guardicore/1965-modify-telemetry-processor
1965 modify telemetry processor
This commit is contained in:
Mike Salvatore
GitHub
commit
bda661c7ae
|
|
@ -3,7 +3,6 @@ from __future__ import annotations
|
|||
from mongoengine import Document, ListField, ReferenceField
|
||||
|
||||
from monkey_island.cc.models import Monkey
|
||||
from monkey_island.cc.services.telemetry.processing.credentials import Credentials
|
||||
|
||||
|
||||
class StolenCredentials(Document):
|
||||
|
|
@ -20,12 +19,3 @@ class StolenCredentials(Document):
|
|||
monkey = ReferenceField(Monkey)
|
||||
identities = ListField()
|
||||
secrets = ListField()
|
||||
|
||||
@staticmethod
|
||||
def from_credentials(credentials: Credentials) -> StolenCredentials:
|
||||
stolen_creds = StolenCredentials()
|
||||
|
||||
stolen_creds.secrets = [secret["credential_type"] for secret in credentials.secrets]
|
||||
stolen_creds.identities = credentials.identities
|
||||
stolen_creds.monkey = Monkey.get_single_monkey_by_guid(credentials.monkey_guid).id
|
||||
return stolen_creds
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from common.config_value_paths import (
|
|||
PBA_LINUX_FILENAME_PATH,
|
||||
PBA_WINDOWS_FILENAME_PATH,
|
||||
SSH_KEYS_PATH,
|
||||
USER_LIST_PATH,
|
||||
)
|
||||
from monkey_island.cc.database import mongo
|
||||
from monkey_island.cc.server_utils.encryption import (
|
||||
|
|
@ -97,52 +96,6 @@ class ConfigService:
|
|||
mongo_key = ".".join(config_key_as_arr)
|
||||
mongo.db.config.update({}, {"$set": {mongo_key: value}})
|
||||
|
||||
# Not added to interface because it's doable by get_config_field + set_config_field
|
||||
@staticmethod
|
||||
def add_item_to_config_set_if_dont_exist(item_path_array, item_value, should_encrypt):
|
||||
item_key = ".".join(item_path_array)
|
||||
items_from_config = ConfigService.get_config_value(item_path_array, should_encrypt)
|
||||
if item_value in items_from_config:
|
||||
return
|
||||
if should_encrypt:
|
||||
if isinstance(item_value, dict):
|
||||
item_value = encrypt_dict(SENSITIVE_SSH_KEY_FIELDS, item_value)
|
||||
else:
|
||||
item_value = get_datastore_encryptor().encrypt(item_value)
|
||||
mongo.db.config.update({}, {"$addToSet": {item_key: item_value}})
|
||||
|
||||
@staticmethod
|
||||
def creds_add_username(username):
|
||||
ConfigService.add_item_to_config_set_if_dont_exist(
|
||||
USER_LIST_PATH, username, should_encrypt=False
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def creds_add_password(password):
|
||||
ConfigService.add_item_to_config_set_if_dont_exist(
|
||||
PASSWORD_LIST_PATH, password, should_encrypt=True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def creds_add_lm_hash(lm_hash):
|
||||
ConfigService.add_item_to_config_set_if_dont_exist(
|
||||
LM_HASH_LIST_PATH, lm_hash, should_encrypt=True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def creds_add_ntlm_hash(ntlm_hash):
|
||||
ConfigService.add_item_to_config_set_if_dont_exist(
|
||||
NTLM_HASH_LIST_PATH, ntlm_hash, should_encrypt=True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ssh_add_keys(public_key, private_key):
|
||||
ConfigService.add_item_to_config_set_if_dont_exist(
|
||||
SSH_KEYS_PATH,
|
||||
{"public_key": public_key, "private_key": private_key},
|
||||
should_encrypt=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _filter_none_values(data):
|
||||
if isinstance(data, dict):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from pymongo import MongoClient
|
|||
|
||||
from common import DIContainer
|
||||
from common.aws import AWSInstance
|
||||
from common.common_consts.telem_categories import TelemCategoryEnum
|
||||
from common.configuration import (
|
||||
DEFAULT_AGENT_CONFIGURATION,
|
||||
DEFAULT_RANSOMWARE_AGENT_CONFIGURATION,
|
||||
|
|
@ -31,6 +32,12 @@ from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
|
|||
from monkey_island.cc.services import AWSService, IslandModeService
|
||||
from monkey_island.cc.services.post_breach_files import PostBreachFilesService
|
||||
from monkey_island.cc.services.run_local_monkey import LocalMonkeyRunService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
|
||||
CredentialsParser,
|
||||
)
|
||||
from monkey_island.cc.services.telemetry.processing.processing import (
|
||||
TELEMETRY_CATEGORY_TO_PROCESSING_FUNC,
|
||||
)
|
||||
from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL
|
||||
|
||||
from . import AuthenticationService, JsonFileUserDatastore
|
||||
|
|
@ -51,6 +58,10 @@ def initialize_services(data_dir: Path) -> DIContainer:
|
|||
_register_repositories(container, data_dir)
|
||||
_register_services(container)
|
||||
|
||||
# Note: A hack to resolve credentials parser
|
||||
# It changes telemetry processing function, this will be refactored!
|
||||
_patch_credentials_parser(container)
|
||||
|
||||
# This is temporary until we get DI all worked out.
|
||||
PostBreachFilesService.initialize(container.resolve(IFileRepository))
|
||||
AuthenticationService.initialize(data_dir, JsonFileUserDatastore(data_dir))
|
||||
|
|
@ -129,3 +140,9 @@ def _register_services(container: DIContainer):
|
|||
container.register_instance(AWSService, container.resolve(AWSService))
|
||||
container.register_instance(LocalMonkeyRunService, container.resolve(LocalMonkeyRunService))
|
||||
container.register_instance(IslandModeService, container.resolve(IslandModeService))
|
||||
|
||||
|
||||
def _patch_credentials_parser(container: DIContainer):
|
||||
TELEMETRY_CATEGORY_TO_PROCESSING_FUNC[TelemCategoryEnum.CREDENTIALS] = container.resolve(
|
||||
CredentialsParser
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
from .credentials import Credentials
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Credentials:
|
||||
identities: Sequence[Mapping]
|
||||
secrets: Sequence[Mapping]
|
||||
monkey_guid: str
|
||||
|
||||
@staticmethod
|
||||
def from_mapping(cred_dict: Mapping[str, Any], monkey_guid: str) -> Credentials:
|
||||
return Credentials(
|
||||
identities=cred_dict["identities"],
|
||||
secrets=cred_dict["secrets"],
|
||||
monkey_guid=monkey_guid,
|
||||
)
|
||||
|
|
@ -1,41 +1,26 @@
|
|||
import logging
|
||||
from itertools import chain
|
||||
from typing import Mapping
|
||||
|
||||
from common.credentials import CredentialComponentType
|
||||
from monkey_island.cc.models import StolenCredentials
|
||||
|
||||
from .credentials import Credentials
|
||||
from .identities.username_processor import process_username
|
||||
from .secrets.lm_hash_processor import process_lm_hash
|
||||
from .secrets.nt_hash_processor import process_nt_hash
|
||||
from .secrets.password_processor import process_password
|
||||
from .secrets.ssh_key_processor import process_ssh_key
|
||||
from common.credentials import Credentials
|
||||
from monkey_island.cc.repository import ICredentialsRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CREDENTIAL_COMPONENT_PROCESSORS = {
|
||||
CredentialComponentType.LM_HASH: process_lm_hash,
|
||||
CredentialComponentType.NT_HASH: process_nt_hash,
|
||||
CredentialComponentType.PASSWORD: process_password,
|
||||
CredentialComponentType.SSH_KEYPAIR: process_ssh_key,
|
||||
CredentialComponentType.USERNAME: process_username,
|
||||
}
|
||||
|
||||
class CredentialsParser:
|
||||
"""
|
||||
This class parses and stores telemetry credentials.
|
||||
"""
|
||||
|
||||
def parse_credentials(telemetry_dict: Mapping):
|
||||
credentials = [
|
||||
Credentials.from_mapping(credential, telemetry_dict["monkey_guid"])
|
||||
for credential in telemetry_dict["data"]
|
||||
]
|
||||
def __init__(self, credentials_repository: ICredentialsRepository):
|
||||
self._credentials_repository = credentials_repository
|
||||
|
||||
for credential in credentials:
|
||||
_store_in_db(credential)
|
||||
for cred_comp in chain(credential.identities, credential.secrets):
|
||||
credential_type = CredentialComponentType[cred_comp["credential_type"]]
|
||||
CREDENTIAL_COMPONENT_PROCESSORS[credential_type](cred_comp, credential)
|
||||
def __call__(self, telemetry_dict):
|
||||
self._parse_credentials(telemetry_dict)
|
||||
|
||||
def _parse_credentials(self, telemetry_dict: Mapping):
|
||||
credentials = [
|
||||
Credentials.from_mapping(credential) for credential in telemetry_dict["data"]
|
||||
]
|
||||
|
||||
def _store_in_db(credentials: Credentials):
|
||||
stolen_cred_doc = StolenCredentials.from_credentials(credentials)
|
||||
stolen_cred_doc.save()
|
||||
self._credentials_repository.save_stolen_credentials(credentials)
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import Mapping
|
||||
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials import Credentials
|
||||
|
||||
|
||||
def process_username(username: Mapping, _: Credentials):
|
||||
ConfigService.creds_add_username(username["username"])
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import Mapping
|
||||
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials import Credentials
|
||||
|
||||
|
||||
def process_lm_hash(lm_hash: Mapping, _: Credentials):
|
||||
ConfigService.creds_add_lm_hash(lm_hash["lm_hash"])
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import Mapping
|
||||
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials import Credentials
|
||||
|
||||
|
||||
def process_nt_hash(nt_hash: Mapping, _: Credentials):
|
||||
ConfigService.creds_add_ntlm_hash(nt_hash["nt_hash"])
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
from typing import Mapping
|
||||
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials import Credentials
|
||||
|
||||
|
||||
def process_password(password: Mapping, _: Credentials):
|
||||
ConfigService.creds_add_password(password["password"])
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
from typing import Mapping
|
||||
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials import Credentials
|
||||
|
||||
|
||||
class SSHKeyProcessingError(ValueError):
|
||||
def __init__(self, msg=""):
|
||||
self.msg = f"Error while processing ssh keypair: {msg}"
|
||||
super().__init__(self.msg)
|
||||
|
||||
|
||||
def process_ssh_key(keypair: Mapping, credentials: Credentials):
|
||||
if len(credentials.identities) != 1:
|
||||
raise SSHKeyProcessingError(
|
||||
f"SSH credentials have {len(credentials.identities)} users associated with it!"
|
||||
)
|
||||
|
||||
if not _contains_both_keys(keypair):
|
||||
raise SSHKeyProcessingError("Private or public key missing")
|
||||
|
||||
ConfigService.ssh_add_keys(
|
||||
public_key=keypair["public_key"],
|
||||
private_key=keypair["private_key"],
|
||||
)
|
||||
|
||||
|
||||
def _contains_both_keys(ssh_key: Mapping) -> bool:
|
||||
try:
|
||||
return ssh_key["public_key"] and ssh_key["private_key"]
|
||||
except KeyError:
|
||||
return False
|
||||
|
|
@ -3,9 +3,6 @@ import logging
|
|||
from common.common_consts.telem_categories import TelemCategoryEnum
|
||||
from monkey_island.cc.models.telemetries import save_telemetry
|
||||
from monkey_island.cc.services.telemetry.processing.aws_info import process_aws_telemetry
|
||||
from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
|
||||
parse_credentials,
|
||||
)
|
||||
from monkey_island.cc.services.telemetry.processing.exploit import process_exploit_telemetry
|
||||
from monkey_island.cc.services.telemetry.processing.post_breach import process_post_breach_telemetry
|
||||
from monkey_island.cc.services.telemetry.processing.scan import process_scan_telemetry
|
||||
|
|
@ -18,7 +15,7 @@ TELEMETRY_CATEGORY_TO_PROCESSING_FUNC = {
|
|||
# `lambda *args, **kwargs: None` is a no-op.
|
||||
TelemCategoryEnum.ATTACK: lambda *args, **kwargs: None,
|
||||
TelemCategoryEnum.AWS_INFO: process_aws_telemetry,
|
||||
TelemCategoryEnum.CREDENTIALS: parse_credentials,
|
||||
TelemCategoryEnum.CREDENTIALS: None, # this is set in monkey_island/cc/services/initialize.py
|
||||
TelemCategoryEnum.EXPLOIT: process_exploit_telemetry,
|
||||
TelemCategoryEnum.POST_BREACH: process_post_breach_telemetry,
|
||||
TelemCategoryEnum.SCAN: process_scan_telemetry,
|
||||
|
|
|
|||
|
|
@ -1,112 +0,0 @@
|
|||
from copy import deepcopy
|
||||
|
||||
import dpath.util
|
||||
import pytest
|
||||
from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials.conftest import (
|
||||
CREDENTIAL_TELEM_TEMPLATE,
|
||||
)
|
||||
|
||||
from common.config_value_paths import (
|
||||
LM_HASH_LIST_PATH,
|
||||
NTLM_HASH_LIST_PATH,
|
||||
PASSWORD_LIST_PATH,
|
||||
USER_LIST_PATH,
|
||||
)
|
||||
from common.credentials import CredentialComponentType
|
||||
from monkey_island.cc.models import StolenCredentials
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
|
||||
parse_credentials,
|
||||
)
|
||||
|
||||
fake_username = "m0nk3y_user"
|
||||
cred_telem_usernames = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
|
||||
cred_telem_usernames["data"] = [
|
||||
{"identities": [{"username": fake_username, "credential_type": "USERNAME"}], "secrets": []}
|
||||
]
|
||||
|
||||
fake_special_username = "$m0nk3y.user"
|
||||
cred_telem_special_usernames = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
|
||||
cred_telem_special_usernames["data"] = [
|
||||
{
|
||||
"identities": [{"username": fake_special_username, "credential_type": "USERNAME"}],
|
||||
"secrets": [],
|
||||
}
|
||||
]
|
||||
|
||||
fake_nt_hash = "c1c58f96cdf212b50837bc11a00be47c"
|
||||
fake_lm_hash = "299BD128C1101FD6"
|
||||
fake_password = "trytostealthis"
|
||||
cred_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
|
||||
cred_telem["data"] = [
|
||||
{
|
||||
"identities": [{"username": fake_username, "credential_type": "USERNAME"}],
|
||||
"secrets": [
|
||||
{"nt_hash": fake_nt_hash, "credential_type": "NT_HASH"},
|
||||
{"lm_hash": fake_lm_hash, "credential_type": "LM_HASH"},
|
||||
{"password": fake_password, "credential_type": "PASSWORD"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
cred_empty_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
|
||||
cred_empty_telem["data"] = [{"identities": [], "secrets": []}]
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
|
||||
def test_cred_username_parsing():
|
||||
parse_credentials(cred_telem_usernames)
|
||||
config = ConfigService.get_config(should_decrypt=True)
|
||||
assert fake_username in dpath.util.get(config, USER_LIST_PATH)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
|
||||
def test_cred_special_username_parsing():
|
||||
parse_credentials(cred_telem_special_usernames)
|
||||
config = ConfigService.get_config(should_decrypt=True)
|
||||
assert fake_special_username in dpath.util.get(config, USER_LIST_PATH)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
|
||||
def test_cred_telemetry_parsing():
|
||||
parse_credentials(cred_telem)
|
||||
config = ConfigService.get_config(should_decrypt=True)
|
||||
assert fake_username in dpath.util.get(config, USER_LIST_PATH)
|
||||
assert fake_nt_hash in dpath.util.get(config, NTLM_HASH_LIST_PATH)
|
||||
assert fake_lm_hash in dpath.util.get(config, LM_HASH_LIST_PATH)
|
||||
assert fake_password in dpath.util.get(config, PASSWORD_LIST_PATH)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
|
||||
def test_cred_storage_in_db():
|
||||
parse_credentials(cred_telem)
|
||||
cred_docs = list(StolenCredentials.objects())
|
||||
assert len(cred_docs) == 1
|
||||
|
||||
stolen_creds = cred_docs[0]
|
||||
assert fake_username == stolen_creds.identities[0]["username"]
|
||||
assert CredentialComponentType.PASSWORD.name in stolen_creds.secrets
|
||||
assert CredentialComponentType.LM_HASH.name in stolen_creds.secrets
|
||||
assert CredentialComponentType.NT_HASH.name in stolen_creds.secrets
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.usefixtures("uses_database", "fake_mongo", "insert_fake_monkey")
|
||||
def test_empty_cred_telemetry_parsing():
|
||||
default_config = deepcopy(ConfigService.get_config(should_decrypt=True))
|
||||
default_usernames = dpath.util.get(default_config, USER_LIST_PATH)
|
||||
default_nt_hashes = dpath.util.get(default_config, NTLM_HASH_LIST_PATH)
|
||||
default_lm_hashes = dpath.util.get(default_config, LM_HASH_LIST_PATH)
|
||||
default_passwords = dpath.util.get(default_config, PASSWORD_LIST_PATH)
|
||||
|
||||
parse_credentials(cred_empty_telem)
|
||||
config = ConfigService.get_config(should_decrypt=True)
|
||||
|
||||
assert default_usernames == dpath.util.get(config, USER_LIST_PATH)
|
||||
assert default_nt_hashes == dpath.util.get(config, NTLM_HASH_LIST_PATH)
|
||||
assert default_lm_hashes == dpath.util.get(config, LM_HASH_LIST_PATH)
|
||||
assert default_passwords == dpath.util.get(config, PASSWORD_LIST_PATH)
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
from copy import deepcopy
|
||||
|
||||
import dpath.util
|
||||
import pytest
|
||||
from tests.unit_tests.monkey_island.cc.services.telemetry.processing.credentials.conftest import (
|
||||
CREDENTIAL_TELEM_TEMPLATE,
|
||||
)
|
||||
|
||||
from common.config_value_paths import SSH_KEYS_PATH, USER_LIST_PATH
|
||||
from monkey_island.cc.services.config import ConfigService
|
||||
from monkey_island.cc.services.telemetry.processing.credentials.credentials_parser import (
|
||||
parse_credentials,
|
||||
)
|
||||
|
||||
fake_private_key = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFlczI1N\n"
|
||||
fake_partial_secret = {"private_key": fake_private_key, "credential_type": "SSH_KEYPAIR"}
|
||||
|
||||
fake_username = "ubuntu"
|
||||
fake_public_key = (
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1u2+50OFRnzOGHpWo69"
|
||||
"tc02oMXudeML7pOl7rqXLmdxuj monkey@krk-wpas5"
|
||||
)
|
||||
fake_secret_full = {
|
||||
"private_key": fake_private_key,
|
||||
"public_key": fake_public_key,
|
||||
"credential_type": "SSH_KEYPAIR",
|
||||
}
|
||||
fake_identity = {"username": fake_username, "credential_type": "USERNAME"}
|
||||
|
||||
ssh_telem = deepcopy(CREDENTIAL_TELEM_TEMPLATE)
|
||||
ssh_telem["data"] = [{"identities": [fake_identity], "secrets": [fake_secret_full]}]
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.usefixtures("uses_encryptor", "uses_database", "fake_mongo", "insert_fake_monkey")
|
||||
def test_ssh_credential_parsing():
|
||||
parse_credentials(ssh_telem)
|
||||
config = ConfigService.get_config(should_decrypt=True)
|
||||
ssh_keypairs = dpath.util.get(config, SSH_KEYS_PATH)
|
||||
assert len(ssh_keypairs) == 1
|
||||
assert ssh_keypairs[0]["private_key"] == fake_private_key
|
||||
assert ssh_keypairs[0]["public_key"] == fake_public_key
|
||||
assert fake_username in dpath.util.get(config, USER_LIST_PATH)
|
||||
Loading…
Reference in New Issue