diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index fb8116c14..eb53d00c8 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -1,6 +1,5 @@ import json import logging -import copy from datetime import datetime import dateutil @@ -9,14 +8,9 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo -from monkey_island.cc.services import mimikatz_utils -from monkey_island.cc.services.config import ConfigService -from monkey_island.cc.services.edge import EdgeService from monkey_island.cc.services.node import NodeService -from monkey_island.cc.encryptor import encryptor -from monkey_island.cc.services.wmi_handler import WMIHandler +from monkey_island.cc.services.telemetry.processing.hooks import TELEMETRY_CATEGORY_TO_PROCESSING_FUNC from monkey_island.cc.models.monkey import Monkey -from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence __author__ = 'Barak' @@ -58,8 +52,8 @@ class Telemetry(flask_restful.Resource): try: NodeService.update_monkey_modify_time(monkey["_id"]) telem_category = telemetry_json.get('telem_category') - if telem_category in TELEM_PROCESS_DICT: - TELEM_PROCESS_DICT[telem_category](telemetry_json) + if telem_category in TELEMETRY_CATEGORY_TO_PROCESSING_FUNC: + TELEMETRY_CATEGORY_TO_PROCESSING_FUNC[telem_category](telemetry_json) else: logger.info('Got unknown type of telemetry: %s' % telem_category) except Exception as ex: @@ -90,216 +84,3 @@ class Telemetry(flask_restful.Resource): x['data']['credentials'][new_user] = x['data']['credentials'].pop(user) return objects - - @staticmethod - def get_edge_by_scan_or_exploit_telemetry(telemetry_json): - dst_ip = telemetry_json['data']['machine']['ip_addr'] - dst_domain_name = telemetry_json['data']['machine']['domain_name'] - src_monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) - dst_node = NodeService.get_monkey_by_ip(dst_ip) - if dst_node is None: - dst_node = NodeService.get_or_create_node(dst_ip, dst_domain_name) - - return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"]) - - @staticmethod - def process_tunnel_telemetry(telemetry_json): - monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])["_id"] - if telemetry_json['data']['proxy'] is not None: - tunnel_host_ip = telemetry_json['data']['proxy'].split(":")[-2].replace("//", "") - NodeService.set_monkey_tunnel(monkey_id, tunnel_host_ip) - else: - NodeService.unset_all_monkey_tunnels(monkey_id) - - @staticmethod - def process_state_telemetry(telemetry_json): - monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) - if telemetry_json['data']['done']: - NodeService.set_monkey_dead(monkey, True) - else: - NodeService.set_monkey_dead(monkey, False) - - @staticmethod - def process_exploit_telemetry(telemetry_json): - edge = Telemetry.get_edge_by_scan_or_exploit_telemetry(telemetry_json) - Telemetry.encrypt_exploit_creds(telemetry_json) - telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started']) - telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished']) - - new_exploit = copy.deepcopy(telemetry_json['data']) - - new_exploit.pop('machine') - new_exploit['timestamp'] = telemetry_json['timestamp'] - - mongo.db.edge.update( - {'_id': edge['_id']}, - {'$push': {'exploits': new_exploit}} - ) - if new_exploit['result']: - EdgeService.set_edge_exploited(edge) - - for attempt in telemetry_json['data']['attempts']: - if attempt['result']: - found_creds = {'user': attempt['user']} - for field in ['password', 'lm_hash', 'ntlm_hash', 'ssh_key']: - if len(attempt[field]) != 0: - found_creds[field] = attempt[field] - NodeService.add_credentials_to_node(edge['to'], found_creds) - - @staticmethod - def process_scan_telemetry(telemetry_json): - edge = Telemetry.get_edge_by_scan_or_exploit_telemetry(telemetry_json) - data = copy.deepcopy(telemetry_json['data']['machine']) - ip_address = data.pop("ip_addr") - domain_name = data.pop("domain_name") - new_scan = \ - { - "timestamp": telemetry_json["timestamp"], - "data": data - } - mongo.db.edge.update( - {"_id": edge["_id"]}, - {"$push": {"scans": new_scan}, - "$set": {"ip_address": ip_address, 'domain_name': domain_name}} - ) - - node = mongo.db.node.find_one({"_id": edge["to"]}) - if node is not None: - scan_os = new_scan["data"]["os"] - if "type" in scan_os: - mongo.db.node.update({"_id": node["_id"]}, - {"$set": {"os.type": scan_os["type"]}}, - upsert=False) - if "version" in scan_os: - mongo.db.node.update({"_id": node["_id"]}, - {"$set": {"os.version": scan_os["version"]}}, - upsert=False) - - @staticmethod - def process_system_info_telemetry(telemetry_json): - Telemetry.process_ssh_info(telemetry_json) - Telemetry.process_credential_info(telemetry_json) - monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') - Telemetry.process_mimikatz_and_wmi_info(monkey_id, telemetry_json) - Telemetry.process_aws_data(monkey_id, telemetry_json) - test_antivirus_existence(telemetry_json) - - @staticmethod - def process_mimikatz_and_wmi_info(monkey_id, telemetry_json): - users_secrets = {} - if 'mimikatz' in telemetry_json['data']: - users_secrets = mimikatz_utils.MimikatzSecrets. \ - extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', '')) - if 'wmi' in telemetry_json['data']: - wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets) - wmi_handler.process_and_handle_wmi_info() - - @staticmethod - def process_aws_data(monkey_id, telemetry_json): - if 'aws' in telemetry_json['data']: - if 'instance_id' in telemetry_json['data']['aws']: - mongo.db.monkey.update_one({'_id': monkey_id}, - {'$set': {'aws_instance_id': telemetry_json['data']['aws']['instance_id']}}) - - @staticmethod - def process_credential_info(telemetry_json): - if 'credentials' in telemetry_json['data']: - creds = telemetry_json['data']['credentials'] - Telemetry.encrypt_system_info_creds(creds) - Telemetry.add_system_info_creds_to_config(creds) - Telemetry.replace_user_dot_with_comma(creds) - - @staticmethod - def process_ssh_info(telemetry_json): - if 'ssh_info' in telemetry_json['data']: - ssh_info = telemetry_json['data']['ssh_info'] - Telemetry.encrypt_system_info_ssh_keys(ssh_info) - if telemetry_json['data']['network_info']['networks']: - # We use user_name@machine_ip as the name of the ssh key stolen, thats why we need ip from telemetry - Telemetry.add_ip_to_ssh_keys(telemetry_json['data']['network_info']['networks'][0], ssh_info) - Telemetry.add_system_info_ssh_keys_to_config(ssh_info) - - @staticmethod - def add_ip_to_ssh_keys(ip, ssh_info): - for key in ssh_info: - key['ip'] = ip['addr'] - - @staticmethod - def process_trace_telemetry(telemetry_json): - # Nothing to do - return - - @staticmethod - def replace_user_dot_with_comma(creds): - for user in creds: - if -1 != user.find('.'): - new_user = user.replace('.', ',') - creds[new_user] = creds.pop(user) - - @staticmethod - def encrypt_system_info_creds(creds): - for user in creds: - for field in ['password', 'lm_hash', 'ntlm_hash']: - if field in creds[user]: - # this encoding is because we might run into passwords which are not pure ASCII - creds[user][field] = encryptor.enc(creds[user][field].encode('utf-8')) - - @staticmethod - def encrypt_system_info_ssh_keys(ssh_info): - for idx, user in enumerate(ssh_info): - for field in ['public_key', 'private_key', 'known_hosts']: - if ssh_info[idx][field]: - ssh_info[idx][field] = encryptor.enc(ssh_info[idx][field].encode('utf-8')) - - @staticmethod - def add_system_info_creds_to_config(creds): - for user in creds: - ConfigService.creds_add_username(user) - if 'password' in creds[user]: - ConfigService.creds_add_password(creds[user]['password']) - if 'lm_hash' in creds[user]: - ConfigService.creds_add_lm_hash(creds[user]['lm_hash']) - if 'ntlm_hash' in creds[user]: - ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) - - @staticmethod - def add_system_info_ssh_keys_to_config(ssh_info): - for user in ssh_info: - ConfigService.creds_add_username(user['name']) - # Public key is useless without private key - if user['public_key'] and user['private_key']: - ConfigService.ssh_add_keys(user['public_key'], user['private_key'], - user['name'], user['ip']) - - @staticmethod - def encrypt_exploit_creds(telemetry_json): - attempts = telemetry_json['data']['attempts'] - for i in range(len(attempts)): - for field in ['password', 'lm_hash', 'ntlm_hash']: - credential = attempts[i][field] - if len(credential) > 0: - attempts[i][field] = encryptor.enc(credential.encode('utf-8')) - - @staticmethod - def process_post_breach_telemetry(telemetry_json): - mongo.db.monkey.update( - {'guid': telemetry_json['monkey_guid']}, - {'$push': {'pba_results': telemetry_json['data']}}) - - @staticmethod - def process_attack_telemetry(telemetry_json): - # No processing required - pass - - -TELEM_PROCESS_DICT = \ - { - 'tunnel': Telemetry.process_tunnel_telemetry, - 'state': Telemetry.process_state_telemetry, - 'exploit': Telemetry.process_exploit_telemetry, - 'scan': Telemetry.process_scan_telemetry, - 'system_info': Telemetry.process_system_info_telemetry, - 'trace': Telemetry.process_trace_telemetry, - 'post_breach': Telemetry.process_post_breach_telemetry, - 'attack': Telemetry.process_attack_telemetry - } diff --git a/monkey/monkey_island/cc/services/telemetry/processing/__init__.py b/monkey/monkey_island/cc/services/telemetry/processing/__init__.py new file mode 100644 index 000000000..d90143c09 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/__init__.py @@ -0,0 +1,7 @@ +# import all implemented hooks, for brevity of hooks.py file +from tunnel import process_tunnel_telemetry +from state import process_state_telemetry +from exploit import process_exploit_telemetry +from scan import process_scan_telemetry +from system_info import process_system_info_telemetry +from post_breach import process_post_breach_telemetry diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py new file mode 100644 index 000000000..98ca76248 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -0,0 +1,45 @@ +import copy + +import dateutil + +from monkey_island.cc.database import mongo +from monkey_island.cc.encryptor import encryptor +from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.node import NodeService +from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry + + +def process_exploit_telemetry(telemetry_json): + edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) + encrypt_exploit_creds(telemetry_json) + telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started']) + telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished']) + + new_exploit = copy.deepcopy(telemetry_json['data']) + + new_exploit.pop('machine') + new_exploit['timestamp'] = telemetry_json['timestamp'] + + mongo.db.edge.update( + {'_id': edge['_id']}, + {'$push': {'exploits': new_exploit}} + ) + if new_exploit['result']: + EdgeService.set_edge_exploited(edge) + + for attempt in telemetry_json['data']['attempts']: + if attempt['result']: + found_creds = {'user': attempt['user']} + for field in ['password', 'lm_hash', 'ntlm_hash', 'ssh_key']: + if len(attempt[field]) != 0: + found_creds[field] = attempt[field] + NodeService.add_credentials_to_node(edge['to'], found_creds) + + +def encrypt_exploit_creds(telemetry_json): + attempts = telemetry_json['data']['attempts'] + for i in range(len(attempts)): + for field in ['password', 'lm_hash', 'ntlm_hash']: + credential = attempts[i][field] + if len(credential) > 0: + attempts[i][field] = encryptor.enc(credential.encode('utf-8')) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/hooks.py b/monkey/monkey_island/cc/services/telemetry/processing/hooks.py new file mode 100644 index 000000000..125bb8b53 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/hooks.py @@ -0,0 +1,14 @@ +from monkey_island.cc.services.telemetry.processing import * + +TELEMETRY_CATEGORY_TO_PROCESSING_FUNC = \ + { + 'tunnel': process_tunnel_telemetry, + 'state': process_state_telemetry, + 'exploit': process_exploit_telemetry, + 'scan': process_scan_telemetry, + 'system_info': process_system_info_telemetry, + 'post_breach': process_post_breach_telemetry, + # `lambda *args, **kwargs: None` is a no-op. + 'trace': lambda *args, **kwargs: None, + 'attack': lambda *args, **kwargs: None, + } diff --git a/monkey/monkey_island/cc/services/telemetry/processing/post_breach.py b/monkey/monkey_island/cc/services/telemetry/processing/post_breach.py new file mode 100644 index 000000000..b086d5ff4 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/post_breach.py @@ -0,0 +1,7 @@ +from monkey_island.cc.database import mongo + + +def process_post_breach_telemetry(telemetry_json): + mongo.db.monkey.update( + {'guid': telemetry_json['monkey_guid']}, + {'$push': {'pba_results': telemetry_json['data']}}) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/scan.py b/monkey/monkey_island/cc/services/telemetry/processing/scan.py new file mode 100644 index 000000000..4e34b9a19 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/scan.py @@ -0,0 +1,33 @@ +import copy + +from monkey_island.cc.database import mongo +from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry + + +def process_scan_telemetry(telemetry_json): + edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) + data = copy.deepcopy(telemetry_json['data']['machine']) + ip_address = data.pop("ip_addr") + domain_name = data.pop("domain_name") + new_scan = \ + { + "timestamp": telemetry_json["timestamp"], + "data": data + } + mongo.db.edge.update( + {"_id": edge["_id"]}, + {"$push": {"scans": new_scan}, + "$set": {"ip_address": ip_address, 'domain_name': domain_name}} + ) + + node = mongo.db.node.find_one({"_id": edge["to"]}) + if node is not None: + scan_os = new_scan["data"]["os"] + if "type" in scan_os: + mongo.db.node.update({"_id": node["_id"]}, + {"$set": {"os.type": scan_os["type"]}}, + upsert=False) + if "version" in scan_os: + mongo.db.node.update({"_id": node["_id"]}, + {"$set": {"os.version": scan_os["version"]}}, + upsert=False) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/state.py b/monkey/monkey_island/cc/services/telemetry/processing/state.py new file mode 100644 index 000000000..e71abacd7 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/state.py @@ -0,0 +1,9 @@ +from monkey_island.cc.services.node import NodeService + + +def process_state_telemetry(telemetry_json): + monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) + if telemetry_json['data']['done']: + NodeService.set_monkey_dead(monkey, True) + else: + NodeService.set_monkey_dead(monkey, False) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py new file mode 100644 index 000000000..ebf11c219 --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -0,0 +1,99 @@ +from monkey_island.cc.database import mongo +from monkey_island.cc.services import mimikatz_utils +from monkey_island.cc.services.node import NodeService +from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence +from monkey_island.cc.services.wmi_handler import WMIHandler +from monkey_island.cc.encryptor import encryptor + + +def process_system_info_telemetry(telemetry_json): + process_ssh_info(telemetry_json) + process_credential_info(telemetry_json) + process_mimikatz_and_wmi_info(telemetry_json) + process_aws_data(telemetry_json) + test_antivirus_existence(telemetry_json) + + +def process_ssh_info(telemetry_json): + if 'ssh_info' in telemetry_json['data']: + ssh_info = telemetry_json['data']['ssh_info'] + encrypt_system_info_ssh_keys(ssh_info) + if telemetry_json['data']['network_info']['networks']: + # We use user_name@machine_ip as the name of the ssh key stolen, thats why we need ip from telemetry + add_ip_to_ssh_keys(telemetry_json['data']['network_info']['networks'][0], ssh_info) + add_system_info_ssh_keys_to_config(ssh_info) + + +def add_system_info_ssh_keys_to_config(ssh_info): + for user in ssh_info: + ConfigService.creds_add_username(user['name']) + # Public key is useless without private key + if user['public_key'] and user['private_key']: + ConfigService.ssh_add_keys(user['public_key'], user['private_key'], + user['name'], user['ip']) + + +def add_ip_to_ssh_keys(ip, ssh_info): + for key in ssh_info: + key['ip'] = ip['addr'] + + +def encrypt_system_info_ssh_keys(ssh_info): + for idx, user in enumerate(ssh_info): + for field in ['public_key', 'private_key', 'known_hosts']: + if ssh_info[idx][field]: + ssh_info[idx][field] = encryptor.enc(ssh_info[idx][field].encode('utf-8')) + + +def process_credential_info(telemetry_json): + if 'credentials' in telemetry_json['data']: + creds = telemetry_json['data']['credentials'] + encrypt_system_info_creds(creds) + add_system_info_creds_to_config(creds) + replace_user_dot_with_comma(creds) + + +def replace_user_dot_with_comma(creds): + for user in creds: + if -1 != user.find('.'): + new_user = user.replace('.', ',') + creds[new_user] = creds.pop(user) + + +def add_system_info_creds_to_config(creds): + for user in creds: + ConfigService.creds_add_username(user) + if 'password' in creds[user]: + ConfigService.creds_add_password(creds[user]['password']) + if 'lm_hash' in creds[user]: + ConfigService.creds_add_lm_hash(creds[user]['lm_hash']) + if 'ntlm_hash' in creds[user]: + ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) + + +def encrypt_system_info_creds(creds): + for user in creds: + for field in ['password', 'lm_hash', 'ntlm_hash']: + if field in creds[user]: + # this encoding is because we might run into passwords which are not pure ASCII + creds[user][field] = encryptor.enc(creds[user][field].encode('utf-8')) + + +def process_mimikatz_and_wmi_info(telemetry_json): + users_secrets = {} + if 'mimikatz' in telemetry_json['data']: + users_secrets = mimikatz_utils.MimikatzSecrets. \ + extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', '')) + if 'wmi' in telemetry_json['data']: + monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') + wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets) + wmi_handler.process_and_handle_wmi_info() + + +def process_aws_data(telemetry_json): + if 'aws' in telemetry_json['data']: + if 'instance_id' in telemetry_json['data']['aws']: + monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') + mongo.db.monkey.update_one({'_id': monkey_id}, + {'$set': {'aws_instance_id': telemetry_json['data']['aws']['instance_id']}}) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/tunnel.py b/monkey/monkey_island/cc/services/telemetry/processing/tunnel.py new file mode 100644 index 000000000..ed57f3c7b --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/tunnel.py @@ -0,0 +1,10 @@ +from monkey_island.cc.services.node import NodeService + + +def process_tunnel_telemetry(telemetry_json): + monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])["_id"] + if telemetry_json['data']['proxy'] is not None: + tunnel_host_ip = telemetry_json['data']['proxy'].split(":")[-2].replace("//", "") + NodeService.set_monkey_tunnel(monkey_id, tunnel_host_ip) + else: + NodeService.unset_all_monkey_tunnels(monkey_id) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/utils.py b/monkey/monkey_island/cc/services/telemetry/processing/utils.py new file mode 100644 index 000000000..9bafb505f --- /dev/null +++ b/monkey/monkey_island/cc/services/telemetry/processing/utils.py @@ -0,0 +1,13 @@ +from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.node import NodeService + + +def get_edge_by_scan_or_exploit_telemetry(telemetry_json): + dst_ip = telemetry_json['data']['machine']['ip_addr'] + dst_domain_name = telemetry_json['data']['machine']['domain_name'] + src_monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) + dst_node = NodeService.get_monkey_by_ip(dst_ip) + if dst_node is None: + dst_node = NodeService.get_or_create_node(dst_ip, dst_domain_name) + + return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"])