p34709852
/
monkey
forked from p15670423/monkey
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Moved all processing code into services/telemetry/processing 

Telemetry class line count: 87. 😄
This commit is contained in:
Shay Nehmad 2019-08-19 14:34:21 +03:00
parent c1ac45470c
commit 757af1c6af
10 changed files with 240 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
monkey/monkey_island/cc/resources/telemetry.py
View File

@ -1,6 +1,5 @@
import json
import logging
import copy
from datetime import datetime
import dateutil
@ -9,14 +8,9 @@ from flask import request
from monkey_island.cc.auth import jwt_required
from monkey_island.cc.database import mongo
from monkey_island.cc.services import mimikatz_utils
from monkey_island.cc.services.config import ConfigService
from monkey_island.cc.services.edge import EdgeService
from monkey_island.cc.services.node import NodeService
from monkey_island.cc.encryptor import encryptor
from monkey_island.cc.services.wmi_handler import WMIHandler
from monkey_island.cc.services.telemetry.processing.hooks import TELEMETRY_CATEGORY_TO_PROCESSING_FUNC
from monkey_island.cc.models.monkey import Monkey
from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence
__author__ = 'Barak'
@ -58,8 +52,8 @@ class Telemetry(flask_restful.Resource):
try:
NodeService.update_monkey_modify_time(monkey["_id"])
telem_category = telemetry_json.get('telem_category')
if telem_category in TELEM_PROCESS_DICT:
TELEM_PROCESS_DICT[telem_category](telemetry_json)
if telem_category in TELEMETRY_CATEGORY_TO_PROCESSING_FUNC:
TELEMETRY_CATEGORY_TO_PROCESSING_FUNC[telem_category](telemetry_json)
else:
logger.info('Got unknown type of telemetry: %s' % telem_category)
except Exception as ex:
@ -90,216 +84,3 @@ class Telemetry(flask_restful.Resource):
x['data']['credentials'][new_user] = x['data']['credentials'].pop(user)
return objects
@staticmethod
def get_edge_by_scan_or_exploit_telemetry(telemetry_json):
dst_ip = telemetry_json['data']['machine']['ip_addr']
dst_domain_name = telemetry_json['data']['machine']['domain_name']
src_monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
dst_node = NodeService.get_monkey_by_ip(dst_ip)
if dst_node is None:
dst_node = NodeService.get_or_create_node(dst_ip, dst_domain_name)
return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"])
@staticmethod
def process_tunnel_telemetry(telemetry_json):
monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])["_id"]
if telemetry_json['data']['proxy'] is not None:
tunnel_host_ip = telemetry_json['data']['proxy'].split(":")[-2].replace("//", "")
NodeService.set_monkey_tunnel(monkey_id, tunnel_host_ip)
else:
NodeService.unset_all_monkey_tunnels(monkey_id)
@staticmethod
def process_state_telemetry(telemetry_json):
monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
if telemetry_json['data']['done']:
NodeService.set_monkey_dead(monkey, True)
else:
NodeService.set_monkey_dead(monkey, False)
@staticmethod
def process_exploit_telemetry(telemetry_json):
edge = Telemetry.get_edge_by_scan_or_exploit_telemetry(telemetry_json)
Telemetry.encrypt_exploit_creds(telemetry_json)
telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started'])
telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished'])
new_exploit = copy.deepcopy(telemetry_json['data'])
new_exploit.pop('machine')
new_exploit['timestamp'] = telemetry_json['timestamp']
mongo.db.edge.update(
{'_id': edge['_id']},
{'$push': {'exploits': new_exploit}}
)
if new_exploit['result']:
EdgeService.set_edge_exploited(edge)
for attempt in telemetry_json['data']['attempts']:
if attempt['result']:
found_creds = {'user': attempt['user']}
for field in ['password', 'lm_hash', 'ntlm_hash', 'ssh_key']:
if len(attempt[field]) != 0:
found_creds[field] = attempt[field]
NodeService.add_credentials_to_node(edge['to'], found_creds)
@staticmethod
def process_scan_telemetry(telemetry_json):
edge = Telemetry.get_edge_by_scan_or_exploit_telemetry(telemetry_json)
data = copy.deepcopy(telemetry_json['data']['machine'])
ip_address = data.pop("ip_addr")
domain_name = data.pop("domain_name")
new_scan = \
{
"timestamp": telemetry_json["timestamp"],
"data": data
}
mongo.db.edge.update(
{"_id": edge["_id"]},
{"$push": {"scans": new_scan},
"$set": {"ip_address": ip_address, 'domain_name': domain_name}}
)
node = mongo.db.node.find_one({"_id": edge["to"]})
if node is not None:
scan_os = new_scan["data"]["os"]
if "type" in scan_os:
mongo.db.node.update({"_id": node["_id"]},
{"$set": {"os.type": scan_os["type"]}},
upsert=False)
if "version" in scan_os:
mongo.db.node.update({"_id": node["_id"]},
{"$set": {"os.version": scan_os["version"]}},
upsert=False)
@staticmethod
def process_system_info_telemetry(telemetry_json):
Telemetry.process_ssh_info(telemetry_json)
Telemetry.process_credential_info(telemetry_json)
monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id')
Telemetry.process_mimikatz_and_wmi_info(monkey_id, telemetry_json)
Telemetry.process_aws_data(monkey_id, telemetry_json)
test_antivirus_existence(telemetry_json)
@staticmethod
def process_mimikatz_and_wmi_info(monkey_id, telemetry_json):
users_secrets = {}
if 'mimikatz' in telemetry_json['data']:
users_secrets = mimikatz_utils.MimikatzSecrets. \
extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', ''))
if 'wmi' in telemetry_json['data']:
wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets)
wmi_handler.process_and_handle_wmi_info()
@staticmethod
def process_aws_data(monkey_id, telemetry_json):
if 'aws' in telemetry_json['data']:
if 'instance_id' in telemetry_json['data']['aws']:
mongo.db.monkey.update_one({'_id': monkey_id},
{'$set': {'aws_instance_id': telemetry_json['data']['aws']['instance_id']}})
@staticmethod
def process_credential_info(telemetry_json):
if 'credentials' in telemetry_json['data']:
creds = telemetry_json['data']['credentials']
Telemetry.encrypt_system_info_creds(creds)
Telemetry.add_system_info_creds_to_config(creds)
Telemetry.replace_user_dot_with_comma(creds)
@staticmethod
def process_ssh_info(telemetry_json):
if 'ssh_info' in telemetry_json['data']:
ssh_info = telemetry_json['data']['ssh_info']
Telemetry.encrypt_system_info_ssh_keys(ssh_info)
if telemetry_json['data']['network_info']['networks']:
# We use user_name@machine_ip as the name of the ssh key stolen, thats why we need ip from telemetry
Telemetry.add_ip_to_ssh_keys(telemetry_json['data']['network_info']['networks'][0], ssh_info)
Telemetry.add_system_info_ssh_keys_to_config(ssh_info)
@staticmethod
def add_ip_to_ssh_keys(ip, ssh_info):
for key in ssh_info:
key['ip'] = ip['addr']
@staticmethod
def process_trace_telemetry(telemetry_json):
# Nothing to do
return
@staticmethod
def replace_user_dot_with_comma(creds):
for user in creds:
if -1 != user.find('.'):
new_user = user.replace('.', ',')
creds[new_user] = creds.pop(user)
@staticmethod
def encrypt_system_info_creds(creds):
for user in creds:
for field in ['password', 'lm_hash', 'ntlm_hash']:
if field in creds[user]:
# this encoding is because we might run into passwords which are not pure ASCII
creds[user][field] = encryptor.enc(creds[user][field].encode('utf-8'))
@staticmethod
def encrypt_system_info_ssh_keys(ssh_info):
for idx, user in enumerate(ssh_info):
for field in ['public_key', 'private_key', 'known_hosts']:
if ssh_info[idx][field]:
ssh_info[idx][field] = encryptor.enc(ssh_info[idx][field].encode('utf-8'))
@staticmethod
def add_system_info_creds_to_config(creds):
for user in creds:
ConfigService.creds_add_username(user)
if 'password' in creds[user]:
ConfigService.creds_add_password(creds[user]['password'])
if 'lm_hash' in creds[user]:
ConfigService.creds_add_lm_hash(creds[user]['lm_hash'])
if 'ntlm_hash' in creds[user]:
ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash'])
@staticmethod
def add_system_info_ssh_keys_to_config(ssh_info):
for user in ssh_info:
ConfigService.creds_add_username(user['name'])
# Public key is useless without private key
if user['public_key'] and user['private_key']:
ConfigService.ssh_add_keys(user['public_key'], user['private_key'],
user['name'], user['ip'])
@staticmethod
def encrypt_exploit_creds(telemetry_json):
attempts = telemetry_json['data']['attempts']
for i in range(len(attempts)):
for field in ['password', 'lm_hash', 'ntlm_hash']:
credential = attempts[i][field]
if len(credential) > 0:
attempts[i][field] = encryptor.enc(credential.encode('utf-8'))
@staticmethod
def process_post_breach_telemetry(telemetry_json):
mongo.db.monkey.update(
{'guid': telemetry_json['monkey_guid']},
{'$push': {'pba_results': telemetry_json['data']}})
@staticmethod
def process_attack_telemetry(telemetry_json):
# No processing required
pass
TELEM_PROCESS_DICT = \
{
'tunnel': Telemetry.process_tunnel_telemetry,
'state': Telemetry.process_state_telemetry,
'exploit': Telemetry.process_exploit_telemetry,
'scan': Telemetry.process_scan_telemetry,
'system_info': Telemetry.process_system_info_telemetry,
'trace': Telemetry.process_trace_telemetry,
'post_breach': Telemetry.process_post_breach_telemetry,
'attack': Telemetry.process_attack_telemetry
}

7
monkey/monkey_island/cc/services/telemetry/processing/__init__.py Normal file
View File

@ -0,0 +1,7 @@
# import all implemented hooks, for brevity of hooks.py file
from tunnel import process_tunnel_telemetry
from state import process_state_telemetry
from exploit import process_exploit_telemetry
from scan import process_scan_telemetry
from system_info import process_system_info_telemetry
from post_breach import process_post_breach_telemetry

45
monkey/monkey_island/cc/services/telemetry/processing/exploit.py Normal file
View File

@ -0,0 +1,45 @@
import copy
import dateutil
from monkey_island.cc.database import mongo
from monkey_island.cc.encryptor import encryptor
from monkey_island.cc.services.edge import EdgeService
from monkey_island.cc.services.node import NodeService
from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry
def process_exploit_telemetry(telemetry_json):
edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json)
encrypt_exploit_creds(telemetry_json)
telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started'])
telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished'])
new_exploit = copy.deepcopy(telemetry_json['data'])
new_exploit.pop('machine')
new_exploit['timestamp'] = telemetry_json['timestamp']
mongo.db.edge.update(
{'_id': edge['_id']},
{'$push': {'exploits': new_exploit}}
)
if new_exploit['result']:
EdgeService.set_edge_exploited(edge)
for attempt in telemetry_json['data']['attempts']:
if attempt['result']:
found_creds = {'user': attempt['user']}
for field in ['password', 'lm_hash', 'ntlm_hash', 'ssh_key']:
if len(attempt[field]) != 0:
found_creds[field] = attempt[field]
NodeService.add_credentials_to_node(edge['to'], found_creds)
def encrypt_exploit_creds(telemetry_json):
attempts = telemetry_json['data']['attempts']
for i in range(len(attempts)):
for field in ['password', 'lm_hash', 'ntlm_hash']:
credential = attempts[i][field]
if len(credential) > 0:
attempts[i][field] = encryptor.enc(credential.encode('utf-8'))

14
monkey/monkey_island/cc/services/telemetry/processing/hooks.py Normal file
View File

@ -0,0 +1,14 @@
from monkey_island.cc.services.telemetry.processing import *
TELEMETRY_CATEGORY_TO_PROCESSING_FUNC = \
{
'tunnel': process_tunnel_telemetry,
'state': process_state_telemetry,
'exploit': process_exploit_telemetry,
'scan': process_scan_telemetry,
'system_info': process_system_info_telemetry,
'post_breach': process_post_breach_telemetry,
# `lambda *args, **kwargs: None` is a no-op.
'trace': lambda *args, **kwargs: None,
'attack': lambda *args, **kwargs: None,
}

7
monkey/monkey_island/cc/services/telemetry/processing/post_breach.py Normal file
View File

@ -0,0 +1,7 @@
from monkey_island.cc.database import mongo
def process_post_breach_telemetry(telemetry_json):
mongo.db.monkey.update(
{'guid': telemetry_json['monkey_guid']},
{'$push': {'pba_results': telemetry_json['data']}})

33
monkey/monkey_island/cc/services/telemetry/processing/scan.py Normal file
View File

@ -0,0 +1,33 @@
import copy
from monkey_island.cc.database import mongo
from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry
def process_scan_telemetry(telemetry_json):
edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json)
data = copy.deepcopy(telemetry_json['data']['machine'])
ip_address = data.pop("ip_addr")
domain_name = data.pop("domain_name")
new_scan = \
{
"timestamp": telemetry_json["timestamp"],
"data": data
}
mongo.db.edge.update(
{"_id": edge["_id"]},
{"$push": {"scans": new_scan},
"$set": {"ip_address": ip_address, 'domain_name': domain_name}}
)
node = mongo.db.node.find_one({"_id": edge["to"]})
if node is not None:
scan_os = new_scan["data"]["os"]
if "type" in scan_os:
mongo.db.node.update({"_id": node["_id"]},
{"$set": {"os.type": scan_os["type"]}},
upsert=False)
if "version" in scan_os:
mongo.db.node.update({"_id": node["_id"]},
{"$set": {"os.version": scan_os["version"]}},
upsert=False)

9
monkey/monkey_island/cc/services/telemetry/processing/state.py Normal file
View File

@ -0,0 +1,9 @@
from monkey_island.cc.services.node import NodeService
def process_state_telemetry(telemetry_json):
monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
if telemetry_json['data']['done']:
NodeService.set_monkey_dead(monkey, True)
else:
NodeService.set_monkey_dead(monkey, False)

99
monkey/monkey_island/cc/services/telemetry/processing/system_info.py Normal file
View File

@ -0,0 +1,99 @@
from monkey_island.cc.database import mongo
from monkey_island.cc.services import mimikatz_utils
from monkey_island.cc.services.node import NodeService
from monkey_island.cc.services.config import ConfigService
from monkey_island.cc.services.telemetry.zero_trust_tests.antivirus_existence import test_antivirus_existence
from monkey_island.cc.services.wmi_handler import WMIHandler
from monkey_island.cc.encryptor import encryptor
def process_system_info_telemetry(telemetry_json):
process_ssh_info(telemetry_json)
process_credential_info(telemetry_json)
process_mimikatz_and_wmi_info(telemetry_json)
process_aws_data(telemetry_json)
test_antivirus_existence(telemetry_json)
def process_ssh_info(telemetry_json):
if 'ssh_info' in telemetry_json['data']:
ssh_info = telemetry_json['data']['ssh_info']
encrypt_system_info_ssh_keys(ssh_info)
if telemetry_json['data']['network_info']['networks']:
# We use user_name@machine_ip as the name of the ssh key stolen, thats why we need ip from telemetry
add_ip_to_ssh_keys(telemetry_json['data']['network_info']['networks'][0], ssh_info)
add_system_info_ssh_keys_to_config(ssh_info)
def add_system_info_ssh_keys_to_config(ssh_info):
for user in ssh_info:
ConfigService.creds_add_username(user['name'])
# Public key is useless without private key
if user['public_key'] and user['private_key']:
ConfigService.ssh_add_keys(user['public_key'], user['private_key'],
user['name'], user['ip'])
def add_ip_to_ssh_keys(ip, ssh_info):
for key in ssh_info:
key['ip'] = ip['addr']
def encrypt_system_info_ssh_keys(ssh_info):
for idx, user in enumerate(ssh_info):
for field in ['public_key', 'private_key', 'known_hosts']:
if ssh_info[idx][field]:
ssh_info[idx][field] = encryptor.enc(ssh_info[idx][field].encode('utf-8'))
def process_credential_info(telemetry_json):
if 'credentials' in telemetry_json['data']:
creds = telemetry_json['data']['credentials']
encrypt_system_info_creds(creds)
add_system_info_creds_to_config(creds)
replace_user_dot_with_comma(creds)
def replace_user_dot_with_comma(creds):
for user in creds:
if -1 != user.find('.'):
new_user = user.replace('.', ',')
creds[new_user] = creds.pop(user)
def add_system_info_creds_to_config(creds):
for user in creds:
ConfigService.creds_add_username(user)
if 'password' in creds[user]:
ConfigService.creds_add_password(creds[user]['password'])
if 'lm_hash' in creds[user]:
ConfigService.creds_add_lm_hash(creds[user]['lm_hash'])
if 'ntlm_hash' in creds[user]:
ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash'])
def encrypt_system_info_creds(creds):
for user in creds:
for field in ['password', 'lm_hash', 'ntlm_hash']:
if field in creds[user]:
# this encoding is because we might run into passwords which are not pure ASCII
creds[user][field] = encryptor.enc(creds[user][field].encode('utf-8'))
def process_mimikatz_and_wmi_info(telemetry_json):
users_secrets = {}
if 'mimikatz' in telemetry_json['data']:
users_secrets = mimikatz_utils.MimikatzSecrets. \
extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', ''))
if 'wmi' in telemetry_json['data']:
monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id')
wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets)
wmi_handler.process_and_handle_wmi_info()
def process_aws_data(telemetry_json):
if 'aws' in telemetry_json['data']:
if 'instance_id' in telemetry_json['data']['aws']:
monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id')
mongo.db.monkey.update_one({'_id': monkey_id},
{'$set': {'aws_instance_id': telemetry_json['data']['aws']['instance_id']}})

10
monkey/monkey_island/cc/services/telemetry/processing/tunnel.py Normal file
View File

@ -0,0 +1,10 @@
from monkey_island.cc.services.node import NodeService
def process_tunnel_telemetry(telemetry_json):
monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])["_id"]
if telemetry_json['data']['proxy'] is not None:
tunnel_host_ip = telemetry_json['data']['proxy'].split(":")[-2].replace("//", "")
NodeService.set_monkey_tunnel(monkey_id, tunnel_host_ip)
else:
NodeService.unset_all_monkey_tunnels(monkey_id)

13
monkey/monkey_island/cc/services/telemetry/processing/utils.py Normal file
View File

@ -0,0 +1,13 @@
from monkey_island.cc.services.edge import EdgeService
from monkey_island.cc.services.node import NodeService
def get_edge_by_scan_or_exploit_telemetry(telemetry_json):
dst_ip = telemetry_json['data']['machine']['ip_addr']
dst_domain_name = telemetry_json['data']['machine']['domain_name']
src_monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
dst_node = NodeService.get_monkey_by_ip(dst_ip)
if dst_node is None:
dst_node = NodeService.get_or_create_node(dst_ip, dst_domain_name)
return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"])