p15693087
/
monkey
forked from p15670423/monkey
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

Seperated logic from entry points 

Node/Edge queries return minimal information necessary. already formatted.
Add MonkeyIsland to NetMap
This commit is contained in:
Itay Mizeretz 2017-09-06 14:49:58 +03:00
parent c143987138
commit 91581d00ab
9 changed files with 258 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
monkey_island/cc/app.py
View File

@ -18,8 +18,6 @@ from cc.resources.root import Root
__author__ = 'Barak'
# TODO: separate logic from resources
def serve_static_file(path):
print 'requested', path
if path.startswith('api/'):
@ -28,6 +26,7 @@ def serve_static_file(path):
def serve_home():
# TODO: remove this or merge with frontend.
return serve_static_file('index.html')
@ -75,7 +74,7 @@ def init_app(mongo_url):
api.add_resource(Telemetry, '/api/telemetry', '/api/telemetry/', '/api/telemetry/<string:monkey_guid>')
api.add_resource(MonkeyConfiguration, '/api/configuration', '/api/configuration/')
api.add_resource(MonkeyDownload, '/api/monkey/download', '/api/monkey/download/',
'/api/monkey/download/<string:path>')
'/api/monkey/download/<string:path>')
api.add_resource(NetMap, '/api/netmap', '/api/netmap/')
api.add_resource(Edge, '/api/netmap/edge', '/api/netmap/edge/')
api.add_resource(Node, '/api/netmap/node', '/api/netmap/node/')

4
monkey_island/cc/island_config.py Normal file
View File

@ -0,0 +1,4 @@
__author__ = 'itay.mizeretz'
ISLAND_PORT = 5000
DEFAULT_MONGO_URL = "mongodb://localhost:27017/monkeyisland"

5
monkey_island/cc/main.py
View File

@ -9,10 +9,9 @@ if BASE_PATH not in sys.path:
from cc.app import init_app
from cc.utils import init_collections, local_ip_addresses
from cc.island_config import DEFAULT_MONGO_URL, ISLAND_PORT
ISLAND_PORT = 5000
DEFAULT_MONGO_URL = "mongodb://localhost:27017/monkeyisland"
# TODO: remove this, and get from global config`
INITIAL_USERNAMES = ['Administrator', 'root', 'user']
INITIAL_PASSWORDS = ["Password1!", "1234", "password", "12345678"]

19
monkey_island/cc/resources/local_run.py
View File

@ -7,8 +7,8 @@ from flask import request, jsonify, make_response
import flask_restful
from cc.resources.monkey_download import get_monkey_executable
from cc.utils import local_ips
from cc.island_config import ISLAND_PORT
from cc.services.node import NodeService
__author__ = 'Barak'
@ -21,7 +21,7 @@ def run_local_monkey(island_address):
# get the monkey executable suitable to run on the server
result = get_monkey_executable(platform.system().lower(), platform.machine().lower())
if not result:
return (False, "OS Type not found")
return False, "OS Type not found"
monkey_path = os.path.join('binaries', result['filename'])
target_path = os.path.join(os.getcwd(), result['filename'])
@ -30,8 +30,8 @@ def run_local_monkey(island_address):
try:
copyfile(monkey_path, target_path)
os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG)
except Exception, exc:
return (False, "Copy file failed: %s" % exc)
except Exception as exc:
return False, "Copy file failed: %s" % exc
# run the monkey
try:
@ -39,16 +39,15 @@ def run_local_monkey(island_address):
if sys.platform == "win32":
args = "".join(args)
pid = subprocess.Popen(args, shell=True).pid
except Exception, exc:
return (False, "popen failed: %s" % exc)
except Exception as exc:
return False, "popen failed: %s" % exc
return (True, "pis: %s" % pid)
return True, "pis: %s" % pid
class LocalRun(flask_restful.Resource):
def get(self):
# TODO implement is_running from db monkeys collection
return jsonify(is_running=False)
return jsonify(is_running=(NodeService.get_monkey_island_monkey() is not None))
def post(self):
body = json.loads(request.data)

8
monkey_island/cc/resources/monkey.py
View File

@ -1,7 +1,7 @@
import json
from datetime import datetime, timedelta
import dateutil
import dateutil.parser
from flask import request
import flask_restful
@ -116,9 +116,9 @@ class Monkey(flask_restful.Resource):
existing_node = mongo.db.node.find_one({"ip_addresses": {"$in": monkey_json["ip_addresses"]}})
if existing_node:
id = existing_node["_id"]
for edge in mongo.db.edge.find({"to": id}):
node_id = existing_node["_id"]
for edge in mongo.db.edge.find({"to": node_id}):
mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}})
mongo.db.node.remove({"_id": id})
mongo.db.node.remove({"_id": node_id})
return {"id": new_monkey_id}

8
monkey_island/cc/resources/netmap.py
View File

@ -1,5 +1,6 @@
import flask_restful
from cc.services.edge import EdgeService
from cc.services.node import NodeService
from cc.database import mongo
@ -11,10 +12,15 @@ class NetMap(flask_restful.Resource):
monkeys = [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})]
nodes = [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})]
edges = [self.edge_to_net_edge(x) for x in mongo.db.edge.find({})]
monkey_island = []
if NodeService.get_monkey_island_monkey() is None:
monkey_island = [NodeService.get_monkey_island_pseudo_net_node()]
# TODO: implement when monkey exists on island
edges += EdgeService.get_monkey_island_pseudo_edges()
return \
{
"nodes": monkeys + nodes,
"nodes": monkeys + nodes + monkey_island,
"edges": edges
}

134
monkey_island/cc/resources/telemetry.py
View File

@ -1,11 +1,14 @@
import json
from datetime import datetime
import traceback
import dateutil
from flask import request
import flask_restful
from cc.database import mongo
from cc.services.edge import EdgeService
from cc.services.node import NodeService
from cc.utils import creds_add_username, creds_add_password
@ -38,74 +41,62 @@ class Telemetry(flask_restful.Resource):
telemetry_json['timestamp'] = datetime.now()
telem_id = mongo.db.telemetry.insert(telemetry_json)
monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
# update exploited monkeys parent
try:
if telemetry_json.get('telem_type') == 'tunnel':
if telemetry_json['data']:
host = telemetry_json['data'].split(":")[-2].replace("//", "")
tunnel_host = mongo.db.monkey.find_one({"ip_addresses": host})
mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']},
{'$set': {'tunnel_guid': tunnel_host.get('guid'),
'modifytime': datetime.now()}},
upsert=False)
else:
mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']},
{'$unset': {'tunnel_guid': ''},
'$set': {'modifytime': datetime.now()}},
upsert=False)
self.process_tunnel_telemetry(telemetry_json)
elif telemetry_json.get('telem_type') == 'state':
if telemetry_json['data']['done']:
mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']},
{'$set': {'dead': True, 'modifytime': datetime.now()}},
upsert=False)
else:
mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']},
{'$set': {'dead': False, 'modifytime': datetime.now()}},
upsert=False)
self.process_state_telemetry(telemetry_json)
elif telemetry_json.get('telem_type') in ['scan', 'exploit']:
dst_ip = telemetry_json['data']['machine']['ip_addr']
src_monkey = mongo.db.monkey.find_one({"guid": telemetry_json['monkey_guid']})
dst_monkey = mongo.db.monkey.find_one({"ip_addresses": dst_ip})
if dst_monkey:
edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_monkey["_id"]})
if edge is None:
edge = self.insert_edge(src_monkey["_id"], dst_monkey["_id"])
else:
dst_node = mongo.db.node.find_one({"ip_addresses": dst_ip})
if dst_node is None:
dst_node_insert_result = mongo.db.node.insert_one({"ip_addresses": [dst_ip]})
dst_node = mongo.db.node.find_one({"_id": dst_node_insert_result.inserted_id})
edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_node["_id"]})
if edge is None:
edge = self.insert_edge(src_monkey["_id"], dst_node["_id"])
if telemetry_json.get('telem_type') == 'scan':
self.add_scan_to_edge(edge, telemetry_json)
else:
self.add_exploit_to_edge(edge, telemetry_json)
except StandardError as e:
pass
# Update credentials DB
try:
if (telemetry_json.get('telem_type') == 'system_info_collection') and (telemetry_json['data'].has_key('credentials')):
creds = telemetry_json['data']['credentials']
for user in creds:
creds_add_username(user)
if creds[user].has_key('password'):
creds_add_password(creds[user]['password'])
self.process_scan_exploit_telemetry(telemetry_json)
elif telemetry_json.get('telem_type') == 'system_info_collection':
self.process_system_info_telemetry(telemetry_json)
NodeService.update_monkey_modify_time(monkey["_id"])
except StandardError as ex:
print("Exception caught while updating DB credentials: %s" % str(ex))
print("Exception caught while processing telemetry: %s" % str(ex))
traceback.print_exc()
return mongo.db.telemetry.find_one_or_404({"_id": telem_id})
def process_tunnel_telemetry(self, telemetry_json):
monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])["_id"]
NodeService.unset_all_monkey_tunnels(monkey_id)
if telemetry_json['data']:
host = telemetry_json['data'].split(":")[-2].replace("//", "")
tunnel_host_id = NodeService.get_monkey_by_ip(host)["_id"]
NodeService.set_monkey_tunnel(monkey_id, tunnel_host_id)
def process_state_telemetry(self, telemetry_json):
monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
if telemetry_json['data']['done']:
NodeService.set_monkey_dead(monkey, True)
else:
NodeService.set_monkey_dead(monkey, False)
def process_scan_exploit_telemetry(self, telemetry_json):
dst_ip = telemetry_json['data']['machine']['ip_addr']
src_monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
dst_node = NodeService.get_monkey_by_ip(dst_ip)
if dst_node is None:
dst_node = NodeService.get_or_create_node(dst_ip)
edge = EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"])
if telemetry_json.get('telem_type') == 'scan':
self.add_scan_to_edge(edge, telemetry_json)
else:
self.add_exploit_to_edge(edge, telemetry_json)
def process_system_info_telemetry(self, telemetry_json):
if 'credentials' in telemetry_json['data']:
creds = telemetry_json['data']['credentials']
for user in creds:
creds_add_username(user)
if 'password' in creds[user]:
creds_add_password(creds[user]['password'])
def add_scan_to_edge(self, edge, telemetry_json):
data = telemetry_json['data']['machine']
data.pop("ip_addr")
@ -120,6 +111,22 @@ class Telemetry(flask_restful.Resource):
{"$push": {"scans": new_scan}}
)
node = mongo.db.node.find_one({"_id": edge["to"]})
if node is not None:
if new_scan["scanner"] == "TcpScanner":
scan_os = new_scan["data"]["os"]
if "type" in scan_os:
mongo.db.node.update({"_id": node["_id"]},
{"$set": {"os.type": scan_os["type"]}},
upsert=False)
if "version" in scan_os:
mongo.db.node.update({"_id": node["_id"]},
{"$set": {"os.version": scan_os["version"]}},
upsert=False)
def add_exploit_to_edge(self, edge, telemetry_json):
data = telemetry_json['data']
data["machine"].pop("ip_addr")
@ -134,12 +141,3 @@ class Telemetry(flask_restful.Resource):
{"$push": {"exploits": new_exploit}}
)
def insert_edge(self, from_id, to_id):
edge_insert_result = mongo.db.edge.insert_one(
{
"from": from_id,
"to": to_id,
"scans": [],
"exploits": []
})
return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id})

59
monkey_island/cc/services/edge.py
View File

@ -6,6 +6,9 @@ __author__ = "itay.mizeretz"
class EdgeService:
def __init__(self):
pass
@staticmethod
def get_displayed_edge_by_id(edge_id):
edge = mongo.db.edge.find({"_id": ObjectId(edge_id)})[0]
@ -26,11 +29,10 @@ class EdgeService:
os = {}
exploits = []
if len(edge["scans"]) > 0:
services = edge["scans"][-1]["data"]["services"]
services = EdgeService.services_to_displayed_services(edge["scans"][-1]["data"]["services"])
os = edge["scans"][-1]["data"]["os"]
for exploit in edge["exploits"]:
new_exploit = EdgeService.exploit_to_displayed_exploit(exploit)
if (len(exploits) > 0) and (exploits[-1]["exploiter"] == exploit["exploiter"]):
@ -66,18 +68,14 @@ class EdgeService:
def exploit_to_displayed_exploit(exploit):
user = ""
password = ""
result = False
# TODO: implement for other exploiters
# TODO: The format that's used today to get the credentials is bad. Change it from monkey side and adapt.
result = exploit["data"]["result"]
if exploit["exploiter"] == "RdpExploiter":
# TODO: check if there could be multiple creds
result = exploit["data"]["result"]
user = exploit["data"]["machine"]["creds"].keys()[0]
password = exploit["data"]["machine"]["creds"][user]
elif exploit["exploiter"] == "SmbExploiter":
result = exploit["data"]["result"]
if result:
user = exploit["data"]["machine"]["cred"].keys()[0]
password = exploit["data"]["machine"]["cred"][user]
@ -91,4 +89,47 @@ class EdgeService:
"user": user,
"password": password,
"result": result,
}
}
@staticmethod
def insert_edge(from_id, to_id):
edge_insert_result = mongo.db.edge.insert_one(
{
"from": from_id,
"to": to_id,
"scans": [],
"exploits": [],
"tunnel": False
})
return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id})
@staticmethod
def get_or_create_edge(edge_from, edge_to):
tunnel_edge = mongo.db.edge.find_one({"from": edge_from, "to": edge_to})
if tunnel_edge is None:
tunnel_edge = EdgeService.insert_edge(edge_from, edge_to)
return tunnel_edge
@staticmethod
def get_monkey_island_pseudo_edges():
edges = []
monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) if "tunnel" not in x]
# TODO: find solution for these ids.
count = 0
for monkey_id in monkey_ids:
count += 1
edges.append(
{
"id": ObjectId(hex(count)[2:].zfill(24)),
"from": monkey_id,
"to": ObjectId("000000000000000000000000")
}
)
return edges
@staticmethod
def services_to_displayed_services(services):
# TODO: Consider returning extended information on services.
return [x + ": " + services[x]["name"] for x in services]

148
monkey_island/cc/services/node.py
View File

@ -1,15 +1,20 @@
import datetime
from bson import ObjectId
from cc.database import mongo
from cc.services.edge import EdgeService
from cc.utils import local_ip_addresses
__author__ = "itay.mizeretz"
class NodeService:
def __init__(self):
pass
@staticmethod
def get_displayed_node_by_id(node_id):
if ObjectId(node_id) == ObjectId("000000000000000000000000"):
return NodeService.get_monkey_island_node()
edges = EdgeService.get_displayed_edges_by_to(node_id)
accessible_from_nodes = []
@ -17,31 +22,28 @@ class NodeService:
new_node = {"id": node_id}
node = mongo.db.node.find_one({"_id": ObjectId(node_id)})
node = NodeService.get_node_by_id(node_id)
if node is None:
monkey = mongo.db.monkey.find_one({"_id": ObjectId(node_id)})
monkey = NodeService.get_monkey_by_id(node_id)
if monkey is None:
return new_node
# node is infected
new_node = NodeService.monkey_to_net_node(monkey)
for key in monkey:
# TODO: do something with tunnel
if key not in ["_id", "modifytime", "parent", "tunnel", "tunnel_guid"]:
if key not in ["_id", "modifytime", "parent", "tunnel", "dead"]:
new_node[key] = monkey[key]
new_node["os"] = NodeService.get_monkey_os(monkey)
new_node["label"] = NodeService.get_monkey_label(monkey)
new_node["group"] = NodeService.get_monkey_group(monkey)
else:
# node is uninfected
new_node = NodeService.node_to_net_node(node)
new_node["ip_addresses"] = node["ip_addresses"]
new_node["group"] = "clean"
for edge in edges:
accessible_from_nodes.append({"id": edge["from"]})
for exploit in edge["exploits"]:
exploit["origin"] = edge["from"]
exploit["origin"] = NodeService.get_monkey_label(NodeService.get_monkey_by_id(edge["from"]))
exploits.append(exploit)
exploits.sort(cmp=NodeService._cmp_exploits_by_timestamp)
@ -50,19 +52,20 @@ class NodeService:
new_node["accessible_from_nodes"] = accessible_from_nodes
if len(edges) > 0:
new_node["services"] = edges[-1]["services"]
new_node["os"] = edges[-1]["os"]["type"]
if "label" not in new_node:
new_node["label"] = edges[-1]["os"]["version"] + " : " + node["ip_addresses"][0]
# TODO: add exploited by
return new_node
@staticmethod
def get_node_label(node):
return node["os"]["version"] + " : " + node["ip_addresses"][0]
@staticmethod
def _cmp_exploits_by_timestamp(exploit_1, exploit_2):
if exploit_1["timestamp"] == exploit_2["timestamp"]:
if exploit_1["start_timestamp"] == exploit_2["start_timestamp"]:
return 0
if exploit_1["timestamp"] > exploit_2["timestamp"]:
if exploit_1["start_timestamp"] > exploit_2["start_timestamp"]:
return 1
return -1
@ -90,6 +93,9 @@ class NodeService:
@staticmethod
def get_monkey_group(monkey):
if len(set(monkey["ip_addresses"]).intersection(local_ip_addresses())) != 0:
return "islandInfected"
return "manuallyInfected" if NodeService.get_monkey_manual_run(monkey) else "infected"
@staticmethod
@ -105,26 +111,100 @@ class NodeService:
@staticmethod
def node_to_net_node(node):
os_version = "undefined"
os_type = "undefined"
found = False
# TODO: Set this as data when received
for edge in mongo.db.edge.find({"to": node["_id"]}):
for scan in edge["scans"]:
if scan["scanner"] != "TcpScanner":
continue
os_type = scan["data"]["os"]["type"]
if "version" in scan["data"]["os"]:
os_version = scan["data"]["os"]["version"]
found = True
break
if found:
break
return \
{
"id": node["_id"],
"label": os_version + " : " + node["ip_addresses"][0],
"label": NodeService.get_node_label(node),
"group": "clean",
"os": os_type
}
"os": node["os"]["type"]
}
@staticmethod
def unset_all_monkey_tunnels(monkey_id):
mongo.db.edge.update(
{"from": monkey_id, 'tunnel': True},
{'$set': {'tunnel': False}},
upsert=False)
@staticmethod
def set_monkey_tunnel(monkey_id, tunnel_host_id):
tunnel_edge = EdgeService.get_or_create_edge(monkey_id, tunnel_host_id)
mongo.db.edge.update({"_id": tunnel_edge["_id"]},
{'$set': {'tunnel': True}},
upsert=False)
@staticmethod
def insert_node(ip_address):
new_node_insert_result = mongo.db.node.insert_one(
{
"ip_addresses": [ip_address],
"os":
{
"type": "unknown",
"version": "unknown"
}
})
return mongo.db.node.find_one({"_id": new_node_insert_result.inserted_id})
@staticmethod
def get_or_create_node(ip_address):
new_node = mongo.db.node.find_one({"ip_addresses": ip_address})
if new_node is None:
new_node = NodeService.insert_node(ip_address)
return new_node
@staticmethod
def get_monkey_by_id(monkey_id):
return mongo.db.monkey.find_one({"_id": ObjectId(monkey_id)})
@staticmethod
def get_monkey_by_guid(monkey_guid):
return mongo.db.monkey.find_one({"guid": monkey_guid})
@staticmethod
def get_monkey_by_ip(ip_address):
return mongo.db.monkey.find_one({"ip_addresses": ip_address})
@staticmethod
def get_node_by_ip(ip_address):
return mongo.db.node.find_one({"ip_addresses": ip_address})
@staticmethod
def get_node_by_id(node_id):
return mongo.db.node.find_one({"_id": ObjectId(node_id)})
@staticmethod
def update_monkey_modify_time(monkey_id):
mongo.db.monkey.update({"_id": monkey_id},
{"$set": {"modifytime": datetime.now()}},
upsert=False)
@staticmethod
def set_monkey_dead(monkey, is_dead):
mongo.db.monkey.update({"guid": monkey['guid']},
{'$set': {'dead': is_dead}},
upsert=False)
@staticmethod
def get_monkey_island_monkey():
ip_addresses = local_ip_addresses()
for ip_address in ip_addresses:
monkey = NodeService.get_monkey_by_ip(ip_address)
if monkey is not None:
return monkey
return None
@staticmethod
def get_monkey_island_pseudo_net_node():
return\
{
"id": ObjectId("000000000000000000000000"),
"label": "MonkeyIsland",
"group": "islandClean",
}
@staticmethod
def get_monkey_island_node():
island_node = NodeService.get_monkey_island_pseudo_net_node()
island_node["ip_addresses"] = local_ip_addresses()
return island_node