From 8163e398049cda83b6594ed22f81807239a57d66 Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Wed, 30 Aug 2017 18:14:24 +0300 Subject: [PATCH] Add edge and node get APIs --- monkey_island/cc/app.py | 8 +- monkey_island/cc/resources/edge.py | 23 ++--- monkey_island/cc/resources/monkey.py | 2 +- monkey_island/cc/resources/netmap.py | 46 +-------- monkey_island/cc/resources/node.py | 18 ++++ monkey_island/cc/resources/telemetry.py | 24 ++++- monkey_island/cc/services/__init__.py | 1 + monkey_island/cc/services/edge.py | 94 ++++++++++++++++++ monkey_island/cc/services/node.py | 126 ++++++++++++++++++++++++ 9 files changed, 275 insertions(+), 67 deletions(-) create mode 100644 monkey_island/cc/resources/node.py create mode 100644 monkey_island/cc/services/__init__.py create mode 100644 monkey_island/cc/services/edge.py create mode 100644 monkey_island/cc/services/node.py diff --git a/monkey_island/cc/app.py b/monkey_island/cc/app.py index cf6d72227..a962bec49 100644 --- a/monkey_island/cc/app.py +++ b/monkey_island/cc/app.py @@ -1,5 +1,5 @@ from datetime import datetime - +import bson from bson.json_util import dumps from flask import Flask, send_from_directory, redirect, make_response import flask_restful @@ -11,12 +11,14 @@ from cc.resources.telemetry import Telemetry from cc.resources.monkey_configuration import MonkeyConfiguration from cc.resources.monkey_download import MonkeyDownload from cc.resources.netmap import NetMap -from cc.resources.edge import Edge, Node +from cc.resources.edge import Edge +from cc.resources.node import Node from cc.resources.root import Root __author__ = 'Barak' +# TODO: separate logic from resources def serve_static_file(path): print 'requested', path @@ -30,7 +32,7 @@ def serve_home(): def normalize_obj(obj): - if obj.has_key('_id') and not obj.has_key('id'): + if '_id' in obj and not 'id' in obj: obj['id'] = obj['_id'] del obj['_id'] diff --git a/monkey_island/cc/resources/edge.py b/monkey_island/cc/resources/edge.py index bc17f6478..8aee77638 100644 --- a/monkey_island/cc/resources/edge.py +++ b/monkey_island/cc/resources/edge.py @@ -3,27 +3,16 @@ from flask import request import flask_restful from cc.database import mongo +from cc.services.edge import EdgeService __author__ = 'Barak' class Edge(flask_restful.Resource): def get(self): - id = request.args.get('id') - to = request.args.get('to') - if id: - edge = mongo.db.edge.find({"_id": ObjectId(id)})[0] - return {"edge": edge} - if to: - edges = mongo.db.edge.find({"to": ObjectId(to)}) - new_edges = [] - # TODO: find better solution for this - for i in range(edges.count()): - new_edges.append(edges[i]) - return {"edges": new_edges} + edge_id = request.args.get('id') + + if edge_id: + return {"edge": EdgeService.get_displayed_edge_by_id(edge_id)} + return {} - - -class Node(flask_restful.Resource): - def get(self): - pass diff --git a/monkey_island/cc/resources/monkey.py b/monkey_island/cc/resources/monkey.py index f8eceaa33..eb936a066 100644 --- a/monkey_island/cc/resources/monkey.py +++ b/monkey_island/cc/resources/monkey.py @@ -121,4 +121,4 @@ class Monkey(flask_restful.Resource): mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}}) mongo.db.node.remove({"_id": id}) - return {new_monkey_id} + return {"id": new_monkey_id} diff --git a/monkey_island/cc/resources/netmap.py b/monkey_island/cc/resources/netmap.py index d45154a8f..f62c3fbe8 100644 --- a/monkey_island/cc/resources/netmap.py +++ b/monkey_island/cc/resources/netmap.py @@ -1,5 +1,6 @@ import flask_restful +from cc.services.node import NodeService from cc.database import mongo __author__ = 'Barak' @@ -7,8 +8,8 @@ __author__ = 'Barak' class NetMap(flask_restful.Resource): def get(self, **kw): - monkeys = [self.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] - nodes = [self.node_to_net_node(x) for x in mongo.db.node.find({})] + monkeys = [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] + nodes = [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})] edges = [self.edge_to_net_edge(x) for x in mongo.db.edge.find({})] return \ @@ -17,47 +18,6 @@ class NetMap(flask_restful.Resource): "edges": edges } - def monkey_to_net_node(self, monkey): - os = "unknown" - if monkey["description"].lower().find("linux") != -1: - os = "linux" - elif monkey["description"].lower().find("windows") != -1: - os = "windows" - - manual_run = (monkey["parent"][0][1] == None) - return \ - { - "id": monkey["_id"], - "label": monkey["hostname"] + " : " + monkey["ip_addresses"][0], - "group": ("manuallyInfected" if manual_run else "infected"), - "os": os, - "dead": monkey["dead"], - } - - def node_to_net_node(self, node): - os_version = "undefined" - os_type = "undefined" - found = False - for edge in mongo.db.edge.find({"to": node["_id"]}): - for scan in edge["scans"]: - if scan["scanner"] != "TcpScanner": - continue - os_type = scan["data"]["os"]["type"] - if scan["data"]["os"].has_key("version"): - os_version = scan["data"]["os"]["version"] - found = True - break - if found: - break - - return \ - { - "id": node["_id"], - "label": os_version + " : " + node["ip_addresses"][0], - "group": "clean", - "os": os_type - } - def edge_to_net_edge(self, edge): return \ { diff --git a/monkey_island/cc/resources/node.py b/monkey_island/cc/resources/node.py new file mode 100644 index 000000000..aaea85463 --- /dev/null +++ b/monkey_island/cc/resources/node.py @@ -0,0 +1,18 @@ +from bson import ObjectId +from flask import request +import flask_restful + +from cc.database import mongo +from cc.services.edge import EdgeService +from cc.services.node import NodeService + +__author__ = 'Barak' + + +class Node(flask_restful.Resource): + def get(self): + node_id = request.args.get('id') + if node_id: + return NodeService.get_displayed_node_by_id(request.args.get('node_id')) + + return {} diff --git a/monkey_island/cc/resources/telemetry.py b/monkey_island/cc/resources/telemetry.py index e015173ff..311039b59 100644 --- a/monkey_island/cc/resources/telemetry.py +++ b/monkey_island/cc/resources/telemetry.py @@ -63,7 +63,7 @@ class Telemetry(flask_restful.Resource): mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, {'$set': {'dead': False, 'modifytime': datetime.now()}}, upsert=False) - elif telemetry_json.get('telem_type') == 'scan': + elif telemetry_json.get('telem_type') in ['scan', 'exploit']: dst_ip = telemetry_json['data']['machine']['ip_addr'] src_monkey = mongo.db.monkey.find_one({"guid": telemetry_json['monkey_guid']}) dst_monkey = mongo.db.monkey.find_one({"ip_addresses": dst_ip}) @@ -84,7 +84,10 @@ class Telemetry(flask_restful.Resource): if edge is None: edge = self.insert_edge(src_monkey["_id"], dst_node["_id"]) - self.add_scan_to_edge(edge, telemetry_json) + if telemetry_json.get('telem_type') == 'scan': + self.add_scan_to_edge(edge, telemetry_json) + else: + self.add_exploit_to_edge(edge, telemetry_json) except StandardError as e: pass @@ -117,11 +120,26 @@ class Telemetry(flask_restful.Resource): {"$push": {"scans": new_scan}} ) + def add_exploit_to_edge(self, edge, telemetry_json): + data = telemetry_json['data'] + data["machine"].pop("ip_addr") + new_exploit = \ + { + "timestamp": telemetry_json["timestamp"], + "data": data, + "exploiter": telemetry_json['data']['exploiter'] + } + mongo.db.edge.update( + {"_id": edge["_id"]}, + {"$push": {"exploits": new_exploit}} + ) + def insert_edge(self, from_id, to_id): edge_insert_result = mongo.db.edge.insert_one( { "from": from_id, "to": to_id, - "scans": [] + "scans": [], + "exploits": [] }) return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) diff --git a/monkey_island/cc/services/__init__.py b/monkey_island/cc/services/__init__.py new file mode 100644 index 000000000..142e51368 --- /dev/null +++ b/monkey_island/cc/services/__init__.py @@ -0,0 +1 @@ +__author__ = 'itay.mizeretz' \ No newline at end of file diff --git a/monkey_island/cc/services/edge.py b/monkey_island/cc/services/edge.py new file mode 100644 index 000000000..12dfd9559 --- /dev/null +++ b/monkey_island/cc/services/edge.py @@ -0,0 +1,94 @@ +from bson import ObjectId + +from cc.database import mongo + +__author__ = "itay.mizeretz" + + +class EdgeService: + @staticmethod + def get_displayed_edge_by_id(edge_id): + edge = mongo.db.edge.find({"_id": ObjectId(edge_id)})[0] + return EdgeService.edge_to_displayed_edge(edge) + + @staticmethod + def get_displayed_edges_by_to(to): + edges = mongo.db.edge.find({"to": ObjectId(to)}) + new_edges = [] + # TODO: find better solution for this + for i in range(edges.count()): + new_edges.append(EdgeService.edge_to_displayed_edge(edges[i])) + return new_edges + + @staticmethod + def edge_to_displayed_edge(edge): + services = {} + os = {} + exploits = [] + if len(edge["scans"]) > 0: + services = edge["scans"][-1]["data"]["services"] + os = edge["scans"][-1]["data"]["os"] + + for exploit in edge["exploits"]: + + new_exploit = EdgeService.exploit_to_displayed_exploit(exploit) + + if (len(exploits) > 0) and (exploits[-1]["exploiter"] == exploit["exploiter"]): + exploit_container = exploits[-1] + else: + exploit_container =\ + { + "exploiter": exploit["exploiter"], + "start_timestamp": exploit["timestamp"], + "end_timestamp": exploit["timestamp"], + "result": False, + "attempts": [] + } + + exploits.append(exploit_container) + + exploit_container["attempts"].append(new_exploit) + if new_exploit["result"]: + exploit_container["result"] = True + exploit_container["end_timestamp"] = new_exploit["timestamp"] + + return \ + { + "id": edge["_id"], + "from": edge["from"], + "to": edge["to"], + "services": services, + "os": os, + "exploits": exploits + } + + @staticmethod + def exploit_to_displayed_exploit(exploit): + user = "" + password = "" + result = False + + # TODO: implement for other exploiters + + if exploit["exploiter"] == "RdpExploiter": + # TODO: check if there could be multiple creds + result = exploit["data"]["result"] + user = exploit["data"]["machine"]["creds"].keys()[0] + password = exploit["data"]["machine"]["creds"][user] + + elif exploit["exploiter"] == "SmbExploiter": + result = exploit["data"]["result"] + if result: + user = exploit["data"]["machine"]["cred"].keys()[0] + password = exploit["data"]["machine"]["cred"][user] + else: + user = exploit["data"]["user"] + password = exploit["data"]["password"] + + return \ + { + "timestamp": exploit["timestamp"], + "user": user, + "password": password, + "result": result, + } \ No newline at end of file diff --git a/monkey_island/cc/services/node.py b/monkey_island/cc/services/node.py new file mode 100644 index 000000000..b42f25a36 --- /dev/null +++ b/monkey_island/cc/services/node.py @@ -0,0 +1,126 @@ +from bson import ObjectId + +from cc.database import mongo +from cc.services.edge import EdgeService + +__author__ = "itay.mizeretz" + + +class NodeService: + + @staticmethod + def get_displayed_node_by_id(node_id): + + edges = EdgeService.get_displayed_edges_by_to(node_id) + accessible_from_nodes = [] + exploits = [] + + new_node = {"id": node_id} + + node = mongo.db.node.find_one({"_id": ObjectId(node_id)}) + if node is None: + monkey = mongo.db.monkey.find_one({"_id": ObjectId(node_id)}) + if monkey is None: + return new_node + + # node is infected + for key in monkey: + # TODO: do something with tunnel + if key not in ["_id", "modifytime", "parent", "tunnel", "tunnel_guid"]: + new_node[key] = monkey[key] + + new_node["os"] = NodeService.get_monkey_os(monkey) + new_node["label"] = NodeService.get_monkey_label(monkey) + new_node["group"] = NodeService.get_monkey_group(monkey) + + else: + # node is uninfected + new_node["ip_addresses"] = node["ip_addresses"] + new_node["group"] = "clean" + + for edge in edges: + accessible_from_nodes.append({"id": edge["from"]}) + for exploit in edge["exploits"]: + exploit["origin"] = edge["from"] + exploits.append(exploit) + + exploits.sort(cmp=NodeService._cmp_exploits_by_timestamp) + + new_node["exploits"] = exploits + new_node["accessible_from_nodes"] = accessible_from_nodes + if len(edges) > 0: + new_node["services"] = edges[-1]["services"] + new_node["os"] = edges[-1]["os"]["type"] + if "label" not in new_node: + new_node["label"] = edges[-1]["os"]["version"] + " : " + node["ip_addresses"][0] + + # TODO: add exploited by + + return new_node + + @staticmethod + def _cmp_exploits_by_timestamp(exploit_1, exploit_2): + if exploit_1["timestamp"] == exploit_2["timestamp"]: + return 0 + if exploit_1["timestamp"] > exploit_2["timestamp"]: + return 1 + return -1 + + @staticmethod + def get_monkey_os(monkey): + os = "unknown" + if monkey["description"].lower().find("linux") != -1: + os = "linux" + elif monkey["description"].lower().find("windows") != -1: + os = "windows" + return os + + @staticmethod + def get_monkey_manual_run(monkey): + # TODO: find better implementation + return monkey["parent"][0][1] == None + + @staticmethod + def get_monkey_label(monkey): + return monkey["hostname"] + " : " + monkey["ip_addresses"][0] + + @staticmethod + def get_monkey_group(monkey): + return "manuallyInfected" if NodeService.get_monkey_manual_run(monkey) else "infected" + + @staticmethod + def monkey_to_net_node(monkey): + return \ + { + "id": monkey["_id"], + "label": NodeService.get_monkey_label(monkey), + "group": NodeService.get_monkey_group(monkey), + "os": NodeService.get_monkey_os(monkey), + "dead": monkey["dead"], + } + + @staticmethod + def node_to_net_node(node): + os_version = "undefined" + os_type = "undefined" + found = False + # TODO: Set this as data when received + for edge in mongo.db.edge.find({"to": node["_id"]}): + for scan in edge["scans"]: + if scan["scanner"] != "TcpScanner": + continue + os_type = scan["data"]["os"]["type"] + if "version" in scan["data"]["os"]: + os_version = scan["data"]["os"]["version"] + found = True + break + if found: + break + + return \ + { + "id": node["_id"], + "label": os_version + " : " + node["ip_addresses"][0], + "group": "clean", + "os": os_type + } \ No newline at end of file