From 91581d00ab1c60e8cbeb416cac46d59308d29842 Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Wed, 6 Sep 2017 14:49:58 +0300 Subject: [PATCH] Seperated logic from entry points Node/Edge queries return minimal information necessary. already formatted. Add MonkeyIsland to NetMap --- monkey_island/cc/app.py | 5 +- monkey_island/cc/island_config.py | 4 + monkey_island/cc/main.py | 5 +- monkey_island/cc/resources/local_run.py | 19 ++- monkey_island/cc/resources/monkey.py | 8 +- monkey_island/cc/resources/netmap.py | 8 +- monkey_island/cc/resources/telemetry.py | 134 +++++++++++---------- monkey_island/cc/services/edge.py | 59 ++++++++-- monkey_island/cc/services/node.py | 148 ++++++++++++++++++------ 9 files changed, 258 insertions(+), 132 deletions(-) create mode 100644 monkey_island/cc/island_config.py diff --git a/monkey_island/cc/app.py b/monkey_island/cc/app.py index 6ca21cab8..21eea154a 100644 --- a/monkey_island/cc/app.py +++ b/monkey_island/cc/app.py @@ -18,8 +18,6 @@ from cc.resources.root import Root __author__ = 'Barak' -# TODO: separate logic from resources - def serve_static_file(path): print 'requested', path if path.startswith('api/'): @@ -28,6 +26,7 @@ def serve_static_file(path): def serve_home(): + # TODO: remove this or merge with frontend. return serve_static_file('index.html') @@ -75,7 +74,7 @@ def init_app(mongo_url): api.add_resource(Telemetry, '/api/telemetry', '/api/telemetry/', '/api/telemetry/') api.add_resource(MonkeyConfiguration, '/api/configuration', '/api/configuration/') api.add_resource(MonkeyDownload, '/api/monkey/download', '/api/monkey/download/', - '/api/monkey/download/') + '/api/monkey/download/') api.add_resource(NetMap, '/api/netmap', '/api/netmap/') api.add_resource(Edge, '/api/netmap/edge', '/api/netmap/edge/') api.add_resource(Node, '/api/netmap/node', '/api/netmap/node/') diff --git a/monkey_island/cc/island_config.py b/monkey_island/cc/island_config.py new file mode 100644 index 000000000..c53d27004 --- /dev/null +++ b/monkey_island/cc/island_config.py @@ -0,0 +1,4 @@ +__author__ = 'itay.mizeretz' + +ISLAND_PORT = 5000 +DEFAULT_MONGO_URL = "mongodb://localhost:27017/monkeyisland" \ No newline at end of file diff --git a/monkey_island/cc/main.py b/monkey_island/cc/main.py index ad3156431..b1b240f8b 100644 --- a/monkey_island/cc/main.py +++ b/monkey_island/cc/main.py @@ -9,10 +9,9 @@ if BASE_PATH not in sys.path: from cc.app import init_app from cc.utils import init_collections, local_ip_addresses +from cc.island_config import DEFAULT_MONGO_URL, ISLAND_PORT -ISLAND_PORT = 5000 -DEFAULT_MONGO_URL = "mongodb://localhost:27017/monkeyisland" - +# TODO: remove this, and get from global config` INITIAL_USERNAMES = ['Administrator', 'root', 'user'] INITIAL_PASSWORDS = ["Password1!", "1234", "password", "12345678"] diff --git a/monkey_island/cc/resources/local_run.py b/monkey_island/cc/resources/local_run.py index ef89d400e..1923f948f 100644 --- a/monkey_island/cc/resources/local_run.py +++ b/monkey_island/cc/resources/local_run.py @@ -7,8 +7,8 @@ from flask import request, jsonify, make_response import flask_restful from cc.resources.monkey_download import get_monkey_executable - -from cc.utils import local_ips +from cc.island_config import ISLAND_PORT +from cc.services.node import NodeService __author__ = 'Barak' @@ -21,7 +21,7 @@ def run_local_monkey(island_address): # get the monkey executable suitable to run on the server result = get_monkey_executable(platform.system().lower(), platform.machine().lower()) if not result: - return (False, "OS Type not found") + return False, "OS Type not found" monkey_path = os.path.join('binaries', result['filename']) target_path = os.path.join(os.getcwd(), result['filename']) @@ -30,8 +30,8 @@ def run_local_monkey(island_address): try: copyfile(monkey_path, target_path) os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG) - except Exception, exc: - return (False, "Copy file failed: %s" % exc) + except Exception as exc: + return False, "Copy file failed: %s" % exc # run the monkey try: @@ -39,16 +39,15 @@ def run_local_monkey(island_address): if sys.platform == "win32": args = "".join(args) pid = subprocess.Popen(args, shell=True).pid - except Exception, exc: - return (False, "popen failed: %s" % exc) + except Exception as exc: + return False, "popen failed: %s" % exc - return (True, "pis: %s" % pid) + return True, "pis: %s" % pid class LocalRun(flask_restful.Resource): def get(self): - # TODO implement is_running from db monkeys collection - return jsonify(is_running=False) + return jsonify(is_running=(NodeService.get_monkey_island_monkey() is not None)) def post(self): body = json.loads(request.data) diff --git a/monkey_island/cc/resources/monkey.py b/monkey_island/cc/resources/monkey.py index eb936a066..90eddd015 100644 --- a/monkey_island/cc/resources/monkey.py +++ b/monkey_island/cc/resources/monkey.py @@ -1,7 +1,7 @@ import json from datetime import datetime, timedelta -import dateutil +import dateutil.parser from flask import request import flask_restful @@ -116,9 +116,9 @@ class Monkey(flask_restful.Resource): existing_node = mongo.db.node.find_one({"ip_addresses": {"$in": monkey_json["ip_addresses"]}}) if existing_node: - id = existing_node["_id"] - for edge in mongo.db.edge.find({"to": id}): + node_id = existing_node["_id"] + for edge in mongo.db.edge.find({"to": node_id}): mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}}) - mongo.db.node.remove({"_id": id}) + mongo.db.node.remove({"_id": node_id}) return {"id": new_monkey_id} diff --git a/monkey_island/cc/resources/netmap.py b/monkey_island/cc/resources/netmap.py index f62c3fbe8..f5ad4d06e 100644 --- a/monkey_island/cc/resources/netmap.py +++ b/monkey_island/cc/resources/netmap.py @@ -1,5 +1,6 @@ import flask_restful +from cc.services.edge import EdgeService from cc.services.node import NodeService from cc.database import mongo @@ -11,10 +12,15 @@ class NetMap(flask_restful.Resource): monkeys = [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] nodes = [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})] edges = [self.edge_to_net_edge(x) for x in mongo.db.edge.find({})] + monkey_island = [] + if NodeService.get_monkey_island_monkey() is None: + monkey_island = [NodeService.get_monkey_island_pseudo_net_node()] + # TODO: implement when monkey exists on island + edges += EdgeService.get_monkey_island_pseudo_edges() return \ { - "nodes": monkeys + nodes, + "nodes": monkeys + nodes + monkey_island, "edges": edges } diff --git a/monkey_island/cc/resources/telemetry.py b/monkey_island/cc/resources/telemetry.py index 311039b59..bfc38c582 100644 --- a/monkey_island/cc/resources/telemetry.py +++ b/monkey_island/cc/resources/telemetry.py @@ -1,11 +1,14 @@ import json from datetime import datetime +import traceback import dateutil from flask import request import flask_restful from cc.database import mongo +from cc.services.edge import EdgeService +from cc.services.node import NodeService from cc.utils import creds_add_username, creds_add_password @@ -38,74 +41,62 @@ class Telemetry(flask_restful.Resource): telemetry_json['timestamp'] = datetime.now() telem_id = mongo.db.telemetry.insert(telemetry_json) + monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) - # update exploited monkeys parent try: if telemetry_json.get('telem_type') == 'tunnel': - if telemetry_json['data']: - host = telemetry_json['data'].split(":")[-2].replace("//", "") - tunnel_host = mongo.db.monkey.find_one({"ip_addresses": host}) - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$set': {'tunnel_guid': tunnel_host.get('guid'), - 'modifytime': datetime.now()}}, - upsert=False) - else: - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$unset': {'tunnel_guid': ''}, - '$set': {'modifytime': datetime.now()}}, - upsert=False) + self.process_tunnel_telemetry(telemetry_json) elif telemetry_json.get('telem_type') == 'state': - if telemetry_json['data']['done']: - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$set': {'dead': True, 'modifytime': datetime.now()}}, - upsert=False) - else: - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$set': {'dead': False, 'modifytime': datetime.now()}}, - upsert=False) + self.process_state_telemetry(telemetry_json) elif telemetry_json.get('telem_type') in ['scan', 'exploit']: - dst_ip = telemetry_json['data']['machine']['ip_addr'] - src_monkey = mongo.db.monkey.find_one({"guid": telemetry_json['monkey_guid']}) - dst_monkey = mongo.db.monkey.find_one({"ip_addresses": dst_ip}) - if dst_monkey: - edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_monkey["_id"]}) - - if edge is None: - edge = self.insert_edge(src_monkey["_id"], dst_monkey["_id"]) - - else: - dst_node = mongo.db.node.find_one({"ip_addresses": dst_ip}) - if dst_node is None: - dst_node_insert_result = mongo.db.node.insert_one({"ip_addresses": [dst_ip]}) - dst_node = mongo.db.node.find_one({"_id": dst_node_insert_result.inserted_id}) - - edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_node["_id"]}) - - if edge is None: - edge = self.insert_edge(src_monkey["_id"], dst_node["_id"]) - - if telemetry_json.get('telem_type') == 'scan': - self.add_scan_to_edge(edge, telemetry_json) - else: - self.add_exploit_to_edge(edge, telemetry_json) - - except StandardError as e: - pass - - # Update credentials DB - try: - if (telemetry_json.get('telem_type') == 'system_info_collection') and (telemetry_json['data'].has_key('credentials')): - creds = telemetry_json['data']['credentials'] - for user in creds: - creds_add_username(user) - - if creds[user].has_key('password'): - creds_add_password(creds[user]['password']) + self.process_scan_exploit_telemetry(telemetry_json) + elif telemetry_json.get('telem_type') == 'system_info_collection': + self.process_system_info_telemetry(telemetry_json) + NodeService.update_monkey_modify_time(monkey["_id"]) except StandardError as ex: - print("Exception caught while updating DB credentials: %s" % str(ex)) + print("Exception caught while processing telemetry: %s" % str(ex)) + traceback.print_exc() return mongo.db.telemetry.find_one_or_404({"_id": telem_id}) + def process_tunnel_telemetry(self, telemetry_json): + monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])["_id"] + NodeService.unset_all_monkey_tunnels(monkey_id) + if telemetry_json['data']: + host = telemetry_json['data'].split(":")[-2].replace("//", "") + tunnel_host_id = NodeService.get_monkey_by_ip(host)["_id"] + NodeService.set_monkey_tunnel(monkey_id, tunnel_host_id) + + def process_state_telemetry(self, telemetry_json): + monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) + if telemetry_json['data']['done']: + NodeService.set_monkey_dead(monkey, True) + else: + NodeService.set_monkey_dead(monkey, False) + + def process_scan_exploit_telemetry(self, telemetry_json): + dst_ip = telemetry_json['data']['machine']['ip_addr'] + src_monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) + dst_node = NodeService.get_monkey_by_ip(dst_ip) + if dst_node is None: + dst_node = NodeService.get_or_create_node(dst_ip) + + edge = EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"]) + + if telemetry_json.get('telem_type') == 'scan': + self.add_scan_to_edge(edge, telemetry_json) + else: + self.add_exploit_to_edge(edge, telemetry_json) + + def process_system_info_telemetry(self, telemetry_json): + if 'credentials' in telemetry_json['data']: + creds = telemetry_json['data']['credentials'] + for user in creds: + creds_add_username(user) + + if 'password' in creds[user]: + creds_add_password(creds[user]['password']) + def add_scan_to_edge(self, edge, telemetry_json): data = telemetry_json['data']['machine'] data.pop("ip_addr") @@ -120,6 +111,22 @@ class Telemetry(flask_restful.Resource): {"$push": {"scans": new_scan}} ) + node = mongo.db.node.find_one({"_id": edge["to"]}) + if node is not None: + if new_scan["scanner"] == "TcpScanner": + scan_os = new_scan["data"]["os"] + if "type" in scan_os: + mongo.db.node.update({"_id": node["_id"]}, + {"$set": {"os.type": scan_os["type"]}}, + upsert=False) + if "version" in scan_os: + mongo.db.node.update({"_id": node["_id"]}, + {"$set": {"os.version": scan_os["version"]}}, + upsert=False) + + + + def add_exploit_to_edge(self, edge, telemetry_json): data = telemetry_json['data'] data["machine"].pop("ip_addr") @@ -134,12 +141,3 @@ class Telemetry(flask_restful.Resource): {"$push": {"exploits": new_exploit}} ) - def insert_edge(self, from_id, to_id): - edge_insert_result = mongo.db.edge.insert_one( - { - "from": from_id, - "to": to_id, - "scans": [], - "exploits": [] - }) - return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) diff --git a/monkey_island/cc/services/edge.py b/monkey_island/cc/services/edge.py index 12dfd9559..e1811bef9 100644 --- a/monkey_island/cc/services/edge.py +++ b/monkey_island/cc/services/edge.py @@ -6,6 +6,9 @@ __author__ = "itay.mizeretz" class EdgeService: + def __init__(self): + pass + @staticmethod def get_displayed_edge_by_id(edge_id): edge = mongo.db.edge.find({"_id": ObjectId(edge_id)})[0] @@ -26,11 +29,10 @@ class EdgeService: os = {} exploits = [] if len(edge["scans"]) > 0: - services = edge["scans"][-1]["data"]["services"] + services = EdgeService.services_to_displayed_services(edge["scans"][-1]["data"]["services"]) os = edge["scans"][-1]["data"]["os"] for exploit in edge["exploits"]: - new_exploit = EdgeService.exploit_to_displayed_exploit(exploit) if (len(exploits) > 0) and (exploits[-1]["exploiter"] == exploit["exploiter"]): @@ -66,18 +68,14 @@ class EdgeService: def exploit_to_displayed_exploit(exploit): user = "" password = "" - result = False # TODO: implement for other exploiters - + # TODO: The format that's used today to get the credentials is bad. Change it from monkey side and adapt. + result = exploit["data"]["result"] if exploit["exploiter"] == "RdpExploiter": - # TODO: check if there could be multiple creds - result = exploit["data"]["result"] user = exploit["data"]["machine"]["creds"].keys()[0] password = exploit["data"]["machine"]["creds"][user] - elif exploit["exploiter"] == "SmbExploiter": - result = exploit["data"]["result"] if result: user = exploit["data"]["machine"]["cred"].keys()[0] password = exploit["data"]["machine"]["cred"][user] @@ -91,4 +89,47 @@ class EdgeService: "user": user, "password": password, "result": result, - } \ No newline at end of file + } + + @staticmethod + def insert_edge(from_id, to_id): + edge_insert_result = mongo.db.edge.insert_one( + { + "from": from_id, + "to": to_id, + "scans": [], + "exploits": [], + "tunnel": False + }) + return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) + + @staticmethod + def get_or_create_edge(edge_from, edge_to): + tunnel_edge = mongo.db.edge.find_one({"from": edge_from, "to": edge_to}) + if tunnel_edge is None: + tunnel_edge = EdgeService.insert_edge(edge_from, edge_to) + + return tunnel_edge + + @staticmethod + def get_monkey_island_pseudo_edges(): + edges = [] + monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) if "tunnel" not in x] + # TODO: find solution for these ids. + count = 0 + for monkey_id in monkey_ids: + count += 1 + edges.append( + { + "id": ObjectId(hex(count)[2:].zfill(24)), + "from": monkey_id, + "to": ObjectId("000000000000000000000000") + } + ) + + return edges + + @staticmethod + def services_to_displayed_services(services): + # TODO: Consider returning extended information on services. + return [x + ": " + services[x]["name"] for x in services] diff --git a/monkey_island/cc/services/node.py b/monkey_island/cc/services/node.py index 67e7602dd..6046726fc 100644 --- a/monkey_island/cc/services/node.py +++ b/monkey_island/cc/services/node.py @@ -1,15 +1,20 @@ +import datetime from bson import ObjectId from cc.database import mongo from cc.services.edge import EdgeService - +from cc.utils import local_ip_addresses __author__ = "itay.mizeretz" class NodeService: + def __init__(self): + pass @staticmethod def get_displayed_node_by_id(node_id): + if ObjectId(node_id) == ObjectId("000000000000000000000000"): + return NodeService.get_monkey_island_node() edges = EdgeService.get_displayed_edges_by_to(node_id) accessible_from_nodes = [] @@ -17,31 +22,28 @@ class NodeService: new_node = {"id": node_id} - node = mongo.db.node.find_one({"_id": ObjectId(node_id)}) + node = NodeService.get_node_by_id(node_id) if node is None: - monkey = mongo.db.monkey.find_one({"_id": ObjectId(node_id)}) + monkey = NodeService.get_monkey_by_id(node_id) if monkey is None: return new_node # node is infected + new_node = NodeService.monkey_to_net_node(monkey) for key in monkey: # TODO: do something with tunnel - if key not in ["_id", "modifytime", "parent", "tunnel", "tunnel_guid"]: + if key not in ["_id", "modifytime", "parent", "tunnel", "dead"]: new_node[key] = monkey[key] - new_node["os"] = NodeService.get_monkey_os(monkey) - new_node["label"] = NodeService.get_monkey_label(monkey) - new_node["group"] = NodeService.get_monkey_group(monkey) - else: # node is uninfected + new_node = NodeService.node_to_net_node(node) new_node["ip_addresses"] = node["ip_addresses"] - new_node["group"] = "clean" for edge in edges: accessible_from_nodes.append({"id": edge["from"]}) for exploit in edge["exploits"]: - exploit["origin"] = edge["from"] + exploit["origin"] = NodeService.get_monkey_label(NodeService.get_monkey_by_id(edge["from"])) exploits.append(exploit) exploits.sort(cmp=NodeService._cmp_exploits_by_timestamp) @@ -50,19 +52,20 @@ class NodeService: new_node["accessible_from_nodes"] = accessible_from_nodes if len(edges) > 0: new_node["services"] = edges[-1]["services"] - new_node["os"] = edges[-1]["os"]["type"] - if "label" not in new_node: - new_node["label"] = edges[-1]["os"]["version"] + " : " + node["ip_addresses"][0] # TODO: add exploited by return new_node + @staticmethod + def get_node_label(node): + return node["os"]["version"] + " : " + node["ip_addresses"][0] + @staticmethod def _cmp_exploits_by_timestamp(exploit_1, exploit_2): - if exploit_1["timestamp"] == exploit_2["timestamp"]: + if exploit_1["start_timestamp"] == exploit_2["start_timestamp"]: return 0 - if exploit_1["timestamp"] > exploit_2["timestamp"]: + if exploit_1["start_timestamp"] > exploit_2["start_timestamp"]: return 1 return -1 @@ -90,6 +93,9 @@ class NodeService: @staticmethod def get_monkey_group(monkey): + if len(set(monkey["ip_addresses"]).intersection(local_ip_addresses())) != 0: + return "islandInfected" + return "manuallyInfected" if NodeService.get_monkey_manual_run(monkey) else "infected" @staticmethod @@ -105,26 +111,100 @@ class NodeService: @staticmethod def node_to_net_node(node): - os_version = "undefined" - os_type = "undefined" - found = False - # TODO: Set this as data when received - for edge in mongo.db.edge.find({"to": node["_id"]}): - for scan in edge["scans"]: - if scan["scanner"] != "TcpScanner": - continue - os_type = scan["data"]["os"]["type"] - if "version" in scan["data"]["os"]: - os_version = scan["data"]["os"]["version"] - found = True - break - if found: - break - return \ { "id": node["_id"], - "label": os_version + " : " + node["ip_addresses"][0], + "label": NodeService.get_node_label(node), "group": "clean", - "os": os_type - } \ No newline at end of file + "os": node["os"]["type"] + } + + @staticmethod + def unset_all_monkey_tunnels(monkey_id): + mongo.db.edge.update( + {"from": monkey_id, 'tunnel': True}, + {'$set': {'tunnel': False}}, + upsert=False) + + @staticmethod + def set_monkey_tunnel(monkey_id, tunnel_host_id): + tunnel_edge = EdgeService.get_or_create_edge(monkey_id, tunnel_host_id) + mongo.db.edge.update({"_id": tunnel_edge["_id"]}, + {'$set': {'tunnel': True}}, + upsert=False) + + @staticmethod + def insert_node(ip_address): + new_node_insert_result = mongo.db.node.insert_one( + { + "ip_addresses": [ip_address], + "os": + { + "type": "unknown", + "version": "unknown" + } + }) + return mongo.db.node.find_one({"_id": new_node_insert_result.inserted_id}) + + @staticmethod + def get_or_create_node(ip_address): + new_node = mongo.db.node.find_one({"ip_addresses": ip_address}) + if new_node is None: + new_node = NodeService.insert_node(ip_address) + return new_node + + @staticmethod + def get_monkey_by_id(monkey_id): + return mongo.db.monkey.find_one({"_id": ObjectId(monkey_id)}) + + @staticmethod + def get_monkey_by_guid(monkey_guid): + return mongo.db.monkey.find_one({"guid": monkey_guid}) + + @staticmethod + def get_monkey_by_ip(ip_address): + return mongo.db.monkey.find_one({"ip_addresses": ip_address}) + + @staticmethod + def get_node_by_ip(ip_address): + return mongo.db.node.find_one({"ip_addresses": ip_address}) + + @staticmethod + def get_node_by_id(node_id): + return mongo.db.node.find_one({"_id": ObjectId(node_id)}) + + @staticmethod + def update_monkey_modify_time(monkey_id): + mongo.db.monkey.update({"_id": monkey_id}, + {"$set": {"modifytime": datetime.now()}}, + upsert=False) + + @staticmethod + def set_monkey_dead(monkey, is_dead): + mongo.db.monkey.update({"guid": monkey['guid']}, + {'$set': {'dead': is_dead}}, + upsert=False) + + @staticmethod + def get_monkey_island_monkey(): + ip_addresses = local_ip_addresses() + for ip_address in ip_addresses: + monkey = NodeService.get_monkey_by_ip(ip_address) + if monkey is not None: + return monkey + return None + + @staticmethod + def get_monkey_island_pseudo_net_node(): + return\ + { + "id": ObjectId("000000000000000000000000"), + "label": "MonkeyIsland", + "group": "islandClean", + } + + @staticmethod + def get_monkey_island_node(): + island_node = NodeService.get_monkey_island_pseudo_net_node() + island_node["ip_addresses"] = local_ip_addresses() + return island_node