From 2a96d23a4acbee6bfe27a29166ae0fb0564fa1fb Mon Sep 17 00:00:00 2001 From: Barak Argaman Date: Fri, 25 Aug 2017 17:47:08 +0300 Subject: [PATCH] organize cc server files --- monkey_island/cc/__init__.py | 1 + monkey_island/cc/app.py | 76 +++ monkey_island/cc/database.py | 5 + monkey_island/cc/main.py | 604 +----------------- monkey_island/cc/resources/__init__.py | 1 + monkey_island/cc/resources/edge.py | 24 + monkey_island/cc/resources/local_run.py | 62 ++ monkey_island/cc/resources/monkey.py | 124 ++++ monkey_island/cc/resources/monkey_download.py | 65 ++ monkey_island/cc/resources/netmap.py | 67 ++ monkey_island/cc/resources/new_config.py | 20 + monkey_island/cc/resources/root.py | 42 ++ monkey_island/cc/resources/telemetry.py | 127 ++++ monkey_island/cc/utils.py | 77 +++ 14 files changed, 704 insertions(+), 591 deletions(-) create mode 100644 monkey_island/cc/__init__.py create mode 100644 monkey_island/cc/app.py create mode 100644 monkey_island/cc/database.py create mode 100644 monkey_island/cc/resources/__init__.py create mode 100644 monkey_island/cc/resources/edge.py create mode 100644 monkey_island/cc/resources/local_run.py create mode 100644 monkey_island/cc/resources/monkey.py create mode 100644 monkey_island/cc/resources/monkey_download.py create mode 100644 monkey_island/cc/resources/netmap.py create mode 100644 monkey_island/cc/resources/new_config.py create mode 100644 monkey_island/cc/resources/root.py create mode 100644 monkey_island/cc/resources/telemetry.py create mode 100644 monkey_island/cc/utils.py diff --git a/monkey_island/cc/__init__.py b/monkey_island/cc/__init__.py new file mode 100644 index 000000000..e593a854b --- /dev/null +++ b/monkey_island/cc/__init__.py @@ -0,0 +1 @@ +__author__ = 'Barak' diff --git a/monkey_island/cc/app.py b/monkey_island/cc/app.py new file mode 100644 index 000000000..4d98749a3 --- /dev/null +++ b/monkey_island/cc/app.py @@ -0,0 +1,76 @@ +from datetime import datetime + +import bson +from flask import Flask, send_from_directory, redirect, make_response +import flask_restful + +from cc.database import mongo +from cc.resources.monkey import Monkey +from cc.resources.local_run import LocalRun +from cc.resources.telemetry import Telemetry +from cc.resources.new_config import NewConfig +from cc.resources.monkey_download import MonkeyDownload +from cc.resources.netmap import NetMap +from cc.resources.edge import Edge +from cc.resources.root import Root + +__author__ = 'Barak' + + +def send_admin(path): + return send_from_directory('admin/ui', path) + + +def send_to_default(): + return redirect('/admin/index.html') + + +def normalize_obj(obj): + if obj.has_key('_id') and not obj.has_key('id'): + obj['id'] = obj['_id'] + del obj['_id'] + + for key, value in obj.items(): + if type(value) is bson.objectid.ObjectId: + obj[key] = str(value) + if type(value) is datetime: + obj[key] = str(value) + if type(value) is dict: + obj[key] = normalize_obj(value) + if type(value) is list: + for i in range(0, len(value)): + if type(value[i]) is dict: + value[i] = normalize_obj(value[i]) + return obj + + +def output_json(obj, code, headers=None): + obj = normalize_obj(obj) + resp = make_response(bson.json_util.dumps(obj), code) + resp.headers.extend(headers or {}) + return resp + + +def init_app(mongo_url): + app = Flask(__name__) + + api = flask_restful.Api(app) + api.representations = {'application/json': output_json} + + app.config['MONGO_URI'] = mongo_url + mongo.init_app(app) + + app.add_url_rule('/', 'send_to_default', send_to_default) + app.add_url_rule('/admin/', 'send_admin', send_admin) + + api.add_resource(Root, '/api') + api.add_resource(Monkey, '/api/monkey', '/api/monkey/', '/api/monkey/') + api.add_resource(LocalRun, '/api/island', '/api/island/') + api.add_resource(Telemetry, '/api/telemetry', '/api/telemetry/', '/api/telemetry/') + api.add_resource(NewConfig, '/api/config/new', '/api/config/new/') + api.add_resource(MonkeyDownload, '/api/monkey/download', '/api/monkey/download/', + '/api/monkey/download/') + api.add_resource(NetMap, '/api/netmap', '/api/netmap/') + api.add_resource(Edge, '/api/edge', '/api/edge/') + + return app diff --git a/monkey_island/cc/database.py b/monkey_island/cc/database.py new file mode 100644 index 000000000..31828705a --- /dev/null +++ b/monkey_island/cc/database.py @@ -0,0 +1,5 @@ +from flask_pymongo import PyMongo + +__author__ = 'Barak' + +mongo = PyMongo() diff --git a/monkey_island/cc/main.py b/monkey_island/cc/main.py index d81f637fd..754f5076f 100644 --- a/monkey_island/cc/main.py +++ b/monkey_island/cc/main.py @@ -2,610 +2,32 @@ from __future__ import print_function # In python 2.7 import os import sys -import array -import struct -from shutil import copyfile -from flask import Flask, request, abort, send_from_directory, redirect -from flask.ext import restful -from flask.ext.pymongo import PyMongo -from flask import make_response -import socket -import bson.json_util -from bson import ObjectId -import json -from datetime import datetime, timedelta -import dateutil.parser + +BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if BASE_PATH not in sys.path: + sys.path.insert(0, BASE_PATH) + +from cc.app import init_app +from cc.utils import init_collections ISLAND_PORT = 5000 - -MONKEY_DOWNLOADS = [ - { - 'type': 'linux', - 'machine': 'x86_64', - 'filename': 'monkey-linux-64', - }, - { - 'type': 'linux', - 'machine': 'i686', - 'filename': 'monkey-linux-32', - }, - { - 'type': 'linux', - 'filename': 'monkey-linux-32', - }, - { - 'type': 'windows', - 'machine': 'x86', - 'filename': 'monkey-windows-32.exe', - }, - { - 'type': 'windows', - 'machine': 'amd64', - 'filename': 'monkey-windows-64.exe', - }, - { - 'type': 'windows', - 'filename': 'monkey-windows-32.exe', - }, -] +DEFAULT_MONGO_URL = "mongodb://localhost:27017/monkeyisland" INITIAL_USERNAMES = ['Administrator', 'root', 'user'] INITIAL_PASSWORDS = ["Password1!", "1234", "password", "12345678"] -MONGO_URL = os.environ.get('MONGO_URL') -if not MONGO_URL: - MONGO_URL = "mongodb://localhost:27017/monkeyisland" - -app = Flask(__name__) -app.config['MONGO_URI'] = MONGO_URL -mongo = PyMongo(app) - - -class Monkey(restful.Resource): - def get(self, guid=None, **kw): - update_dead_monkeys() # refresh monkeys status - if not guid: - guid = request.args.get('guid') - timestamp = request.args.get('timestamp') - - if guid: - monkey_json = mongo.db.monkey.find_one_or_404({"guid": guid}) - monkey_json['config']['exploit_user_list'] = \ - map(lambda x: x['username'], mongo.db.usernames.find({}, {'_id': 0, 'username': 1}).sort([('count', -1)])) - monkey_json['config']['exploit_password_list'] = \ - map(lambda x: x['password'], mongo.db.passwords.find({}, {'_id': 0, 'password': 1}).sort([('count', -1)])) - return monkey_json - else: - result = {'timestamp': datetime.now().isoformat()} - find_filter = {} - if timestamp is not None: - find_filter['modifytime'] = {'$gt': dateutil.parser.parse(timestamp)} - result['objects'] = [x for x in mongo.db.monkey.find(find_filter)] - return result - - def patch(self, guid): - monkey_json = json.loads(request.data) - update = {"$set": {'modifytime': datetime.now()}} - - if 'keepalive' in monkey_json: - update['$set']['keepalive'] = dateutil.parser.parse(monkey_json['keepalive']) - else: - update['$set']['keepalive'] = datetime.now() - if 'config' in monkey_json: - update['$set']['config'] = monkey_json['config'] - if 'tunnel' in monkey_json: - update['$set']['tunnel'] = monkey_json['tunnel'] - if 'config_error' in monkey_json: - update['$set']['config_error'] = monkey_json['config_error'] - - return mongo.db.monkey.update({"guid": guid}, update, upsert=False) - - def post(self, **kw): - monkey_json = json.loads(request.data) - if 'keepalive' in monkey_json: - monkey_json['keepalive'] = dateutil.parser.parse(monkey_json['keepalive']) - else: - monkey_json['keepalive'] = datetime.now() - - monkey_json['modifytime'] = datetime.now() - - # if new monkey telem, change config according to "new monkeys" config. - db_monkey = mongo.db.monkey.find_one({"guid": monkey_json["guid"]}) - if not db_monkey: - new_config = mongo.db.config.find_one({'name': 'newconfig'}) or {} - monkey_json['config'] = monkey_json.get('config', {}) - monkey_json['config'].update(new_config) - else: - db_config = db_monkey.get('config', {}) - if 'current_server' in db_config: - del db_config['current_server'] - monkey_json.get('config', {}).update(db_config) - - # try to find new monkey parent - parent = monkey_json.get('parent') - parent_to_add = (monkey_json.get('guid'), None) # default values in case of manual run - if parent and parent != monkey_json.get('guid'): # current parent is known - exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, - 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}, - 'monkey_guid': {'$eq': parent}})] - if 1 == len(exploit_telem): - parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) - else: - parent_to_add = (parent, None) - elif (not parent or parent == monkey_json.get('guid')) and 'ip_addresses' in monkey_json: - exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, - 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}})] - - if 1 == len(exploit_telem): - parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) - - if not db_monkey: - monkey_json['parent'] = [parent_to_add] - else: - monkey_json['parent'] = db_monkey.get('parent') + [parent_to_add] - - mongo.db.monkey.update({"guid": monkey_json["guid"]}, - {"$set": monkey_json}, - upsert=True) - - # Merge existing scanned node with new monkey - - new_monkey_id = mongo.db.monkey.find_one({"guid": monkey_json["guid"]})["_id"] - - existing_node = mongo.db.node.find_one({"ip_addresses": {"$in": monkey_json["ip_addresses"]}}) - - if existing_node: - id = existing_node["_id"] - for edge in mongo.db.edge.find({"to": id}): - mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}}) - mongo.db.node.remove({"_id": id}) - - return {new_monkey_id} - - -class Telemetry(restful.Resource): - def get(self, **kw): - monkey_guid = request.args.get('monkey_guid') - telem_type = request.args.get('telem_type') - timestamp = request.args.get('timestamp') - if "null" == timestamp: # special case to avoid ugly JS code... - timestamp = None - - result = {'timestamp': datetime.now().isoformat()} - find_filter = {} - - if monkey_guid: - find_filter["monkey_guid"] = {'$eq': monkey_guid} - if telem_type: - find_filter["telem_type"] = {'$eq': telem_type} - if timestamp: - find_filter['timestamp'] = {'$gt': dateutil.parser.parse(timestamp)} - - result['objects'] = [x for x in mongo.db.telemetry.find(find_filter)] - return result - - def post(self): - telemetry_json = json.loads(request.data) - telemetry_json['timestamp'] = datetime.now() - - telem_id = mongo.db.telemetry.insert(telemetry_json) - - # update exploited monkeys parent - try: - if telemetry_json.get('telem_type') == 'tunnel': - if telemetry_json['data']: - host = telemetry_json['data'].split(":")[-2].replace("//", "") - tunnel_host = mongo.db.monkey.find_one({"ip_addresses": host}) - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$set': {'tunnel_guid': tunnel_host.get('guid'), - 'modifytime': datetime.now()}}, - upsert=False) - else: - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$unset': {'tunnel_guid': ''}, - '$set': {'modifytime': datetime.now()}}, - upsert=False) - elif telemetry_json.get('telem_type') == 'state': - if telemetry_json['data']['done']: - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$set': {'dead': True, 'modifytime': datetime.now()}}, - upsert=False) - else: - mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$set': {'dead': False, 'modifytime': datetime.now()}}, - upsert=False) - elif telemetry_json.get('telem_type') == 'scan': - dst_ip = telemetry_json['data']['machine']['ip_addr'] - src_monkey = mongo.db.monkey.find_one({"guid": telemetry_json['monkey_guid']}) - dst_monkey = mongo.db.monkey.find_one({"ip_addresses": dst_ip}) - if dst_monkey: - edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_monkey["_id"]}) - - if edge is None: - edge = self.insert_edge(src_monkey["_id"], dst_monkey["_id"]) - - else: - dst_node = mongo.db.node.find_one({"ip_addresses": dst_ip}) - if dst_node is None: - dst_node_insert_result = mongo.db.node.insert_one({"ip_addresses": [dst_ip]}) - dst_node = mongo.db.node.find_one({"_id": dst_node_insert_result.inserted_id}) - - edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_node["_id"]}) - - if edge is None: - edge = self.insert_edge(src_monkey["_id"], dst_node["_id"]) - - self.add_scan_to_edge(edge, telemetry_json) - - except StandardError as e: - pass - - # Update credentials DB - try: - if (telemetry_json.get('telem_type') == 'system_info_collection') and (telemetry_json['data'].has_key('credentials')): - creds = telemetry_json['data']['credentials'] - for user in creds: - creds_add_username(user) - - if creds[user].has_key('password'): - creds_add_password(creds[user]['password']) - except StandardError as ex: - print("Exception caught while updating DB credentials: %s" % str(ex)) - - return mongo.db.telemetry.find_one_or_404({"_id": telem_id}) - - def add_scan_to_edge(self, edge, telemetry_json): - data = telemetry_json['data']['machine'] - data.pop("ip_addr") - new_scan = \ - { - "timestamp": telemetry_json["timestamp"], - "data": data, - "scanner": telemetry_json['data']['scanner'] - } - mongo.db.edge.update( - {"_id": edge["_id"]}, - {"$push": {"scans": new_scan}} - ) - - def insert_edge(self, from_id, to_id): - edge_insert_result = mongo.db.edge.insert_one( - { - "from": from_id, - "to": to_id, - "scans": [] - }) - return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) - - -class LocalRun(restful.Resource): - def get(self): - req_type = request.args.get('type') - if req_type == "interfaces": - return {"interfaces": local_ips()} - else: - return {"message": "unknown action"} - - def post(self): - action_json = json.loads(request.data) - if 'action' in action_json: - if action_json["action"] == "monkey" and action_json.get("island_address") is not None: - return {"res": run_local_monkey(action_json.get("island_address"))} - - return {"res": (False, "Unknown action")} - - -class NewConfig(restful.Resource): - def get(self): - config = mongo.db.config.find_one({'name': 'newconfig'}) or {} - if 'name' in config: - del config['name'] - return config - - def post(self): - config_json = json.loads(request.data) - return mongo.db.config.update({'name': 'newconfig'}, {"$set": config_json}, upsert=True) - - -class MonkeyDownload(restful.Resource): - def get(self, path): - return send_from_directory('binaries', path) - - def post(self): - host_json = json.loads(request.data) - host_os = host_json.get('os') - if host_os: - result = get_monkey_executable(host_os.get('type'), host_os.get('machine')) - - if result: - real_path = os.path.join('binaries', result['filename']) - if os.path.isfile(real_path): - result['size'] = os.path.getsize(real_path) - return result - - return {} - - -class Root(restful.Resource): - def get(self, action=None): - if not action: - action = request.args.get('action') - if not action: - return { - 'status': 'OK', - 'mongo': str(mongo.db), - } - elif action == "reset": - mongo.db.config.drop() - mongo.db.monkey.drop() - mongo.db.telemetry.drop() - mongo.db.usernames.drop() - mongo.db.passwords.drop() - mongo.db.node.drop() - mongo.db.edge.drop() - init_db() - return { - 'status': 'OK', - } - elif action == "killall": - mongo.db.monkey.update({}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}}, upsert=False, - multi=True) - return { - 'status': 'OK', - } - else: - return {'status': 'BAD', - 'reason': 'unknown action'} - - -class NetMap(restful.Resource): - def get(self, **kw): - monkeys = [self.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] - nodes = [self.node_to_net_node(x) for x in mongo.db.node.find({})] - edges = [self.edge_to_net_edge(x) for x in mongo.db.edge.find({})] - - return \ - { - "nodes": monkeys + nodes, - "edges": edges - } - - def monkey_to_net_node(self, monkey): - os = "unknown" - if monkey["description"].lower().find("linux") != -1: - os = "linux" - elif monkey["description"].lower().find("windows") != -1: - os = "windows" - - manual_run = (monkey["parent"][0][1] == None) - return \ - { - "id": monkey["_id"], - "label": monkey["hostname"] + " : " + monkey["ip_addresses"][0], - "group": ("manuallyInfected" if manual_run else "infected"), - "os": os, - "dead": monkey["dead"], - } - - def node_to_net_node(self, node): - os_version = "undefined" - os_type = "undefined" - found = False - for edge in mongo.db.edge.find({"to": node["_id"]}): - for scan in edge["scans"]: - if scan["scanner"] != "TcpScanner": - continue - os_type = scan["data"]["os"]["type"] - if scan["data"]["os"].has_key("version"): - os_version = scan["data"]["os"]["version"] - found = True - break - if found: - break - - return \ - { - "id": node["_id"], - "label": os_version + " : " + node["ip_addresses"][0], - "group": "clean", - "os": os_type - } - def edge_to_net_edge(self, edge): - return \ - { - "id": edge["_id"], - "from": edge["from"], - "to": edge["to"] - } - - -class Edge(restful.Resource): - def get(self): - id = request.args.get('id') - to = request.args.get('to') - if id: - edge = mongo.db.edge.find({"_id": ObjectId(id)})[0] - return {"edge": edge} - if to: - edges = mongo.db.edge.find({"to": ObjectId(to)}) - new_edges = [] - # TODO: find better solution for this - for i in range(edges.count()): - new_edges.append(edges[i]) - return {"edges": new_edges} - return {} - -def normalize_obj(obj): - if obj.has_key('_id') and not obj.has_key('id'): - obj['id'] = obj['_id'] - del obj['_id'] - - for key, value in obj.items(): - if type(value) is bson.objectid.ObjectId: - obj[key] = str(value) - if type(value) is datetime: - obj[key] = str(value) - if type(value) is dict: - obj[key] = normalize_obj(value) - if type(value) is list: - for i in range(0, len(value)): - if type(value[i]) is dict: - value[i] = normalize_obj(value[i]) - return obj - - -def output_json(obj, code, headers=None): - obj = normalize_obj(obj) - resp = make_response(bson.json_util.dumps(obj), code) - resp.headers.extend(headers or {}) - return resp - - -def update_dead_monkeys(): - # Update dead monkeys only if no living monkey transmitted keepalive in the last 10 minutes - if mongo.db.monkey.find_one({'dead': {'$ne': True}, 'keepalive': {'$gte': datetime.now() - timedelta(minutes=10)}}): - return - - mongo.db.monkey.update( - {'keepalive': {'$lte': datetime.now() - timedelta(minutes=10)}, 'dead': {'$ne': True}}, - {'$set': {'dead': True, 'modifytime': datetime.now()}}, upsert=False, multi=True) - - -def get_monkey_executable(host_os, machine): - for download in MONKEY_DOWNLOADS: - if host_os == download.get('type') and machine == download.get('machine'): - return download - return None - - -def run_local_monkey(island_address): - import platform - import subprocess - import stat - - # get the monkey executable suitable to run on the server - result = get_monkey_executable(platform.system().lower(), platform.machine().lower()) - if not result: - return (False, "OS Type not found") - - monkey_path = os.path.join('binaries', result['filename']) - target_path = os.path.join(os.getcwd(), result['filename']) - - # copy the executable to temp path (don't run the monkey from its current location as it may delete itself) - try: - copyfile(monkey_path, target_path) - os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG) - except Exception, exc: - return (False, "Copy file failed: %s" % exc) - - # run the monkey - try: - args = ["%s m0nk3y -s %s:%s" % (target_path, island_address, ISLAND_PORT)] - if sys.platform == "win32": - args = "".join(args) - pid = subprocess.Popen(args, shell=True).pid - except Exception, exc: - return (False, "popen failed: %s" % exc) - - return (True, "pis: %s" % pid) - -def creds_add_username(username): - mongo.db.usernames.update( - {'username': username}, - {'$inc': {'count': 1}}, - upsert=True - ) - -def creds_add_password(password): - mongo.db.passwords.update( - {'password': password}, - {'$inc': {'count': 1}}, - upsert=True - ) - -### Local ips function -if sys.platform == "win32": - def local_ips(): - local_hostname = socket.gethostname() - return socket.gethostbyname_ex(local_hostname)[2] -else: - import fcntl - def local_ips(): - result = [] - try: - is_64bits = sys.maxsize > 2 ** 32 - struct_size = 40 if is_64bits else 32 - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - max_possible = 8 # initial value - while True: - struct_bytes = max_possible * struct_size - names = array.array('B', '\0' * struct_bytes) - outbytes = struct.unpack('iL', fcntl.ioctl( - s.fileno(), - 0x8912, # SIOCGIFCONF - struct.pack('iL', struct_bytes, names.buffer_info()[0]) - ))[0] - if outbytes == struct_bytes: - max_possible *= 2 - else: - break - namestr = names.tostring() - - for i in range(0, outbytes, struct_size): - addr = socket.inet_ntoa(namestr[i + 20:i + 24]) - if not addr.startswith('127'): - result.append(addr) - # name of interface is (namestr[i:i+16].split('\0', 1)[0] - finally: - return result - - -### End of local ips function - -@app.route('/admin/') -def send_admin(path): - return send_from_directory('admin/ui', path) - - -@app.route("/") -def send_to_default(): - return redirect('/admin/index.html') - - -def init_db(): - if not "usernames" in mongo.db.collection_names(): - mongo.db.usernames.create_index([( "username", 1 )], unique= True) - for username in INITIAL_USERNAMES: - creds_add_username(username) - - if not "passwords" in mongo.db.collection_names(): - mongo.db.passwords.create_index([( "password", 1 )], unique= True) - for password in INITIAL_PASSWORDS: - creds_add_password(password) - -DEFAULT_REPRESENTATIONS = {'application/json': output_json} -api = restful.Api(app) -api.representations = DEFAULT_REPRESENTATIONS - -api.add_resource(Root, '/api') -api.add_resource(Monkey, '/api/monkey', '/api/monkey/', '/api/monkey/') -api.add_resource(LocalRun, '/api/island', '/api/island/') -api.add_resource(Telemetry, '/api/telemetry', '/api/telemetry/', '/api/telemetry/') -api.add_resource(NewConfig, '/api/config/new', '/api/config/new/') -api.add_resource(MonkeyDownload, '/api/monkey/download', '/api/monkey/download/', '/api/monkey/download/') -api.add_resource(NetMap, '/api/netmap', '/api/netmap/') -api.add_resource(Edge, '/api/edge', '/api/edge/') if __name__ == '__main__': from tornado.wsgi import WSGIContainer from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop + app = init_app(os.environ.get('MONGO_URL', DEFAULT_MONGO_URL)) with app.app_context(): - init_db() - http_server = HTTPServer(WSGIContainer(app), ssl_options={'certfile': 'server.crt', 'keyfile': 'server.key'}) + init_collections(INITIAL_USERNAMES, INITIAL_PASSWORDS) + http_server = HTTPServer(WSGIContainer(app), + ssl_options={'certfile': os.environ.get('SERVER_CRT', 'server.crt'), + 'keyfile': os.environ.get('SERVER_KEY', 'server.key')}) http_server.listen(ISLAND_PORT) IOLoop.instance().start() # app.run(host='0.0.0.0', debug=True, ssl_context=('server.crt', 'server.key')) diff --git a/monkey_island/cc/resources/__init__.py b/monkey_island/cc/resources/__init__.py new file mode 100644 index 000000000..e593a854b --- /dev/null +++ b/monkey_island/cc/resources/__init__.py @@ -0,0 +1 @@ +__author__ = 'Barak' diff --git a/monkey_island/cc/resources/edge.py b/monkey_island/cc/resources/edge.py new file mode 100644 index 000000000..22d7e307a --- /dev/null +++ b/monkey_island/cc/resources/edge.py @@ -0,0 +1,24 @@ +from bson import ObjectId +from flask import request +import flask_restful + +from cc.database import mongo + +__author__ = 'Barak' + + +class Edge(flask_restful.Resource): + def get(self): + id = request.args.get('id') + to = request.args.get('to') + if id: + edge = mongo.db.edge.find({"_id": ObjectId(id)})[0] + return {"edge": edge} + if to: + edges = mongo.db.edge.find({"to": ObjectId(to)}) + new_edges = [] + # TODO: find better solution for this + for i in range(edges.count()): + new_edges.append(edges[i]) + return {"edges": new_edges} + return {} diff --git a/monkey_island/cc/resources/local_run.py b/monkey_island/cc/resources/local_run.py new file mode 100644 index 000000000..c1d78267c --- /dev/null +++ b/monkey_island/cc/resources/local_run.py @@ -0,0 +1,62 @@ +import json +import os +from shutil import copyfile + +import sys +from flask import request +import flask_restful + +from cc.resources.monkey_download import get_monkey_executable + +from cc.utils import local_ips + +__author__ = 'Barak' + + +def run_local_monkey(island_address): + import platform + import subprocess + import stat + + # get the monkey executable suitable to run on the server + result = get_monkey_executable(platform.system().lower(), platform.machine().lower()) + if not result: + return (False, "OS Type not found") + + monkey_path = os.path.join('binaries', result['filename']) + target_path = os.path.join(os.getcwd(), result['filename']) + + # copy the executable to temp path (don't run the monkey from its current location as it may delete itself) + try: + copyfile(monkey_path, target_path) + os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG) + except Exception, exc: + return (False, "Copy file failed: %s" % exc) + + # run the monkey + try: + args = ["%s m0nk3y -s %s:%s" % (target_path, island_address, ISLAND_PORT)] + if sys.platform == "win32": + args = "".join(args) + pid = subprocess.Popen(args, shell=True).pid + except Exception, exc: + return (False, "popen failed: %s" % exc) + + return (True, "pis: %s" % pid) + + +class LocalRun(flask_restful.Resource): + def get(self): + req_type = request.args.get('type') + if req_type == "interfaces": + return {"interfaces": local_ips()} + else: + return {"message": "unknown action"} + + def post(self): + action_json = json.loads(request.data) + if 'action' in action_json: + if action_json["action"] == "monkey" and action_json.get("island_address") is not None: + return {"res": run_local_monkey(action_json.get("island_address"))} + + return {"res": (False, "Unknown action")} diff --git a/monkey_island/cc/resources/monkey.py b/monkey_island/cc/resources/monkey.py new file mode 100644 index 000000000..f8eceaa33 --- /dev/null +++ b/monkey_island/cc/resources/monkey.py @@ -0,0 +1,124 @@ +import json +from datetime import datetime, timedelta + +import dateutil +from flask import request +import flask_restful + +from cc.database import mongo + +__author__ = 'Barak' + + +def update_dead_monkeys(): + # Update dead monkeys only if no living monkey transmitted keepalive in the last 10 minutes + if mongo.db.monkey.find_one({'dead': {'$ne': True}, 'keepalive': {'$gte': datetime.now() - timedelta(minutes=10)}}): + return + + mongo.db.monkey.update( + {'keepalive': {'$lte': datetime.now() - timedelta(minutes=10)}, 'dead': {'$ne': True}}, + {'$set': {'dead': True, 'modifytime': datetime.now()}}, upsert=False, multi=True) + + +class Monkey(flask_restful.Resource): + def get(self, guid=None, **kw): + update_dead_monkeys() # refresh monkeys status + if not guid: + guid = request.args.get('guid') + timestamp = request.args.get('timestamp') + + if guid: + monkey_json = mongo.db.monkey.find_one_or_404({"guid": guid}) + monkey_json['config']['exploit_user_list'] = \ + map(lambda x: x['username'], mongo.db.usernames.find({}, {'_id': 0, 'username': 1}).sort([('count', -1)])) + monkey_json['config']['exploit_password_list'] = \ + map(lambda x: x['password'], mongo.db.passwords.find({}, {'_id': 0, 'password': 1}).sort([('count', -1)])) + return monkey_json + else: + result = {'timestamp': datetime.now().isoformat()} + find_filter = {} + if timestamp is not None: + find_filter['modifytime'] = {'$gt': dateutil.parser.parse(timestamp)} + result['objects'] = [x for x in mongo.db.monkey.find(find_filter)] + return result + + def patch(self, guid): + monkey_json = json.loads(request.data) + update = {"$set": {'modifytime': datetime.now()}} + + if 'keepalive' in monkey_json: + update['$set']['keepalive'] = dateutil.parser.parse(monkey_json['keepalive']) + else: + update['$set']['keepalive'] = datetime.now() + if 'config' in monkey_json: + update['$set']['config'] = monkey_json['config'] + if 'tunnel' in monkey_json: + update['$set']['tunnel'] = monkey_json['tunnel'] + if 'config_error' in monkey_json: + update['$set']['config_error'] = monkey_json['config_error'] + + return mongo.db.monkey.update({"guid": guid}, update, upsert=False) + + def post(self, **kw): + monkey_json = json.loads(request.data) + if 'keepalive' in monkey_json: + monkey_json['keepalive'] = dateutil.parser.parse(monkey_json['keepalive']) + else: + monkey_json['keepalive'] = datetime.now() + + monkey_json['modifytime'] = datetime.now() + + # if new monkey telem, change config according to "new monkeys" config. + db_monkey = mongo.db.monkey.find_one({"guid": monkey_json["guid"]}) + if not db_monkey: + new_config = mongo.db.config.find_one({'name': 'newconfig'}) or {} + monkey_json['config'] = monkey_json.get('config', {}) + monkey_json['config'].update(new_config) + else: + db_config = db_monkey.get('config', {}) + if 'current_server' in db_config: + del db_config['current_server'] + monkey_json.get('config', {}).update(db_config) + + # try to find new monkey parent + parent = monkey_json.get('parent') + parent_to_add = (monkey_json.get('guid'), None) # default values in case of manual run + if parent and parent != monkey_json.get('guid'): # current parent is known + exploit_telem = [x for x in + mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}, + 'monkey_guid': {'$eq': parent}})] + if 1 == len(exploit_telem): + parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) + else: + parent_to_add = (parent, None) + elif (not parent or parent == monkey_json.get('guid')) and 'ip_addresses' in monkey_json: + exploit_telem = [x for x in + mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}})] + + if 1 == len(exploit_telem): + parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) + + if not db_monkey: + monkey_json['parent'] = [parent_to_add] + else: + monkey_json['parent'] = db_monkey.get('parent') + [parent_to_add] + + mongo.db.monkey.update({"guid": monkey_json["guid"]}, + {"$set": monkey_json}, + upsert=True) + + # Merge existing scanned node with new monkey + + new_monkey_id = mongo.db.monkey.find_one({"guid": monkey_json["guid"]})["_id"] + + existing_node = mongo.db.node.find_one({"ip_addresses": {"$in": monkey_json["ip_addresses"]}}) + + if existing_node: + id = existing_node["_id"] + for edge in mongo.db.edge.find({"to": id}): + mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}}) + mongo.db.node.remove({"_id": id}) + + return {new_monkey_id} diff --git a/monkey_island/cc/resources/monkey_download.py b/monkey_island/cc/resources/monkey_download.py new file mode 100644 index 000000000..6025d899b --- /dev/null +++ b/monkey_island/cc/resources/monkey_download.py @@ -0,0 +1,65 @@ +import json + +import os +from flask import request, send_from_directory +import flask_restful + +__author__ = 'Barak' + + +MONKEY_DOWNLOADS = [ + { + 'type': 'linux', + 'machine': 'x86_64', + 'filename': 'monkey-linux-64', + }, + { + 'type': 'linux', + 'machine': 'i686', + 'filename': 'monkey-linux-32', + }, + { + 'type': 'linux', + 'filename': 'monkey-linux-32', + }, + { + 'type': 'windows', + 'machine': 'x86', + 'filename': 'monkey-windows-32.exe', + }, + { + 'type': 'windows', + 'machine': 'amd64', + 'filename': 'monkey-windows-64.exe', + }, + { + 'type': 'windows', + 'filename': 'monkey-windows-32.exe', + }, +] + + +def get_monkey_executable(host_os, machine): + for download in MONKEY_DOWNLOADS: + if host_os == download.get('type') and machine == download.get('machine'): + return download + return None + + +class MonkeyDownload(flask_restful.Resource): + def get(self, path): + return send_from_directory('binaries', path) + + def post(self): + host_json = json.loads(request.data) + host_os = host_json.get('os') + if host_os: + result = get_monkey_executable(host_os.get('type'), host_os.get('machine')) + + if result: + real_path = os.path.join('binaries', result['filename']) + if os.path.isfile(real_path): + result['size'] = os.path.getsize(real_path) + return result + + return {} diff --git a/monkey_island/cc/resources/netmap.py b/monkey_island/cc/resources/netmap.py new file mode 100644 index 000000000..d45154a8f --- /dev/null +++ b/monkey_island/cc/resources/netmap.py @@ -0,0 +1,67 @@ +import flask_restful + +from cc.database import mongo + +__author__ = 'Barak' + + +class NetMap(flask_restful.Resource): + def get(self, **kw): + monkeys = [self.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] + nodes = [self.node_to_net_node(x) for x in mongo.db.node.find({})] + edges = [self.edge_to_net_edge(x) for x in mongo.db.edge.find({})] + + return \ + { + "nodes": monkeys + nodes, + "edges": edges + } + + def monkey_to_net_node(self, monkey): + os = "unknown" + if monkey["description"].lower().find("linux") != -1: + os = "linux" + elif monkey["description"].lower().find("windows") != -1: + os = "windows" + + manual_run = (monkey["parent"][0][1] == None) + return \ + { + "id": monkey["_id"], + "label": monkey["hostname"] + " : " + monkey["ip_addresses"][0], + "group": ("manuallyInfected" if manual_run else "infected"), + "os": os, + "dead": monkey["dead"], + } + + def node_to_net_node(self, node): + os_version = "undefined" + os_type = "undefined" + found = False + for edge in mongo.db.edge.find({"to": node["_id"]}): + for scan in edge["scans"]: + if scan["scanner"] != "TcpScanner": + continue + os_type = scan["data"]["os"]["type"] + if scan["data"]["os"].has_key("version"): + os_version = scan["data"]["os"]["version"] + found = True + break + if found: + break + + return \ + { + "id": node["_id"], + "label": os_version + " : " + node["ip_addresses"][0], + "group": "clean", + "os": os_type + } + + def edge_to_net_edge(self, edge): + return \ + { + "id": edge["_id"], + "from": edge["from"], + "to": edge["to"] + } diff --git a/monkey_island/cc/resources/new_config.py b/monkey_island/cc/resources/new_config.py new file mode 100644 index 000000000..4d15d1611 --- /dev/null +++ b/monkey_island/cc/resources/new_config.py @@ -0,0 +1,20 @@ +import json + +from flask import request +import flask_restful + +from cc.database import mongo + +__author__ = 'Barak' + + +class NewConfig(flask_restful.Resource): + def get(self): + config = mongo.db.config.find_one({'name': 'newconfig'}) or {} + if 'name' in config: + del config['name'] + return config + + def post(self): + config_json = json.loads(request.data) + return mongo.db.config.update({'name': 'newconfig'}, {"$set": config_json}, upsert=True) diff --git a/monkey_island/cc/resources/root.py b/monkey_island/cc/resources/root.py new file mode 100644 index 000000000..2d7af1eb6 --- /dev/null +++ b/monkey_island/cc/resources/root.py @@ -0,0 +1,42 @@ +from datetime import datetime + +from flask import request +import flask_restful + +from cc.database import mongo + +from cc.utils import init_collections + +__author__ = 'Barak' + + +class Root(flask_restful.Resource): + def get(self, action=None): + if not action: + action = request.args.get('action') + if not action: + return { + 'status': 'OK', + 'mongo': str(mongo.db), + } + elif action == "reset": + mongo.db.config.drop() + mongo.db.monkey.drop() + mongo.db.telemetry.drop() + mongo.db.usernames.drop() + mongo.db.passwords.drop() + mongo.db.node.drop() + mongo.db.edge.drop() + init_collections() + return { + 'status': 'OK', + } + elif action == "killall": + mongo.db.monkey.update({}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}}, upsert=False, + multi=True) + return { + 'status': 'OK', + } + else: + return {'status': 'BAD', + 'reason': 'unknown action'} diff --git a/monkey_island/cc/resources/telemetry.py b/monkey_island/cc/resources/telemetry.py new file mode 100644 index 000000000..e015173ff --- /dev/null +++ b/monkey_island/cc/resources/telemetry.py @@ -0,0 +1,127 @@ +import json +from datetime import datetime + +import dateutil +from flask import request +import flask_restful + +from cc.database import mongo + +from cc.utils import creds_add_username, creds_add_password + +__author__ = 'Barak' + + +class Telemetry(flask_restful.Resource): + def get(self, **kw): + monkey_guid = request.args.get('monkey_guid') + telem_type = request.args.get('telem_type') + timestamp = request.args.get('timestamp') + if "null" == timestamp: # special case to avoid ugly JS code... + timestamp = None + + result = {'timestamp': datetime.now().isoformat()} + find_filter = {} + + if monkey_guid: + find_filter["monkey_guid"] = {'$eq': monkey_guid} + if telem_type: + find_filter["telem_type"] = {'$eq': telem_type} + if timestamp: + find_filter['timestamp'] = {'$gt': dateutil.parser.parse(timestamp)} + + result['objects'] = [x for x in mongo.db.telemetry.find(find_filter)] + return result + + def post(self): + telemetry_json = json.loads(request.data) + telemetry_json['timestamp'] = datetime.now() + + telem_id = mongo.db.telemetry.insert(telemetry_json) + + # update exploited monkeys parent + try: + if telemetry_json.get('telem_type') == 'tunnel': + if telemetry_json['data']: + host = telemetry_json['data'].split(":")[-2].replace("//", "") + tunnel_host = mongo.db.monkey.find_one({"ip_addresses": host}) + mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, + {'$set': {'tunnel_guid': tunnel_host.get('guid'), + 'modifytime': datetime.now()}}, + upsert=False) + else: + mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, + {'$unset': {'tunnel_guid': ''}, + '$set': {'modifytime': datetime.now()}}, + upsert=False) + elif telemetry_json.get('telem_type') == 'state': + if telemetry_json['data']['done']: + mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, + {'$set': {'dead': True, 'modifytime': datetime.now()}}, + upsert=False) + else: + mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, + {'$set': {'dead': False, 'modifytime': datetime.now()}}, + upsert=False) + elif telemetry_json.get('telem_type') == 'scan': + dst_ip = telemetry_json['data']['machine']['ip_addr'] + src_monkey = mongo.db.monkey.find_one({"guid": telemetry_json['monkey_guid']}) + dst_monkey = mongo.db.monkey.find_one({"ip_addresses": dst_ip}) + if dst_monkey: + edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_monkey["_id"]}) + + if edge is None: + edge = self.insert_edge(src_monkey["_id"], dst_monkey["_id"]) + + else: + dst_node = mongo.db.node.find_one({"ip_addresses": dst_ip}) + if dst_node is None: + dst_node_insert_result = mongo.db.node.insert_one({"ip_addresses": [dst_ip]}) + dst_node = mongo.db.node.find_one({"_id": dst_node_insert_result.inserted_id}) + + edge = mongo.db.edge.find_one({"from": src_monkey["_id"], "to": dst_node["_id"]}) + + if edge is None: + edge = self.insert_edge(src_monkey["_id"], dst_node["_id"]) + + self.add_scan_to_edge(edge, telemetry_json) + + except StandardError as e: + pass + + # Update credentials DB + try: + if (telemetry_json.get('telem_type') == 'system_info_collection') and (telemetry_json['data'].has_key('credentials')): + creds = telemetry_json['data']['credentials'] + for user in creds: + creds_add_username(user) + + if creds[user].has_key('password'): + creds_add_password(creds[user]['password']) + except StandardError as ex: + print("Exception caught while updating DB credentials: %s" % str(ex)) + + return mongo.db.telemetry.find_one_or_404({"_id": telem_id}) + + def add_scan_to_edge(self, edge, telemetry_json): + data = telemetry_json['data']['machine'] + data.pop("ip_addr") + new_scan = \ + { + "timestamp": telemetry_json["timestamp"], + "data": data, + "scanner": telemetry_json['data']['scanner'] + } + mongo.db.edge.update( + {"_id": edge["_id"]}, + {"$push": {"scans": new_scan}} + ) + + def insert_edge(self, from_id, to_id): + edge_insert_result = mongo.db.edge.insert_one( + { + "from": from_id, + "to": to_id, + "scans": [] + }) + return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) diff --git a/monkey_island/cc/utils.py b/monkey_island/cc/utils.py new file mode 100644 index 000000000..30530470d --- /dev/null +++ b/monkey_island/cc/utils.py @@ -0,0 +1,77 @@ +import socket +import sys + +import array + +import struct +from cc.database import mongo + +__author__ = 'Barak' + + +# data structures + +def creds_add_username(username): + mongo.db.usernames.update( + {'username': username}, + {'$inc': {'count': 1}}, + upsert=True + ) + + +def creds_add_password(password): + mongo.db.passwords.update( + {'password': password}, + {'$inc': {'count': 1}}, + upsert=True + ) + + +def init_collections(usernames, passwords): + if "usernames" not in mongo.db.collection_names(): + mongo.db.usernames.create_index([("username", 1)], unique=True) + for username in usernames: + creds_add_username(username) + + if "passwords" not in mongo.db.collection_names(): + mongo.db.passwords.create_index([("password", 1)], unique=True) + for password in passwords: + creds_add_password(password) + + +# Local ips function +if sys.platform == "win32": + def local_ips(): + local_hostname = socket.gethostname() + return socket.gethostbyname_ex(local_hostname)[2] +else: + import fcntl + def local_ips(): + result = [] + try: + is_64bits = sys.maxsize > 2 ** 32 + struct_size = 40 if is_64bits else 32 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + max_possible = 8 # initial value + while True: + struct_bytes = max_possible * struct_size + names = array.array('B', '\0' * struct_bytes) + outbytes = struct.unpack('iL', fcntl.ioctl( + s.fileno(), + 0x8912, # SIOCGIFCONF + struct.pack('iL', struct_bytes, names.buffer_info()[0]) + ))[0] + if outbytes == struct_bytes: + max_possible *= 2 + else: + break + namestr = names.tostring() + + for i in range(0, outbytes, struct_size): + addr = socket.inet_ntoa(namestr[i + 20:i + 24]) + if not addr.startswith('127'): + result.append(addr) + # name of interface is (namestr[i:i+16].split('\0', 1)[0] + finally: + return result +# End of local ips function