From fc95dccf2222b11183b56bafd37dec3542a935a6 Mon Sep 17 00:00:00 2001 From: itsikkes Date: Sun, 24 Jul 2016 01:04:42 +0300 Subject: [PATCH 1/3] bug fixes and features added support for multi-parent, auto mark dead monkeys,; UI: get exploit details, can select edges, show num pf monkeys alive, show failed exploit attempts; some bug fixes; --- chaos_monkey/monkey.py | 5 +- monkey_island/cc/admin/ui/index.html | 25 +- monkey_island/cc/admin/ui/js/monkeys-admin.js | 251 +++++++++++++----- monkey_island/cc/main.py | 76 +++--- 4 files changed, 254 insertions(+), 103 deletions(-) diff --git a/chaos_monkey/monkey.py b/chaos_monkey/monkey.py index be7184ad3..482211b81 100644 --- a/chaos_monkey/monkey.py +++ b/chaos_monkey/monkey.py @@ -162,6 +162,9 @@ class ChaosMonkey(object): break else: LOG.info("Failed exploiting %r with exploiter %s", machine, exploiter.__class__.__name__) + ControlClient.send_telemetry('exploit', {'result': False, 'machine': machine.__dict__, + 'exploiter': exploiter.__class__.__name__}) + except Exception, exc: LOG.error("Exception while attacking %s using %s: %s", machine, exploiter.__class__.__name__, exc) @@ -169,7 +172,7 @@ class ChaosMonkey(object): if successful_exploiter: self._exploited_machines.add(machine) - ControlClient.send_telemetry('exploit', {'machine': machine.__dict__, + ControlClient.send_telemetry('exploit', {'result': True, 'machine': machine.__dict__, 'exploiter': successful_exploiter.__class__.__name__}) LOG.info("Successfully propagated to %s using %s", diff --git a/monkey_island/cc/admin/ui/index.html b/monkey_island/cc/admin/ui/index.html index d1c79e1d5..c2a8eb2a1 100644 --- a/monkey_island/cc/admin/ui/index.html +++ b/monkey_island/cc/admin/ui/index.html @@ -83,8 +83,9 @@
- Num of Monkeys: ( by exploiting)
- Num of Hosts Detected:
+ Num of Monkeys: ( exploiting were done)
+ Monkyes Alive:
+ Num of Hosts Not Exploited:
Num of Tunnels Used:
@@ -132,7 +133,7 @@
- +
@@ -188,6 +189,24 @@
+ +
+
+
+ Reset Database +
+
+ + + +
+
+
+ +
diff --git a/monkey_island/cc/admin/ui/js/monkeys-admin.js b/monkey_island/cc/admin/ui/js/monkeys-admin.js index b98edc16a..e009b46b7 100644 --- a/monkey_island/cc/admin/ui/js/monkeys-admin.js +++ b/monkey_island/cc/admin/ui/js/monkeys-admin.js @@ -62,12 +62,13 @@ function initAdmin() { nodes = []; edges = []; + createNodes(); createEdges(); createTunnels(); createScanned(); var data = { - nodes: createNodes(), + nodes: nodes, edges: edges }; @@ -76,7 +77,10 @@ function initAdmin() { var options = { layout: { improvedLayout: false - } + }/*, + physics: { + enabled: true + }*/ }; // Using jQuery to get the element does not work with vis.js library @@ -196,7 +200,19 @@ function updateMonkeys() { else { monkeys.push(new_monkeys[i]); - nodes.push(createMonkeyNode(new_monkeys[i])); + var exiting_scan = undefined; + for (var j=0; jFrom: " + parent.hostname + "
To: " + monkey.hostname; + edges.push({from: parent.id, to: monkey.id, arrows:'middle', type: EDGE_TYPE_PARENT, title: title, /*label: exploit, font: {color: 'red', size: 10, align: 'top'},*/ color: EDGE_COLOR_PARENT}); + if (removeEdge([parent.id, monkey.id, EDGE_TYPE_SCAN])) { + numOfScanLinks--; + } + numOfParentLinks++; + } } } } @@ -327,24 +387,20 @@ function createScanned() { var scan = scans[i]; var monkey = getMonkeyByGuid(scan.monkey_guid); - //Check if we already exploited this machine from another PoV, if so no point in scanning. - if (null != getMonkeyByIP(scan.data.machine.ip_addr)) { - //if so, make sure we don't already have such a node - nodes = nodes.filter(function (node) { - return (node.id != ip_addr); - }); - continue; - } + // And check if we've already added this scanned machine + var machineNode = getMonkeyByIP(scan.data.machine.ip_addr); - //And check if we've already added this scanned machine - var machineNode = getScannedByIP(scan.data.machine.ip_addr) if (null == machineNode) { - machineNode = createMachineNode(scan.data.machine); - scannedMachines.push(machineNode); - nodes.push(machineNode); + machineNode = getScannedByIP(scan.data.machine.ip_addr); + + if (null == machineNode) { + machineNode = createMachineNode(scan.data.machine); + scannedMachines.push(machineNode); + nodes.push(machineNode); + } } - if(!edgeExists([monkey.id, machineNode.id, EDGE_TYPE_SCAN])) { + if(!edgeExists([monkey.id, machineNode.id, EDGE_TYPE_SCAN]) && !edgeExists([monkey.id, machineNode.id, EDGE_TYPE_PARENT])) { edges.push({from: monkey.id, to: machineNode.id, arrows:'middle', type: EDGE_TYPE_SCAN, color: EDGE_COLOR_SCAN}); numOfScanLinks++; } @@ -372,10 +428,34 @@ function buildMonkeyDescription(monkey) { } html += " " + monkey.keepalive + "
" + - "
"; + "
"; + html += ""; + + + if (monkey.parent != null) { + html += "
" + html += ""; } return html; @@ -386,6 +466,11 @@ function updateCounters() { $('#infoNumOfHosts').html(scannedMachines.length); $('#infoNumOfParents').html(numOfParentLinks); $('#infoNumOfTunnels').html(numOfTunnelLinks); + var numOfAlive = monkeys.length; + for (var i=0;i 0) { + if ((properties.nodes.length > 0) && getMonkey(properties.nodes[0])){ onNodeSelect(properties.nodes); } else { - var content = "No selection" + var content = "Monkey not selected" $("#selectionInfo").html(content); $('#monkey-config').hide() $('#btnConfigLoad, #btnConfigUpdate').hide(); $('#monkey-enabled').hide(); telemTable.clear(); telemTable.draw(); - } - if (properties.edges.length > 0) { - onEdgeSelect(properties.edges); + if (properties.edges.length > 0) { + onEdgeSelect(properties.edges); + } } } @@ -483,33 +568,32 @@ function onNodeSelect(nodeId) { if (monkey) { htmlContent = buildMonkeyDescription(monkey); $("#monkeySearch").val(monkey.hostname); + $("#selectionInfo").html(htmlContent); + $('#monkey-config').show() + $('#btnConfigLoad, #btnConfigUpdate').show(); + + loadMonkeyConfig(); + + if (monkey.config.alive) { + $("[name='chboxMonkeyEnabled']").bootstrapSwitch('state', true, true); + } + else { + $("[name='chboxMonkeyEnabled']").bootstrapSwitch('state', false, true); + } + $('#monkey-enabled').show(); + + $.getJSON('/api/telemetry?monkey_guid=' + monkey.guid, function(json) { + telemTable.clear(); + var telemetries = json.objects; + + for (var i = 0; i < telemetries.length; i++) { + telemTable.row.add([telemetries[i].timestamp, telemetries[i].telem_type, JSON.stringify(telemetries[i].data)]); + } + + telemTable.draw(); + }); } - $("#selectionInfo").html(htmlContent); - $('#monkey-config').show() - $('#btnConfigLoad, #btnConfigUpdate').show(); - - loadMonkeyConfig(); - - if (monkey.config.alive) { - $("[name='chboxMonkeyEnabled']").bootstrapSwitch('state', true); - } - else { - $("[name='chboxMonkeyEnabled']").bootstrapSwitch('state', false); - } - $('#monkey-enabled').show(); - - $.getJSON('/api/telemetry?monkey_guid=' + monkey.guid, function(json) { - telemTable.clear(); - var telemetries = json.objects; - - for (var i = 0; i < telemetries.length; i++) { - telemTable.row.add([telemetries[i].timestamp, telemetries[i].telem_type, JSON.stringify(telemetries[i].data)]); - } - - telemTable.draw(); - }); - network.selectNodes([nodeId]); } @@ -518,7 +602,32 @@ function onNodeSelect(nodeId) { */ function onEdgeSelect(edge) { var edge = getEdge(edge); + var monkey = getMonkey(edge.from); + if (!monkey) {return;}; + var target = undefined; + if (edge.type == 'scan') { + target = getScannedByIP(edge.to) + } + else { + target = getMonkey(edge.to) + } + + $.getJSON(jsonFileTelemetry + '?monkey_guid=' + monkey.guid, function(json) { + telemTable.clear(); + var telemetries = json.objects; + + for (var i = 0; i < telemetries.length; i++) { + var telem = telemetries[i] + if (telem.telem_type == 'scan' || telem.telem_type == 'exploit') { + if (((edge.type == 'scan') && (telem.data.machine.ip_addr == target.id)) || + ((edge.type == 'parent') && (0 <= $.inArray(telem.data.machine.ip_addr, target.ip_addresses)))) { + telemTable.row.add([telemetries[i].timestamp, telemetries[i].telem_type, JSON.stringify(telemetries[i].data)]); + } + } + } + telemTable.draw(); + }); } function toggleMonkeyEnabled(event, state) { @@ -728,24 +837,24 @@ function getMonkeyByGuid(guid) { if (monkeys[i].guid == guid) { return monkeys[i]; } - } + } + return null; } function getMonkeyByIP(ip) { - for (var i = 0; i < monkeys.length; i++) { + for (var i = 0; i < monkeys.length; i++) { var monkey = monkeys[i]; - for (var j = 0; j< monkey.ip_addresses; j++) { - if (monkeys[i].ip == ip) { - return monkeys[i]; + for (var j = 0; j< monkey.ip_addresses.length; j++) { + if (monkey.ip_addresses[j] == ip) { + return monkey; } } } return null; } -function getScannedByIP(ip) -{ - for (var i = 0; i < scannedMachines.length; i++) { +function getScannedByIP(ip) { + for (var i = 0; i < scannedMachines.length; i++) { var machine = scannedMachines[i]; if (machine.id == ip) { return machine @@ -801,13 +910,23 @@ function edgeExists(link) { var to = edges[i].to; var type = edges[i].type; if (from == link[0] && to == link[1] && type == link[2]) { - return true; + return edges[i]; } } } - - +function removeEdge(link) { + for (var i = 0; i < edges.length; i++) { + var from = edges[i].from; + var to = edges[i].to; + var type = edges[i].type; + if (from == link[0] && to == link[1] && type == link[2]) { + edges.splice(i, 1); + return true; + } + } + return false; +} /** * Clears the value in the local storage diff --git a/monkey_island/cc/main.py b/monkey_island/cc/main.py index 97799c303..9dfe6a78b 100644 --- a/monkey_island/cc/main.py +++ b/monkey_island/cc/main.py @@ -5,7 +5,7 @@ from flask.ext.pymongo import PyMongo from flask import make_response import bson.json_util import json -from datetime import datetime +from datetime import datetime, timedelta import dateutil.parser MONKEY_DOWNLOADS = [ @@ -50,6 +50,7 @@ mongo = PyMongo(app) class Monkey(restful.Resource): def get(self, guid=None, **kw): + update_dead_monkeys() # refresh monkeys status if not guid: guid = request.args.get('guid') timestamp = request.args.get('timestamp') @@ -59,7 +60,7 @@ class Monkey(restful.Resource): else: result = {'timestamp': datetime.now().isoformat()} find_filter = {} - if None != timestamp: + if timestamp is not None: find_filter['modifytime'] = {'$gt': dateutil.parser.parse(timestamp)} result['objects'] = [x for x in mongo.db.monkey.find(find_filter)] return result @@ -67,7 +68,7 @@ class Monkey(restful.Resource): def patch(self, guid): monkey_json = json.loads(request.data) update = {"$set": {'modifytime': datetime.now()}} - + if monkey_json.has_key('keepalive'): update['$set']['keepalive'] = dateutil.parser.parse(monkey_json['keepalive']) else: @@ -76,7 +77,7 @@ class Monkey(restful.Resource): update['$set']['config'] = monkey_json['config'] if monkey_json.has_key('tunnel'): update['$set']['tunnel'] = monkey_json['tunnel'] - + return mongo.db.monkey.update({"guid": guid}, update, upsert=False) def post(self, **kw): @@ -88,7 +89,7 @@ class Monkey(restful.Resource): monkey_json['modifytime'] = datetime.now() - # if new monkey, change config according to "new monkeys" config. + # if new monkey telem, change config according to "new monkeys" config. db_monkey = mongo.db.monkey.find_one({"guid": monkey_json["guid"]}) if not db_monkey: new_config = mongo.db.config.find_one({'name': 'newconfig'}) or {} @@ -99,18 +100,31 @@ class Monkey(restful.Resource): if db_config.has_key('current_server'): del db_config['current_server'] monkey_json.get('config', {}).update(db_config) - - if not monkey_json.has_key('parent') and db_monkey.get('parent'): - monkey_json['parent'] = db_monkey.get('parent') # try to find new monkey parent parent = monkey_json.get('parent') - if (not parent or parent == monkey_json.get('guid')) and monkey_json.has_key('ip_addresses'): + parent_to_add = (monkey_json.get('guid'), None) # default values in case of manual run + if parent and parent != monkey_json.get('guid'): # current parent is known exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.machine.ip_addr': - {'$in': monkey_json['ip_addresses']}})] + mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}, + 'monkey_guid': {'$eq': parent}})] if 1 == len(exploit_telem): - monkey_json['parent'] = exploit_telem[0].get('monkey_guid') + parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) + else: + parent_to_add = (parent, None) + elif (not parent or parent == monkey_json.get('guid')) and monkey_json.has_key('ip_addresses'): + exploit_telem = [x for x in + mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}})] + + if 1 == len(exploit_telem): + parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) + + if not db_monkey: + monkey_json['parent'] = [parent_to_add] + else: + monkey_json['parent'] = db_monkey.get('parent') + [parent_to_add] return mongo.db.monkey.update({"guid": monkey_json["guid"]}, {"$set": monkey_json}, @@ -122,7 +136,7 @@ class Telemetry(restful.Resource): monkey_guid = request.args.get('monkey_guid') telem_type = request.args.get('telem_type') timestamp = request.args.get('timestamp') - if "null" == timestamp: #special case to avoid ugly JS code... + if "null" == timestamp: # special case to avoid ugly JS code... timestamp = None result = {'timestamp': datetime.now().isoformat()} @@ -146,37 +160,26 @@ class Telemetry(restful.Resource): # update exploited monkeys parent try: - if telemetry_json.get('telem_type') == 'exploit': - update_parent = [] - for monkey in mongo.db.monkey.find({"ip_addresses": - {'$elemMatch': - {'$eq': telemetry_json['data']['machine']['ip_addr']}}}): - parent = monkey.get('parent') - if parent == monkey.get('guid') or not parent: - update_parent.append(monkey) - if 1 == len(update_parent): - update_parent[0]['parent'] = telemetry_json['monkey_guid'] - mongo.db.monkey.update({"guid": update_parent[0]['guid']}, {"$set": update_parent[0]}, upsert=False) - elif telemetry_json.get('telem_type') == 'tunnel': + if telemetry_json.get('telem_type') == 'tunnel': if telemetry_json['data']: host = telemetry_json['data'].split(":")[-2].replace("//", "") tunnel_host = mongo.db.monkey.find_one({"ip_addresses": host}) mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, {'$set': {'tunnel_guid': tunnel_host.get('guid')}}, - upsert=True) + upsert=False) else: mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, - {'$unset': {'tunnel_guid':''}}, - upsert=True) + {'$unset': {'tunnel_guid': ''}}, + upsert=False) elif telemetry_json.get('telem_type') == 'state': if telemetry_json['data']['done']: mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, {'$set': {'dead': True}}, - upsert=True) + upsert=False) else: mongo.db.monkey.update({"guid": telemetry_json['monkey_guid']}, {'$set': {'dead': False}}, - upsert=True) + upsert=False) except: pass @@ -232,15 +235,15 @@ def normalize_obj(obj): obj['id'] = obj['_id'] del obj['_id'] - for key,value in obj.items(): + for key, value in obj.items(): if type(value) is bson.objectid.ObjectId: obj[key] = str(value) if type(value) is datetime: - obj[key] = str(value) + obj[key] = str(value) if type(value) is dict: obj[key] = normalize_obj(value) if type(value) is list: - for i in range(0,len(value)): + for i in range(0, len(value)): if type(value[i]) is dict: value[i] = normalize_obj(value[i]) return obj @@ -253,10 +256,17 @@ def output_json(obj, code, headers=None): return resp +def update_dead_monkeys(): + mongo.db.monkey.update( + {'keepalive': {'$lte': datetime.now() - timedelta(minutes=10)}, 'dead': {'$ne': True}}, + {'$set': {'dead': True, 'modifytime': datetime.now()}}, upsert=False) + + @app.route('/admin/') def send_admin(path): return send_from_directory('admin/ui', path) + DEFAULT_REPRESENTATIONS = {'application/json': output_json} api = restful.Api(app) api.representations = DEFAULT_REPRESENTATIONS From 4ed667d805fd60e4bdb31b8a197b83253846d6f7 Mon Sep 17 00:00:00 2001 From: itsikkes Date: Sun, 24 Jul 2016 16:27:56 +0300 Subject: [PATCH 2/3] implemented reset DB from UI --- monkey_island/cc/admin/ui/js/monkeys-admin.js | 20 ++++++++++------ monkey_island/cc/main.py | 23 +++++++++++++++---- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/monkey_island/cc/admin/ui/js/monkeys-admin.js b/monkey_island/cc/admin/ui/js/monkeys-admin.js index e009b46b7..c58b4f54e 100644 --- a/monkey_island/cc/admin/ui/js/monkeys-admin.js +++ b/monkey_island/cc/admin/ui/js/monkeys-admin.js @@ -77,10 +77,7 @@ function initAdmin() { var options = { layout: { improvedLayout: false - }/*, - physics: { - enabled: true - }*/ + } }; // Using jQuery to get the element does not work with vis.js library @@ -213,7 +210,6 @@ function updateMonkeys() { else { convertScanNodeToMonkey(exiting_scan, new_monkeys[i]); } - updateCounters(); } } @@ -224,6 +220,7 @@ function updateMonkeys() { refreshDrawing(); } createScanned(); + updateCounters(); }); } @@ -803,8 +800,17 @@ function resetDB() { url : '/api?action=reset', type : 'GET', success : function(response, textStatus, jqXhr) { - console.log("DB was successfully reset!"); - location.reload(); + console.log(response); + if (response.status != 'OK') { + BootstrapDialog.show({ + title: "Reset DB", + message: "The following error occured: " + response.reason + }); + } + else { + console.log("DB was successfully reset!"); + location.reload(); + } }, error : function(jqXHR, textStatus, errorThrown) { // log the error to the console diff --git a/monkey_island/cc/main.py b/monkey_island/cc/main.py index 9dfe6a78b..ea4c55b35 100644 --- a/monkey_island/cc/main.py +++ b/monkey_island/cc/main.py @@ -223,11 +223,24 @@ class MonkeyDownload(restful.Resource): class Root(restful.Resource): - def get(self): - return { - 'status': 'OK', - 'mongo': str(mongo.db), - } + def get(self, action=None): + if not action: + action = request.args.get('action') + if not action: + return { + 'status': 'OK', + 'mongo': str(mongo.db), + } + elif action=="reset": + mongo.db.config.drop() + mongo.db.monkey.drop() + mongo.db.telemetry.drop() + return { + 'status': 'OK', + } + else: + return {'status': 'BAD', + 'reason': 'unknown action'} def normalize_obj(obj): From 16c1d75d013b1c38b99671312cc5514f10a910d7 Mon Sep 17 00:00:00 2001 From: itsikkes Date: Sun, 24 Jul 2016 17:23:57 +0300 Subject: [PATCH 3/3] Update business readme --- monkey_business/readme.txt | 49 ++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/monkey_business/readme.txt b/monkey_business/readme.txt index 6f456df9e..b00d16469 100644 --- a/monkey_business/readme.txt +++ b/monkey_business/readme.txt @@ -1,4 +1,45 @@ -dependencies: -sudo pip install pyVmomi -sudo pip install celery -sudo pip install -U celery[mongodb] \ No newline at end of file +How to install Monkey Business server: + +---------------- On Linux ----------------: +1. Create the following directories: + sudo mkdir /var/monkey_business + sudo chmod 777 /var/monkey_business + mkdir -p /var/monkey_business/bin/mongodb + mkdir -p /var/monkey_business/db + mkdir -p /var/monkey_business/cc + +2. Install the following packages: + sudo pip install flask + sudo pip install Flask-Pymongo + sudo pip install Flask-Restful + sudo pip install python-dateutil + sudo pip install pyVmomi + sudo pip install celery + sudo pip install -U celery[mongodb] + +4. Download MongoDB and extract it to /var/monkey_business/bin/mongodb + for debian64 - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-debian71-3.0.7.tgz + for ubuntu64 14.10 - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1410-clang-3.0.7.tgz + find more at - https://www.mongodb.org/downloads#production + untar.gz with: tar -zxvf filename.tar.gz -C /var/monkey_business/bin/mongodb + (make sure the content of the mongo folder is in this directory, meaning this path exists: + /var/monkey_business/bin/mongodb/bin) + +5. install OpenSSL + sudo apt-get install openssl + +6. Generate SSL Certificate, Run create_certificate.sh (located under /linux) + +7. Copy monkey business server to /var/monkey_business: + cp -r [monkey_island_source]/cc /var/monkey_business/ + + +How to run: +1. run run.sh + * This performs: + DB startup: + /var/monkey_business/bin/mongodb/bin/mongod --dbpath db --fork --logpath db.log + Jobs worker startup: + nohup celery -A tasks_manager worker --loglevel=info + Main Web Server startup: + nohup python main.py \ No newline at end of file