From 0ed2f74824426224c7ef48508e4191c409e90fb8 Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Tue, 13 Feb 2018 12:05:01 +0200 Subject: [PATCH 01/64] Add encryptor --- monkey_island/cc/encryptor.py | 48 +++++++++++++++++++ .../monkey_island_pip_requirements.txt | 1 + monkey_island/requirements.txt | 3 +- 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 monkey_island/cc/encryptor.py diff --git a/monkey_island/cc/encryptor.py b/monkey_island/cc/encryptor.py new file mode 100644 index 000000000..7af0840d0 --- /dev/null +++ b/monkey_island/cc/encryptor.py @@ -0,0 +1,48 @@ +import base64 +import os + +from Crypto import Random +from Crypto.Cipher import AES + +__author__ = "itay.mizeretz" + + +class Encryptor: + _BLOCK_SIZE = 32 + _DB_PASSWORD_FILENAME = "mongo_key.bin" + + def __init__(self): + self._load_key() + + def _init_key(self): + self._cipher_key = Random.new().read(self._BLOCK_SIZE) + with open(self._DB_PASSWORD_FILENAME, 'wb') as f: + f.write(self._cipher_key) + + def _load_existing_key(self): + with open(self._DB_PASSWORD_FILENAME, 'rb') as f: + self._cipher_key = f.read() + + def _load_key(self): + if os.path.exists(self._DB_PASSWORD_FILENAME): + self._load_existing_key() + else: + self._init_key() + + def _pad(self, message): + return message + (self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) * chr( + self._BLOCK_SIZE - (len(message) % self._BLOCK_SIZE)) + + def _unpad(self, message): + return message[0:-ord(message[len(message) - 1])] + + def enc(self, message): + cipher_iv = Random.new().read(AES.block_size) + cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) + return base64.b64encode(cipher_iv + cipher.encrypt(self._pad(message))) + + def dec(self, enc_message): + enc_message = base64.b64decode(enc_message) + cipher_iv = enc_message[0:AES.block_size] + cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) + return self._unpad(cipher.decrypt(enc_message[AES.block_size:])) diff --git a/monkey_island/deb-package/monkey_island_pip_requirements.txt b/monkey_island/deb-package/monkey_island_pip_requirements.txt index 404aad8b0..582efc5f2 100644 --- a/monkey_island/deb-package/monkey_island_pip_requirements.txt +++ b/monkey_island/deb-package/monkey_island_pip_requirements.txt @@ -12,4 +12,5 @@ jsonschema netifaces ipaddress enum34 +PyCrypto virtualenv \ No newline at end of file diff --git a/monkey_island/requirements.txt b/monkey_island/requirements.txt index 9d8bfbfb8..18098eec0 100644 --- a/monkey_island/requirements.txt +++ b/monkey_island/requirements.txt @@ -11,4 +11,5 @@ Flask-Restful jsonschema netifaces ipaddress -enum34 \ No newline at end of file +enum34 +PyCrypto \ No newline at end of file From 29e85100d2dce87c58f4c4b1df6092c92c989311 Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Tue, 13 Feb 2018 16:29:24 +0200 Subject: [PATCH 02/64] Add global encryptor --- monkey_island/cc/encryptor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monkey_island/cc/encryptor.py b/monkey_island/cc/encryptor.py index 7af0840d0..90009d1b0 100644 --- a/monkey_island/cc/encryptor.py +++ b/monkey_island/cc/encryptor.py @@ -46,3 +46,6 @@ class Encryptor: cipher_iv = enc_message[0:AES.block_size] cipher = AES.new(self._cipher_key, AES.MODE_CBC, cipher_iv) return self._unpad(cipher.decrypt(enc_message[AES.block_size:])) + + +encryptor = Encryptor() From 06a2e4f18dcf372f5339be3c8f87403bca16eecb Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Tue, 13 Feb 2018 16:34:37 +0200 Subject: [PATCH 03/64] encrypt credentials in config+telemetry --- .../cc/resources/monkey_configuration.py | 2 +- monkey_island/cc/resources/telemetry.py | 53 +++-- monkey_island/cc/services/config.py | 213 +++++++++++------- 3 files changed, 170 insertions(+), 98 deletions(-) diff --git a/monkey_island/cc/resources/monkey_configuration.py b/monkey_island/cc/resources/monkey_configuration.py index 6d622b1cd..c3c9f51cb 100644 --- a/monkey_island/cc/resources/monkey_configuration.py +++ b/monkey_island/cc/resources/monkey_configuration.py @@ -18,6 +18,6 @@ class MonkeyConfiguration(flask_restful.Resource): if config_json.has_key('reset'): ConfigService.reset_config() else: - ConfigService.update_config(config_json) + ConfigService.update_config(config_json, should_encrypt=True) return self.get() diff --git a/monkey_island/cc/resources/telemetry.py b/monkey_island/cc/resources/telemetry.py index 94c4046b5..6f69ecc2f 100644 --- a/monkey_island/cc/resources/telemetry.py +++ b/monkey_island/cc/resources/telemetry.py @@ -11,6 +11,7 @@ from cc.database import mongo from cc.services.config import ConfigService from cc.services.edge import EdgeService from cc.services.node import NodeService +from cc.encryptor import encryptor __author__ = 'Barak' @@ -121,6 +122,8 @@ class Telemetry(flask_restful.Resource): if new_exploit['result']: EdgeService.set_edge_exploited(edge) + Telemetry.encrypt_exploit_creds(telemetry_json) + for attempt in telemetry_json['data']['attempts']: if attempt['result']: found_creds = {'user': attempt['user']} @@ -163,25 +166,49 @@ class Telemetry(flask_restful.Resource): def process_system_info_telemetry(telemetry_json): if 'credentials' in telemetry_json['data']: creds = telemetry_json['data']['credentials'] - for user in creds: - ConfigService.creds_add_username(user) - if 'password' in creds[user]: - ConfigService.creds_add_password(creds[user]['password']) - if 'lm_hash' in creds[user]: - ConfigService.creds_add_lm_hash(creds[user]['lm_hash']) - if 'ntlm_hash' in creds[user]: - ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) - - for user in creds: - if -1 != user.find('.'): - new_user = user.replace('.', ',') - creds[new_user] = creds.pop(user) + Telemetry.encrypt_system_info_creds(creds) + Telemetry.add_system_info_creds_to_config(creds) + Telemetry.replace_user_dot_with_comma(creds) @staticmethod def process_trace_telemetry(telemetry_json): # Nothing to do return + @staticmethod + def replace_user_dot_with_comma(creds): + for user in creds: + if -1 != user.find('.'): + new_user = user.replace('.', ',') + creds[new_user] = creds.pop(user) + + @staticmethod + def encrypt_system_info_creds(creds): + for user in creds: + for field in ['password', 'lm_hash', 'ntlm_hash']: + if field in creds[user]: + creds[user][field] = encryptor.enc(creds[user][field]) + + @staticmethod + def add_system_info_creds_to_config(creds): + for user in creds: + ConfigService.creds_add_username(user) + if 'password' in creds[user]: + ConfigService.creds_add_password(creds[user]['password']) + if 'lm_hash' in creds[user]: + ConfigService.creds_add_lm_hash(creds[user]['lm_hash']) + if 'ntlm_hash' in creds[user]: + ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) + + @staticmethod + def encrypt_exploit_creds(telemetry_json): + attempts = telemetry_json['data']['attempts'] + for i in range(len(attempts)): + for field in ['password', 'lm_hash', 'ntlm_hash']: + credential = attempts[i][field] + if len(credential) > 0: + attempts[i][field] = encryptor.enc(credential) + TELEM_PROCESS_DICT = \ { diff --git a/monkey_island/cc/services/config.py b/monkey_island/cc/services/config.py index ea755312f..878520217 100644 --- a/monkey_island/cc/services/config.py +++ b/monkey_island/cc/services/config.py @@ -1,6 +1,9 @@ -from cc.database import mongo +import copy + from jsonschema import Draft4Validator, validators +from cc.database import mongo +from cc.encryptor import encryptor from cc.island_config import ISLAND_PORT from cc.utils import local_ip_addresses @@ -17,60 +20,60 @@ SCHEMA = { "type": "string", "anyOf": [ { - "type": "string", - "enum": [ - "SmbExploiter" - ], - "title": "SMB Exploiter" + "type": "string", + "enum": [ + "SmbExploiter" + ], + "title": "SMB Exploiter" }, { - "type": "string", - "enum": [ - "WmiExploiter" - ], - "title": "WMI Exploiter" + "type": "string", + "enum": [ + "WmiExploiter" + ], + "title": "WMI Exploiter" }, { - "type": "string", - "enum": [ - "RdpExploiter" - ], - "title": "RDP Exploiter (UNSAFE)" + "type": "string", + "enum": [ + "RdpExploiter" + ], + "title": "RDP Exploiter (UNSAFE)" }, { - "type": "string", - "enum": [ - "Ms08_067_Exploiter" - ], - "title": "MS08-067 Exploiter (UNSAFE)" + "type": "string", + "enum": [ + "Ms08_067_Exploiter" + ], + "title": "MS08-067 Exploiter (UNSAFE)" }, { - "type": "string", - "enum": [ - "SSHExploiter" - ], - "title": "SSH Exploiter" + "type": "string", + "enum": [ + "SSHExploiter" + ], + "title": "SSH Exploiter" }, { - "type": "string", - "enum": [ - "ShellShockExploiter" - ], - "title": "ShellShock Exploiter" + "type": "string", + "enum": [ + "ShellShockExploiter" + ], + "title": "ShellShock Exploiter" }, { - "type": "string", - "enum": [ - "SambaCryExploiter" - ], - "title": "SambaCry Exploiter" + "type": "string", + "enum": [ + "SambaCryExploiter" + ], + "title": "SambaCry Exploiter" }, { - "type": "string", - "enum": [ - "ElasticGroovyExploiter" - ], - "title": "ElasticGroovy Exploiter" + "type": "string", + "enum": [ + "ElasticGroovyExploiter" + ], + "title": "ElasticGroovy Exploiter" }, ] }, @@ -79,46 +82,46 @@ SCHEMA = { "type": "string", "anyOf": [ { - "type": "string", - "enum": [ - "SMBFinger" - ], - "title": "SMBFinger" + "type": "string", + "enum": [ + "SMBFinger" + ], + "title": "SMBFinger" }, { - "type": "string", - "enum": [ - "SSHFinger" - ], - "title": "SSHFinger" + "type": "string", + "enum": [ + "SSHFinger" + ], + "title": "SSHFinger" }, { - "type": "string", - "enum": [ - "PingScanner" - ], - "title": "PingScanner" + "type": "string", + "enum": [ + "PingScanner" + ], + "title": "PingScanner" }, { - "type": "string", - "enum": [ - "HTTPFinger" - ], - "title": "HTTPFinger" + "type": "string", + "enum": [ + "HTTPFinger" + ], + "title": "HTTPFinger" }, { - "type": "string", - "enum": [ - "MySQLFinger" - ], - "title": "MySQLFinger" + "type": "string", + "enum": [ + "MySQLFinger" + ], + "title": "MySQLFinger" }, { - "type": "string", - "enum": [ - "ElasticFinger" - ], - "title": "ElasticFinger" + "type": "string", + "enum": [ + "ElasticFinger" + ], + "title": "ElasticFinger" } ] } @@ -794,29 +797,42 @@ SCHEMA = { } } +ENCRYPTED_CONFIG_ARRAYS = \ + [ + ['basic', 'credentials', 'exploit_password_list'], + ['internal', 'exploits', 'exploit_lm_hash_list'], + ['internal', 'exploits', 'exploit_ntlm_hash_list'] + ] + class ConfigService: + default_config = None + def __init__(self): pass @staticmethod - def get_config(is_initial_config=False): + def get_config(is_initial_config=False, should_decrypt=True): config = mongo.db.config.find_one({'name': 'initial' if is_initial_config else 'newconfig'}) or {} for field in ('name', '_id'): config.pop(field, None) + if should_decrypt and len(config) > 0: + ConfigService.decrypt_config(config) return config @staticmethod - def get_config_value(config_key_as_arr, is_initial_config=False): - config_key = reduce(lambda x, y: x+'.'+y, config_key_as_arr) + def get_config_value(config_key_as_arr, is_initial_config=False, should_decrypt=True): + config_key = reduce(lambda x, y: x + '.' + y, config_key_as_arr) config = mongo.db.config.find_one({'name': 'initial' if is_initial_config else 'newconfig'}, {config_key: 1}) for config_key_part in config_key_as_arr: config = config[config_key_part] + if should_decrypt and (config_key_as_arr in ENCRYPTED_CONFIG_ARRAYS): + config = [encryptor.dec(x) for x in config] return config @staticmethod - def get_flat_config(is_initial_config=False): - config_json = ConfigService.get_config(is_initial_config) + def get_flat_config(is_initial_config=False, should_decrypt=True): + config_json = ConfigService.get_config(is_initial_config, should_decrypt) flat_config_json = {} for i in config_json: for j in config_json[i]: @@ -860,27 +876,38 @@ class ConfigService: ConfigService.add_item_to_config_set('internal.exploits.exploit_ntlm_hash_list', ntlm_hash) @staticmethod - def update_config(config_json): + def update_config(config_json, should_encrypt): + if should_encrypt: + ConfigService.encrypt_config(config_json) mongo.db.config.update({'name': 'newconfig'}, {"$set": config_json}, upsert=True) @staticmethod - def get_default_config(): - defaultValidatingDraft4Validator = ConfigService._extend_config_with_default(Draft4Validator) - config = {} - defaultValidatingDraft4Validator(SCHEMA).validate(config) + def init_default_config(): + if ConfigService.default_config is None: + defaultValidatingDraft4Validator = ConfigService._extend_config_with_default(Draft4Validator) + config = {} + defaultValidatingDraft4Validator(SCHEMA).validate(config) + ConfigService.default_config = config + + @staticmethod + def get_default_config(should_decrypt=True): + ConfigService.init_default_config() + config = copy.deepcopy(ConfigService.default_config) + if not should_decrypt: + ConfigService.encrypt_config(config) return config @staticmethod def init_config(): - if ConfigService.get_config() != {}: + if ConfigService.get_config(should_decrypt=False) != {}: return ConfigService.reset_config() @staticmethod def reset_config(): - config = ConfigService.get_default_config() + config = ConfigService.get_default_config(should_decrypt=False) ConfigService.set_server_ips_in_config(config) - ConfigService.update_config(config) + ConfigService.update_config(config, should_encrypt=False) @staticmethod def set_server_ips_in_config(config): @@ -922,3 +949,21 @@ class ConfigService: return validators.extend( validator_class, {"properties": set_defaults}, ) + + @staticmethod + def decrypt_config(config): + ConfigService._encrypt_config(config, True) + + @staticmethod + def encrypt_config(config): + ConfigService._encrypt_config(config, False) + + @staticmethod + def _encrypt_config(config, is_decrypt=False): + for config_arr_as_array in ENCRYPTED_CONFIG_ARRAYS: + config_arr = config + for config_key_part in config_arr_as_array: + config_arr = config_arr[config_key_part] + + for i in range(len(config_arr)): + config_arr[i] = encryptor.dec(config_arr[i]) if is_decrypt else encryptor.enc(config_arr[i]) From dbe7a6a378b87e25e61b17a01ae76aaff5026a4c Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Wed, 14 Feb 2018 15:50:53 +0200 Subject: [PATCH 04/64] Add log sending logic to monkey Add log processing logic to monkey island backend --- chaos_monkey/control.py | 16 +++++++++ chaos_monkey/main.py | 9 +++--- chaos_monkey/monkey.py | 13 ++++++++ chaos_monkey/utils.py | 14 ++++++++ monkey_island/cc/app.py | 13 +++++--- monkey_island/cc/resources/log.py | 29 +++++++++++++++++ monkey_island/cc/resources/root.py | 3 +- monkey_island/cc/services/log.py | 52 ++++++++++++++++++++++++++++++ monkey_island/cc/services/node.py | 6 +++- 9 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 chaos_monkey/utils.py create mode 100644 monkey_island/cc/resources/log.py create mode 100644 monkey_island/cc/services/log.py diff --git a/chaos_monkey/control.py b/chaos_monkey/control.py index b4f2769cd..fef37de1f 100644 --- a/chaos_monkey/control.py +++ b/chaos_monkey/control.py @@ -1,3 +1,4 @@ +import base64 import json import logging import platform @@ -111,6 +112,21 @@ class ControlClient(object): LOG.warn("Error connecting to control server %s: %s", WormConfiguration.current_server, exc) + @staticmethod + def send_log(log): + if not WormConfiguration.current_server: + return + try: + telemetry = {'monkey_guid': GUID, 'log': base64.b64encode(log)} + reply = requests.post("https://%s/api/log" % (WormConfiguration.current_server,), + data=json.dumps(telemetry), + headers={'content-type': 'application/json'}, + verify=False, + proxies=ControlClient.proxies) + except Exception as exc: + LOG.warn("Error connecting to control server %s: %s", + WormConfiguration.current_server, exc) + @staticmethod def load_control_config(): if not WormConfiguration.current_server: diff --git a/chaos_monkey/main.py b/chaos_monkey/main.py index c53232b2c..ef57492cc 100644 --- a/chaos_monkey/main.py +++ b/chaos_monkey/main.py @@ -12,6 +12,7 @@ from config import WormConfiguration, EXTERNAL_CONFIG_FILE from dropper import MonkeyDrops from model import MONKEY_ARG, DROPPER_ARG from monkey import ChaosMonkey +import utils if __name__ == "__main__": sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -78,12 +79,10 @@ def main(): try: if MONKEY_ARG == monkey_mode: - log_path = os.path.expandvars( - WormConfiguration.monkey_log_path_windows) if sys.platform == "win32" else WormConfiguration.monkey_log_path_linux + log_path = utils.get_monkey_log_path() monkey_cls = ChaosMonkey elif DROPPER_ARG == monkey_mode: - log_path = os.path.expandvars( - WormConfiguration.dropper_log_path_windows) if sys.platform == "win32" else WormConfiguration.dropper_log_path_linux + log_path = utils.get_dropper_log_path() monkey_cls = MonkeyDrops else: return True @@ -91,6 +90,8 @@ def main(): return True if WormConfiguration.use_file_logging: + if os.path.exists(log_path): + os.remove(log_path) LOG_CONFIG['handlers']['file']['filename'] = log_path LOG_CONFIG['root']['handlers'].append('file') else: diff --git a/chaos_monkey/monkey.py b/chaos_monkey/monkey.py index 79012dc39..79e8bf3ec 100644 --- a/chaos_monkey/monkey.py +++ b/chaos_monkey/monkey.py @@ -6,6 +6,7 @@ import sys import time import tunnel +import utils from config import WormConfiguration from control import ControlClient from model import DELAY_DELETE_CMD @@ -226,6 +227,8 @@ class ChaosMonkey(object): firewall.close() + self.send_log() + self._singleton.unlock() if WormConfiguration.self_delete_in_cleanup and -1 == sys.executable.find('python'): @@ -244,3 +247,13 @@ class ChaosMonkey(object): LOG.error("Exception in self delete: %s", exc) LOG.info("Monkey is shutting down") + + def send_log(self): + monkey_log_path = utils.get_monkey_log_path() + if os.path.exists(monkey_log_path): + with open(monkey_log_path, 'r') as f: + log = f.read() + else: + log = '' + + ControlClient.send_log(log) diff --git a/chaos_monkey/utils.py b/chaos_monkey/utils.py new file mode 100644 index 000000000..d95407341 --- /dev/null +++ b/chaos_monkey/utils.py @@ -0,0 +1,14 @@ +import os +import sys + +from config import WormConfiguration + + +def get_monkey_log_path(): + return os.path.expandvars(WormConfiguration.monkey_log_path_windows) if sys.platform == "win32" \ + else WormConfiguration.monkey_log_path_linux + + +def get_dropper_log_path(): + return os.path.expandvars(WormConfiguration.dropper_log_path_windows) if sys.platform == "win32" \ + else WormConfiguration.dropper_log_path_linux diff --git a/monkey_island/cc/app.py b/monkey_island/cc/app.py index 9c85f6230..2d8041eb0 100644 --- a/monkey_island/cc/app.py +++ b/monkey_island/cc/app.py @@ -1,22 +1,24 @@ from datetime import datetime + import bson -from bson.json_util import dumps -from flask import Flask, send_from_directory, redirect, make_response import flask_restful +from bson.json_util import dumps +from flask import Flask, send_from_directory, make_response from werkzeug.exceptions import NotFound from cc.database import mongo from cc.resources.client_run import ClientRun -from cc.resources.monkey import Monkey +from cc.resources.edge import Edge from cc.resources.local_run import LocalRun -from cc.resources.telemetry import Telemetry +from cc.resources.log import Log +from cc.resources.monkey import Monkey from cc.resources.monkey_configuration import MonkeyConfiguration from cc.resources.monkey_download import MonkeyDownload from cc.resources.netmap import NetMap -from cc.resources.edge import Edge from cc.resources.node import Node from cc.resources.report import Report from cc.resources.root import Root +from cc.resources.telemetry import Telemetry from cc.resources.telemetry_feed import TelemetryFeed from cc.services.config import ConfigService @@ -91,5 +93,6 @@ def init_app(mongo_url): api.add_resource(Node, '/api/netmap/node', '/api/netmap/node/') api.add_resource(Report, '/api/report', '/api/report/') api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/') + api.add_resource(Log, '/api/log', '/api/log/') return app diff --git a/monkey_island/cc/resources/log.py b/monkey_island/cc/resources/log.py new file mode 100644 index 000000000..5a308b5dd --- /dev/null +++ b/monkey_island/cc/resources/log.py @@ -0,0 +1,29 @@ +import json + +import flask_restful +from bson import ObjectId +from flask import request + +from cc.database import mongo +from cc.services.log import LogService +from cc.services.node import NodeService + +__author__ = "itay.mizeretz" + + +class Log(flask_restful.Resource): + def get(self): + monkey_id = request.args.get('id') + exists_monkey_id = request.args.get('exists') + if monkey_id: + return LogService.get_log_by_monkey_id(ObjectId(monkey_id)) + else: + return LogService.log_exists(ObjectId(exists_monkey_id)) + + def post(self): + telemetry_json = json.loads(request.data) + + monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])['_id'] + log_id = LogService.add_log(monkey_id, telemetry_json['log']) + + return mongo.db.log.find_one_or_404({"_id": log_id}) diff --git a/monkey_island/cc/resources/root.py b/monkey_island/cc/resources/root.py index 25d7dfed7..f725163aa 100644 --- a/monkey_island/cc/resources/root.py +++ b/monkey_island/cc/resources/root.py @@ -33,7 +33,8 @@ class Root(flask_restful.Resource): @staticmethod def reset_db(): - [mongo.db[x].drop() for x in ['config', 'monkey', 'telemetry', 'node', 'edge', 'report']] + [mongo.db[x].drop() for x in + ['config', 'monkey', 'telemetry', 'node', 'edge', 'report', 'log', 'fs.chunks', 'fs.files']] ConfigService.init_config() return jsonify(status='OK') diff --git a/monkey_island/cc/services/log.py b/monkey_island/cc/services/log.py new file mode 100644 index 000000000..2783a1dfa --- /dev/null +++ b/monkey_island/cc/services/log.py @@ -0,0 +1,52 @@ +from datetime import datetime + +import gridfs + +import cc.services.node +from cc.database import mongo + +__author__ = "itay.mizeretz" + + +class LogService: + def __init__(self): + pass + + @staticmethod + def get_log_by_monkey_id(monkey_id): + log = mongo.db.log.find_one({'monkey_id': monkey_id}) + if log: + fs = gridfs.GridFS(mongo.db) + log_file = fs.get(log['file_id']) + monkey_label = cc.services.node.NodeService.get_monkey_label( + cc.services.node.NodeService.get_monkey_by_id(log['monkey_id'])) + return \ + { + 'monkey_label': monkey_label, + 'log': log_file.read(), + 'timestamp': log['timestamp'] + } + + @staticmethod + def remove_logs_by_monkey_id(monkey_id): + fs = gridfs.GridFS(mongo.db) + for log in mongo.db.log.find({'monkey_id': monkey_id}): + fs.delete(log['file_id']) + mongo.db.log.delete_many({'monkey_id': monkey_id}) + + @staticmethod + def add_log(monkey_id, log_data, timestamp=datetime.now()): + LogService.remove_logs_by_monkey_id(monkey_id) + fs = gridfs.GridFS(mongo.db) + file_id = fs.put(log_data) + return mongo.db.log.insert( + { + 'monkey_id': monkey_id, + 'file_id': file_id, + 'timestamp': timestamp + } + ) + + @staticmethod + def log_exists(monkey_id): + return mongo.db.log.find_one({'monkey_id': monkey_id}) is not None diff --git a/monkey_island/cc/services/node.py b/monkey_island/cc/services/node.py index 47cfba8d9..47cd9cd21 100644 --- a/monkey_island/cc/services/node.py +++ b/monkey_island/cc/services/node.py @@ -1,9 +1,12 @@ from datetime import datetime, timedelta + from bson import ObjectId +import cc.services.log from cc.database import mongo from cc.services.edge import EdgeService from cc.utils import local_ip_addresses + __author__ = "itay.mizeretz" @@ -54,6 +57,7 @@ class NodeService: else: new_node["services"] = [] + new_node['has_log'] = cc.services.log.LogService.log_exists(ObjectId(node_id)) return new_node @staticmethod @@ -241,7 +245,7 @@ class NodeService: @staticmethod def get_monkey_island_pseudo_net_node(): - return\ + return \ { "id": NodeService.get_monkey_island_pseudo_id(), "label": "MonkeyIsland", From 86a0e47d15afde8be46ccdadb5eaebe751b7556d Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Wed, 14 Feb 2018 15:51:22 +0200 Subject: [PATCH 05/64] Add log downloading from map --- monkey_island/cc/ui/package.json | 1 + .../map/preview-pane/PreviewPane.js | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/monkey_island/cc/ui/package.json b/monkey_island/cc/ui/package.json index 5ee2e5389..27b536365 100644 --- a/monkey_island/cc/ui/package.json +++ b/monkey_island/cc/ui/package.json @@ -63,6 +63,7 @@ "dependencies": { "bootstrap": "^3.3.7", "core-js": "^2.5.1", + "downloadjs": "^1.4.7", "fetch": "^1.1.0", "js-file-download": "^0.4.1", "normalize.css": "^4.0.0", diff --git a/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js b/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js index 842440149..56c6d0e75 100644 --- a/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js +++ b/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js @@ -2,6 +2,7 @@ import React from 'react'; import {Icon} from 'react-fa'; import Toggle from 'react-toggle'; import {OverlayTrigger, Tooltip} from 'react-bootstrap'; +import download from 'downloadjs' class PreviewPaneComponent extends React.Component { @@ -88,6 +89,34 @@ class PreviewPaneComponent extends React.Component { ); } + downloadLog(asset) { + + fetch('/api/log?id=' + asset.id) + .then(res => res.json()) + .then(res => { + let timestamp = res['timestamp']; + timestamp = timestamp.substr(0, timestamp.indexOf('.')); + let filename = res['monkey_label'].split(':').join('-') + ' - ' + timestamp + '.log'; + download(atob(res['log']), filename, 'text/plain'); + }); + + } + + downloadLogRow(asset) { + return ( + + + Download Log + + + this.downloadLog(asset)}>Download + + + ); + } + exploitsTimeline(asset) { if (asset.exploits.length === 0) { return (
); @@ -140,6 +169,7 @@ class PreviewPaneComponent extends React.Component { {this.servicesRow(asset)} {this.accessibleRow(asset)} {this.forceKillRow(asset)} + {this.downloadLogRow(asset)} {this.exploitsTimeline(asset)} From 70766e7358556bec80524b25c010cbc62734e5ff Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Wed, 14 Feb 2018 16:58:58 +0200 Subject: [PATCH 06/64] Save some space --- monkey_island/cc/resources/log.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey_island/cc/resources/log.py b/monkey_island/cc/resources/log.py index 5a308b5dd..e49c7024d 100644 --- a/monkey_island/cc/resources/log.py +++ b/monkey_island/cc/resources/log.py @@ -24,6 +24,8 @@ class Log(flask_restful.Resource): telemetry_json = json.loads(request.data) monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])['_id'] - log_id = LogService.add_log(monkey_id, telemetry_json['log']) + # This is base64 so no data will be lost. this'll take 2 time less space. + log_data = str(telemetry_json['log']) + log_id = LogService.add_log(monkey_id, log_data) return mongo.db.log.find_one_or_404({"_id": log_id}) From aa02d8945dc32121e12f7fb530ca79d5e74ca107 Mon Sep 17 00:00:00 2001 From: Itay Mizeretz Date: Mon, 19 Feb 2018 17:22:48 +0200 Subject: [PATCH 07/64] Replace base64 with string escaping --- chaos_monkey/control.py | 3 +- .../map/preview-pane/PreviewPane.js | 60 +++++++++++-------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/chaos_monkey/control.py b/chaos_monkey/control.py index fef37de1f..dd1814133 100644 --- a/chaos_monkey/control.py +++ b/chaos_monkey/control.py @@ -1,4 +1,3 @@ -import base64 import json import logging import platform @@ -117,7 +116,7 @@ class ControlClient(object): if not WormConfiguration.current_server: return try: - telemetry = {'monkey_guid': GUID, 'log': base64.b64encode(log)} + telemetry = {'monkey_guid': GUID, 'log': json.dumps(log)} reply = requests.post("https://%s/api/log" % (WormConfiguration.current_server,), data=json.dumps(telemetry), headers={'content-type': 'application/json'}, diff --git a/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js b/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js index 56c6d0e75..b7f055103 100644 --- a/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js +++ b/monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js @@ -82,22 +82,34 @@ class PreviewPaneComponent extends React.Component { this.forceKill(e, asset)} /> + onChange={(e) => this.forceKill(e, asset)}/> ); } - downloadLog(asset) { + unescapeLog(st) { + return st.substr(1, st.length - 2) // remove quotation marks on beginning and end of string. + .replace(/\\n/g, "\n") + .replace(/\\r/g, "\r") + .replace(/\\t/g, "\t") + .replace(/\\b/g, "\b") + .replace(/\\f/g, "\f") + .replace(/\\"/g, '\"') + .replace(/\\'/g, "\'") + .replace(/\\&/g, "\&"); + } + downloadLog(asset) { fetch('/api/log?id=' + asset.id) .then(res => res.json()) .then(res => { let timestamp = res['timestamp']; timestamp = timestamp.substr(0, timestamp.indexOf('.')); let filename = res['monkey_label'].split(':').join('-') + ' - ' + timestamp + '.log'; - download(atob(res['log']), filename, 'text/plain'); + let logContent = this.unescapeLog(res['log']); + download(logContent, filename, 'text/plain'); }); } @@ -119,7 +131,7 @@ class PreviewPaneComponent extends React.Component { exploitsTimeline(asset) { if (asset.exploits.length === 0) { - return (
); + return (
); } return ( @@ -129,9 +141,9 @@ class PreviewPaneComponent extends React.Component { {this.generateToolTip('Timeline of exploit attempts. Red is successful. Gray is unsuccessful')}