Merge pull request #87 from guardicore/feature/send-raw-log

Feature/send raw log

infection_monkey/config.py

@@ -106,6 +106,7 @@ class Configuration(object):
     dropper_log_path_linux = '/tmp/user-1562'
     monkey_log_path_windows = '%temp%\\~df1563.tmp'
     monkey_log_path_linux = '/tmp/user-1563'
+    send_log_to_server = True
 
     ###########################
     # dropper config

infection_monkey/control.py

@@ -111,6 +111,21 @@ class ControlClient(object):
             LOG.warn("Error connecting to control server %s: %s",
                      WormConfiguration.current_server, exc)
 
+    @staticmethod
+    def send_log(log):
+        if not WormConfiguration.current_server:
+            return
+        try:
+            telemetry = {'monkey_guid': GUID, 'log': json.dumps(log)}
+            reply = requests.post("https://%s/api/log" % (WormConfiguration.current_server,),
+                                  data=json.dumps(telemetry),
+                                  headers={'content-type': 'application/json'},
+                                  verify=False,
+                                  proxies=ControlClient.proxies)
+        except Exception as exc:
+            LOG.warn("Error connecting to control server %s: %s",
+                     WormConfiguration.current_server, exc)
+
     @staticmethod
     def load_control_config():
         if not WormConfiguration.current_server:

infection_monkey/example.conf

@@ -48,6 +48,7 @@
     "max_iterations": 3,
     "monkey_log_path_windows": "%temp%\\~df1563.tmp",
     "monkey_log_path_linux": "/tmp/user-1563",
+    "send_log_to_server": true,
     "ms08_067_exploit_attempts": 5,
     "ms08_067_remote_user_add": "Monkey_IUSER_SUPPORT",
     "ms08_067_remote_user_pass": "Password1!",

infection_monkey/main.py

@@ -12,6 +12,7 @@ from config import WormConfiguration, EXTERNAL_CONFIG_FILE
 from dropper import MonkeyDrops
 from model import MONKEY_ARG, DROPPER_ARG
 from monkey import InfectionMonkey
+import utils
 
 if __name__ == "__main__":
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -78,12 +79,10 @@ def main():
 
     try:
         if MONKEY_ARG == monkey_mode:
-            log_path = os.path.expandvars(
-                WormConfiguration.monkey_log_path_windows) if sys.platform == "win32" else WormConfiguration.monkey_log_path_linux
+            log_path = utils.get_monkey_log_path()
             monkey_cls = InfectionMonkey
         elif DROPPER_ARG == monkey_mode:
-            log_path = os.path.expandvars(
-                WormConfiguration.dropper_log_path_windows) if sys.platform == "win32" else WormConfiguration.dropper_log_path_linux
+            log_path = utils.get_dropper_log_path()
             monkey_cls = MonkeyDrops
         else:
             return True
@@ -91,6 +90,8 @@ def main():
         return True
 
     if WormConfiguration.use_file_logging:
+        if os.path.exists(log_path):
+            os.remove(log_path)
         LOG_CONFIG['handlers']['file']['filename'] = log_path
         LOG_CONFIG['root']['handlers'].append('file')
     else:
@@ -120,6 +121,8 @@ def main():
                 json.dump(json_dict, config_fo, skipkeys=True, sort_keys=True, indent=4, separators=(',', ': '))
 
         return True
+    except Exception:
+        LOG.exception("Exception thrown from monkey's start function")
     finally:
         monkey.cleanup()
 

infection_monkey/monkey.py

@@ -6,6 +6,7 @@ import sys
 import time
 
 import tunnel
+import utils
 from config import WormConfiguration
 from control import ControlClient
 from model import DELAY_DELETE_CMD
@@ -226,6 +227,9 @@ class InfectionMonkey(object):
 
         firewall.close()
 
+        if WormConfiguration.send_log_to_server:
+            self.send_log()
+
         self._singleton.unlock()
 
         if WormConfiguration.self_delete_in_cleanup and -1 == sys.executable.find('python'):
@@ -244,3 +248,13 @@ class InfectionMonkey(object):
                 LOG.error("Exception in self delete: %s", exc)
 
         LOG.info("Monkey is shutting down")
+
+    def send_log(self):
+        monkey_log_path = utils.get_monkey_log_path()
+        if os.path.exists(monkey_log_path):
+            with open(monkey_log_path, 'r') as f:
+                log = f.read()
+        else:
+            log = ''
+
+        ControlClient.send_log(log)

infection_monkey/utils.py

@@ -0,0 +1,14 @@
+import os
+import sys
+
+from config import WormConfiguration
+
+
+def get_monkey_log_path():
+    return os.path.expandvars(WormConfiguration.monkey_log_path_windows) if sys.platform == "win32" \
+        else WormConfiguration.monkey_log_path_linux
+
+
+def get_dropper_log_path():
+    return os.path.expandvars(WormConfiguration.dropper_log_path_windows) if sys.platform == "win32" \
+        else WormConfiguration.dropper_log_path_linux

monkey_island/cc/app.py

@@ -8,11 +8,12 @@ from flask import Flask, send_from_directory, make_response
 from werkzeug.exceptions import NotFound
 
 from cc.auth import init_jwt
-from cc.database import mongo
+from cc.database import mongo, database
 from cc.environment.environment import env
 from cc.resources.client_run import ClientRun
 from cc.resources.edge import Edge
 from cc.resources.local_run import LocalRun
+from cc.resources.log import Log
 from cc.resources.monkey import Monkey
 from cc.resources.monkey_configuration import MonkeyConfiguration
 from cc.resources.monkey_download import MonkeyDownload
@@ -83,6 +84,7 @@ def init_app(mongo_url):
     mongo.init_app(app)
 
     with app.app_context():
+        database.init()
         ConfigService.init_config()
 
     app.add_url_rule('/', 'serve_home', serve_home)
@@ -101,5 +103,6 @@ def init_app(mongo_url):
     api.add_resource(Node, '/api/netmap/node', '/api/netmap/node/')
     api.add_resource(Report, '/api/report', '/api/report/')
     api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/')
+    api.add_resource(Log, '/api/log', '/api/log/')
 
     return app

monkey_island/cc/database.py

@@ -1,5 +1,5 @@
-from flask_pymongo import PyMongo
-from flask_pymongo import MongoClient
+import gridfs
+from flask_pymongo import MongoClient, PyMongo
 from pymongo.errors import ServerSelectionTimeoutError
 
 __author__ = 'Barak'
@@ -7,6 +7,17 @@ __author__ = 'Barak'
 mongo = PyMongo()
 
 
+class Database:
+    def __init__(self):
+        self.gridfs = None
+
+    def init(self):
+        self.gridfs = gridfs.GridFS(mongo.db)
+
+
+database = Database()
+
+
 def is_db_server_up(mongo_url):
     client = MongoClient(mongo_url, serverSelectionTimeoutMS=100)
     try:

monkey_island/cc/resources/log.py

@@ -0,0 +1,34 @@
+import json
+
+import flask_restful
+from bson import ObjectId
+from flask import request
+
+from cc.auth import jwt_required
+from cc.database import mongo
+from cc.services.log import LogService
+from cc.services.node import NodeService
+
+__author__ = "itay.mizeretz"
+
+
+class Log(flask_restful.Resource):
+    @jwt_required()
+    def get(self):
+        monkey_id = request.args.get('id')
+        exists_monkey_id = request.args.get('exists')
+        if monkey_id:
+            return LogService.get_log_by_monkey_id(ObjectId(monkey_id))
+        else:
+            return LogService.log_exists(ObjectId(exists_monkey_id))
+
+    # Used by monkey. can't secure.
+    def post(self):
+        telemetry_json = json.loads(request.data)
+
+        monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])['_id']
+        # This shouldn't contain any unicode characters. this'll take 2 time less space.
+        log_data = str(telemetry_json['log'])
+        log_id = LogService.add_log(monkey_id, log_data)
+
+        return mongo.db.log.find_one_or_404({"_id": log_id})

monkey_island/cc/resources/root.py

@@ -36,7 +36,7 @@ class Root(flask_restful.Resource):
 
     @staticmethod
     def reset_db():
-        [mongo.db[x].drop() for x in ['config', 'monkey', 'telemetry', 'node', 'edge', 'report']]
+        [mongo.db[x].drop() for x in mongo.db.collection_names()]
         ConfigService.init_config()
         return jsonify(status='OK')
 

monkey_island/cc/services/config.py

@@ -483,6 +483,12 @@ SCHEMA = {
                             "type": "string",
                             "default": "%temp%\\~df1563.tmp",
                             "description": "The fullpath of the monkey log file on Windows"
+                        },
+                        "send_log_to_server": {
+                            "title": "Send log to server",
+                            "type": "boolean",
+                            "default": True,
+                            "description": "Determines whether the monkey sends its log to the Monkey Island server"
                         }
                     }
                 },

monkey_island/cc/services/log.py

@@ -0,0 +1,48 @@
+from datetime import datetime
+
+import cc.services.node
+from cc.database import mongo, database
+
+__author__ = "itay.mizeretz"
+
+
+class LogService:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_log_by_monkey_id(monkey_id):
+        log = mongo.db.log.find_one({'monkey_id': monkey_id})
+        if log:
+            log_file = database.gridfs.get(log['file_id'])
+            monkey_label = cc.services.node.NodeService.get_monkey_label(
+                cc.services.node.NodeService.get_monkey_by_id(log['monkey_id']))
+            return \
+                {
+                    'monkey_label': monkey_label,
+                    'log': log_file.read(),
+                    'timestamp': log['timestamp']
+                }
+
+    @staticmethod
+    def remove_logs_by_monkey_id(monkey_id):
+        log = mongo.db.log.find_one({'monkey_id': monkey_id})
+        if log is not None:
+            database.gridfs.delete(log['file_id'])
+            mongo.db.log.delete_one({'monkey_id': monkey_id})
+
+    @staticmethod
+    def add_log(monkey_id, log_data, timestamp=datetime.now()):
+        LogService.remove_logs_by_monkey_id(monkey_id)
+        file_id = database.gridfs.put(log_data)
+        return mongo.db.log.insert(
+            {
+                'monkey_id': monkey_id,
+                'file_id': file_id,
+                'timestamp': timestamp
+            }
+        )
+
+    @staticmethod
+    def log_exists(monkey_id):
+        return mongo.db.log.find_one({'monkey_id': monkey_id}) is not None

monkey_island/cc/services/node.py

@@ -1,9 +1,12 @@
 from datetime import datetime, timedelta
+
 from bson import ObjectId
 
+import cc.services.log
 from cc.database import mongo
 from cc.services.edge import EdgeService
 from cc.utils import local_ip_addresses
+
 __author__ = "itay.mizeretz"
 
 
@@ -54,6 +57,7 @@ class NodeService:
         else:
             new_node["services"] = []
 
+        new_node['has_log'] = cc.services.log.LogService.log_exists(ObjectId(node_id))
         return new_node
 
     @staticmethod
@@ -241,7 +245,7 @@ class NodeService:
 
     @staticmethod
     def get_monkey_island_pseudo_net_node():
-        return\
+        return \
             {
                 "id": NodeService.get_monkey_island_pseudo_id(),
                 "label": "MonkeyIsland",

monkey_island/cc/ui/package.json

@@ -63,6 +63,7 @@
   "dependencies": {
     "bootstrap": "^3.3.7",
     "core-js": "^2.5.1",
+    "downloadjs": "^1.4.7",
     "fetch": "^1.1.0",
     "js-file-download": "^0.4.1",
     "json-loader": "^0.5.7",

monkey_island/cc/ui/src/components/map/preview-pane/PreviewPane.js

@@ -2,6 +2,7 @@ import React from 'react';
 import {Icon} from 'react-fa';
 import Toggle from 'react-toggle';
 import {OverlayTrigger, Tooltip} from 'react-bootstrap';
+import download from 'downloadjs'
 import AuthComponent from '../../AuthComponent';
 
 class PreviewPaneComponent extends AuthComponent {
@@ -82,16 +83,56 @@ class PreviewPaneComponent extends AuthComponent {
         </th>
         <td>
           <Toggle id={asset.id} checked={!asset.config.alive} icons={false} disabled={asset.dead}
-                  onChange={(e) => this.forceKill(e, asset)} />
+                  onChange={(e) => this.forceKill(e, asset)}/>
 
         </td>
       </tr>
     );
   }
 
+  unescapeLog(st) {
+    return st.substr(1, st.length - 2) // remove quotation marks on beginning and end of string.
+        .replace(/\\n/g, "\n")
+        .replace(/\\r/g, "\r")
+        .replace(/\\t/g, "\t")
+        .replace(/\\b/g, "\b")
+        .replace(/\\f/g, "\f")
+        .replace(/\\"/g, '\"')
+        .replace(/\\'/g, "\'")
+        .replace(/\\&/g, "\&");
+  }
+
+  downloadLog(asset) {
+    fetch('/api/log?id=' + asset.id)
+      .then(res => res.json())
+      .then(res => {
+        let timestamp = res['timestamp'];
+        timestamp = timestamp.substr(0, timestamp.indexOf('.'));
+        let filename = res['monkey_label'].split(':').join('-') + ' - ' + timestamp + '.log';
+        let logContent = this.unescapeLog(res['log']);
+        download(logContent, filename, 'text/plain');
+      });
+
+  }
+
+  downloadLogRow(asset) {
+    return (
+      <tr>
+        <th>
+          Download Log
+        </th>
+        <td>
+          <a type="button" className="btn btn-primary"
+             disabled={!asset.has_log}
+             onClick={() => this.downloadLog(asset)}>Download</a>
+        </td>
+      </tr>
+    );
+  }
+
   exploitsTimeline(asset) {
     if (asset.exploits.length === 0) {
-      return (<div />);
+      return (<div/>);
     }
 
     return (
@@ -101,9 +142,9 @@ class PreviewPaneComponent extends AuthComponent {
           {this.generateToolTip('Timeline of exploit attempts. Red is successful. Gray is unsuccessful')}
         </h4>
         <ul className="timeline">
-          { asset.exploits.map(exploit =>
+          {asset.exploits.map(exploit =>
             <li key={exploit.timestamp}>
-              <div className={'bullet ' + (exploit.result ? 'bad' : '')} />
+              <div className={'bullet ' + (exploit.result ? 'bad' : '')}/>
               <div>{new Date(exploit.timestamp).toLocaleString()}</div>
               <div>{exploit.origin}</div>
               <div>{exploit.exploiter}</div>
@@ -119,10 +160,10 @@ class PreviewPaneComponent extends AuthComponent {
       <div>
         <table className="table table-condensed">
           <tbody>
-            {this.osRow(asset)}
-            {this.ipsRow(asset)}
-            {this.servicesRow(asset)}
-            {this.accessibleRow(asset)}
+          {this.osRow(asset)}
+          {this.ipsRow(asset)}
+          {this.servicesRow(asset)}
+          {this.accessibleRow(asset)}
           </tbody>
         </table>
         {this.exploitsTimeline(asset)}
@@ -135,12 +176,13 @@ class PreviewPaneComponent extends AuthComponent {
       <div>
         <table className="table table-condensed">
           <tbody>
-            {this.osRow(asset)}
-            {this.statusRow(asset)}
-            {this.ipsRow(asset)}
-            {this.servicesRow(asset)}
-            {this.accessibleRow(asset)}
-            {this.forceKillRow(asset)}
+          {this.osRow(asset)}
+          {this.statusRow(asset)}
+          {this.ipsRow(asset)}
+          {this.servicesRow(asset)}
+          {this.accessibleRow(asset)}
+          {this.forceKillRow(asset)}
+          {this.downloadLogRow(asset)}
           </tbody>
         </table>
         {this.exploitsTimeline(asset)}
@@ -173,9 +215,9 @@ class PreviewPaneComponent extends AuthComponent {
             <div>
               <h4 style={{'marginTop': '2em'}}>Timeline</h4>
               <ul className="timeline">
-                { edge.exploits.map(exploit =>
+                {edge.exploits.map(exploit =>
                   <li key={exploit.timestamp}>
-                    <div className={'bullet ' + (exploit.result ? 'bad' : '')} />
+                    <div className={'bullet ' + (exploit.result ? 'bad' : '')}/>
                     <div>{new Date(exploit.timestamp).toLocaleString()}</div>
                     <div>{exploit.origin}</div>
                     <div>{exploit.exploiter}</div>
@@ -206,8 +248,8 @@ class PreviewPaneComponent extends AuthComponent {
           this.infectedAssetInfo(this.props.item) : this.assetInfo(this.props.item);
         break;
       case 'island_edge':
-      info = this.islandEdgeInfo();
-      break;
+        info = this.islandEdgeInfo();
+        break;
     }
 
     let label = '';
@@ -221,12 +263,12 @@ class PreviewPaneComponent extends AuthComponent {
 
     return (
       <div className="preview-pane">
-        { !info ?
+        {!info ?
           <span>
-            <Icon name="hand-o-left" style={{'marginRight': '0.5em'}} />
+            <Icon name="hand-o-left" style={{'marginRight': '0.5em'}}/>
             Select an item on the map for a detailed look
           </span>
-        :
+          :
           <div>
             <h3>
               {label}