Merge pull request #139 from guardicore/feature/Adding_logs_to_monkey_island

Feature/adding logs to monkey island

monkey_island/cc/app.py

@@ -14,6 +14,7 @@ from cc.resources.client_run import ClientRun
 from cc.resources.edge import Edge
 from cc.resources.local_run import LocalRun
 from cc.resources.log import Log
+from cc.resources.island_logs import IslandLog
 from cc.resources.monkey import Monkey
 from cc.resources.monkey_configuration import MonkeyConfiguration
 from cc.resources.monkey_download import MonkeyDownload
@@ -104,5 +105,6 @@ def init_app(mongo_url):
     api.add_resource(Report, '/api/report', '/api/report/')
     api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/')
     api.add_resource(Log, '/api/log', '/api/log/')
+    api.add_resource(IslandLog, '/api/log/island/download', '/api/log/island/download/')
 
     return app

monkey_island/cc/environment/environment.py

@@ -1,7 +1,11 @@
 import json
+import logging
 import standard
 import aws
 
+logger = logging.getLogger(__name__)
+
+
 ENV_DICT = {
     'standard': standard.StandardEnvironment,
     'aws': aws.AwsEnvironment
@@ -18,6 +22,7 @@ def load_env_from_file():
 try:
     __env_type = load_env_from_file()
     env = ENV_DICT[__env_type]()
+    logger.info('Monkey\'s env is: {0}'.format(env.__class__.__name__))
 except Exception:
-    print('Failed initializing environment: %s' % __env_type)
+    logger.error('Failed initializing environment', exc_info=True)
     raise

monkey_island/cc/island_logger.py

@@ -0,0 +1,26 @@
+import os
+import json
+import logging.config
+
+
+__author__ = 'Maor.Rayzin'
+
+
+def json_setup_logging(default_path='logging.json', default_level=logging.INFO, env_key='LOG_CFG'):
+    """
+    Setup the logging configuration
+    :param default_path: the default log configuration file path
+    :param default_level: Default level to log from
+    :param env_key: SYS ENV key to use for external configuration file path
+    :return:
+    """
+    path = default_path
+    value = os.getenv(env_key, None)
+    if value:
+        path = value
+    if os.path.exists(path):
+        with open(path, 'rt') as f:
+            config = json.load(f)
+        logging.config.dictConfig(config)
+    else:
+        logging.basicConfig(level=default_level)

monkey_island/cc/island_logger_default_config.json

@@ -0,0 +1,33 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "format": "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s"
+        }
+    },
+
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "info.log",
+            "maxBytes": 10485760,
+            "backupCount": 20,
+            "encoding": "utf8"
+        }
+    },
+
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

monkey_island/cc/main.py

@@ -2,19 +2,25 @@ from __future__ import print_function  # In python 2.7
 
 import os
 import sys
-
 import time
+import logging
 
 BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if BASE_PATH not in sys.path:
     sys.path.insert(0, BASE_PATH)
 
+from cc.island_logger import json_setup_logging
+# This is here in order to catch EVERYTHING, some functions are being called on imports the log init needs to be on top.
+json_setup_logging(default_path='island_logger_default_config.json', default_level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
 from cc.app import init_app
 from cc.utils import local_ip_addresses
 from cc.environment.environment import env
 from cc.database import is_db_server_up
 
-if __name__ == '__main__':
+
+def main():
     from tornado.wsgi import WSGIContainer
     from tornado.httpserver import HTTPServer
     from tornado.ioloop import IOLoop
@@ -22,7 +28,7 @@ if __name__ == '__main__':
     mongo_url = os.environ.get('MONGO_URL', env.get_mongo_url())
 
     while not is_db_server_up(mongo_url):
-        print('Waiting for MongoDB server')
+        logger.info('Waiting for MongoDB server')
         time.sleep(1)
 
     app = init_app(mongo_url)
@@ -33,6 +39,10 @@ if __name__ == '__main__':
                                  ssl_options={'certfile': os.environ.get('SERVER_CRT', 'server.crt'),
                                               'keyfile': os.environ.get('SERVER_KEY', 'server.key')})
         http_server.listen(env.get_island_port())
-        print('Monkey Island Server is running on https://{}:{}'.format(local_ip_addresses()[0], env.get_island_port()))
+        logger.info(
+            'Monkey Island Server is running on https://{}:{}'.format(local_ip_addresses()[0], env.get_island_port()))
         IOLoop.instance().start()
 
+
+if __name__ == '__main__':
+    main()

monkey_island/cc/resources/client_run.py

@@ -1,3 +1,4 @@
+import logging
 from flask import request, jsonify
 import flask_restful
 
@@ -5,6 +6,8 @@ from cc.services.node import NodeService
 
 __author__ = 'itay.mizeretz'
 
+logger = logging.getLogger(__name__)
+
 
 class ClientRun(flask_restful.Resource):
     def get(self):
@@ -17,6 +20,7 @@ class ClientRun(flask_restful.Resource):
         if monkey is not None:
             is_monkey_running = not monkey["dead"]
         else:
+            logger.info("Monkey is not running")
             is_monkey_running = False
 
         return jsonify(is_running=is_monkey_running)

monkey_island/cc/resources/island_logs.py

@@ -0,0 +1,19 @@
+import logging
+
+import flask_restful
+
+from cc.auth import jwt_required
+from cc.services.island_logs import IslandLogService
+
+__author__ = "Maor.Rayzin"
+
+logger = logging.getLogger(__name__)
+
+
+class IslandLog(flask_restful.Resource):
+    @jwt_required()
+    def get(self):
+        try:
+            return IslandLogService.get_log_file()
+        except Exception as e:
+            logger.error('Monkey Island logs failed to download', exc_info=True)

monkey_island/cc/resources/local_run.py

@@ -13,6 +13,8 @@ from cc.utils import local_ip_addresses
 
 __author__ = 'Barak'
 
+import logging
+logger = logging.getLogger(__name__)
 
 def run_local_monkey():
     import platform
@@ -32,6 +34,7 @@ def run_local_monkey():
         copyfile(monkey_path, target_path)
         os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG)
     except Exception as exc:
+        logger.error('Copy file failed', exc_info=True)
         return False, "Copy file failed: %s" % exc
 
     # run the monkey
@@ -41,6 +44,7 @@ def run_local_monkey():
             args = "".join(args)
         pid = subprocess.Popen(args, shell=True).pid
     except Exception as exc:
+        logger.error('popen failed', exc_info=True)
         return False, "popen failed: %s" % exc
 
     return True, "pis: %s" % pid

monkey_island/cc/resources/monkey_download.py

@@ -1,3 +1,4 @@
+import logging
 import json
 
 import os
@@ -6,6 +7,8 @@ import flask_restful
 
 __author__ = 'Barak'
 
+logger = logging.getLogger(__name__)
+
 
 MONKEY_DOWNLOADS = [
     {
@@ -42,7 +45,10 @@ MONKEY_DOWNLOADS = [
 def get_monkey_executable(host_os, machine):
     for download in MONKEY_DOWNLOADS:
         if host_os == download.get('type') and machine == download.get('machine'):
+            logger.info('Monkey exec found for os: {0} and machine: {1}'.format(host_os, machine))
             return download
+    logger.warning('No monkey executables could be found for the host os or machine or both: host_os: {0}, machine: {1}'
+                   .format(host_os, machine))
     return None
 
 

monkey_island/cc/resources/root.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import logging
 
 import flask_restful
 from flask import request, make_response, jsonify
@@ -12,6 +13,8 @@ from cc.utils import local_ip_addresses
 
 __author__ = 'Barak'
 
+logger = logging.getLogger(__name__)
+
 
 class Root(flask_restful.Resource):
 
@@ -42,6 +45,7 @@ class Root(flask_restful.Resource):
         # We can't drop system collections.
         [mongo.db[x].drop() for x in mongo.db.collection_names() if not x.startswith('system.')]
         ConfigService.init_config()
+        logger.info('DB was reset')
         return jsonify(status='OK')
 
     @staticmethod
@@ -50,6 +54,7 @@ class Root(flask_restful.Resource):
         mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}},
                                upsert=False,
                                multi=True)
+        logger.info('Kill all monkeys was called')
         return jsonify(status='OK')
 
     @staticmethod
@@ -59,6 +64,7 @@ class Root(flask_restful.Resource):
         infection_done = NodeService.is_monkey_finished_running()
         if not infection_done:
             report_done = False
+            logger.info('Report generation cannot be completed, infection is not done.')
         else:
             report_done = ReportService.is_report_generated()
         return dict(run_server=True, run_monkey=is_any_exists, infection_done=infection_done, report_done=report_done)

monkey_island/cc/resources/telemetry.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import traceback
 import copy
 from datetime import datetime
@@ -17,6 +18,9 @@ from cc.encryptor import encryptor
 __author__ = 'Barak'
 
 
+logger = logging.getLogger(__name__)
+
+
 class Telemetry(flask_restful.Resource):
     @jwt_required()
     def get(self, **kw):
@@ -52,10 +56,9 @@ class Telemetry(flask_restful.Resource):
             if telem_type in TELEM_PROCESS_DICT:
                 TELEM_PROCESS_DICT[telem_type](telemetry_json)
             else:
-                print('Got unknown type of telemetry: %s' % telem_type)
+                logger.info('Got unknown type of telemetry: %s' % telem_type)
         except Exception as ex:
-            print("Exception caught while processing telemetry: %s" % str(ex))
-            traceback.print_exc()
+            logger.error("Exception caught while processing telemetry", exc_info=True)
 
         telem_id = mongo.db.telemetry.insert(telemetry_json)
         return mongo.db.telemetry.find_one_or_404({"_id": telem_id})

monkey_island/cc/services/config.py

@@ -1,6 +1,7 @@
 import copy
 import collections
 import functools
+import logging
 from jsonschema import Draft4Validator, validators
 from six import string_types
 
@@ -11,6 +12,8 @@ from cc.utils import local_ip_addresses
 
 __author__ = "itay.mizeretz"
 
+logger = logging.getLogger(__name__)
+
 WARNING_SIGN = u" \u26A0"
 
 SCHEMA = {
@@ -894,6 +897,7 @@ class ConfigService:
         if should_encrypt:
             ConfigService.encrypt_config(config_json)
         mongo.db.config.update({'name': 'newconfig'}, {"$set": config_json}, upsert=True)
+        logger.info('monkey config was updated')
 
     @staticmethod
     def init_default_config():
@@ -909,6 +913,7 @@ class ConfigService:
         config = copy.deepcopy(ConfigService.default_config)
         if should_encrypt:
             ConfigService.encrypt_config(config)
+        logger.info("Default config was called")
         return config
 
     @staticmethod
@@ -922,6 +927,7 @@ class ConfigService:
         config = ConfigService.get_default_config(True)
         ConfigService.set_server_ips_in_config(config)
         ConfigService.update_config(config, should_encrypt=False)
+        logger.info('Monkey config reset was called')
 
     @staticmethod
     def set_server_ips_in_config(config):
@@ -938,6 +944,7 @@ class ConfigService:
         initial_config['name'] = 'initial'
         initial_config.pop('_id')
         mongo.db.config.insert(initial_config)
+        logger.info('Monkey config was inserted to mongo and saved')
 
     @staticmethod
     def _extend_config_with_default(validator_class):

monkey_island/cc/services/island_logs.py

@@ -0,0 +1,32 @@
+import logging
+__author__ = "Maor.Rayzin"
+
+logger = logging.getLogger(__name__)
+
+
+class IslandLogService:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_log_file():
+        """
+        This static function is a helper function for the monkey island log download function.
+        It finds the logger handlers and checks if one of them is a fileHandler of any kind by checking if the handler
+        has the property handler.baseFilename.
+        :return:
+        a dict with the log file content.
+        """
+        logger_handlers = logger.parent.handlers
+        for handler in logger_handlers:
+            if hasattr(handler, 'baseFilename'):
+                logger.info('Log file found: {0}'.format(handler.baseFilename))
+                log_file_path = handler.baseFilename
+                with open(log_file_path, 'rt') as f:
+                    log_file = f.read()
+                return {
+                    'log_file': log_file
+                }
+
+        logger.warning('No log file could be found, check logger config.')
+        return None

monkey_island/cc/services/report.py

@@ -1,4 +1,5 @@
 import ipaddress
+import logging
 from enum import Enum
 
 from six import text_type
@@ -12,6 +13,9 @@ from cc.utils import local_ip_addresses, get_subnets
 __author__ = "itay.mizeretz"
 
 
+logger = logging.getLogger(__name__)
+
+
 class ReportService:
     def __init__(self):
         pass
@@ -79,6 +83,8 @@ class ReportService:
         creds = ReportService.get_azure_creds()
         machines = set([instance['origin'] for instance in creds])
 
+        logger.info('Azure issues generated for reporting')
+
         return [
             {
                 'type': 'azure_password',
@@ -105,6 +111,8 @@ class ReportService:
             }
             for node in nodes]
 
+        logger.info('Scanned nodes generated for reporting')
+
         return nodes
 
     @staticmethod
@@ -126,6 +134,8 @@ class ReportService:
             }
             for monkey in exploited]
 
+        logger.info('Exploited nodes generated for reporting')
+
         return exploited
 
     @staticmethod
@@ -149,6 +159,7 @@ class ReportService:
                             'origin': origin
                         }
                     )
+        logger.info('Stolen creds generated for reporting')
         return creds
 
     @staticmethod
@@ -169,6 +180,8 @@ class ReportService:
             azure_leaked_users = [{'username': user.replace(',', '.'), 'type': 'Clear Password',
                                    'origin': origin} for user in azure_users]
             creds.extend(azure_leaked_users)
+
+        logger.info('Azure machines creds generated for reporting')
         return creds
 
     @staticmethod
@@ -320,6 +333,7 @@ class ReportService:
             if machine not in issues_dict:
                 issues_dict[machine] = []
             issues_dict[machine].append(issue)
+        logger.info('Issues generated for reporting')
         return issues_dict
 
     @staticmethod
@@ -407,6 +421,7 @@ class ReportService:
             {'name': 'generated_report'},
             {'$set': {'value': True}},
             upsert=True)
+        logger.info("Report marked as generated.")
 
     @staticmethod
     def get_report():

monkey_island/cc/ui/src/components/pages/TelemetryPage.js

@@ -1,8 +1,9 @@
 import React from 'react';
-import {Col} from 'react-bootstrap';
+import {Button, Col} from 'react-bootstrap';
 import JSONTree from 'react-json-tree'
 import {DataTable} from 'react-data-components';
 import AuthComponent from '../AuthComponent';
+import download from 'downloadjs'
 
 const renderJson = (val) => <JSONTree data={val} level={1} theme="eighties" invertTheme={true} />;
 const renderTime = (val) => val.split('.')[0];
@@ -28,21 +29,47 @@ class TelemetryPageComponent extends AuthComponent {
       .then(res => this.setState({data: res.objects}));
   };
 
+downloadIslandLog = () => {
+    this.authFetch('/api/log/island/download')
+      .then(res => res.json())
+      .then(res => {
+        let filename = 'Island_log'
+        let logContent = (res['log_file']);
+        download(logContent, filename, 'text/plain');
+      });
+    };
+
   render() {
     return (
-      <Col xs={12} lg={8}>
-        <h1 className="page-title">Log</h1>
-        <div className="data-table-container">
-          <DataTable
-            keys="name"
-            columns={columns}
-            initialData={this.state.data}
-            initialPageLength={20}
-            initialSortBy={{ prop: 'timestamp', order: 'descending' }}
-            pageLengthOptions={[ 20, 50, 100 ]}
-          />
-        </div>
-      </Col>
+    <div>
+      <div>
+        <Col xs={12} lg={8}>
+          <h1 className="page-title">Log</h1>
+          <div className="data-table-container">
+            <DataTable
+              keys="name"
+              columns={columns}
+              initialData={this.state.data}
+              initialPageLength={20}
+              initialSortBy={{ prop: 'timestamp', order: 'descending' }}
+              pageLengthOptions={[ 20, 50, 100 ]}
+            />
+          </div>
+        </Col>
+      </div>
+      <div>
+        <Col xs={12} lg={8}>
+          <h1 className="page-title"> Monkey Island Logs </h1>
+          <div className="text-center" style={{marginBottom: '20px'}}>
+            <p style={{'marginBottom': '2em', 'fontSize': '1.2em'}}> Download Monkey Island internal log file </p>
+            <Button bsSize="large" onClick={()=> {
+              this.downloadIslandLog();
+              }}>
+              <i className="glyphicon glyphicon-download"/> Download </Button>
+          </div>
+        </Col>
+      </div>
+    </div>
     );
   }
 }