Merge pull request #139 from guardicore/feature/Adding_logs_to_monkey_island

Feature/adding logs to monkey island
This commit is contained in:
Daniel Goldberg 2018-06-05 14:06:23 +03:00 committed by GitHub
commit d77704b3e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 914 additions and 413 deletions

View File

@ -14,6 +14,7 @@ from cc.resources.client_run import ClientRun
from cc.resources.edge import Edge
from cc.resources.local_run import LocalRun
from cc.resources.log import Log
from cc.resources.island_logs import IslandLog
from cc.resources.monkey import Monkey
from cc.resources.monkey_configuration import MonkeyConfiguration
from cc.resources.monkey_download import MonkeyDownload
@ -104,5 +105,6 @@ def init_app(mongo_url):
api.add_resource(Report, '/api/report', '/api/report/')
api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/')
api.add_resource(Log, '/api/log', '/api/log/')
api.add_resource(IslandLog, '/api/log/island/download', '/api/log/island/download/')
return app

View File

@ -1,7 +1,11 @@
import json
import logging
import standard
import aws
logger = logging.getLogger(__name__)
ENV_DICT = {
'standard': standard.StandardEnvironment,
'aws': aws.AwsEnvironment
@ -18,6 +22,7 @@ def load_env_from_file():
try:
__env_type = load_env_from_file()
env = ENV_DICT[__env_type]()
logger.info('Monkey\'s env is: {0}'.format(env.__class__.__name__))
except Exception:
print('Failed initializing environment: %s' % __env_type)
logger.error('Failed initializing environment', exc_info=True)
raise

View File

@ -0,0 +1,26 @@
import os
import json
import logging.config
__author__ = 'Maor.Rayzin'
def json_setup_logging(default_path='logging.json', default_level=logging.INFO, env_key='LOG_CFG'):
"""
Setup the logging configuration
:param default_path: the default log configuration file path
:param default_level: Default level to log from
:param env_key: SYS ENV key to use for external configuration file path
:return:
"""
path = default_path
value = os.getenv(env_key, None)
if value:
path = value
if os.path.exists(path):
with open(path, 'rt') as f:
config = json.load(f)
logging.config.dictConfig(config)
else:
logging.basicConfig(level=default_level)

View File

@ -0,0 +1,33 @@
{
"version": 1,
"disable_existing_loggers": false,
"formatters": {
"simple": {
"format": "%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s"
}
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"level": "DEBUG",
"formatter": "simple",
"stream": "ext://sys.stdout"
},
"info_file_handler": {
"class": "logging.handlers.RotatingFileHandler",
"level": "INFO",
"formatter": "simple",
"filename": "info.log",
"maxBytes": 10485760,
"backupCount": 20,
"encoding": "utf8"
}
},
"root": {
"level": "INFO",
"handlers": ["console", "info_file_handler"]
}
}

View File

@ -2,19 +2,25 @@ from __future__ import print_function # In python 2.7
import os
import sys
import time
import logging
BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if BASE_PATH not in sys.path:
sys.path.insert(0, BASE_PATH)
from cc.island_logger import json_setup_logging
# This is here in order to catch EVERYTHING, some functions are being called on imports the log init needs to be on top.
json_setup_logging(default_path='island_logger_default_config.json', default_level=logging.DEBUG)
logger = logging.getLogger(__name__)
from cc.app import init_app
from cc.utils import local_ip_addresses
from cc.environment.environment import env
from cc.database import is_db_server_up
if __name__ == '__main__':
def main():
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
@ -22,7 +28,7 @@ if __name__ == '__main__':
mongo_url = os.environ.get('MONGO_URL', env.get_mongo_url())
while not is_db_server_up(mongo_url):
print('Waiting for MongoDB server')
logger.info('Waiting for MongoDB server')
time.sleep(1)
app = init_app(mongo_url)
@ -33,6 +39,10 @@ if __name__ == '__main__':
ssl_options={'certfile': os.environ.get('SERVER_CRT', 'server.crt'),
'keyfile': os.environ.get('SERVER_KEY', 'server.key')})
http_server.listen(env.get_island_port())
print('Monkey Island Server is running on https://{}:{}'.format(local_ip_addresses()[0], env.get_island_port()))
logger.info(
'Monkey Island Server is running on https://{}:{}'.format(local_ip_addresses()[0], env.get_island_port()))
IOLoop.instance().start()
if __name__ == '__main__':
main()

View File

@ -1,3 +1,4 @@
import logging
from flask import request, jsonify
import flask_restful
@ -5,6 +6,8 @@ from cc.services.node import NodeService
__author__ = 'itay.mizeretz'
logger = logging.getLogger(__name__)
class ClientRun(flask_restful.Resource):
def get(self):
@ -17,6 +20,7 @@ class ClientRun(flask_restful.Resource):
if monkey is not None:
is_monkey_running = not monkey["dead"]
else:
logger.info("Monkey is not running")
is_monkey_running = False
return jsonify(is_running=is_monkey_running)

View File

@ -0,0 +1,19 @@
import logging
import flask_restful
from cc.auth import jwt_required
from cc.services.island_logs import IslandLogService
__author__ = "Maor.Rayzin"
logger = logging.getLogger(__name__)
class IslandLog(flask_restful.Resource):
@jwt_required()
def get(self):
try:
return IslandLogService.get_log_file()
except Exception as e:
logger.error('Monkey Island logs failed to download', exc_info=True)

View File

@ -13,6 +13,8 @@ from cc.utils import local_ip_addresses
__author__ = 'Barak'
import logging
logger = logging.getLogger(__name__)
def run_local_monkey():
import platform
@ -32,6 +34,7 @@ def run_local_monkey():
copyfile(monkey_path, target_path)
os.chmod(target_path, stat.S_IRWXU | stat.S_IRWXG)
except Exception as exc:
logger.error('Copy file failed', exc_info=True)
return False, "Copy file failed: %s" % exc
# run the monkey
@ -41,6 +44,7 @@ def run_local_monkey():
args = "".join(args)
pid = subprocess.Popen(args, shell=True).pid
except Exception as exc:
logger.error('popen failed', exc_info=True)
return False, "popen failed: %s" % exc
return True, "pis: %s" % pid

View File

@ -1,3 +1,4 @@
import logging
import json
import os
@ -6,6 +7,8 @@ import flask_restful
__author__ = 'Barak'
logger = logging.getLogger(__name__)
MONKEY_DOWNLOADS = [
{
@ -42,7 +45,10 @@ MONKEY_DOWNLOADS = [
def get_monkey_executable(host_os, machine):
for download in MONKEY_DOWNLOADS:
if host_os == download.get('type') and machine == download.get('machine'):
logger.info('Monkey exec found for os: {0} and machine: {1}'.format(host_os, machine))
return download
logger.warning('No monkey executables could be found for the host os or machine or both: host_os: {0}, machine: {1}'
.format(host_os, machine))
return None

View File

@ -1,4 +1,5 @@
from datetime import datetime
import logging
import flask_restful
from flask import request, make_response, jsonify
@ -12,6 +13,8 @@ from cc.utils import local_ip_addresses
__author__ = 'Barak'
logger = logging.getLogger(__name__)
class Root(flask_restful.Resource):
@ -42,6 +45,7 @@ class Root(flask_restful.Resource):
# We can't drop system collections.
[mongo.db[x].drop() for x in mongo.db.collection_names() if not x.startswith('system.')]
ConfigService.init_config()
logger.info('DB was reset')
return jsonify(status='OK')
@staticmethod
@ -50,6 +54,7 @@ class Root(flask_restful.Resource):
mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}},
upsert=False,
multi=True)
logger.info('Kill all monkeys was called')
return jsonify(status='OK')
@staticmethod
@ -59,6 +64,7 @@ class Root(flask_restful.Resource):
infection_done = NodeService.is_monkey_finished_running()
if not infection_done:
report_done = False
logger.info('Report generation cannot be completed, infection is not done.')
else:
report_done = ReportService.is_report_generated()
return dict(run_server=True, run_monkey=is_any_exists, infection_done=infection_done, report_done=report_done)

View File

@ -1,4 +1,5 @@
import json
import logging
import traceback
import copy
from datetime import datetime
@ -17,6 +18,9 @@ from cc.encryptor import encryptor
__author__ = 'Barak'
logger = logging.getLogger(__name__)
class Telemetry(flask_restful.Resource):
@jwt_required()
def get(self, **kw):
@ -52,10 +56,9 @@ class Telemetry(flask_restful.Resource):
if telem_type in TELEM_PROCESS_DICT:
TELEM_PROCESS_DICT[telem_type](telemetry_json)
else:
print('Got unknown type of telemetry: %s' % telem_type)
logger.info('Got unknown type of telemetry: %s' % telem_type)
except Exception as ex:
print("Exception caught while processing telemetry: %s" % str(ex))
traceback.print_exc()
logger.error("Exception caught while processing telemetry", exc_info=True)
telem_id = mongo.db.telemetry.insert(telemetry_json)
return mongo.db.telemetry.find_one_or_404({"_id": telem_id})

View File

@ -1,6 +1,7 @@
import copy
import collections
import functools
import logging
from jsonschema import Draft4Validator, validators
from six import string_types
@ -11,6 +12,8 @@ from cc.utils import local_ip_addresses
__author__ = "itay.mizeretz"
logger = logging.getLogger(__name__)
WARNING_SIGN = u" \u26A0"
SCHEMA = {
@ -894,6 +897,7 @@ class ConfigService:
if should_encrypt:
ConfigService.encrypt_config(config_json)
mongo.db.config.update({'name': 'newconfig'}, {"$set": config_json}, upsert=True)
logger.info('monkey config was updated')
@staticmethod
def init_default_config():
@ -909,6 +913,7 @@ class ConfigService:
config = copy.deepcopy(ConfigService.default_config)
if should_encrypt:
ConfigService.encrypt_config(config)
logger.info("Default config was called")
return config
@staticmethod
@ -922,6 +927,7 @@ class ConfigService:
config = ConfigService.get_default_config(True)
ConfigService.set_server_ips_in_config(config)
ConfigService.update_config(config, should_encrypt=False)
logger.info('Monkey config reset was called')
@staticmethod
def set_server_ips_in_config(config):
@ -938,6 +944,7 @@ class ConfigService:
initial_config['name'] = 'initial'
initial_config.pop('_id')
mongo.db.config.insert(initial_config)
logger.info('Monkey config was inserted to mongo and saved')
@staticmethod
def _extend_config_with_default(validator_class):

View File

@ -0,0 +1,32 @@
import logging
__author__ = "Maor.Rayzin"
logger = logging.getLogger(__name__)
class IslandLogService:
def __init__(self):
pass
@staticmethod
def get_log_file():
"""
This static function is a helper function for the monkey island log download function.
It finds the logger handlers and checks if one of them is a fileHandler of any kind by checking if the handler
has the property handler.baseFilename.
:return:
a dict with the log file content.
"""
logger_handlers = logger.parent.handlers
for handler in logger_handlers:
if hasattr(handler, 'baseFilename'):
logger.info('Log file found: {0}'.format(handler.baseFilename))
log_file_path = handler.baseFilename
with open(log_file_path, 'rt') as f:
log_file = f.read()
return {
'log_file': log_file
}
logger.warning('No log file could be found, check logger config.')
return None

View File

@ -1,4 +1,5 @@
import ipaddress
import logging
from enum import Enum
from six import text_type
@ -12,6 +13,9 @@ from cc.utils import local_ip_addresses, get_subnets
__author__ = "itay.mizeretz"
logger = logging.getLogger(__name__)
class ReportService:
def __init__(self):
pass
@ -79,6 +83,8 @@ class ReportService:
creds = ReportService.get_azure_creds()
machines = set([instance['origin'] for instance in creds])
logger.info('Azure issues generated for reporting')
return [
{
'type': 'azure_password',
@ -105,6 +111,8 @@ class ReportService:
}
for node in nodes]
logger.info('Scanned nodes generated for reporting')
return nodes
@staticmethod
@ -126,6 +134,8 @@ class ReportService:
}
for monkey in exploited]
logger.info('Exploited nodes generated for reporting')
return exploited
@staticmethod
@ -149,6 +159,7 @@ class ReportService:
'origin': origin
}
)
logger.info('Stolen creds generated for reporting')
return creds
@staticmethod
@ -169,6 +180,8 @@ class ReportService:
azure_leaked_users = [{'username': user.replace(',', '.'), 'type': 'Clear Password',
'origin': origin} for user in azure_users]
creds.extend(azure_leaked_users)
logger.info('Azure machines creds generated for reporting')
return creds
@staticmethod
@ -320,6 +333,7 @@ class ReportService:
if machine not in issues_dict:
issues_dict[machine] = []
issues_dict[machine].append(issue)
logger.info('Issues generated for reporting')
return issues_dict
@staticmethod
@ -407,6 +421,7 @@ class ReportService:
{'name': 'generated_report'},
{'$set': {'value': True}},
upsert=True)
logger.info("Report marked as generated.")
@staticmethod
def get_report():

File diff suppressed because it is too large Load Diff

View File

@ -1,8 +1,9 @@
import React from 'react';
import {Col} from 'react-bootstrap';
import {Button, Col} from 'react-bootstrap';
import JSONTree from 'react-json-tree'
import {DataTable} from 'react-data-components';
import AuthComponent from '../AuthComponent';
import download from 'downloadjs'
const renderJson = (val) => <JSONTree data={val} level={1} theme="eighties" invertTheme={true} />;
const renderTime = (val) => val.split('.')[0];
@ -28,8 +29,20 @@ class TelemetryPageComponent extends AuthComponent {
.then(res => this.setState({data: res.objects}));
};
downloadIslandLog = () => {
this.authFetch('/api/log/island/download')
.then(res => res.json())
.then(res => {
let filename = 'Island_log'
let logContent = (res['log_file']);
download(logContent, filename, 'text/plain');
});
};
render() {
return (
<div>
<div>
<Col xs={12} lg={8}>
<h1 className="page-title">Log</h1>
<div className="data-table-container">
@ -43,6 +56,20 @@ class TelemetryPageComponent extends AuthComponent {
/>
</div>
</Col>
</div>
<div>
<Col xs={12} lg={8}>
<h1 className="page-title"> Monkey Island Logs </h1>
<div className="text-center" style={{marginBottom: '20px'}}>
<p style={{'marginBottom': '2em', 'fontSize': '1.2em'}}> Download Monkey Island internal log file </p>
<Button bsSize="large" onClick={()=> {
this.downloadIslandLog();
}}>
<i className="glyphicon glyphicon-download"/> Download </Button>
</div>
</Col>
</div>
</div>
);
}
}