Merge remote-tracking branch 'upstream/develop' into attack_module_load

# Conflicts:
#	monkey/monkey_island/cc/ui/src/components/attack/techniques/Helpers.js

deployment_scripts/README.md

@@ -13,9 +13,10 @@ Don't forget to add python to PATH or do so while installing it via this script.
 
 ## Linux
 
-You must have root permissions, but there is no need to run the script as root.<br>
+You must have root permissions, but don't run the script as root.<br>
 Launch deploy_linux.sh from scripts directory.<br>
-First argument is an empty directory (script can create one) and second is branch you want to clone.
+First argument should be an empty directory (script can create one, default is ./infection_monkey) and second is the branch you want to clone (develop by default).
+Choose a directory where you have all the relevant permissions, for e.g. /home/your_username
 Example usages:<br>
 ./deploy_linux.sh (deploys under ./infection_monkey)<br>
 ./deploy_linux.sh "/home/test/monkey" (deploys under /home/test/monkey)<br>

monkey/infection_monkey/monkey.py

@@ -255,6 +255,7 @@ class InfectionMonkey(object):
         if WormConfiguration.self_delete_in_cleanup \
                 and -1 == sys.executable.find('python'):
             try:
+                status = None
                 if "win32" == sys.platform:
                     from _subprocess import SW_HIDE, STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
                     startupinfo = subprocess.STARTUPINFO()
@@ -265,10 +266,12 @@ class InfectionMonkey(object):
                                      close_fds=True, startupinfo=startupinfo)
                 else:
                     os.remove(sys.executable)
-                    T1107Telem(ScanStatus.USED, sys.executable).send()
+                    status = ScanStatus.USED
             except Exception as exc:
                 LOG.error("Exception in self delete: %s", exc)
-                T1107Telem(ScanStatus.SCANNED, sys.executable).send()
+                status = ScanStatus.SCANNED
+            if status:
+                T1107Telem(status, sys.executable).send()
 
     def send_log(self):
         monkey_log_path = utils.get_monkey_log_path()

monkey/monkey_island/cc/consts.py

@@ -3,3 +3,4 @@ import os
 __author__ = 'itay.mizeretz'
 
 MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), 'monkey_island')
+DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5

monkey/monkey_island/cc/models/monkey.py

@@ -1,11 +1,11 @@
 """
 Define a Document Schema for the Monkey document.
 """
-import mongoengine
 from mongoengine import Document, StringField, ListField, BooleanField, EmbeddedDocumentField, ReferenceField, \
-    DateTimeField
+    DateTimeField, DynamicField, DoesNotExist
 
-from monkey_island.cc.models.monkey_ttl import MonkeyTtl
+from monkey_island.cc.models.monkey_ttl import MonkeyTtl, create_monkey_ttl_document
+from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS
 
 
 class Monkey(Document):
@@ -26,24 +26,37 @@ class Monkey(Document):
     ip_addresses = ListField(StringField())
     keepalive = DateTimeField()
     modifytime = DateTimeField()
-    # TODO change this to an embedded document as well - RN it's an unnamed tuple which is confusing.
+    # TODO make "parent" an embedded document, so this can be removed and the schema explained (and validated) verbosly.
-    parent = ListField(ListField(StringField()))
+    # This is a temporary fix, since mongoengine doesn't allow for lists of strings to be null
+    # (even with required=False of null=True).
+    # See relevant issue: https://github.com/MongoEngine/mongoengine/issues/1904
+    parent = ListField(ListField(DynamicField()))
     config_error = BooleanField()
     critical_services = ListField(StringField())
     pba_results = ListField()
     ttl_ref = ReferenceField(MonkeyTtl)
+    tunnel = ReferenceField("self")
 
     # LOGIC
     @staticmethod
     def get_single_monkey_by_id(db_id):
         try:
-            return Monkey.objects(id=db_id)[0]
+            return Monkey.objects.get(id=db_id)
-        except IndexError:
+        except DoesNotExist as ex:
-            raise MonkeyNotFoundError("id: {0}".format(str(db_id)))
+            raise MonkeyNotFoundError("info: {0} | id: {1}".format(ex.message, str(db_id)))
+
+    @staticmethod
+    def get_single_monkey_by_guid(monkey_guid):
+        try:
+            return Monkey.objects.get(guid=monkey_guid)
+        except DoesNotExist as ex:
+            raise MonkeyNotFoundError("info: {0} | guid: {1}".format(ex.message, str(monkey_guid)))
 
     @staticmethod
     def get_latest_modifytime():
+        if Monkey.objects.count() > 0:
             return Monkey.objects.order_by('-modifytime').first().modifytime
+        return None
 
     def is_dead(self):
         monkey_is_dead = False
@@ -54,7 +67,7 @@ class Monkey(Document):
                 if MonkeyTtl.objects(id=self.ttl_ref.id).count() == 0:
                     # No TTLs - monkey has timed out. The monkey is MIA.
                     monkey_is_dead = True
-            except (mongoengine.DoesNotExist, AttributeError):
+            except (DoesNotExist, AttributeError):
                 # Trying to dereference unknown document - the monkey is MIA.
                 monkey_is_dead = True
         return monkey_is_dead
@@ -67,6 +80,10 @@ class Monkey(Document):
             os = "windows"
         return os
 
+    def renew_ttl(self, duration=DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS):
+        self.ttl_ref = create_monkey_ttl_document(duration)
+        self.save()
+
 
 class MonkeyNotFoundError(Exception):
     pass

monkey/monkey_island/cc/models/monkey_ttl.py

@@ -38,3 +38,16 @@ class MonkeyTtl(Document):
     }
 
     expire_at = DateTimeField()
+
+
+def create_monkey_ttl_document(expiry_duration_in_seconds):
+    """
+    Create a new Monkey TTL document and save it as a document.
+    :param expiry_duration_in_seconds:  How long should the TTL last for. THIS IS A LOWER BOUND - depends on mongodb
+    performance.
+    :return: The TTL document. To get its ID use `.id`.
+    """
+    # The TTL data uses the new `models` module which depends on mongoengine.
+    current_ttl = MonkeyTtl.create_ttl_expire_in(expiry_duration_in_seconds)
+    current_ttl.save()
+    return current_ttl

monkey/monkey_island/cc/models/test_monkey.py

@@ -46,6 +46,19 @@ class TestMonkey(IslandTestCase):
         self.assertTrue(mia_monkey.is_dead())
         self.assertFalse(alive_monkey.is_dead())
 
+    def test_ttl_renewal(self):
+        self.fail_if_not_testing_env()
+        self.clean_monkey_db()
+
+        # Arrange
+        monkey = Monkey(guid=str(uuid.uuid4()))
+        monkey.save()
+        self.assertIsNone(monkey.ttl_ref)
+
+        # act + assert
+        monkey.renew_ttl()
+        self.assertIsNotNone(monkey.ttl_ref)
+
     def test_get_single_monkey_by_id(self):
         self.fail_if_not_testing_env()
         self.clean_monkey_db()

monkey/monkey_island/cc/resources/monkey.py

@@ -5,26 +5,17 @@ import dateutil.parser
 import flask_restful
 from flask import request
 
+from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS
 from monkey_island.cc.database import mongo
-from monkey_island.cc.models.monkey_ttl import MonkeyTtl
+from monkey_island.cc.models.monkey_ttl import create_monkey_ttl_document
 from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.services.node import NodeService
 
-MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5
-
 __author__ = 'Barak'
 
 # TODO: separate logic from interface
 
 
-def create_monkey_ttl():
-    # The TTL data uses the new `models` module which depends on mongoengine.
-    current_ttl = MonkeyTtl.create_ttl_expire_in(MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)
-    current_ttl.save()
-    ttlid = current_ttl.id
-    return ttlid
-
-
 class Monkey(flask_restful.Resource):
 
     # Used by monkey. can't secure.
@@ -58,8 +49,8 @@ class Monkey(flask_restful.Resource):
             tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "")
             NodeService.set_monkey_tunnel(monkey["_id"], tunnel_host_ip)
 
-        ttlid = create_monkey_ttl()
+        ttl = create_monkey_ttl_document(DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)
-        update['$set']['ttl_ref'] = ttlid
+        update['$set']['ttl_ref'] = ttl.id
 
         return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False)
 
@@ -120,7 +111,8 @@ class Monkey(flask_restful.Resource):
             tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "")
             monkey_json.pop('tunnel')
 
-        monkey_json['ttl_ref'] = create_monkey_ttl()
+        ttl = create_monkey_ttl_document(DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)
+        monkey_json['ttl_ref'] = ttl.id
 
         mongo.db.monkey.update({"guid": monkey_json["guid"]},
                                {"$set": monkey_json},

monkey/monkey_island/cc/resources/telemetry.py

@@ -15,10 +15,10 @@ from monkey_island.cc.services.edge import EdgeService
 from monkey_island.cc.services.node import NodeService
 from monkey_island.cc.encryptor import encryptor
 from monkey_island.cc.services.wmi_handler import WMIHandler
+from monkey_island.cc.models.monkey import Monkey
 
 __author__ = 'Barak'
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -49,6 +49,9 @@ class Telemetry(flask_restful.Resource):
         telemetry_json = json.loads(request.data)
         telemetry_json['timestamp'] = datetime.now()
 
+        # Monkey communicated, so it's alive. Update the TTL.
+        Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']).renew_ttl()
+
         monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid'])
 
         try:
@@ -59,7 +62,7 @@ class Telemetry(flask_restful.Resource):
             else:
                 logger.info('Got unknown type of telemetry: %s' % telem_category)
         except Exception as ex:
-            logger.error("Exception caught while processing telemetry", exc_info=True)
+            logger.error("Exception caught while processing telemetry. Info: {}".format(ex.message), exc_info=True)
 
         telem_id = mongo.db.telemetry.insert(telemetry_json)
         return mongo.db.telemetry.find_one_or_404({"_id": telem_id})
@@ -188,7 +191,7 @@ class Telemetry(flask_restful.Resource):
             Telemetry.add_system_info_creds_to_config(creds)
             Telemetry.replace_user_dot_with_comma(creds)
         if 'mimikatz' in telemetry_json['data']:
-            users_secrets = mimikatz_utils.MimikatzSecrets.\
+            users_secrets = mimikatz_utils.MimikatzSecrets. \
                 extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', ''))
         if 'wmi' in telemetry_json['data']:
             wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets)

monkey/monkey_island/cc/resources/telemetry_feed.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 
 import dateutil
@@ -9,6 +10,8 @@ from monkey_island.cc.auth import jwt_required
 from monkey_island.cc.database import mongo
 from monkey_island.cc.services.node import NodeService
 
+logger = logging.getLogger(__name__)
+
 __author__ = 'itay.mizeretz'
 
 
@@ -23,11 +26,15 @@ class TelemetryFeed(flask_restful.Resource):
 
         telemetries = telemetries.sort([('timestamp', flask_pymongo.ASCENDING)])
 
+        try:
             return \
                 {
                     'telemetries': [TelemetryFeed.get_displayed_telemetry(telem) for telem in telemetries],
                     'timestamp': datetime.now().isoformat()
                 }
+        except KeyError as err:
+            logger.error("Failed parsing telemetries. Error: {0}.".format(err.message))
+            return {'telemetries': [], 'timestamp': datetime.now().isoformat()}
 
     @staticmethod
     def get_displayed_telemetry(telem):

monkey/monkey_island/cc/services/config_schema.py

@@ -389,7 +389,7 @@ SCHEMA = {
                         "self_delete_in_cleanup": {
                             "title": "Self delete on cleanup",
                             "type": "boolean",
-                            "default": False,
+                            "default": True,
                             "description": "Should the monkey delete its executable when going down"
                         },
                         "use_file_logging": {

monkey/monkey_island/cc/services/report.py

@@ -373,8 +373,13 @@ class ReportService:
 
     @staticmethod
     def get_exploits():
+        query = [{'$match': {'telem_category': 'exploit', 'data.result': True}},
+                 {'$group': {'_id': {'ip_address': '$data.machine.ip_addr'},
+                             'data': {'$first': '$$ROOT'},
+                             }},
+                 {"$replaceRoot": {"newRoot": "$data"}}]
         exploits = []
-        for exploit in mongo.db.telemetry.find({'telem_category': 'exploit', 'data.result': True}):
+        for exploit in mongo.db.telemetry.aggregate(query):
             new_exploit = ReportService.process_exploit(exploit)
             if new_exploit not in exploits:
                 exploits.append(new_exploit)

monkey/monkey_island/cc/ui/src/components/attack/techniques/Helpers.js

@@ -36,7 +36,7 @@ export function getUsageColumns() {
           style: { 'whiteSpace': 'unset' }}]
     }])}
 
-export const scanStatus = {
+export const ScanStatus = {
     UNSCANNED: 0,
     SCANNED: 1,
     USED: 2

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1003.js

@@ -2,7 +2,7 @@ import React from 'react';
 import '../../../styles/Collapse.scss'
 import '../../report-components/StolenPasswords'
 import StolenPasswordsComponent from "../../report-components/StolenPasswords";
-import {scanStatus} from "./Helpers"
+import {ScanStatus} from "./Helpers"
 
 
 class T1003 extends React.Component {
@@ -16,7 +16,7 @@ class T1003 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status === scanStatus.USED ?
+        {this.props.data.status === ScanStatus.USED ?
           <StolenPasswordsComponent data={this.props.reportData.glance.stolen_creds.concat(this.props.reportData.glance.ssh_keys)}/>
           : ""}
       </div>

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1059.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachine, scanStatus } from "./Helpers"
+import { renderMachine, ScanStatus } from "./Helpers"
 
 
 class T1059 extends React.Component {
@@ -25,7 +25,7 @@ class T1059 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status === scanStatus.USED ?
+        {this.props.data.status === ScanStatus.USED ?
           <ReactTable
               columns={T1059.getCommandColumns()}
               data={this.props.data.cmds}

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1075.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachine, scanStatus } from "./Helpers"
+import { renderMachine, ScanStatus } from "./Helpers"
 
 
 class T1075 extends React.Component {
@@ -34,7 +34,7 @@ class T1075 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status !== scanStatus.UNSCANNED ?
+        {this.props.data.status === ScanStatus.USED ?
           <ReactTable
               columns={T1075.getHashColumns()}
               data={this.props.data.successful_logins}

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1082.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachineFromSystemData, scanStatus } from "./Helpers"
+import { renderMachineFromSystemData, ScanStatus } from "./Helpers"
 
 
 class T1082 extends React.Component {
@@ -33,7 +33,7 @@ class T1082 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status === scanStatus.USED ?
+        {this.props.data.status === ScanStatus.USED ?
           <ReactTable
               columns={T1082.getSystemInfoColumns()}
               data={this.props.data.system_info}

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1086.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachine, scanStatus } from "./Helpers"
+import { renderMachine, ScanStatus } from "./Helpers"
 
 
 class T1086 extends React.Component {
@@ -25,7 +25,7 @@ class T1086 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status === scanStatus.USED ?
+        {this.props.data.status === ScanStatus.USED ?
           <ReactTable
               columns={T1086.getPowershellColumns()}
               data={this.props.data.cmds}

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1107.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachineFromSystemData, scanStatus } from "./Helpers"
+import { renderMachineFromSystemData, ScanStatus } from "./Helpers"
 
 
 class T1107 extends React.Component {
@@ -11,7 +11,7 @@ class T1107 extends React.Component {
   }
 
   static renderDelete(status){
-    if(status === scanStatus.USED){
+    if(status === ScanStatus.USED){
       return <span>Yes</span>
     } else {
       return <span>No</span>

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1110.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachine, scanStatus } from "./Helpers"
+import { renderMachine, ScanStatus } from "./Helpers"
 
 
 class T1110 extends React.Component {
@@ -32,7 +32,7 @@ class T1110 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status !== scanStatus.UNSCANNED ?
+        {this.props.data.status !== ScanStatus.UNSCANNED ?
           <ReactTable
             columns={T1110.getServiceColumns()}
             data={this.props.data.services}

monkey/monkey_island/cc/ui/src/components/attack/techniques/T1145.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import '../../../styles/Collapse.scss'
 import ReactTable from "react-table";
-import { renderMachineFromSystemData, scanStatus } from "./Helpers"
+import { renderMachineFromSystemData, ScanStatus } from "./Helpers"
 
 
 class T1145 extends React.Component {
@@ -38,7 +38,7 @@ class T1145 extends React.Component {
       <div>
         <div>{this.props.data.message}</div>
         <br/>
-        {this.props.data.status === scanStatus.USED ?
+        {this.props.data.status === ScanStatus.USED ?
           <ReactTable
               columns={T1145.getKeysInfoColumns()}
               data={this.props.data.ssh_info}

monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js

@@ -241,7 +241,7 @@ class RunMonkeyPageComponent extends AuthComponent {
         <div style={{'marginTop': '1em', 'marginBottom': '1em'}}>
           <p className="alert alert-info">
             <i className="glyphicon glyphicon-info-sign" style={{'marginRight': '5px'}}/>
-            Not sure what this is? Not seeing your AWS EC2 instances? <a href="https://github.com/guardicore/monkey/wiki/Monkey-Island:-Running-the-monkey-on-AWS-EC2-instances">Read the documentation</a>!
+            Not sure what this is? Not seeing your AWS EC2 instances? <a href="https://github.com/guardicore/monkey/wiki/Monkey-Island:-Running-the-monkey-on-AWS-EC2-instances" target="_blank">Read the documentation</a>!
           </p>
         </div>
         {

monkey/monkey_island/cc/ui/src/components/report-components/AttackReport.js

@@ -4,7 +4,7 @@ import {ReactiveGraph} from 'components/reactive-graph/ReactiveGraph';
 import {edgeGroupToColor, options} from 'components/map/MapOptions';
 import '../../styles/Collapse.scss';
 import AuthComponent from '../AuthComponent';
-import {scanStatus} from "../attack/techniques/Helpers";
+import {ScanStatus} from "../attack/techniques/Helpers";
 import Collapse from '@kunukn/react-collapse';
 import T1210 from '../attack/techniques/T1210';
 import T1197 from '../attack/techniques/T1197';
@@ -84,9 +84,9 @@ class AttackReportPageComponent extends AuthComponent {
 
   getComponentClass(tech_id){
     switch (this.state.report[tech_id].status) {
-      case scanStatus.SCANNED:
+      case ScanStatus.SCANNED:
         return 'collapse-info';
-      case scanStatus.USED:
+      case ScanStatus.USED:
         return 'collapse-danger';
       default:
         return 'collapse-default';

monkey/monkey_island/scripts/island_password_hasher.py

@@ -0,0 +1,23 @@
+"""
+Utility script for running a string through SHA3_512 hash.
+Used for Monkey Island password hash, see
+https://github.com/guardicore/monkey/wiki/Enabling-Monkey-Island-Password-Protection
+for more details.
+"""
+
+import argparse
+from Crypto.Hash import SHA3_512
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("string_to_sha", help="The string to do sha for")
+    args = parser.parse_args()
+
+    h = SHA3_512.new()
+    h.update(args.string_to_sha)
+    print(h.hexdigest())
+
+
+if __name__ == '__main__':
+    main()