Refactored ZT events sending and display on report to improve performance and UX

monkey/monkey_island/cc/app.py

@@ -32,6 +32,7 @@ from monkey_island.cc.resources.pba_file_upload import FileUpload
 from monkey_island.cc.resources.attack.attack_config import AttackConfiguration
 from monkey_island.cc.resources.attack.attack_report import AttackReport
 from monkey_island.cc.resources.bootloader import Bootloader
+from monkey_island.cc.resources.zero_trust.finding_event import ZeroTrustFindingEvent
 from monkey_island.cc.services.database import Database
 from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService
 from monkey_island.cc.services.representations import output_json
@@ -107,6 +108,7 @@ def init_api_resources(api):
         Report,
         '/api/report/<string:report_type>',
         '/api/report/<string:report_type>/<string:report_data>')
+    api.add_resource(ZeroTrustFindingEvent, '/api/zero-trust/finding-event/<string:finding_id>')
 
     api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/')
     api.add_resource(Log, '/api/log', '/api/log/')

monkey/monkey_island/cc/resources/reporting/report.py

@@ -6,6 +6,7 @@ from flask import jsonify
 from monkey_island.cc.auth import jwt_required
 from monkey_island.cc.services.reporting.report import ReportService
 from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
+from monkey_island.cc.testing.profiler_decorator import profile
 
 ZERO_TRUST_REPORT_TYPE = "zero_trust"
 SECURITY_REPORT_TYPE = "security"
@@ -21,6 +22,7 @@ __author__ = ["itay.mizeretz", "shay.nehmad"]
 class Report(flask_restful.Resource):
 
     @jwt_required()
+    @profile()
     def get(self, report_type=SECURITY_REPORT_TYPE, report_data=None):
         if report_type == SECURITY_REPORT_TYPE:
             return ReportService.get_report()

monkey/monkey_island/cc/resources/zero_trust/finding_event.py

@@ -0,0 +1,14 @@
+import flask_restful
+import json
+
+from monkey_island.cc.auth import jwt_required
+from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
+from monkey_island.cc.testing.profiler_decorator import profile
+
+
+class ZeroTrustFindingEvent(flask_restful.Resource):
+
+    @jwt_required()
+    @profile()
+    def get(self, finding_id: str):
+        return {'events_json': json.dumps(ZeroTrustService.get_events_by_finding(finding_id), default=str)}

monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py

@@ -1,6 +1,7 @@
 import common.data.zero_trust_consts as zero_trust_consts
 from monkey_island.cc.models.zero_trust.finding import Finding
 from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
+import monkey_island.cc.services.reporting.zero_trust_service
 from monkey_island.cc.testing.IslandTestCase import IslandTestCase
 
 EXPECTED_DICT = {
@@ -316,6 +317,12 @@ class TestZeroTrustService(IslandTestCase):
 
         self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected)
 
+    def test_get_events_without_overlap(self):
+        monkey_island.cc.services.reporting.zero_trust_service.EVENT_FETCH_CNT = 5
+        self.assertListEqual([], ZeroTrustService._ZeroTrustService__get_events_without_overlap(5, [1, 2, 3]))
+        self.assertListEqual([3], ZeroTrustService._ZeroTrustService__get_events_without_overlap(6, [1, 2, 3]))
+        self.assertListEqual([1, 2, 3, 4, 5], ZeroTrustService._ZeroTrustService__get_events_without_overlap(10, [1, 2, 3, 4, 5]))
+
 
 def compare_lists_no_order(s, t):
     t = list(t)  # make a mutable copy

monkey/monkey_island/cc/services/reporting/zero_trust_service.py

@@ -1,9 +1,14 @@
-import json
+from typing import List
 
 import common.data.zero_trust_consts as zero_trust_consts
+from bson.objectid import ObjectId
 
 from monkey_island.cc.models.zero_trust.finding import Finding
 
+# How many events of a single finding to return to UI.
+# 50 will return 50 latest and 50 oldest events from a finding
+EVENT_FETCH_CNT = 50
+
 
 class ZeroTrustService(object):
     @staticmethod
@@ -104,25 +109,43 @@ class ZeroTrustService(object):
 
     @staticmethod
     def get_all_findings():
-        all_findings = Finding.objects()
+        pipeline = [{'$match': {}},
+                    {'$addFields': {'oldest_events': {'$slice': ['$events', EVENT_FETCH_CNT]},
+                                    'latest_events': {'$slice': ['$events', -1*EVENT_FETCH_CNT]},
+                                    'event_count': {'$size': '$events'}}},
+                    {'$unset': ['events']}]
+        all_findings = list(Finding.objects.aggregate(*pipeline))
+        for finding in all_findings:
+            finding['latest_events'] = ZeroTrustService.__get_events_without_overlap(finding['event_count'],
+                                                                                     finding['latest_events'])
+
         enriched_findings = [ZeroTrustService.__get_enriched_finding(f) for f in all_findings]
         return enriched_findings
 
     @staticmethod
-    def __get_enriched_finding(finding):
+    def __get_events_without_overlap(event_count: int, events: List[object]) -> List[object]:
-        test_info = zero_trust_consts.TESTS_MAP[finding.test]
+        overlap_count = event_count - EVENT_FETCH_CNT
-        enriched_finding = {
+        if overlap_count >= EVENT_FETCH_CNT:
-            "test": test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding.status],
+            return events
-            "test_key": finding.test,
+        elif overlap_count <= 0:
-            "pillars": test_info[zero_trust_consts.PILLARS_KEY],
+            return []
-            "status": finding.status,
+        else:
-            "events": ZeroTrustService.__get_events_as_dict(finding.events)
+            return events[ -overlap_count :]
-        }
-        return enriched_finding
 
     @staticmethod
-    def __get_events_as_dict(events):
+    def __get_enriched_finding(finding):
-        return [json.loads(event.to_json()) for event in events]
+        test_info = zero_trust_consts.TESTS_MAP[finding['test']]
+        enriched_finding = {
+            'finding_id': str(finding['_id']),
+            'test': test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding['status']],
+            'test_key': finding['test'],
+            'pillars': test_info[zero_trust_consts.PILLARS_KEY],
+            'status': finding['status'],
+            'latest_events': finding['latest_events'],
+            'oldest_events': finding['oldest_events'],
+            'event_count': finding['event_count']
+        }
+        return enriched_finding
 
     @staticmethod
     def get_statuses_to_pillars():
@@ -153,3 +176,11 @@ class ZeroTrustService(object):
             if grade[status] > 0:
                 return status
         return zero_trust_consts.STATUS_UNEXECUTED
+
+    @staticmethod
+    def get_events_by_finding(finding_id: str) -> List[object]:
+        pipeline = [{'$match': {'_id': ObjectId(finding_id)}},
+                    {'$unwind': '$events'},
+                    {'$project': {'events': '$events'}},
+                    {'$replaceRoot': {'newRoot': '$events'}}]
+        return list(Finding.objects.aggregate(*pipeline))

monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js

@@ -24,7 +24,12 @@ export default class EventsButton extends Component {
 
   render() {
     return <Fragment>
-      <EventsModal events={this.props.events} showEvents={this.state.isShow} hideCallback={this.hide}
+      <EventsModal finding_id={this.props.finding_id}
+                   latest_events={this.props.latest_events}
+                   oldest_events={this.props.oldest_events}
+                   event_count={this.props.event_count}
+                   showEvents={this.state.isShow}
+                   hideCallback={this.hide}
                    exportFilename={this.props.exportFilename}/>
       <div className="text-center" style={{'display': 'grid'}}>
         <Button className="btn btn-primary btn-lg" onClick={this.show}>
@@ -35,12 +40,14 @@ export default class EventsButton extends Component {
   }
 
   createEventsAmountBadge() {
-    const eventsAmountBadgeContent = this.props.events.length > 9 ? '9+' : this.props.events.length;
+    const eventsAmountBadgeContent = this.props.event_count > 9 ? '9+' : this.props.event_count;
     return <Badge>{eventsAmountBadgeContent}</Badge>;
   }
 }
 
 EventsButton.propTypes = {
-  events: PropTypes.array,
+  latest_events: PropTypes.array,
+  oldest_events: PropTypes.array,
+  event_count: PropTypes.number,
   exportFilename: PropTypes.string
 };

monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsModal.js

@@ -1,13 +1,17 @@
 import React, {Component} from 'react';
-import {Badge, Modal} from 'react-bootstrap';
+import {Modal} from 'react-bootstrap';
 import EventsTimeline from './EventsTimeline';
 import * as PropTypes from 'prop-types';
 import saveJsonToFile from '../../utils/SaveJsonToFile';
 import EventsModalButtons from './EventsModalButtons';
-import Pluralize from 'pluralize'
+import AuthComponent from '../../AuthComponent';
-import {statusToLabelType} from './StatusLabel';
+import Pluralize from 'pluralize';
+import SkippedEventsTimeline from "./SkippedEventsTimeline";
 
-export default class EventsModal extends Component {
+const FINDING_EVENTS_URL = '/api/zero-trust/finding-event/';
+
+
+export default class EventsModal extends AuthComponent {
   constructor(props) {
     super(props);
   }
@@ -22,12 +26,20 @@ export default class EventsModal extends Component {
             </h3>
             <hr/>
             <p>
-              There {Pluralize('is', this.props.events.length)} {<div
+              There {Pluralize('is', this.props.event_count)} {<div
-              className={'label label-primary'}>{this.props.events.length}</div>} {Pluralize('event', this.props.events.length)} associated
+              className={'label label-primary'}>{this.props.event_count}</div>}
+              {Pluralize('event', this.props.event_count)} associated
               with this finding.
+              {<div className={'label label-primary'}>
+                {this.props.latest_events.length + this.props.oldest_events.length}
+              </div>} {Pluralize('is', this.props.event_count)} displayed below.
+              All events can be exported to json.
             </p>
-            {this.props.events.length > 5 ? this.renderButtons() : null}
+            {this.props.event_count > 5 ? this.renderButtons() : null}
-            <EventsTimeline events={this.props.events}/>
+            <EventsTimeline events={this.props.oldest_events}/>
+            {this.props.event_count > this.props.latest_events.length+this.props.oldest_events.length ?
+              this.renderSkippedEventsTimeline() : null}
+            <EventsTimeline events={this.props.latest_events}/>
             {this.renderButtons()}
           </Modal.Body>
         </Modal>
@@ -35,13 +47,23 @@ export default class EventsModal extends Component {
     );
   }
 
+  renderSkippedEventsTimeline(){
+    return <div className={'skipped-events-timeline'}>
+      <SkippedEventsTimeline
+              skipped_count={this.props.event_count - this.props.latest_events.length+this.props.oldest_events.length}/>
+    </div>
+  }
+
   renderButtons() {
     return <EventsModalButtons
       onClickClose={() => this.props.hideCallback()}
       onClickExport={() => {
-        const dataToSave = this.props.events;
+        let full_url = FINDING_EVENTS_URL + this.props.finding_id;
-        const filename = this.props.exportFilename;
+        this.authFetch(full_url).then(res => res.json()).then(res => {
-        saveJsonToFile(dataToSave, filename);
+          const dataToSave = res.events_json;
+          const filename = this.props.exportFilename;
+          saveJsonToFile(dataToSave, filename);
+        });
       }}/>;
   }
 }

monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsTimeline.js

@@ -17,7 +17,7 @@ export default class EventsTimeline extends Component {
         <Timeline style={{fontSize: '100%'}}>
           {
             this.props.events.map((event, index) => {
-              const event_time = new Date(event.timestamp['$date']).toString();
+              const event_time = new Date(event.timestamp).toString();
               return (<TimelineEvent
                 key={index}
                 createdAt={event_time}

monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/FindingsTable.js

@@ -18,7 +18,11 @@ const columns = [
       {
         Header: 'Events', id: 'events',
         accessor: x => {
-          return <EventsButton events={x.events} exportFilename={'Events_' + x.test_key}/>;
+          return <EventsButton finding_id={x.finding_id}
+                               latest_events={x.latest_events}
+                               oldest_events={x.oldest_events}
+                               event_count={x.event_count}
+                               exportFilename={'Events_' + x.test_key}/>;
         },
         maxWidth: EVENTS_COLUMN_MAX_WIDTH
       },

monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/SkippedEventsTimeline.js

@@ -0,0 +1,26 @@
+import React, {Component} from 'react';
+import {Timeline, TimelineEvent} from 'react-event-timeline';
+import { faArrowsAltV } from '@fortawesome/free-solid-svg-icons';
+import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import * as PropTypes from 'prop-types';
+
+
+export default class SkippedEventsTimeline extends Component {
+
+  render() {
+    return (
+      <div>
+        <Timeline style={{fontSize: '100%'}}>
+          <TimelineEvent
+            bubbleStyle={{border: '2px solid #ffcc00'}}
+            title='Events in between are not displayed, but can be exported to JSON.'
+            icon={<FontAwesomeIcon className={'timeline-event-icon'} icon={faArrowsAltV}/>} >
+            {this.props.skipped_count} events not displayed.
+          </TimelineEvent>
+        </Timeline>
+      </div>
+    );
+  }
+}
+
+SkippedEventsTimeline.propTypes = {skipped_count: PropTypes.number};

monkey/monkey_island/linux/install_mongo.sh

@@ -17,9 +17,6 @@ elif [[ ${os_version_monkey} == "Ubuntu 18.04"* ]]; then
 elif [[ ${os_version_monkey} == "Ubuntu 19.10"* ]]; then
   echo Detected Ubuntu 19.10
   export tgz_url="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.2.3.tgz"
-elif [[ ${os_version_monkey} == "Debian GNU/Linux 8"* ]]; then
-  echo Detected Debian 8
-  export tgz_url="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-debian81-4.0.16.tgz"
 elif [[ ${os_version_monkey} == "Debian GNU/Linux 9"* ]]; then
   echo Detected Debian 9
   export tgz_url="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-debian92-4.2.3.tgz"