From 571682fff9cea192bd37cde04d63e3adb577cb43 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 6 May 2020 16:52:50 +0300 Subject: [PATCH] Refactored ZT events sending and display on report to improve performance and UX --- monkey/monkey_island/cc/app.py | 2 + .../cc/resources/reporting/report.py | 2 + .../cc/resources/zero_trust/finding_event.py | 14 +++++ .../reporting/test_zero_trust_service.py | 7 +++ .../services/reporting/zero_trust_service.py | 59 ++++++++++++++----- .../zerotrust/EventsButton.js | 13 +++- .../zerotrust/EventsModal.js | 44 ++++++++++---- .../zerotrust/EventsTimeline.js | 2 +- .../zerotrust/FindingsTable.js | 6 +- .../zerotrust/SkippedEventsTimeline.js | 26 ++++++++ monkey/monkey_island/linux/install_mongo.sh | 3 - 11 files changed, 145 insertions(+), 33 deletions(-) create mode 100644 monkey/monkey_island/cc/resources/zero_trust/finding_event.py create mode 100644 monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/SkippedEventsTimeline.js diff --git a/monkey/monkey_island/cc/app.py b/monkey/monkey_island/cc/app.py index 3a1134930..13aac018a 100644 --- a/monkey/monkey_island/cc/app.py +++ b/monkey/monkey_island/cc/app.py @@ -32,6 +32,7 @@ from monkey_island.cc.resources.pba_file_upload import FileUpload from monkey_island.cc.resources.attack.attack_config import AttackConfiguration from monkey_island.cc.resources.attack.attack_report import AttackReport from monkey_island.cc.resources.bootloader import Bootloader +from monkey_island.cc.resources.zero_trust.finding_event import ZeroTrustFindingEvent from monkey_island.cc.services.database import Database from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService from monkey_island.cc.services.representations import output_json @@ -107,6 +108,7 @@ def init_api_resources(api): Report, '/api/report/', '/api/report//') + api.add_resource(ZeroTrustFindingEvent, '/api/zero-trust/finding-event/') api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/') api.add_resource(Log, '/api/log', '/api/log/') diff --git a/monkey/monkey_island/cc/resources/reporting/report.py b/monkey/monkey_island/cc/resources/reporting/report.py index 961e745a8..6770512e6 100644 --- a/monkey/monkey_island/cc/resources/reporting/report.py +++ b/monkey/monkey_island/cc/resources/reporting/report.py @@ -6,6 +6,7 @@ from flask import jsonify from monkey_island.cc.auth import jwt_required from monkey_island.cc.services.reporting.report import ReportService from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService +from monkey_island.cc.testing.profiler_decorator import profile ZERO_TRUST_REPORT_TYPE = "zero_trust" SECURITY_REPORT_TYPE = "security" @@ -21,6 +22,7 @@ __author__ = ["itay.mizeretz", "shay.nehmad"] class Report(flask_restful.Resource): @jwt_required() + @profile() def get(self, report_type=SECURITY_REPORT_TYPE, report_data=None): if report_type == SECURITY_REPORT_TYPE: return ReportService.get_report() diff --git a/monkey/monkey_island/cc/resources/zero_trust/finding_event.py b/monkey/monkey_island/cc/resources/zero_trust/finding_event.py new file mode 100644 index 000000000..73cfa7f4c --- /dev/null +++ b/monkey/monkey_island/cc/resources/zero_trust/finding_event.py @@ -0,0 +1,14 @@ +import flask_restful +import json + +from monkey_island.cc.auth import jwt_required +from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService +from monkey_island.cc.testing.profiler_decorator import profile + + +class ZeroTrustFindingEvent(flask_restful.Resource): + + @jwt_required() + @profile() + def get(self, finding_id: str): + return {'events_json': json.dumps(ZeroTrustService.get_events_by_finding(finding_id), default=str)} diff --git a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py index 328be2e00..403967d8f 100644 --- a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py @@ -1,6 +1,7 @@ import common.data.zero_trust_consts as zero_trust_consts from monkey_island.cc.models.zero_trust.finding import Finding from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService +import monkey_island.cc.services.reporting.zero_trust_service from monkey_island.cc.testing.IslandTestCase import IslandTestCase EXPECTED_DICT = { @@ -316,6 +317,12 @@ class TestZeroTrustService(IslandTestCase): self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected) + def test_get_events_without_overlap(self): + monkey_island.cc.services.reporting.zero_trust_service.EVENT_FETCH_CNT = 5 + self.assertListEqual([], ZeroTrustService._ZeroTrustService__get_events_without_overlap(5, [1, 2, 3])) + self.assertListEqual([3], ZeroTrustService._ZeroTrustService__get_events_without_overlap(6, [1, 2, 3])) + self.assertListEqual([1, 2, 3, 4, 5], ZeroTrustService._ZeroTrustService__get_events_without_overlap(10, [1, 2, 3, 4, 5])) + def compare_lists_no_order(s, t): t = list(t) # make a mutable copy diff --git a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py index 5a2045da5..7ecac4e7f 100644 --- a/monkey/monkey_island/cc/services/reporting/zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/zero_trust_service.py @@ -1,9 +1,14 @@ -import json +from typing import List import common.data.zero_trust_consts as zero_trust_consts +from bson.objectid import ObjectId from monkey_island.cc.models.zero_trust.finding import Finding +# How many events of a single finding to return to UI. +# 50 will return 50 latest and 50 oldest events from a finding +EVENT_FETCH_CNT = 50 + class ZeroTrustService(object): @staticmethod @@ -104,25 +109,43 @@ class ZeroTrustService(object): @staticmethod def get_all_findings(): - all_findings = Finding.objects() + pipeline = [{'$match': {}}, + {'$addFields': {'oldest_events': {'$slice': ['$events', EVENT_FETCH_CNT]}, + 'latest_events': {'$slice': ['$events', -1*EVENT_FETCH_CNT]}, + 'event_count': {'$size': '$events'}}}, + {'$unset': ['events']}] + all_findings = list(Finding.objects.aggregate(*pipeline)) + for finding in all_findings: + finding['latest_events'] = ZeroTrustService.__get_events_without_overlap(finding['event_count'], + finding['latest_events']) + enriched_findings = [ZeroTrustService.__get_enriched_finding(f) for f in all_findings] return enriched_findings @staticmethod - def __get_enriched_finding(finding): - test_info = zero_trust_consts.TESTS_MAP[finding.test] - enriched_finding = { - "test": test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding.status], - "test_key": finding.test, - "pillars": test_info[zero_trust_consts.PILLARS_KEY], - "status": finding.status, - "events": ZeroTrustService.__get_events_as_dict(finding.events) - } - return enriched_finding + def __get_events_without_overlap(event_count: int, events: List[object]) -> List[object]: + overlap_count = event_count - EVENT_FETCH_CNT + if overlap_count >= EVENT_FETCH_CNT: + return events + elif overlap_count <= 0: + return [] + else: + return events[ -overlap_count :] @staticmethod - def __get_events_as_dict(events): - return [json.loads(event.to_json()) for event in events] + def __get_enriched_finding(finding): + test_info = zero_trust_consts.TESTS_MAP[finding['test']] + enriched_finding = { + 'finding_id': str(finding['_id']), + 'test': test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding['status']], + 'test_key': finding['test'], + 'pillars': test_info[zero_trust_consts.PILLARS_KEY], + 'status': finding['status'], + 'latest_events': finding['latest_events'], + 'oldest_events': finding['oldest_events'], + 'event_count': finding['event_count'] + } + return enriched_finding @staticmethod def get_statuses_to_pillars(): @@ -153,3 +176,11 @@ class ZeroTrustService(object): if grade[status] > 0: return status return zero_trust_consts.STATUS_UNEXECUTED + + @staticmethod + def get_events_by_finding(finding_id: str) -> List[object]: + pipeline = [{'$match': {'_id': ObjectId(finding_id)}}, + {'$unwind': '$events'}, + {'$project': {'events': '$events'}}, + {'$replaceRoot': {'newRoot': '$events'}}] + return list(Finding.objects.aggregate(*pipeline)) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js b/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js index 08c83babd..94d92b27a 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js @@ -24,7 +24,12 @@ export default class EventsButton extends Component { render() { return -