p15670423
/
monkey
1
0
Fork 5
Code Issues Pull Requests 3 Projects Releases Wiki Activity

Refactored ZT events sending and display on report to improve performance and UX

This commit is contained in:
VakarisZ 2020-05-06 16:52:50 +03:00
parent 4073e2f41f
commit 571682fff9
11 changed files with 145 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
monkey/monkey_island/cc/app.py
View File

@ -32,6 +32,7 @@ from monkey_island.cc.resources.pba_file_upload import FileUpload
from monkey_island.cc.resources.attack.attack_config import AttackConfiguration
from monkey_island.cc.resources.attack.attack_report import AttackReport
from monkey_island.cc.resources.bootloader import Bootloader
from monkey_island.cc.resources.zero_trust.finding_event import ZeroTrustFindingEvent
from monkey_island.cc.services.database import Database
from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService
from monkey_island.cc.services.representations import output_json
@ -107,6 +108,7 @@ def init_api_resources(api):
Report,
'/api/report/<string:report_type>',
'/api/report/<string:report_type>/<string:report_data>')
api.add_resource(ZeroTrustFindingEvent, '/api/zero-trust/finding-event/<string:finding_id>')
api.add_resource(TelemetryFeed, '/api/telemetry-feed', '/api/telemetry-feed/')
api.add_resource(Log, '/api/log', '/api/log/')

2
monkey/monkey_island/cc/resources/reporting/report.py
View File

@ -6,6 +6,7 @@ from flask import jsonify
from monkey_island.cc.auth import jwt_required
from monkey_island.cc.services.reporting.report import ReportService
from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
from monkey_island.cc.testing.profiler_decorator import profile
ZERO_TRUST_REPORT_TYPE = "zero_trust"
SECURITY_REPORT_TYPE = "security"
@ -21,6 +22,7 @@ __author__ = ["itay.mizeretz", "shay.nehmad"]
class Report(flask_restful.Resource):
@jwt_required()
@profile()
def get(self, report_type=SECURITY_REPORT_TYPE, report_data=None):
if report_type == SECURITY_REPORT_TYPE:
return ReportService.get_report()

14
monkey/monkey_island/cc/resources/zero_trust/finding_event.py Normal file
View File

@ -0,0 +1,14 @@
import flask_restful
import json
from monkey_island.cc.auth import jwt_required
from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
from monkey_island.cc.testing.profiler_decorator import profile
class ZeroTrustFindingEvent(flask_restful.Resource):
@jwt_required()
@profile()
def get(self, finding_id: str):
return {'events_json': json.dumps(ZeroTrustService.get_events_by_finding(finding_id), default=str)}

7
monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py
View File

@ -1,6 +1,7 @@
import common.data.zero_trust_consts as zero_trust_consts
from monkey_island.cc.models.zero_trust.finding import Finding
from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
import monkey_island.cc.services.reporting.zero_trust_service
from monkey_island.cc.testing.IslandTestCase import IslandTestCase
EXPECTED_DICT = {
@ -316,6 +317,12 @@ class TestZeroTrustService(IslandTestCase):
self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected)
def test_get_events_without_overlap(self):
monkey_island.cc.services.reporting.zero_trust_service.EVENT_FETCH_CNT = 5
self.assertListEqual([], ZeroTrustService._ZeroTrustService__get_events_without_overlap(5, [1, 2, 3]))
self.assertListEqual([3], ZeroTrustService._ZeroTrustService__get_events_without_overlap(6, [1, 2, 3]))
self.assertListEqual([1, 2, 3, 4, 5], ZeroTrustService._ZeroTrustService__get_events_without_overlap(10, [1, 2, 3, 4, 5]))
def compare_lists_no_order(s, t):
t = list(t) # make a mutable copy

59
monkey/monkey_island/cc/services/reporting/zero_trust_service.py
View File

@ -1,9 +1,14 @@
import json
from typing import List
import common.data.zero_trust_consts as zero_trust_consts
from bson.objectid import ObjectId
from monkey_island.cc.models.zero_trust.finding import Finding
# How many events of a single finding to return to UI.
# 50 will return 50 latest and 50 oldest events from a finding
EVENT_FETCH_CNT = 50
class ZeroTrustService(object):
@staticmethod
@ -104,25 +109,43 @@ class ZeroTrustService(object):
@staticmethod
def get_all_findings():
all_findings = Finding.objects()
pipeline = [{'$match': {}},
{'$addFields': {'oldest_events': {'$slice': ['$events', EVENT_FETCH_CNT]},
'latest_events': {'$slice': ['$events', -1*EVENT_FETCH_CNT]},
'event_count': {'$size': '$events'}}},
{'$unset': ['events']}]
all_findings = list(Finding.objects.aggregate(*pipeline))
for finding in all_findings:
finding['latest_events'] = ZeroTrustService.__get_events_without_overlap(finding['event_count'],
finding['latest_events'])
enriched_findings = [ZeroTrustService.__get_enriched_finding(f) for f in all_findings]
return enriched_findings
@staticmethod
def __get_enriched_finding(finding):
test_info = zero_trust_consts.TESTS_MAP[finding.test]
enriched_finding = {
"test": test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding.status],
"test_key": finding.test,
"pillars": test_info[zero_trust_consts.PILLARS_KEY],
"status": finding.status,
"events": ZeroTrustService.__get_events_as_dict(finding.events)
}
return enriched_finding
def __get_events_without_overlap(event_count: int, events: List[object]) -> List[object]:
overlap_count = event_count - EVENT_FETCH_CNT
if overlap_count >= EVENT_FETCH_CNT:
return events
elif overlap_count <= 0:
return []
else:
return events[ -overlap_count :]
@staticmethod
def __get_events_as_dict(events):
return [json.loads(event.to_json()) for event in events]
def __get_enriched_finding(finding):
test_info = zero_trust_consts.TESTS_MAP[finding['test']]
enriched_finding = {
'finding_id': str(finding['_id']),
'test': test_info[zero_trust_consts.FINDING_EXPLANATION_BY_STATUS_KEY][finding['status']],
'test_key': finding['test'],
'pillars': test_info[zero_trust_consts.PILLARS_KEY],
'status': finding['status'],
'latest_events': finding['latest_events'],
'oldest_events': finding['oldest_events'],
'event_count': finding['event_count']
}
return enriched_finding
@staticmethod
def get_statuses_to_pillars():
@ -153,3 +176,11 @@ class ZeroTrustService(object):
if grade[status] > 0:
return status
return zero_trust_consts.STATUS_UNEXECUTED
@staticmethod
def get_events_by_finding(finding_id: str) -> List[object]:
pipeline = [{'$match': {'_id': ObjectId(finding_id)}},
{'$unwind': '$events'},
{'$project': {'events': '$events'}},
{'$replaceRoot': {'newRoot': '$events'}}]
return list(Finding.objects.aggregate(*pipeline))

13
monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsButton.js
View File

@ -24,7 +24,12 @@ export default class EventsButton extends Component {
render() {
return <Fragment>
<EventsModal events={this.props.events} showEvents={this.state.isShow} hideCallback={this.hide}
<EventsModal finding_id={this.props.finding_id}
latest_events={this.props.latest_events}
oldest_events={this.props.oldest_events}
event_count={this.props.event_count}
showEvents={this.state.isShow}
hideCallback={this.hide}
exportFilename={this.props.exportFilename}/>
<div className="text-center" style={{'display': 'grid'}}>
<Button className="btn btn-primary btn-lg" onClick={this.show}>
@ -35,12 +40,14 @@ export default class EventsButton extends Component {
}
createEventsAmountBadge() {
const eventsAmountBadgeContent = this.props.events.length > 9 ? '9+' : this.props.events.length;
const eventsAmountBadgeContent = this.props.event_count > 9 ? '9+' : this.props.event_count;
return <Badge>{eventsAmountBadgeContent}</Badge>;
}
}
EventsButton.propTypes = {
events: PropTypes.array,
latest_events: PropTypes.array,
oldest_events: PropTypes.array,
event_count: PropTypes.number,
exportFilename: PropTypes.string
};

44
monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsModal.js
View File

@ -1,13 +1,17 @@
import React, {Component} from 'react';
import {Badge, Modal} from 'react-bootstrap';
import {Modal} from 'react-bootstrap';
import EventsTimeline from './EventsTimeline';
import * as PropTypes from 'prop-types';
import saveJsonToFile from '../../utils/SaveJsonToFile';
import EventsModalButtons from './EventsModalButtons';
import Pluralize from 'pluralize'
import {statusToLabelType} from './StatusLabel';
import AuthComponent from '../../AuthComponent';
import Pluralize from 'pluralize';
import SkippedEventsTimeline from "./SkippedEventsTimeline";
export default class EventsModal extends Component {
const FINDING_EVENTS_URL = '/api/zero-trust/finding-event/';
export default class EventsModal extends AuthComponent {
constructor(props) {
super(props);
}
@ -22,12 +26,20 @@ export default class EventsModal extends Component {
</h3>
<hr/>
<p>
There {Pluralize('is', this.props.events.length)} {<div
className={'label label-primary'}>{this.props.events.length}</div>} {Pluralize('event', this.props.events.length)} associated
There {Pluralize('is', this.props.event_count)} {<div
className={'label label-primary'}>{this.props.event_count}</div>}
{Pluralize('event', this.props.event_count)} associated
with this finding.
{<div className={'label label-primary'}>
{this.props.latest_events.length + this.props.oldest_events.length}
</div>} {Pluralize('is', this.props.event_count)} displayed below.
All events can be exported to json.
</p>
{this.props.events.length > 5 ? this.renderButtons() : null}
<EventsTimeline events={this.props.events}/>
{this.props.event_count > 5 ? this.renderButtons() : null}
<EventsTimeline events={this.props.oldest_events}/>
{this.props.event_count > this.props.latest_events.length+this.props.oldest_events.length ?
this.renderSkippedEventsTimeline() : null}
<EventsTimeline events={this.props.latest_events}/>
{this.renderButtons()}
</Modal.Body>
</Modal>
@ -35,13 +47,23 @@ export default class EventsModal extends Component {
);
}
renderSkippedEventsTimeline(){
return <div className={'skipped-events-timeline'}>
<SkippedEventsTimeline
skipped_count={this.props.event_count - this.props.latest_events.length+this.props.oldest_events.length}/>
</div>
}
renderButtons() {
return <EventsModalButtons
onClickClose={() => this.props.hideCallback()}
onClickExport={() => {
const dataToSave = this.props.events;
const filename = this.props.exportFilename;
saveJsonToFile(dataToSave, filename);
let full_url = FINDING_EVENTS_URL + this.props.finding_id;
this.authFetch(full_url).then(res => res.json()).then(res => {
const dataToSave = res.events_json;
const filename = this.props.exportFilename;
saveJsonToFile(dataToSave, filename);
});
}}/>;
}
}

2
monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/EventsTimeline.js
View File

@ -17,7 +17,7 @@ export default class EventsTimeline extends Component {
<Timeline style={{fontSize: '100%'}}>
{
this.props.events.map((event, index) => {
const event_time = new Date(event.timestamp['$date']).toString();
const event_time = new Date(event.timestamp).toString();
return (<TimelineEvent
key={index}
createdAt={event_time}

6
monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/FindingsTable.js
View File

@ -18,7 +18,11 @@ const columns = [
{
Header: 'Events', id: 'events',
accessor: x => {
return <EventsButton events={x.events} exportFilename={'Events_' + x.test_key}/>;
return <EventsButton finding_id={x.finding_id}
latest_events={x.latest_events}
oldest_events={x.oldest_events}
event_count={x.event_count}
exportFilename={'Events_' + x.test_key}/>;
},
maxWidth: EVENTS_COLUMN_MAX_WIDTH
},

26
monkey/monkey_island/cc/ui/src/components/report-components/zerotrust/SkippedEventsTimeline.js Normal file
View File

@ -0,0 +1,26 @@
import React, {Component} from 'react';
import {Timeline, TimelineEvent} from 'react-event-timeline';
import { faArrowsAltV } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import * as PropTypes from 'prop-types';
export default class SkippedEventsTimeline extends Component {
render() {
return (
<div>
<Timeline style={{fontSize: '100%'}}>
<TimelineEvent
bubbleStyle={{border: '2px solid #ffcc00'}}
title='Events in between are not displayed, but can be exported to JSON.'
icon={<FontAwesomeIcon className={'timeline-event-icon'} icon={faArrowsAltV}/>} >
{this.props.skipped_count} events not displayed.
</TimelineEvent>
</Timeline>
</div>
);
}
}
SkippedEventsTimeline.propTypes = {skipped_count: PropTypes.number};

3
monkey/monkey_island/linux/install_mongo.sh
View File

@ -17,9 +17,6 @@ elif [[ ${os_version_monkey} == "Ubuntu 18.04"* ]]; then
elif [[ ${os_version_monkey} == "Ubuntu 19.10"* ]]; then
echo Detected Ubuntu 19.10
export tgz_url="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.2.3.tgz"
elif [[ ${os_version_monkey} == "Debian GNU/Linux 8"* ]]; then
echo Detected Debian 8
export tgz_url="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-debian81-4.0.16.tgz"
elif [[ ${os_version_monkey} == "Debian GNU/Linux 9"* ]]; then
echo Detected Debian 9
export tgz_url="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-debian92-4.2.3.tgz"