Backend ScoutSuite backend code, which handles ScoutSuite data reception, parsing and storing

This commit is contained in:
VakarisZ 2020-09-18 10:13:27 +03:00
parent 0b9b89f639
commit 4440027699
20 changed files with 286 additions and 61 deletions

View File

@ -22,6 +22,11 @@ STATUS_FAILED = "Failed"
# Don't change order! The statuses are ordered by importance/severity.
ORDERED_TEST_STATUSES = [STATUS_FAILED, STATUS_VERIFY, STATUS_PASSED, STATUS_UNEXECUTED]
MONKEY_FINDING = "monkey_finding"
SCOUTSUITE_FINDING = "scoutsuite_finding"
FINDING_TYPES = [MONKEY_FINDING, SCOUTSUITE_FINDING]
TEST_DATA_ENDPOINT_ELASTIC = "unencrypted_data_endpoint_elastic"
TEST_DATA_ENDPOINT_HTTP = "unencrypted_data_endpoint_http"
TEST_MACHINE_EXPLOITED = "machine_exploited"
@ -31,6 +36,7 @@ TEST_MALICIOUS_ACTIVITY_TIMELINE = "malicious_activity_timeline"
TEST_SEGMENTATION = "segmentation"
TEST_TUNNELING = "tunneling"
TEST_COMMUNICATE_AS_NEW_USER = "communicate_as_new_user"
TEST_SCOUTSUITE_PERMISSIVE_FIREWALL_RULES = "scoutsuite_permissive_firewall_rules"
TESTS = (
TEST_SEGMENTATION,
TEST_MALICIOUS_ACTIVITY_TIMELINE,
@ -40,7 +46,8 @@ TESTS = (
TEST_DATA_ENDPOINT_HTTP,
TEST_DATA_ENDPOINT_ELASTIC,
TEST_TUNNELING,
TEST_COMMUNICATE_AS_NEW_USER
TEST_COMMUNICATE_AS_NEW_USER,
TEST_SCOUTSUITE_PERMISSIVE_FIREWALL_RULES
)
PRINCIPLE_DATA_TRANSIT = "data_transit"
@ -165,6 +172,16 @@ TESTS_MAP = {
PILLARS_KEY: [PEOPLE, NETWORKS, VISIBILITY_ANALYTICS],
POSSIBLE_STATUSES_KEY: [STATUS_UNEXECUTED, STATUS_FAILED, STATUS_PASSED]
},
TEST_SCOUTSUITE_PERMISSIVE_FIREWALL_RULES: {
TEST_EXPLANATION_KEY: "ScoutSuite assessed cloud firewall rules and settings.",
FINDING_EXPLANATION_BY_STATUS_KEY: {
STATUS_FAILED: "ScoutSuite found overly permissive firewall rules.",
STATUS_PASSED: "ScoutSuite found no problems with cloud firewall rules."
},
PRINCIPLE_KEY: PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES,
PILLARS_KEY: [NETWORKS],
POSSIBLE_STATUSES_KEY: [STATUS_UNEXECUTED, STATUS_FAILED, STATUS_PASSED]
},
}
EVENT_TYPE_MONKEY_NETWORK = "monkey_network"

View File

@ -1,5 +1,10 @@
# abstract, static method decorator
# noinspection PyPep8Naming
import operator
from functools import reduce
from typing import List
class abstractstatic(staticmethod):
__slots__ = ()
@ -8,3 +13,15 @@ class abstractstatic(staticmethod):
function.__isabstractmethod__ = True
__isabstractmethod__ = True
def _get_value_by_path(data, path: List[str]):
return reduce(operator.getitem, path, data)
def get_object_value_by_path(data_object: object, path: List[str]):
return _get_value_by_path(data_object, path)
def get_dict_value_by_path(data_dict: dict, path: List[str]):
return _get_value_by_path(data_dict, path)

View File

@ -2,16 +2,15 @@
"""
Define a Document Schema for Zero Trust findings.
"""
from typing import List
from typing import Union
from mongoengine import Document, StringField, GenericLazyReferenceField
import common.common_consts.zero_trust_consts as zero_trust_consts
# Dummy import for mongoengine.
# noinspection PyUnresolvedReferences
from monkey_island.cc.models.zero_trust.event import Event
from monkey_island.cc.models.zero_trust.monkey_finding_details import MonkeyFindingDetails
from monkey_island.cc.models.zero_trust.scoutsuite_finding_details import ScoutsuiteFindingDetails
from monkey_island.cc.models.zero_trust.scoutsuite_finding_details import ScoutSuiteFindingDetails
class Finding(Document):
@ -35,7 +34,8 @@ class Finding(Document):
# SCHEMA
test = StringField(required=True, choices=zero_trust_consts.TESTS)
status = StringField(required=True, choices=zero_trust_consts.ORDERED_TEST_STATUSES)
details = GenericLazyReferenceField(choices=[MonkeyFindingDetails, ScoutsuiteFindingDetails], required=True)
type = StringField(required=True, choices=zero_trust_consts.FINDING_TYPES)
details = GenericLazyReferenceField(choices=[MonkeyFindingDetails, ScoutSuiteFindingDetails], required=True)
# http://docs.mongoengine.org/guide/defining-documents.html#document-inheritance
meta = {'allow_inheritance': True}
@ -48,11 +48,19 @@ class Finding(Document):
# Creation methods
@staticmethod
def save_finding(test: str, status: str, detail_ref):
finding = Finding(test=test,
status=status,
details=detail_ref)
def save_finding(test: str,
status: str,
detail_ref: Union[MonkeyFindingDetails, ScoutSuiteFindingDetails]):
temp_finding = Finding(test=test,
status=status,
details=detail_ref,
type=Finding._get_finding_type_by_details(detail_ref))
temp_finding.save()
return temp_finding
finding.save()
return finding
@staticmethod
def _get_finding_type_by_details(details: Union[MonkeyFindingDetails, ScoutSuiteFindingDetails]) -> str:
if type(details) == MonkeyFindingDetails:
return zero_trust_consts.MONKEY_FINDING
else:
return zero_trust_consts.SCOUTSUITE_FINDING

View File

@ -0,0 +1,19 @@
from mongoengine import Document, DynamicField
class ScoutSuiteDataJson(Document):
"""
This model is a container for ScoutSuite report data dump.
"""
# SCHEMA
scoutsuite_data = DynamicField(required=True)
# LOGIC
@staticmethod
def add_scoutsuite_data(scoutsuite_data: str) -> None:
current_data = ScoutSuiteDataJson.objects()
if not current_data:
current_data = ScoutSuiteDataJson()
current_data.scoutsuite_data = scoutsuite_data
current_data.save()

View File

@ -1,17 +0,0 @@
from datetime import datetime
from mongoengine import DateTimeField, EmbeddedDocument, StringField
class ScoutsuiteFinding(EmbeddedDocument):
# SCHEMA
temp = StringField(required=True)
# LOGIC
@staticmethod
def create_scoutsuite_finding(title, message, event_type, timestamp=None):
scoutsuite_finding = ScoutsuiteFinding()
scoutsuite_finding.temp = "temp"
return scoutsuite_finding

View File

@ -1,11 +1,9 @@
from typing import List
from mongoengine import Document, EmbeddedDocumentListField
from mongoengine import DateTimeField, Document, StringField, EmbeddedDocumentListField
from monkey_island.cc.models.zero_trust.scoutsuite_finding import ScoutsuiteFinding
from monkey_island.cc.models.zero_trust.scoutsuite_rule import ScoutSuiteRule
class ScoutsuiteFindingDetails(Document):
class ScoutSuiteFindingDetails(Document):
"""
This model represents additional information about monkey finding:
Events if monkey finding
@ -13,7 +11,9 @@ class ScoutsuiteFindingDetails(Document):
"""
# SCHEMA
scoutsuite_findings = EmbeddedDocumentListField(document_type=ScoutsuiteFinding, required=False)
scoutsuite_rules = EmbeddedDocumentListField(document_type=ScoutSuiteRule, required=False)
def add_scoutsuite_findings(self, scoutsuite_findings: List[ScoutsuiteFinding]) -> None:
self.update(push_all__scoutsuite_findings=scoutsuite_findings)
def add_rule(self, rule: ScoutSuiteRule) -> None:
if rule not in self.scoutsuite_rules:
self.scoutsuite_rules.append(rule)
self.save()

View File

@ -0,0 +1,26 @@
from mongoengine import StringField, EmbeddedDocument, ListField, \
IntField
from monkey_island.cc.services.zero_trust.scoutsuite.consts import rule_consts
class ScoutSuiteRule(EmbeddedDocument):
"""
This model represents additional information about monkey finding:
Events if monkey finding
Scoutsuite findings if scoutsuite finding
"""
# SCHEMA
description = StringField(required=True)
path = StringField(required=True)
level = StringField(required=True, options=rule_consts.RULE_LEVELS)
items = ListField()
dashboard_name = StringField(required=True)
checked_items = IntField(min_value=0)
flagged_items = IntField(min_value=0)
service = StringField(required=True)
rationale = StringField(required=True)
remediation = StringField(required=False)
compliance = StringField(required=False)
references = ListField(required=False)

View File

@ -1,11 +1,12 @@
import http.client
import flask_restful
from flask import jsonify
from flask import jsonify, Response
from monkey_island.cc.models.zero_trust.scoutsuite_data_json import ScoutSuiteDataJson
from monkey_island.cc.resources.auth.auth import jwt_required
from monkey_island.cc.services.reporting.report import ReportService
from monkey_island.cc.services.zero_trust.monkey_finding_service import MonkeyFindingService
from monkey_island.cc.services.zero_trust.finding_service import FindingService
from monkey_island.cc.services.zero_trust.zero_trust_service import \
ZeroTrustService
@ -16,6 +17,7 @@ REPORT_TYPES = [SECURITY_REPORT_TYPE, ZERO_TRUST_REPORT_TYPE]
REPORT_DATA_PILLARS = "pillars"
REPORT_DATA_FINDINGS = "findings"
REPORT_DATA_PRINCIPLES_STATUS = "principles"
REPORT_DATA_SCOUTSUITE = "scoutsuite"
__author__ = ["itay.mizeretz", "shay.nehmad"]
@ -36,6 +38,12 @@ class Report(flask_restful.Resource):
elif report_data == REPORT_DATA_PRINCIPLES_STATUS:
return jsonify(ZeroTrustService.get_principles_status())
elif report_data == REPORT_DATA_FINDINGS:
return jsonify(MonkeyFindingService.get_all_monkey_findings())
return jsonify(FindingService.get_all_findings())
elif report_data == REPORT_DATA_SCOUTSUITE:
try:
data = ScoutSuiteDataJson.objects.get().scoutsuite_data
except Exception:
data = {}
return Response(data, mimetype='application/json')
flask_restful.abort(http.client.NOT_FOUND)

View File

@ -1,15 +1,32 @@
import json
from monkey_island.cc.database import mongo
from monkey_island.cc.models.zero_trust.scoutsuite_data_json import ScoutSuiteDataJson
from monkey_island.cc.services.zero_trust.scoutsuite.consts.findings_list import SCOUTSUITE_FINDINGS
from monkey_island.cc.services.zero_trust.scoutsuite.data_parsing.rule_parsing import RuleParser
from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_finding_service import ScoutSuiteFindingService
from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_rule_service import ScoutSuiteRuleService
def process_scoutsuite_telemetry(telemetry_json):
# Encode data to json, because mongo can't save it as document (invalid document keys)
telemetry_json['data'] = json.dumps(telemetry_json['data'])
ScoutSuiteDataJson.add_scoutsuite_data(telemetry_json['data'])
scoutsuite_data = json.loads(telemetry_json['data'])['data']
create_scoutsuite_findings(scoutsuite_data)
update_data(telemetry_json)
def create_scoutsuite_findings(scoutsuite_data):
for finding in SCOUTSUITE_FINDINGS:
for rule in finding.rules:
rule_data = RuleParser.get_rule_data(scoutsuite_data, rule)
rule = ScoutSuiteRuleService.get_rule_from_rule_data(rule_data)
ScoutSuiteFindingService.process_rule(finding, rule)
def update_data(telemetry_json):
mongo.db.scoutsuite.update(
mongo.db.scoutsuite.insert_one(
{'guid': telemetry_json['monkey_guid']},
{'$push': {'results': telemetry_json['data']}})
{'results': telemetry_json['data']})

View File

@ -18,9 +18,11 @@ class EventsService:
'latest_events': {'$slice': ['$events', -1 * EVENT_FETCH_CNT]},
'event_count': {'$size': '$events'}}},
{'$unset': ['events']}]
details = MonkeyFindingDetails.objects.aggregate(*pipeline).next()
details['latest_events'] = EventsService._get_events_without_overlap(details['event_count'],
details['latest_events'])
details = list(MonkeyFindingDetails.objects.aggregate(*pipeline))
if details:
details = details[0]
details['latest_events'] = EventsService._get_events_without_overlap(details['event_count'],
details['latest_events'])
return details
@staticmethod

View File

@ -2,13 +2,24 @@ from typing import List
from common.common_consts import zero_trust_consts
from monkey_island.cc.models.zero_trust.finding import Finding
from monkey_island.cc.services.zero_trust.events_service import EventsService
class FindingService:
@staticmethod
def get_all_findings() -> List[Finding]:
return list(Finding.objects)
findings = list(Finding.objects)
details = []
for i in range(len(findings)):
if findings[i].type == zero_trust_consts.MONKEY_FINDING:
details = EventsService.fetch_events_for_display(findings[i].details.id)
elif findings[i].type == zero_trust_consts.SCOUTSUITE_FINDING:
details = findings[i].details.fetch().to_mongo()
findings[i] = findings[i].to_mongo()
findings[i] = FindingService.get_enriched_finding(findings[i])
findings[i]['details'] = details
return findings
@staticmethod
def get_enriched_finding(finding):
@ -19,5 +30,6 @@ class FindingService:
'test_key': finding['test'],
'pillars': test_info[zero_trust_consts.PILLARS_KEY],
'status': finding['status'],
'type': finding['type']
}
return enriched_finding

View File

@ -6,7 +6,6 @@ from common.common_consts import zero_trust_consts
from monkey_island.cc.models.zero_trust.event import Event
from monkey_island.cc.models.zero_trust.finding import Finding
from monkey_island.cc.models.zero_trust.monkey_finding_details import MonkeyFindingDetails
from monkey_island.cc.services.zero_trust.finding_service import FindingService
class MonkeyFindingService:
@ -38,17 +37,7 @@ class MonkeyFindingService:
@staticmethod
def add_events(finding: Finding, events: List[Event]):
finding.details.fetch().add_events(events)
@staticmethod
def get_all_monkey_findings():
findings = FindingService.get_all_findings()
for i in range(len(findings)):
details = MonkeyFindingService.fetch_events_for_display(findings[i].details.id)
findings[i] = findings[i].to_mongo()
findings[i] = FindingService.get_enriched_finding(findings[i])
findings[i]['details'] = details
return findings
finding.details.fetch().add_events(events).save()
@staticmethod
def get_events_by_finding(finding_id: str) -> List[object]:

View File

@ -0,0 +1,19 @@
from common.common_consts import zero_trust_consts
from common.common_consts.zero_trust_consts import NETWORKS
from monkey_island.cc.services.zero_trust.scoutsuite.consts.ec2_rules import EC2Rules
class PERMISSIVE_FIREWALL_RULES:
rules = [EC2Rules.SECURITY_GROUP_ALL_PORTS_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_TCP_PORT_TO_ALL,
EC2Rules.SECURITY_GROUP_OPENS_UDP_PORT_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_RDP_PORT_TO_ALL,
EC2Rules.SECURITY_GROUP_OPENS_SSH_PORT_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_MYSQL_PORT_TO_ALL,
EC2Rules.SECURITY_GROUP_OPENS_MSSQL_PORT_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_MONGODB_PORT_TO_ALL,
EC2Rules.SECURITY_GROUP_OPENS_ORACLE_DB_PORT_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_POSTGRESQL_PORT_TO_ALL,
EC2Rules.SECURITY_GROUP_OPENS_NFS_PORT_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_SMTP_PORT_TO_ALL,
EC2Rules.SECURITY_GROUP_OPENS_DNS_PORT_TO_ALL, EC2Rules.SECURITY_GROUP_OPENS_ALL_PORTS_TO_SELF,
EC2Rules.SECURITY_GROUP_OPENS_ALL_PORTS, EC2Rules.SECURITY_GROUP_OPENS_PLAINTEXT_PORT_FTP,
EC2Rules.SECURITY_GROUP_OPENS_PLAINTEXT_PORT_TELNET, EC2Rules.SECURITY_GROUP_OPENS_PORT_RANGE]
pillars = [NETWORKS]
test = zero_trust_consts.TEST_SCOUTSUITE_PERMISSIVE_FIREWALL_RULES

View File

@ -0,0 +1,3 @@
from monkey_island.cc.services.zero_trust.scoutsuite.consts.findings import PERMISSIVE_FIREWALL_RULES
SCOUTSUITE_FINDINGS = [PERMISSIVE_FIREWALL_RULES]

View File

@ -0,0 +1,4 @@
RULE_LEVEL_DANGER = 'danger'
RULE_LEVEL_WARNING = 'warning'
RULE_LEVELS = (RULE_LEVEL_DANGER, RULE_LEVEL_WARNING)

View File

@ -0,0 +1,9 @@
from enum import Enum
SERVICES = 'services'
FINDINGS = 'findings'
class SERVICE_TYPES(Enum):
EC2 = 'ec2'

View File

@ -0,0 +1,65 @@
from typing import List
from common.common_consts import zero_trust_consts
from monkey_island.cc.models.zero_trust.finding import Finding
from monkey_island.cc.models.zero_trust.scoutsuite_finding_details import ScoutSuiteFindingDetails
from monkey_island.cc.models.zero_trust.scoutsuite_rule import ScoutSuiteRule
from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_rule_service import ScoutSuiteRuleService
class ScoutSuiteFindingService:
@staticmethod
# TODO add type hinting like finding: Union[SCOUTSUITE_FINDINGS]?
def process_rule(finding, rule: ScoutSuiteRule):
existing_findings = Finding.objects(test=finding.test, type=zero_trust_consts.SCOUTSUITE_FINDING)
assert (len(existing_findings) < 2), "More than one finding exists for {}".format(finding.test)
if len(existing_findings) == 0:
ScoutSuiteFindingService.create_new_finding_from_rule(finding, rule)
else:
ScoutSuiteFindingService.add_rule(existing_findings[0], rule)
@staticmethod
def create_new_finding_from_rule(finding, rule: ScoutSuiteRule):
details = ScoutSuiteFindingDetails()
details.scoutsuite_rules = [rule]
details.save()
status = ScoutSuiteFindingService.get_finding_status_from_rules(details.scoutsuite_rules)
Finding.save_finding(finding.test, status, details)
@staticmethod
def get_finding_status_from_rules(rules: List[ScoutSuiteRule]) -> str:
if len(rules) == 0:
return zero_trust_consts.STATUS_UNEXECUTED
elif filter(lambda x: ScoutSuiteRuleService.is_rule_dangerous(x), rules):
return zero_trust_consts.STATUS_FAILED
elif filter(lambda x: ScoutSuiteRuleService.is_rule_warning(x), rules):
return zero_trust_consts.STATUS_VERIFY
else:
return zero_trust_consts.STATUS_PASSED
@staticmethod
def add_rule(finding: Finding, rule: ScoutSuiteRule):
ScoutSuiteFindingService.change_finding_status_by_rule(finding, rule)
finding.save()
finding.details.fetch().add_rule(rule)
@staticmethod
def change_finding_status_by_rule(finding: Finding, rule: ScoutSuiteRule):
rule_status = ScoutSuiteFindingService.get_finding_status_from_rules([rule])
finding_status = finding.status
new_finding_status = ScoutSuiteFindingService.get_finding_status_from_rule_status(finding_status, rule_status)
if finding_status != new_finding_status:
finding.status = new_finding_status
@staticmethod
def get_finding_status_from_rule_status(finding_status: str, rule_status: str) -> str:
if finding_status == zero_trust_consts.STATUS_FAILED or rule_status == zero_trust_consts.STATUS_FAILED:
return zero_trust_consts.STATUS_FAILED
elif finding_status == zero_trust_consts.STATUS_VERIFY or rule_status == zero_trust_consts.STATUS_VERIFY:
return zero_trust_consts.STATUS_VERIFY
elif finding_status == zero_trust_consts.STATUS_PASSED or rule_status == zero_trust_consts.STATUS_PASSED:
return zero_trust_consts.STATUS_PASSED
else:
return zero_trust_consts.STATUS_UNEXECUTED

View File

@ -0,0 +1,30 @@
from monkey_island.cc.models.zero_trust.scoutsuite_rule import ScoutSuiteRule
from monkey_island.cc.services.zero_trust.scoutsuite.consts import rule_consts
class ScoutSuiteRuleService:
@staticmethod
def get_rule_from_rule_data(rule_data: dict) -> ScoutSuiteRule:
rule = ScoutSuiteRule()
rule.description = rule_data['description']
rule.path = rule_data['path']
rule.level = rule_data['level']
rule.items = rule_data['items']
rule.dashboard_name = rule_data['dashboard_name']
rule.checked_items = rule_data['checked_items']
rule.flagged_items = rule_data['flagged_items']
rule.service = rule_data['service']
rule.rationale = rule_data['rationale']
rule.remediation = rule_data['remediation']
rule.compliance = rule_data['compliance']
rule.references = rule_data['references']
return rule
@staticmethod
def is_rule_dangerous(rule: ScoutSuiteRule):
return rule.level == rule_consts.RULE_LEVEL_DANGER and len(rule.items) != 0
@staticmethod
def is_rule_warning(rule: ScoutSuiteRule):
return rule.level == rule_consts.RULE_LEVEL_WARNING and len(rule.items) != 0

View File

@ -1,3 +0,0 @@
class ScoutsuiteFindingService:
pass