From c6fd7ae5e8d63cb728aec3471128f9a1443a7605 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Mar 2021 09:11:45 +0200 Subject: [PATCH 1/6] Refactored ScoutSuite into a separate package rather than submodule --- .gitmodules | 3 --- monkey/common/cloud/scoutsuite | 1 - monkey/infection_monkey/monkey.spec | 27 ++++++++++++------- monkey/infection_monkey/requirements.txt | 1 + .../scoutsuite_collector/__init__.py | 15 ----------- .../scoutsuite_collector/scoutsuite_api.py | 5 ---- .../scoutsuite_collector.py | 13 ++++----- .../telemetry/scoutsuite_telem.py | 4 +-- .../zero_trust/scoutsuite/__init__.py | 13 --------- .../scoutsuite/scoutsuite_auth_service.py | 2 +- monkey/monkey_island/requirements.txt | 1 + 11 files changed, 29 insertions(+), 56 deletions(-) delete mode 160000 monkey/common/cloud/scoutsuite delete mode 100644 monkey/infection_monkey/system_info/collectors/scoutsuite_collector/__init__.py delete mode 100644 monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_api.py delete mode 100644 monkey/monkey_island/cc/services/zero_trust/scoutsuite/__init__.py diff --git a/.gitmodules b/.gitmodules index b898f160a..2fb33dd37 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,3 @@ [submodule "docs/themes/learn"] path = docs/themes/learn url = https://github.com/guardicode/hugo-theme-learn.git -[submodule "monkey/infection_monkey/system_info/collectors/scoutsuite"] - path = monkey/common/cloud/scoutsuite - url = https://github.com/guardicode/ScoutSuite.git diff --git a/monkey/common/cloud/scoutsuite b/monkey/common/cloud/scoutsuite deleted file mode 160000 index 9de1e78ba..000000000 --- a/monkey/common/cloud/scoutsuite +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9de1e78ba475f925c66c5b645564ec9eb08e2309 diff --git a/monkey/infection_monkey/monkey.spec b/monkey/infection_monkey/monkey.spec index 51bd4bb83..2031ee92a 100644 --- a/monkey/infection_monkey/monkey.spec +++ b/monkey/infection_monkey/monkey.spec @@ -1,15 +1,17 @@ # -*- mode: python -*- import os -import sys import platform - +import sys __author__ = 'itay.mizeretz' +from PyInstaller.utils.hooks import collect_data_files + block_cipher = None def main(): + print(collect_data_files('policyuniverse')) a = Analysis(['main.py'], pathex=['..'], hiddenimports=get_hidden_imports(), @@ -17,7 +19,7 @@ def main(): runtime_hooks=None, binaries=None, datas=[ - ("../common/BUILD", "/common") + ("../common/BUILD", "/common") ], excludes=None, win_no_prefer_redirects=None, @@ -48,7 +50,7 @@ def is_windows(): def is_32_bit(): - return sys.maxsize <= 2**32 + return sys.maxsize <= 2 ** 32 def get_bin_folder(): @@ -79,7 +81,12 @@ def get_linux_only_binaries(): def get_hidden_imports(): - return ['_cffi_backend', 'queue', '_mssql'] if is_windows() else ['_cffi_backend','_mssql'] + imports = ['ScoutSuite'] + if is_windows(): + imports.extend(['_cffi_backend', 'queue', '_mssql']) + else: + imports.extend(['_cffi_backend', '_mssql']) + return imports def get_sc_binaries(): @@ -94,15 +101,15 @@ def get_traceroute_binaries(): def get_monkey_filename(): name = 'monkey-' if is_windows(): - name = name+"windows-" + name = name + "windows-" else: - name = name+"linux-" + name = name + "linux-" if is_32_bit(): - name = name+"32" + name = name + "32" else: - name = name+"64" + name = name + "64" if is_windows(): - name = name+".exe" + name = name + ".exe" return name diff --git a/monkey/infection_monkey/requirements.txt b/monkey/infection_monkey/requirements.txt index dc0ab227e..e478095ac 100644 --- a/monkey/infection_monkey/requirements.txt +++ b/monkey/infection_monkey/requirements.txt @@ -16,3 +16,4 @@ pypykatz==0.3.12 pysmb==1.2.5 requests>=2.24 wmi==1.5.1 ; sys_platform == 'win32' +git+https://github.com/guardicode/ScoutSuite diff --git a/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/__init__.py b/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/__init__.py deleted file mode 100644 index 97e736b4b..000000000 --- a/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -import pkgutil -import sys -from pathlib import PurePath - -_scoutsuite_api_package = pkgutil.get_loader('common.cloud.scoutsuite.ScoutSuite.__main__') - - -def _add_scoutsuite_to_python_path(): - scoutsuite_path = PurePath(_scoutsuite_api_package.path).parent.parent.__str__() - sys.path.append(scoutsuite_path) - - -# Add ScoutSuite to python path because this way -# we don't need to change any imports in ScoutSuite code -_add_scoutsuite_to_python_path() diff --git a/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_api.py b/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_api.py deleted file mode 100644 index 88ef32293..000000000 --- a/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_api.py +++ /dev/null @@ -1,5 +0,0 @@ -import common.cloud.scoutsuite.ScoutSuite.api_run as scoutsuite_api - - -def run(*args, **kwargs): - return scoutsuite_api.run(*args, **kwargs) diff --git a/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_collector.py b/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_collector.py index c637e3593..79aabea56 100644 --- a/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_collector.py +++ b/monkey/infection_monkey/system_info/collectors/scoutsuite_collector/scoutsuite_collector.py @@ -1,8 +1,9 @@ import logging from typing import Union -import infection_monkey.system_info.collectors.scoutsuite_collector.scoutsuite_api as scoutsuite_api -from common.cloud.scoutsuite.ScoutSuite.providers.base.provider import BaseProvider +import ScoutSuite.api_run +from ScoutSuite.providers.base.provider import BaseProvider + from common.cloud.scoutsuite_consts import CloudProviders from common.utils.exceptions import ScoutSuiteScanError from infection_monkey.config import WormConfiguration @@ -22,10 +23,10 @@ def scan_cloud_security(cloud_type: CloudProviders): def run_scoutsuite(cloud_type: str) -> Union[BaseProvider, dict]: - return scoutsuite_api.run(provider=cloud_type, - aws_access_key_id=WormConfiguration.aws_access_key_id, - aws_secret_access_key=WormConfiguration.aws_secret_access_key, - aws_session_token=WormConfiguration.aws_session_token) + return ScoutSuite.api_run.run(provider=cloud_type, + aws_access_key_id=WormConfiguration.aws_access_key_id, + aws_secret_access_key=WormConfiguration.aws_secret_access_key, + aws_session_token=WormConfiguration.aws_session_token) def send_scoutsuite_run_results(run_results: BaseProvider): diff --git a/monkey/infection_monkey/telemetry/scoutsuite_telem.py b/monkey/infection_monkey/telemetry/scoutsuite_telem.py index 16cf47bdd..ba112f8b9 100644 --- a/monkey/infection_monkey/telemetry/scoutsuite_telem.py +++ b/monkey/infection_monkey/telemetry/scoutsuite_telem.py @@ -1,5 +1,5 @@ -from common.cloud.scoutsuite.ScoutSuite.output.result_encoder import ScoutJsonEncoder -from common.cloud.scoutsuite.ScoutSuite.providers.base.provider import BaseProvider +from ScoutSuite.output.result_encoder import ScoutJsonEncoder +from ScoutSuite.providers.base.provider import BaseProvider from common.common_consts.telem_categories import TelemCategoryEnum from infection_monkey.telemetry.base_telem import BaseTelem diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/__init__.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/__init__.py deleted file mode 100644 index e8a36338b..000000000 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -import pkgutil -import sys -from pathlib import PurePath - -_scoutsuite_api_package = pkgutil.get_loader('common.cloud.scoutsuite.ScoutSuite.__main__') - - -def _add_scoutsuite_to_python_path(): - scoutsuite_path = PurePath(_scoutsuite_api_package.path).parent.parent.__str__() - sys.path.append(scoutsuite_path) - - -_add_scoutsuite_to_python_path() diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py index b5d405234..701598168 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/scoutsuite_auth_service.py @@ -14,7 +14,7 @@ def is_cloud_authentication_setup(provider: CloudProviders) -> Tuple[bool, str]: if is_aws_keys_setup(): return True, "AWS keys already setup." - import common.cloud.scoutsuite.ScoutSuite.providers.aws.authentication_strategy as auth_strategy + import ScoutSuite.providers.aws.authentication_strategy as auth_strategy try: profile = auth_strategy.AWSAuthenticationStrategy().authenticate() return True, f" Profile \"{profile.session.profile_name}\" is already setup. " diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index 3cb3a4e42..ddbf2418a 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -26,5 +26,6 @@ tqdm>=4.47 virtualenv>=20.0.26 werkzeug>=1.0.1 wheel>=0.34.2 +git+https://github.com/guardicode/ScoutSuite pyjwt>=1.5.1 # not directly required, pinned by Snyk to avoid a vulnerability From 2925815fd3d6a098244b5c45925a95660e1a538c Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 17 Mar 2021 10:20:17 +0200 Subject: [PATCH 2/6] Removed ScoutSuite from travis, since it's no longer a submodule --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 668d9cdc3..8ac8db204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,6 @@ install: - pip install flake8 pytest pytest-cov dlint isort # for next stages - pip install coverage # for code coverage - pip install -r monkey/infection_monkey/requirements.txt # for unit tests -- pip install -r monkey/common/cloud/scoutsuite/requirements.txt - pip install pipdeptree # Fail builds on possible conflicting dependencies. - pipdeptree --warn fail @@ -56,7 +55,7 @@ install: script: # Check Python code ## Check syntax errors and fail the build if any are found. -- flake8 ./monkey --exclude=monkey/common/cloud/scoutsuite --config=./ci_scripts/flake8_syntax_check.ini +- flake8 ./monkey --config=./ci_scripts/flake8_syntax_check.ini ## Warn about linter issues. ### --exit-zero forces Flake8 to use the exit status code 0 even if there are errors, which means this will NOT fail the build. From 9e27a93a3b0fd1b079257e90206dbd12a87663b0 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 17 Mar 2021 12:45:08 +0200 Subject: [PATCH 3/6] Minor spec file style improvement --- monkey/infection_monkey/monkey.spec | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monkey/infection_monkey/monkey.spec b/monkey/infection_monkey/monkey.spec index 2031ee92a..f9b656611 100644 --- a/monkey/infection_monkey/monkey.spec +++ b/monkey/infection_monkey/monkey.spec @@ -18,9 +18,7 @@ def main(): hookspath=['./pyinstaller_hooks'], runtime_hooks=None, binaries=None, - datas=[ - ("../common/BUILD", "/common") - ], + datas=[("../common/BUILD", "/common")], excludes=None, win_no_prefer_redirects=None, win_private_assemblies=None, From aaf6a33f928e917c85a5f81d55428fe7be2e6918 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Mar 2021 16:51:21 +0200 Subject: [PATCH 4/6] Small profiling decorator improvement --- monkey/monkey_island/cc/test_common/profiling/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/test_common/profiling/README.md b/monkey/monkey_island/cc/test_common/profiling/README.md index 1c1446b2f..d0cb92bfa 100644 --- a/monkey/monkey_island/cc/test_common/profiling/README.md +++ b/monkey/monkey_island/cc/test_common/profiling/README.md @@ -2,7 +2,7 @@ To profile specific methods on island a `@profile(sort_args=['cumulative'], print_args=[100])` decorator can be used. -Use it as any other decorator. After decorated method is used, a file will appear in a +Use it as a parameterised decorator(`@profile()`). After decorated method is used, a file will appear in a directory provided in `profiler_decorator.py`. Filename describes the path of the method that was profiled. For example if method `monkey_island/cc/resources/netmap.get` was profiled, then the results of this profiling will appear in From 2c1e89c7b35a58809998621e62ea1f1e62506485 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 17 Mar 2021 10:02:13 +0200 Subject: [PATCH 5/6] ScoutSuite performance upgrades. --- monkey/common/utils/code_utils.py | 10 ++++++++++ .../cc/services/telemetry/processing/scoutsuite.py | 7 ++++--- .../zero_trust/scoutsuite/data_parsing/rule_parser.py | 5 ++--- .../rule_path_building/abstract_rule_path_creator.py | 4 ++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/monkey/common/utils/code_utils.py b/monkey/common/utils/code_utils.py index 214e6d108..d9ad573b1 100644 --- a/monkey/common/utils/code_utils.py +++ b/monkey/common/utils/code_utils.py @@ -1,5 +1,8 @@ # abstract, static method decorator # noinspection PyPep8Naming +from typing import List + + class abstractstatic(staticmethod): __slots__ = () @@ -8,3 +11,10 @@ class abstractstatic(staticmethod): function.__isabstractmethod__ = True __isabstractmethod__ = True + + +def get_value_from_dict(dict_data: dict, path: List[str]): + current_data = dict_data + for key in path: + current_data = current_data[key] + return current_data diff --git a/monkey/monkey_island/cc/services/telemetry/processing/scoutsuite.py b/monkey/monkey_island/cc/services/telemetry/processing/scoutsuite.py index 8ee4737e8..9160861ea 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/scoutsuite.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/scoutsuite.py @@ -3,6 +3,7 @@ import json from monkey_island.cc.database import mongo from monkey_island.cc.models.zero_trust.scoutsuite_data_json import ScoutSuiteRawDataJson from monkey_island.cc.services.zero_trust.scoutsuite.consts.scoutsuite_findings_list import SCOUTSUITE_FINDINGS +from monkey_island.cc.services.zero_trust.scoutsuite.consts.service_consts import SERVICES from monkey_island.cc.services.zero_trust.scoutsuite.data_parsing.rule_parser import RuleParser from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_rule_service import ScoutSuiteRuleService from monkey_island.cc.services.zero_trust.scoutsuite.scoutsuite_zt_finding_service import ScoutSuiteZTFindingService @@ -13,14 +14,14 @@ def process_scoutsuite_telemetry(telemetry_json): telemetry_json['data'] = json.dumps(telemetry_json['data']) ScoutSuiteRawDataJson.add_scoutsuite_data(telemetry_json['data']) scoutsuite_data = json.loads(telemetry_json['data'])['data'] - create_scoutsuite_findings(scoutsuite_data) + create_scoutsuite_findings(scoutsuite_data[SERVICES]) update_data(telemetry_json) -def create_scoutsuite_findings(scoutsuite_data): +def create_scoutsuite_findings(cloud_services: dict): for finding in SCOUTSUITE_FINDINGS: for rule in finding.rules: - rule_data = RuleParser.get_rule_data(scoutsuite_data, rule) + rule_data = RuleParser.get_rule_data(cloud_services, rule) rule = ScoutSuiteRuleService.get_rule_from_rule_data(rule_data) ScoutSuiteZTFindingService.process_rule(finding, rule) diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_parser.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_parser.py index e07431541..935f1c989 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_parser.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_parser.py @@ -1,7 +1,6 @@ from enum import Enum -import dpath.util - +from common.utils.code_utils import get_value_from_dict from common.utils.exceptions import RulePathCreatorNotFound from monkey_island.cc.services.zero_trust.scoutsuite.data_parsing.rule_path_building.rule_path_creators_list import \ RULE_PATH_CREATORS_LIST @@ -23,7 +22,7 @@ class RuleParser: @staticmethod def get_rule_data(scoutsuite_data: dict, rule_name: Enum) -> dict: rule_path = RuleParser._get_rule_path(rule_name) - return dpath.util.get(scoutsuite_data, rule_path) + return get_value_from_dict(scoutsuite_data, rule_path) @staticmethod def _get_rule_path(rule_name: Enum): diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/abstract_rule_path_creator.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/abstract_rule_path_creator.py index b4767124b..ee7f7c38b 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/abstract_rule_path_creator.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/abstract_rule_path_creator.py @@ -3,7 +3,7 @@ from enum import Enum from typing import List, Type from monkey_island.cc.services.zero_trust.scoutsuite.consts.rule_names.rule_name_enum import RuleNameEnum -from monkey_island.cc.services.zero_trust.scoutsuite.consts.service_consts import FINDINGS, SERVICES, SERVICE_TYPES +from monkey_island.cc.services.zero_trust.scoutsuite.consts.service_consts import FINDINGS, SERVICE_TYPES class AbstractRulePathCreator(ABC): @@ -21,4 +21,4 @@ class AbstractRulePathCreator(ABC): @classmethod def build_rule_path(cls, rule_name: Enum) -> List[str]: assert(rule_name in cls.supported_rules) - return [SERVICES, cls.service_type.value, FINDINGS, rule_name.value] + return [cls.service_type.value, FINDINGS, rule_name.value] From 4cd105abe4f02bc15dbd4bdde419232d0f67aff1 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 18 Mar 2021 09:31:45 +0200 Subject: [PATCH 6/6] Fixed ScoutSuite unit test --- .../zero_trust/scoutsuite/data_parsing/test_rule_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/test_rule_parser.py b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/test_rule_parser.py index 5a7572eb0..afe14c54c 100644 --- a/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/test_rule_parser.py +++ b/monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/test_rule_parser.py @@ -4,6 +4,7 @@ import pytest from common.utils.exceptions import RulePathCreatorNotFound from monkey_island.cc.services.zero_trust.scoutsuite.consts.rule_names.ec2_rules import EC2Rules +from monkey_island.cc.services.zero_trust.scoutsuite.consts.service_consts import SERVICES from monkey_island.cc.services.zero_trust.scoutsuite.data_parsing.rule_parser import RuleParser from monkey_island.cc.services.zero_trust.test_common.raw_scoutsute_data import RAW_SCOUTSUITE_DATA @@ -28,9 +29,9 @@ EXPECTED_RESULT = {'description': 'Security Group Opens All Ports to All', def test_get_rule_data(): # Test proper parsing of the raw data to rule - results = RuleParser.get_rule_data(RAW_SCOUTSUITE_DATA, ALL_PORTS_OPEN) + results = RuleParser.get_rule_data(RAW_SCOUTSUITE_DATA[SERVICES], ALL_PORTS_OPEN) assert results == EXPECTED_RESULT with pytest.raises(RulePathCreatorNotFound): - RuleParser.get_rule_data(RAW_SCOUTSUITE_DATA, ExampleRules.NON_EXSISTENT_RULE) + RuleParser.get_rule_data(RAW_SCOUTSUITE_DATA[SERVICES], ExampleRules.NON_EXSISTENT_RULE) pass