From 91a431517abd9506aea564b4d24ca680dcd3b0fa Mon Sep 17 00:00:00 2001 From: vakarisz Date: Tue, 12 Apr 2022 14:59:19 +0300 Subject: [PATCH] BB: Use grouped tests Grouping tests will allow us to run more tests at once --- envs/monkey_zoo/blackbox/test_blackbox.py | 140 +-------- .../blackbox/test_blackbox_in_depth.py | 296 ++++++++++++++++++ 2 files changed, 305 insertions(+), 131 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/test_blackbox_in_depth.py diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 31cbdd379..452a7ef81 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -8,39 +8,14 @@ from typing_extensions import Type from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer from envs.monkey_zoo.blackbox.analyzers.zerologon_analyzer import ZerologonAnalyzer from envs.monkey_zoo.blackbox.config_templates.config_template import ConfigTemplate -from envs.monkey_zoo.blackbox.config_templates.hadoop import Hadoop -from envs.monkey_zoo.blackbox.config_templates.log4j_logstash import Log4jLogstash -from envs.monkey_zoo.blackbox.config_templates.log4j_solr import Log4jSolr -from envs.monkey_zoo.blackbox.config_templates.log4j_tomcat import Log4jTomcat -from envs.monkey_zoo.blackbox.config_templates.mssql import Mssql -from envs.monkey_zoo.blackbox.config_templates.performance import Performance -from envs.monkey_zoo.blackbox.config_templates.powershell import PowerShell -from envs.monkey_zoo.blackbox.config_templates.powershell_credentials_reuse import ( - PowerShellCredentialsReuse, -) -from envs.monkey_zoo.blackbox.config_templates.smb_mimikatz import SmbMimikatz -from envs.monkey_zoo.blackbox.config_templates.smb_pth import SmbPth -from envs.monkey_zoo.blackbox.config_templates.ssh import Ssh -from envs.monkey_zoo.blackbox.config_templates.tunneling import Tunneling -from envs.monkey_zoo.blackbox.config_templates.wmi_mimikatz import WmiMimikatz -from envs.monkey_zoo.blackbox.config_templates.wmi_pth import WmiPth -from envs.monkey_zoo.blackbox.config_templates.zerologon import Zerologon +from envs.monkey_zoo.blackbox.config_templates.grouped.depth_1_a import Depth1A +from envs.monkey_zoo.blackbox.config_templates.grouped.depth_1_b import Depth1B +from envs.monkey_zoo.blackbox.config_templates.grouped.depth_4_a import Depth4A from envs.monkey_zoo.blackbox.gcp_test_machine_list import GCP_TEST_MACHINE_LIST from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest -from envs.monkey_zoo.blackbox.tests.performance.map_generation_from_telemetries import ( - MapGenerationFromTelemetryTest, -) -from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest -from envs.monkey_zoo.blackbox.tests.performance.report_generation_from_telemetries import ( - ReportGenerationFromTelemetryTest, -) -from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import ( - TelemetryPerformanceTest, -) from envs.monkey_zoo.blackbox.utils.gcp_machine_handlers import ( initialize_gcp_client, start_machines, @@ -153,72 +128,17 @@ class TestMonkeyBlackbox: def get_log_dir_path(): return os.path.abspath(LOG_DIR_PATH) - def test_ssh_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, Ssh, "SSH_exploiter_and_keys") + def test_depth_1_a(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Depth1A, "Depth1A test suite") - def test_hadoop_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, Hadoop, "Hadoop_exploiter", 6 * 60) - - def test_mssql_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, Mssql, "MSSQL_exploiter") - - def test_powershell_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, PowerShell, "PowerShell_Remoting_exploiter" - ) - - @pytest.mark.skip_powershell_reuse - def test_powershell_exploiter_credentials_reuse(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, - PowerShellCredentialsReuse, - "PowerShell_Remoting_exploiter_credentials_reuse", - ) - - def test_smb_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, SmbMimikatz, "SMB_exploiter_mimikatz" - ) - - def test_smb_pth(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, SmbPth, "SMB_PTH") - - def test_log4j_solr_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, Log4jSolr, "Log4Shell_Solr_exploiter" - ) - - def test_log4j_tomcat_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, Log4jTomcat, "Log4Shell_tomcat_exploiter" - ) - - def test_log4j_logstash_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, Log4jLogstash, "Log4Shell_logstash_exploiter" - ) - - def test_tunneling(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, Tunneling, "Tunneling_exploiter", 3 * 60 - ) - - def test_wmi_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_exploitation_test( - island_client, WmiMimikatz, "WMI_exploiter,_mimikatz" - ) - - def test_wmi_pth(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, WmiPth, "WMI_PTH") - - def test_zerologon_exploiter(self, island_client): + def test_depth_1_b(self, island_client): test_name = "Zerologon_exploiter" expected_creds = [ "Administrator", "aad3b435b51404eeaad3b435b51404ee", "2864b62ea4496934a5d6e86f50b834a5", ] - raw_config = IslandConfigParser.get_raw_config(Zerologon, island_client) + raw_config = IslandConfigParser.get_raw_config(Depth1B, island_client) zero_logon_analyzer = ZerologonAnalyzer(island_client, expected_creds) communication_analyzer = CommunicationAnalyzer( island_client, IslandConfigParser.get_ips_of_targets(raw_config) @@ -235,47 +155,5 @@ class TestMonkeyBlackbox: log_handler=log_handler, ).run() - @pytest.mark.skip( - reason="Perfomance test that creates env from fake telemetries is faster, use that instead." - ) - def test_report_generation_performance(self, island_client, quick_performance_tests): - """ - This test includes the SSH + Hadoop + MSSQL machines all in one test - for a total of 8 machines including the Monkey Island. - - Is has 2 analyzers - the regular one which checks all the Monkeys - and the Timing one which checks how long the report took to execute - """ - if not quick_performance_tests: - TestMonkeyBlackbox.run_performance_test( - ReportGenerationTest, island_client, Performance, timeout_in_seconds=10 * 60 - ) - else: - LOGGER.error("This test doesn't support 'quick_performance_tests' option.") - assert False - - @pytest.mark.skip( - reason="Perfomance test that creates env from fake telemetries is faster, use that instead." - ) - def test_map_generation_performance(self, island_client, quick_performance_tests): - if not quick_performance_tests: - TestMonkeyBlackbox.run_performance_test( - MapGenerationTest, island_client, "PERFORMANCE.conf", timeout_in_seconds=10 * 60 - ) - else: - LOGGER.error("This test doesn't support 'quick_performance_tests' option.") - assert False - - @pytest.mark.run_performance_tests - def test_report_generation_from_fake_telemetries(self, island_client, quick_performance_tests): - ReportGenerationFromTelemetryTest(island_client, quick_performance_tests).run() - - @pytest.mark.run_performance_tests - def test_map_generation_from_fake_telemetries(self, island_client, quick_performance_tests): - MapGenerationFromTelemetryTest(island_client, quick_performance_tests).run() - - @pytest.mark.run_performance_tests - def test_telem_performance(self, island_client, quick_performance_tests): - TelemetryPerformanceTest( - island_client, quick_performance_tests - ).test_telemetry_performance() + def test_depth_4_a(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Depth4A, "Depth4A test suite") diff --git a/envs/monkey_zoo/blackbox/test_blackbox_in_depth.py b/envs/monkey_zoo/blackbox/test_blackbox_in_depth.py new file mode 100644 index 000000000..42d2e28b7 --- /dev/null +++ b/envs/monkey_zoo/blackbox/test_blackbox_in_depth.py @@ -0,0 +1,296 @@ +import logging +import os +from time import sleep + +import pytest +from typing_extensions import Type + +from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer +from envs.monkey_zoo.blackbox.analyzers.zerologon_analyzer import ZerologonAnalyzer +from envs.monkey_zoo.blackbox.config_templates.config_template import ConfigTemplate +from envs.monkey_zoo.blackbox.config_templates.single_tests.drupal import Drupal +from envs.monkey_zoo.blackbox.config_templates.single_tests.hadoop import Hadoop +from envs.monkey_zoo.blackbox.config_templates.single_tests.log4j_logstash import Log4jLogstash +from envs.monkey_zoo.blackbox.config_templates.single_tests.log4j_solr import Log4jSolr +from envs.monkey_zoo.blackbox.config_templates.single_tests.log4j_tomcat import Log4jTomcat +from envs.monkey_zoo.blackbox.config_templates.single_tests.mssql import Mssql +from envs.monkey_zoo.blackbox.config_templates.single_tests.performance import Performance +from envs.monkey_zoo.blackbox.config_templates.single_tests.powershell import PowerShell +from envs.monkey_zoo.blackbox.config_templates.single_tests.powershell_credentials_reuse import ( + PowerShellCredentialsReuse, +) +from envs.monkey_zoo.blackbox.config_templates.single_tests.smb_mimikatz import SmbMimikatz +from envs.monkey_zoo.blackbox.config_templates.single_tests.smb_pth import SmbPth +from envs.monkey_zoo.blackbox.config_templates.single_tests.ssh import Ssh +from envs.monkey_zoo.blackbox.config_templates.single_tests.struts2 import Struts2 +from envs.monkey_zoo.blackbox.config_templates.single_tests.tunneling import Tunneling +from envs.monkey_zoo.blackbox.config_templates.single_tests.weblogic import Weblogic +from envs.monkey_zoo.blackbox.config_templates.single_tests.wmi_mimikatz import WmiMimikatz +from envs.monkey_zoo.blackbox.config_templates.single_tests.wmi_pth import WmiPth +from envs.monkey_zoo.blackbox.config_templates.single_tests.zerologon import Zerologon +from envs.monkey_zoo.blackbox.gcp_test_machine_list import GCP_TEST_MACHINE_LIST +from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.map_generation_from_telemetries import ( + MapGenerationFromTelemetryTest, +) +from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.report_generation_from_telemetries import ( + ReportGenerationFromTelemetryTest, +) +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import ( + TelemetryPerformanceTest, +) +from envs.monkey_zoo.blackbox.utils.gcp_machine_handlers import ( + initialize_gcp_client, + start_machines, + stop_machines, +) +from monkey_island.cc.services.mode.mode_enum import IslandModeEnum + +DEFAULT_TIMEOUT_SECONDS = 2 * 60 +MACHINE_BOOTUP_WAIT_SECONDS = 30 +LOG_DIR_PATH = "./logs" +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True, scope="session") +def GCPHandler(request, no_gcp): + if not no_gcp: + try: + initialize_gcp_client() + start_machines(GCP_TEST_MACHINE_LIST) + except Exception as e: + LOGGER.error("GCP Handler failed to initialize: %s." % e) + pytest.exit("Encountered an error while starting GCP machines. Stopping the tests.") + wait_machine_bootup() + + def fin(): + stop_machines(GCP_TEST_MACHINE_LIST) + + request.addfinalizer(fin) + + +@pytest.fixture(autouse=True, scope="session") +def delete_logs(): + LOGGER.info("Deleting monkey logs before new tests.") + TestLogsHandler.delete_log_folder_contents(TestMonkeyBlackbox.get_log_dir_path()) + + +def wait_machine_bootup(): + sleep(MACHINE_BOOTUP_WAIT_SECONDS) + + +@pytest.fixture(scope="class") +def island_client(island, quick_performance_tests): + client_established = False + try: + island_client_object = MonkeyIslandClient(island) + client_established = island_client_object.get_api_status() + except Exception: + logging.exception("Got an exception while trying to establish connection to the Island.") + finally: + if not client_established: + pytest.exit("BB tests couldn't establish communication to the island.") + if not quick_performance_tests: + island_client_object.reset_env() + island_client_object.set_scenario(IslandModeEnum.ADVANCED.value) + yield island_client_object + + +@pytest.mark.usefixtures("island_client") +# noinspection PyUnresolvedReferences +class TestMonkeyBlackbox: + @staticmethod + def run_exploitation_test( + island_client: MonkeyIslandClient, + config_template: Type[ConfigTemplate], + test_name: str, + timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS, + ): + raw_config = IslandConfigParser.get_raw_config(config_template, island_client) + analyzer = CommunicationAnalyzer( + island_client, IslandConfigParser.get_ips_of_targets(raw_config) + ) + log_handler = TestLogsHandler( + test_name, island_client, TestMonkeyBlackbox.get_log_dir_path() + ) + ExploitationTest( + name=test_name, + island_client=island_client, + raw_config=raw_config, + analyzers=[analyzer], + timeout=timeout_in_seconds, + log_handler=log_handler, + ).run() + + @staticmethod + def run_performance_test( + performance_test_class, + island_client, + config_template, + timeout_in_seconds, + break_on_timeout=False, + ): + raw_config = IslandConfigParser.get_raw_config(config_template, island_client) + log_handler = TestLogsHandler( + performance_test_class.TEST_NAME, island_client, TestMonkeyBlackbox.get_log_dir_path() + ) + analyzers = [ + CommunicationAnalyzer(island_client, IslandConfigParser.get_ips_of_targets(raw_config)) + ] + performance_test_class( + island_client=island_client, + raw_config=raw_config, + analyzers=analyzers, + timeout=timeout_in_seconds, + log_handler=log_handler, + break_on_timeout=break_on_timeout, + ).run() + + @staticmethod + def get_log_dir_path(): + return os.path.abspath(LOG_DIR_PATH) + + def test_ssh_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Ssh, "SSH_exploiter_and_keys") + + def test_hadoop_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Hadoop, "Hadoop_exploiter", 6 * 60) + + def test_mssql_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Mssql, "MSSQL_exploiter") + + def test_powershell_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, PowerShell, "PowerShell_Remoting_exploiter" + ) + + @pytest.mark.skip_powershell_reuse + def test_powershell_exploiter_credentials_reuse(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, + PowerShellCredentialsReuse, + "PowerShell_Remoting_exploiter_credentials_reuse", + ) + + def test_smb_and_mimikatz_exploiters(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, SmbMimikatz, "SMB_exploiter_mimikatz" + ) + + def test_smb_pth(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, SmbPth, "SMB_PTH") + + @pytest.mark.skip(reason="Drupal exploiter is deprecated") + def test_drupal_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Drupal, "Drupal_exploiter") + + @pytest.mark.skip(reason="Struts2 exploiter is deprecated") + def test_struts_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Struts2, "Struts2_exploiter") + + @pytest.mark.skip(reason="Weblogic exploiter is deprecated") + def test_weblogic_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, Weblogic, "Weblogic_exploiter") + + def test_log4j_solr_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, Log4jSolr, "Log4Shell_Solr_exploiter" + ) + + def test_log4j_tomcat_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, Log4jTomcat, "Log4Shell_tomcat_exploiter" + ) + + def test_log4j_logstash_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, Log4jLogstash, "Log4Shell_logstash_exploiter" + ) + + def test_tunneling(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, Tunneling, "Tunneling_exploiter", 3 * 60 + ) + + def test_wmi_and_mimikatz_exploiters(self, island_client): + TestMonkeyBlackbox.run_exploitation_test( + island_client, WmiMimikatz, "WMI_exploiter,_mimikatz" + ) + + def test_wmi_pth(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, WmiPth, "WMI_PTH") + + def test_zerologon_exploiter(self, island_client): + test_name = "Zerologon_exploiter" + expected_creds = [ + "Administrator", + "aad3b435b51404eeaad3b435b51404ee", + "2864b62ea4496934a5d6e86f50b834a5", + ] + raw_config = IslandConfigParser.get_raw_config(Zerologon, island_client) + zero_logon_analyzer = ZerologonAnalyzer(island_client, expected_creds) + communication_analyzer = CommunicationAnalyzer( + island_client, IslandConfigParser.get_ips_of_targets(raw_config) + ) + log_handler = TestLogsHandler( + test_name, island_client, TestMonkeyBlackbox.get_log_dir_path() + ) + ExploitationTest( + name=test_name, + island_client=island_client, + raw_config=raw_config, + analyzers=[zero_logon_analyzer, communication_analyzer], + timeout=DEFAULT_TIMEOUT_SECONDS, + log_handler=log_handler, + ).run() + + @pytest.mark.skip( + reason="Perfomance test that creates env from fake telemetries is faster, use that instead." + ) + def test_report_generation_performance(self, island_client, quick_performance_tests): + """ + This test includes the SSH + Hadoop + MSSQL machines all in one test + for a total of 8 machines including the Monkey Island. + + Is has 2 analyzers - the regular one which checks all the Monkeys + and the Timing one which checks how long the report took to execute + """ + if not quick_performance_tests: + TestMonkeyBlackbox.run_performance_test( + ReportGenerationTest, island_client, Performance, timeout_in_seconds=10 * 60 + ) + else: + LOGGER.error("This test doesn't support 'quick_performance_tests' option.") + assert False + + @pytest.mark.skip( + reason="Perfomance test that creates env from fake telemetries is faster, use that instead." + ) + def test_map_generation_performance(self, island_client, quick_performance_tests): + if not quick_performance_tests: + TestMonkeyBlackbox.run_performance_test( + MapGenerationTest, island_client, "PERFORMANCE.conf", timeout_in_seconds=10 * 60 + ) + else: + LOGGER.error("This test doesn't support 'quick_performance_tests' option.") + assert False + + @pytest.mark.run_performance_tests + def test_report_generation_from_fake_telemetries(self, island_client, quick_performance_tests): + ReportGenerationFromTelemetryTest(island_client, quick_performance_tests).run() + + @pytest.mark.run_performance_tests + def test_map_generation_from_fake_telemetries(self, island_client, quick_performance_tests): + MapGenerationFromTelemetryTest(island_client, quick_performance_tests).run() + + @pytest.mark.run_performance_tests + def test_telem_performance(self, island_client, quick_performance_tests): + TelemetryPerformanceTest( + island_client, quick_performance_tests + ).test_telemetry_performance()