From 303dda1621bd4f4cf95dc4be05eac7beaabc7767 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 30 Mar 2020 10:45:42 +0300 Subject: [PATCH 1/4] Refactoring current tests --- .../analyzers/performance_analyzer.py | 42 ++------ envs/monkey_zoo/blackbox/tests/basic_test.py | 100 +----------------- .../monkey_zoo/blackbox/tests/exploitation.py | 95 +++++++++++++++++ .../blackbox/tests/performance/__init__.py | 0 .../tests/performance/map_generation.py | 0 .../tests/performance/report_generation.py | 81 ++++++++++++++ 6 files changed, 190 insertions(+), 128 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/exploitation.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/__init__.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/map_generation.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/report_generation.py diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index 3e1c48199..b47b7d748 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -15,7 +15,7 @@ REPORT_URLS = [ "api/report/zero_trust/pillars" ] -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class PerformanceAnalyzer(Analyzer): @@ -24,36 +24,14 @@ class PerformanceAnalyzer(Analyzer): self.break_if_took_too_long = break_if_took_too_long self.island_client = island_client - def analyze_test_results(self) -> bool: - if not self.island_client.is_all_monkeys_dead(): - raise RuntimeError("Can't test report times since not all Monkeys have died.") - # Collect timings for all pages - self.island_client.clear_caches() - report_resource_to_response_time = {} - for url in REPORT_URLS: - report_resource_to_response_time[url] = self.island_client.get_elapsed_for_get_request(url) - # Calculate total time and check each page - single_page_time_less_then_max = True - total_time = timedelta() - for page, elapsed in report_resource_to_response_time.items(): - logger.info(f"page {page} took {str(elapsed)}") - total_time += elapsed - if elapsed > MAX_ALLOWED_SINGLE_PAGE_TIME: - single_page_time_less_then_max = False - - total_time_less_then_max = total_time < MAX_ALLOWED_TOTAL_TIME - - logger.info(f"total time is {str(total_time)}") - - performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max - - if self.break_if_took_too_long and not performance_is_good_enough: - logger.warning( - "Calling breakpoint - pausing to enable investigation of island. Type 'c' to continue once you're done " - "investigating. Type 'p timings' and 'p total_time' to see performance information." - ) - breakpoint() - - return performance_is_good_enough + def get_elapsed_for_get_request(self, url): + response = self.island_client.requests.get(url) + if response.ok: + LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") + return response.elapsed + else: + LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") + # instead of raising for status, mark failed responses as maxtime + return timedelta.max() diff --git a/envs/monkey_zoo/blackbox/tests/basic_test.py b/envs/monkey_zoo/blackbox/tests/basic_test.py index a5e71c64c..fa722ffb7 100644 --- a/envs/monkey_zoo/blackbox/tests/basic_test.py +++ b/envs/monkey_zoo/blackbox/tests/basic_test.py @@ -1,100 +1,8 @@ -from time import sleep - -import logging - -from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer - -MAX_TIME_FOR_MONKEYS_TO_DIE = 5 * 60 -WAIT_TIME_BETWEEN_REQUESTS = 10 -TIME_FOR_MONKEY_PROCESS_TO_FINISH = 40 -DELAY_BETWEEN_ANALYSIS = 3 -LOGGER = logging.getLogger(__name__) +import abc -class BasicTest(object): - - def __init__(self, name, island_client, config_parser, analyzers, timeout, post_exec_analyzers, log_handler): - self.name = name - self.island_client = island_client - self.config_parser = config_parser - self.analyzers = analyzers - self.post_exec_analyzers = post_exec_analyzers - self.timeout = timeout - self.log_handler = log_handler +class BasicTest(abc.ABC): + @abc.abstractmethod def run(self): - self.island_client.import_config(self.config_parser.config_raw) - self.print_test_starting_info() - try: - self.island_client.run_monkey_local() - self.test_until_timeout() - finally: - self.island_client.kill_all_monkeys() - self.wait_until_monkeys_die() - self.wait_for_monkey_process_to_finish() - self.test_post_exec_analyzers() - self.parse_logs() - self.island_client.reset_env() - - def print_test_starting_info(self): - LOGGER.info("Started {} test".format(self.name)) - LOGGER.info("Machines participating in test: " + ", ".join(self.config_parser.get_ips_of_targets())) - print("") - - def test_until_timeout(self): - timer = TestTimer(self.timeout) - while not timer.is_timed_out(): - if self.all_analyzers_pass(): - self.log_success(timer) - return - sleep(DELAY_BETWEEN_ANALYSIS) - LOGGER.debug("Waiting until all analyzers passed. Time passed: {}".format(timer.get_time_taken())) - self.log_failure(timer) - assert False - - def log_success(self, timer): - LOGGER.info(self.get_analyzer_logs()) - LOGGER.info("{} test passed, time taken: {:.1f} seconds.".format(self.name, timer.get_time_taken())) - - def log_failure(self, timer): - LOGGER.info(self.get_analyzer_logs()) - LOGGER.error("{} test failed because of timeout. Time taken: {:.1f} seconds.".format(self.name, - timer.get_time_taken())) - - def all_analyzers_pass(self): - analyzers_results = [analyzer.analyze_test_results() for analyzer in self.analyzers] - return all(analyzers_results) - - def get_analyzer_logs(self): - log = "" - for analyzer in self.analyzers: - log += "\n" + analyzer.log.get_contents() - return log - - def wait_until_monkeys_die(self): - time_passed = 0 - while not self.island_client.is_all_monkeys_dead() and time_passed < MAX_TIME_FOR_MONKEYS_TO_DIE: - sleep(WAIT_TIME_BETWEEN_REQUESTS) - time_passed += WAIT_TIME_BETWEEN_REQUESTS - LOGGER.debug("Waiting for all monkeys to die. Time passed: {}".format(time_passed)) - if time_passed > MAX_TIME_FOR_MONKEYS_TO_DIE: - LOGGER.error("Some monkeys didn't die after the test, failing") - assert False - - def parse_logs(self): - LOGGER.info("Parsing test logs:") - self.log_handler.parse_test_logs() - - @staticmethod - def wait_for_monkey_process_to_finish(): - """ - There is a time period when monkey is set to dead, but the process is still closing. - If we try to launch monkey during that time window monkey will fail to start, that's - why test needs to wait a bit even after all monkeys are dead. - """ - LOGGER.debug("Waiting for Monkey process to close...") - sleep(TIME_FOR_MONKEY_PROCESS_TO_FINISH) - - def test_post_exec_analyzers(self): - post_exec_analyzers_results = [analyzer.analyze_test_results() for analyzer in self.post_exec_analyzers] - assert all(post_exec_analyzers_results) + pass diff --git a/envs/monkey_zoo/blackbox/tests/exploitation.py b/envs/monkey_zoo/blackbox/tests/exploitation.py new file mode 100644 index 000000000..e731d8f90 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/exploitation.py @@ -0,0 +1,95 @@ +from time import sleep + +import logging + +from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest + +MAX_TIME_FOR_MONKEYS_TO_DIE = 5 * 60 +WAIT_TIME_BETWEEN_REQUESTS = 10 +TIME_FOR_MONKEY_PROCESS_TO_FINISH = 40 +DELAY_BETWEEN_ANALYSIS = 3 +LOGGER = logging.getLogger(__name__) + + +class ExploitationTest(BasicTest): + + def __init__(self, name, island_client, config_parser, analyzers, timeout, log_handler): + self.name = name + self.island_client = island_client + self.config_parser = config_parser + self.analyzers = analyzers + self.timeout = timeout + self.log_handler = log_handler + + def run(self): + self.island_client.import_config(self.config_parser.config_raw) + self.print_test_starting_info() + try: + self.island_client.run_monkey_local() + self.test_until_timeout() + finally: + self.island_client.kill_all_monkeys() + self.wait_until_monkeys_die() + self.wait_for_monkey_process_to_finish() + self.parse_logs() + self.island_client.reset_env() + + def print_test_starting_info(self): + LOGGER.info("Started {} test".format(self.name)) + LOGGER.info("Machines participating in test: " + ", ".join(self.config_parser.get_ips_of_targets())) + print("") + + def test_until_timeout(self): + timer = TestTimer(self.timeout) + while not timer.is_timed_out(): + if self.all_analyzers_pass(): + self.log_success(timer) + return + sleep(DELAY_BETWEEN_ANALYSIS) + LOGGER.debug("Waiting until all analyzers passed. Time passed: {}".format(timer.get_time_taken())) + self.log_failure(timer) + assert False + + def log_success(self, timer): + LOGGER.info(self.get_analyzer_logs()) + LOGGER.info("{} test passed, time taken: {:.1f} seconds.".format(self.name, timer.get_time_taken())) + + def log_failure(self, timer): + LOGGER.info(self.get_analyzer_logs()) + LOGGER.error("{} test failed because of timeout. Time taken: {:.1f} seconds.".format(self.name, + timer.get_time_taken())) + + def all_analyzers_pass(self): + analyzers_results = [analyzer.analyze_test_results() for analyzer in self.analyzers] + return all(analyzers_results) + + def get_analyzer_logs(self): + log = "" + for analyzer in self.analyzers: + log += "\n" + analyzer.log.get_contents() + return log + + def wait_until_monkeys_die(self): + time_passed = 0 + while not self.island_client.is_all_monkeys_dead() and time_passed < MAX_TIME_FOR_MONKEYS_TO_DIE: + sleep(WAIT_TIME_BETWEEN_REQUESTS) + time_passed += WAIT_TIME_BETWEEN_REQUESTS + LOGGER.debug("Waiting for all monkeys to die. Time passed: {}".format(time_passed)) + if time_passed > MAX_TIME_FOR_MONKEYS_TO_DIE: + LOGGER.error("Some monkeys didn't die after the test, failing") + assert False + + def parse_logs(self): + LOGGER.info("Parsing test logs:") + self.log_handler.parse_test_logs() + + @staticmethod + def wait_for_monkey_process_to_finish(): + """ + There is a time period when monkey is set to dead, but the process is still closing. + If we try to launch monkey during that time window monkey will fail to start, that's + why test needs to wait a bit even after all monkeys are dead. + """ + LOGGER.debug("Waiting for Monkey process to close...") + sleep(TIME_FOR_MONKEY_PROCESS_TO_FINISH) diff --git a/envs/monkey_zoo/blackbox/tests/performance/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py new file mode 100644 index 000000000..a4d3c718a --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py @@ -0,0 +1,81 @@ +import logging +from datetime import timedelta + +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest + +MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) + +REPORT_URLS = [ + "api/report/security", + "api/attack/report", + "api/report/zero_trust/findings", + "api/report/zero_trust/principles", + "api/report/zero_trust/pillars" +] + +LOGGER = logging.getLogger(__name__) + + +class ReportGenerationTest(BasicTest): + + def __init__(self, name, island_client, config_parser, analyzers, + timeout, log_handler, break_if_took_too_long=False): + self.name = name + self.island_client = island_client + self.config_parser = config_parser + self.exploitation_test = ExploitationTest(name, island_client, config_parser, analyzers, timeout, log_handler) + self.break_if_took_too_long = break_if_took_too_long + + def test_report_generation_performance(self) -> bool: + if not self.island_client.is_all_monkeys_dead(): + raise RuntimeError("Can't test report times since not all Monkeys have died.") + + # Collect timings for all pages + self.island_client.clear_caches() + report_resource_to_response_time = {} + for url in REPORT_URLS: + report_resource_to_response_time[url] = self.island_client.get_elapsed_for_get_request(url) + + # Calculate total time and check each page + single_page_time_less_then_max = True + total_time = timedelta() + for page, elapsed in report_resource_to_response_time.items(): + LOGGER.info(f"page {page} took {str(elapsed)}") + total_time += elapsed + if elapsed > MAX_ALLOWED_SINGLE_PAGE_TIME: + single_page_time_less_then_max = False + + total_time_less_then_max = total_time < MAX_ALLOWED_TOTAL_TIME + + LOGGER.info(f"total time is {str(total_time)}") + + performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max + + if self.break_if_took_too_long and not performance_is_good_enough: + LOGGER.warning( + "Calling breakpoint - pausing to enable investigation of island. Type 'c' to continue once you're done " + "investigating. Type 'p timings' and 'p total_time' to see performance information." + ) + breakpoint() + + return performance_is_good_enough + + def run(self): + self.island_client.import_config(self.config_parser.config_raw) + self.exploitation_test.print_test_starting_info() + try: + self.island_client.run_monkey_local() + self.exploitation_test.test_until_timeout() + finally: + self.island_client.kill_all_monkeys() + self.exploitation_test.wait_until_monkeys_die() + self.exploitation_test.wait_for_monkey_process_to_finish() + self.test_post_exec_analyzers() + self.exploitation_test.parse_logs() + self.island_client.reset_env() + + def test_post_exec_analyzers(self): + post_exec_analyzers_results = [analyzer.analyze_test_results() for analyzer in self.post_exec_analyzers] + assert all(post_exec_analyzers_results) From 0fc56150589bcf24962915e04f59b0859797459b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 9 Apr 2020 18:23:01 +0300 Subject: [PATCH 2/4] Further improving and refactoring performance test code --- .../analyzers/performance_analyzer.py | 52 ++++++++-------- .../island_client/monkey_island_client.py | 10 --- .../tests/performance/map_generation.py | 47 ++++++++++++++ .../tests/performance/performance_test.py | 42 +++++++++++++ .../performance/performance_test_config.py | 12 ++++ .../tests/performance/report_generation.py | 62 ++++--------------- 6 files changed, 140 insertions(+), 85 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/performance_test.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index b47b7d748..d9067eeee 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -1,37 +1,39 @@ import logging from datetime import timedelta +from typing import Dict from envs.monkey_zoo.blackbox.analyzers.analyzer import Analyzer -from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient - -MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) -MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) - -REPORT_URLS = [ - "api/report/security", - "api/attack/report", - "api/report/zero_trust/findings", - "api/report/zero_trust/principles", - "api/report/zero_trust/pillars" -] - +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig LOGGER = logging.getLogger(__name__) class PerformanceAnalyzer(Analyzer): - def __init__(self, island_client: MonkeyIslandClient, break_if_took_too_long=False): - self.break_if_took_too_long = break_if_took_too_long - self.island_client = island_client + def __init__(self, performance_test_config: PerformanceTestConfig, endpoint_timings: Dict[str, timedelta]): + self.performance_test_config = performance_test_config + self.endpoint_timings = endpoint_timings + def analyze_test_results(self): + # Calculate total time and check each page + single_page_time_less_then_max = True + total_time = timedelta() + for page, elapsed in self.endpoint_timings.items(): + LOGGER.info(f"page {page} took {str(elapsed)}") + total_time += elapsed + if elapsed > self.performance_test_config.max_allowed_single_page_time: + single_page_time_less_then_max = False + total_time_less_then_max = total_time < self.performance_test_config.max_allowed_total_time - def get_elapsed_for_get_request(self, url): - response = self.island_client.requests.get(url) - if response.ok: - LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") - return response.elapsed - else: - LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") - # instead of raising for status, mark failed responses as maxtime - return timedelta.max() + LOGGER.info(f"total time is {str(total_time)}") + + performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max + + if self.performance_test_config.break_on_timeout and not performance_is_good_enough: + LOGGER.warning( + "Calling breakpoint - pausing to enable investigation of island. Type 'c' to continue once you're done " + "investigating. Type 'p timings' and 'p total_time' to see performance information." + ) + breakpoint() + + return performance_is_good_enough diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index 27a54222b..93780bf3b 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -96,13 +96,3 @@ class MonkeyIslandClient(object): response = self.requests.get("api/test/clear_caches") response.raise_for_status() return response - - def get_elapsed_for_get_request(self, url): - response = self.requests.get(url) - if response.ok: - LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") - return response.elapsed - else: - LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") - # instead of raising for status, mark failed responses as maxtime - return timedelta.max() diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py index e69de29bb..b8f1b416e 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py @@ -0,0 +1,47 @@ +from datetime import timedelta + +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest + +MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) + +REPORT_RESOURCES = [ + "api/report/security", + "api/attack/report", + "api/report/zero_trust/findings", + "api/report/zero_trust/principles", + "api/report/zero_trust/pillars" +] + + +class MapGenerationTest(BasicTest): + + def __init__(self, name, island_client, config_parser, analyzers, + timeout, log_handler, break_on_timeout=False): + self.name = name + self.island_client = island_client + self.config_parser = config_parser + self.exploitation_test = ExploitationTest(name, island_client, config_parser, analyzers, timeout, log_handler) + self.break_on_timeout = break_on_timeout + + def run(self): + self.island_client.import_config(self.config_parser.config_raw) + self.exploitation_test.print_test_starting_info() + try: + self.island_client.run_monkey_local() + self.exploitation_test.test_until_timeout() + finally: + self.island_client.kill_all_monkeys() + self.exploitation_test.wait_until_monkeys_die() + self.exploitation_test.wait_for_monkey_process_to_finish() + performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, + max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, + endpoints_to_test=REPORT_RESOURCES, + break_on_timeout=self.break_on_timeout) + performance_test = PerformanceTest("Report generation test", performance_config, self.exploitation_test) + performance_test.run() + self.exploitation_test.parse_logs() + self.island_client.reset_env() diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test.py new file mode 100644 index 000000000..294626986 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test.py @@ -0,0 +1,42 @@ +import logging +from datetime import timedelta + +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer + + +LOGGER = logging.getLogger(__name__) + + +class PerformanceTest(BasicTest): + + def __init__(self, name, test_config: PerformanceTestConfig, exploitation_test: ExploitationTest): + self.name = name + self.test_config = test_config + self.exploitation_test = exploitation_test + + def run(self) -> bool: + if not self.exploitation_test.island_client.is_all_monkeys_dead(): + raise RuntimeError("Can't test report times since not all Monkeys have died.") + + # Collect timings for all pages + self.exploitation_test.island_client.clear_caches() + endpoint_timings = {} + for endpoint in self.test_config.endpoints_to_test: + endpoint_timings[endpoint] = self.exploitation_test.island_client.get_elapsed_for_get_request(endpoint) + + analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings) + + return analyzer.analyze_test_results() + + def get_elapsed_for_get_request(self, url): + response = self.exploitation_test.island_client.requests.get(url) + if response.ok: + LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") + return response.elapsed + else: + LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") + # instead of raising for status, mark failed responses as maxtime + return timedelta.max() diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py new file mode 100644 index 000000000..8ed2b5a62 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py @@ -0,0 +1,12 @@ +from datetime import timedelta +from typing import List + + +class PerformanceTestConfig: + + def __init__(self, max_allowed_single_page_time: timedelta, max_allowed_total_time: timedelta, + endpoints_to_test: List[str], break_on_timeout=False): + self.max_allowed_single_page_time = max_allowed_single_page_time + self.max_allowed_total_time = max_allowed_total_time + self.endpoints_to_test = endpoints_to_test + self.break_on_timeout = break_on_timeout diff --git a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py index a4d3c718a..8c98a8289 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py @@ -1,66 +1,27 @@ -import logging from datetime import timedelta from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) -REPORT_URLS = [ - "api/report/security", - "api/attack/report", - "api/report/zero_trust/findings", - "api/report/zero_trust/principles", - "api/report/zero_trust/pillars" +MAP_RESOURCES = [ + "api/report/security" ] -LOGGER = logging.getLogger(__name__) - class ReportGenerationTest(BasicTest): def __init__(self, name, island_client, config_parser, analyzers, - timeout, log_handler, break_if_took_too_long=False): + timeout, log_handler, break_on_timeout=False): self.name = name self.island_client = island_client self.config_parser = config_parser self.exploitation_test = ExploitationTest(name, island_client, config_parser, analyzers, timeout, log_handler) - self.break_if_took_too_long = break_if_took_too_long - - def test_report_generation_performance(self) -> bool: - if not self.island_client.is_all_monkeys_dead(): - raise RuntimeError("Can't test report times since not all Monkeys have died.") - - # Collect timings for all pages - self.island_client.clear_caches() - report_resource_to_response_time = {} - for url in REPORT_URLS: - report_resource_to_response_time[url] = self.island_client.get_elapsed_for_get_request(url) - - # Calculate total time and check each page - single_page_time_less_then_max = True - total_time = timedelta() - for page, elapsed in report_resource_to_response_time.items(): - LOGGER.info(f"page {page} took {str(elapsed)}") - total_time += elapsed - if elapsed > MAX_ALLOWED_SINGLE_PAGE_TIME: - single_page_time_less_then_max = False - - total_time_less_then_max = total_time < MAX_ALLOWED_TOTAL_TIME - - LOGGER.info(f"total time is {str(total_time)}") - - performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max - - if self.break_if_took_too_long and not performance_is_good_enough: - LOGGER.warning( - "Calling breakpoint - pausing to enable investigation of island. Type 'c' to continue once you're done " - "investigating. Type 'p timings' and 'p total_time' to see performance information." - ) - breakpoint() - - return performance_is_good_enough + self.break_on_timeout = break_on_timeout def run(self): self.island_client.import_config(self.config_parser.config_raw) @@ -72,10 +33,11 @@ class ReportGenerationTest(BasicTest): self.island_client.kill_all_monkeys() self.exploitation_test.wait_until_monkeys_die() self.exploitation_test.wait_for_monkey_process_to_finish() - self.test_post_exec_analyzers() + performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, + max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, + endpoints_to_test=REPORT_RESOURCES, + break_on_timeout=self.break_on_timeout) + performance_test = PerformanceTest("Report generation test", performance_config, self.exploitation_test) + performance_test.run() self.exploitation_test.parse_logs() self.island_client.reset_env() - - def test_post_exec_analyzers(self): - post_exec_analyzers_results = [analyzer.analyze_test_results() for analyzer in self.post_exec_analyzers] - assert all(post_exec_analyzers_results) From 367017a6b6d7c5cd085df8a5d87d5adbe1e26d97 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 10 Apr 2020 14:32:39 +0300 Subject: [PATCH 3/4] Refactored and further improved performance test code structure --- envs/monkey_zoo/blackbox/test_blackbox.py | 121 +++++++++--------- .../performance/endpoint_performance_test.py | 42 ++++++ .../tests/performance/map_generation.py | 48 +++---- .../tests/performance/performance_test.py | 46 ++----- .../performance/performance_test_workflow.py | 29 +++++ .../tests/performance/report_generation.py | 47 +++---- 6 files changed, 180 insertions(+), 153 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index e2da6a992..2c7ade076 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -4,12 +4,12 @@ import logging import pytest from time import sleep -from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser +from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers -from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler DEFAULT_TIMEOUT_SECONDS = 5*60 @@ -24,7 +24,7 @@ LOGGER = logging.getLogger(__name__) @pytest.fixture(autouse=True, scope='session') def GCPHandler(request): GCPHandler = gcp_machine_handlers.GCPHandler() - GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) wait_machine_bootup() def fin(): @@ -55,34 +55,32 @@ def island_client(island): class TestMonkeyBlackbox(object): @staticmethod - def run_basic_test(island_client, conf_filename, test_name, timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS): + def run_exploitation_test(island_client, conf_filename, test_name, timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS): config_parser = IslandConfigParser(conf_filename) analyzer = CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets()) log_handler = TestLogsHandler(test_name, island_client, TestMonkeyBlackbox.get_log_dir_path()) - BasicTest( + ExploitationTest( name=test_name, island_client=island_client, config_parser=config_parser, analyzers=[analyzer], timeout=timeout_in_seconds, - post_exec_analyzers=[], log_handler=log_handler).run() @staticmethod - def run_performance_test(island_client, conf_filename, test_name, timeout_in_seconds): + def run_performance_test(performance_test_class, island_client, + conf_filename, timeout_in_seconds, break_on_timeout=False): config_parser = IslandConfigParser(conf_filename) - log_handler = TestLogsHandler(test_name, island_client, TestMonkeyBlackbox.get_log_dir_path()) - BasicTest( - name=test_name, - island_client=island_client, - config_parser=config_parser, - analyzers=[CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets())], - timeout=timeout_in_seconds, - post_exec_analyzers=[PerformanceAnalyzer( - island_client, - break_if_took_too_long=False - )], - log_handler=log_handler).run() + log_handler = TestLogsHandler(performance_test_class.TEST_NAME, + island_client, + TestMonkeyBlackbox.get_log_dir_path()) + analyzer = CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets()) + performance_test_class.__init__(island_client=island_client, + config_parser=config_parser, + analyzers=analyzer, + timeout=timeout_in_seconds, + log_handler=log_handler, + break_on_timeout=break_on_timeout).run() @staticmethod def get_log_dir_path(): @@ -91,44 +89,44 @@ class TestMonkeyBlackbox(object): def test_server_online(self, island_client): assert island_client.get_api_status() is not None - def test_ssh_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") + #def test_ssh_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") +# + #def test_hadoop_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) +# + #def test_mssql_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") +# + #def test_smb_and_mimikatz_exploiters(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") +# + #def test_smb_pth(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") +# + #def test_elastic_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") +# + #def test_struts_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") +# + #def test_weblogic_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") +# + #def test_shellshock_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") +# + #@pytest.mark.xfail(reason="Test fails randomly - still investigating.") + #def test_tunneling(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 10 * 60) +# + #def test_wmi_and_mimikatz_exploiters(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") +# + #def test_wmi_pth(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") - def test_hadoop_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6*60) - - def test_mssql_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "MSSQL.conf", "MSSQL_exploiter") - - def test_smb_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") - - def test_smb_pth(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "SMB_PTH.conf", "SMB_PTH") - - def test_elastic_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "ELASTIC.conf", "Elastic_exploiter") - - def test_struts_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") - - def test_weblogic_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") - - def test_shellshock_exploiter(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") - - @pytest.mark.xfail(reason="Test fails randomly - still investigating.") - def test_tunneling(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 10*60) - - def test_wmi_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") - - def test_wmi_pth(self, island_client): - TestMonkeyBlackbox.run_basic_test(island_client, "WMI_PTH.conf", "WMI_PTH") - - def test_performance(self, island_client): + def test_report_generation_performance(self, island_client): """ This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test for a total of 8 machines including the Monkey Island. @@ -136,8 +134,9 @@ class TestMonkeyBlackbox(object): Is has 2 analyzers - the regular one which checks all the Monkeys and the Timing one which checks how long the report took to execute """ - TestMonkeyBlackbox.run_performance_test( - island_client, - "PERFORMANCE.conf", - "test_report_performance", - timeout_in_seconds=10*60) + TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, + island_client, + "PERFORMANCE.conf", + timeout_in_seconds=10*60) + + diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py new file mode 100644 index 000000000..dc2b6b0c7 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -0,0 +1,42 @@ +import logging +from datetime import timedelta + +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer + + +LOGGER = logging.getLogger(__name__) + + +class EndpointPerformanceTest(BasicTest): + + def __init__(self, name, test_config: PerformanceTestConfig, island_client: MonkeyIslandClient): + self.name = name + self.test_config = test_config + self.island_client = island_client + + def run(self) -> bool: + if not self.island_client.is_all_monkeys_dead(): + raise RuntimeError("Can't test report times since not all Monkeys have died.") + + # Collect timings for all pages + self.island_client.clear_caches() + endpoint_timings = {} + for endpoint in self.test_config.endpoints_to_test: + endpoint_timings[endpoint] = self.get_elapsed_for_get_request(endpoint) + + analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings) + + return analyzer.analyze_test_results() + + def get_elapsed_for_get_request(self, url): + response = self.island_client.requests.get(url) + if response.ok: + LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") + return response.elapsed + else: + LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") + # instead of raising for status, mark failed responses as maxtime + return timedelta.max() diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py index b8f1b416e..782bf82fe 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py @@ -1,47 +1,35 @@ from datetime import timedelta -from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) -REPORT_RESOURCES = [ - "api/report/security", - "api/attack/report", - "api/report/zero_trust/findings", - "api/report/zero_trust/principles", - "api/report/zero_trust/pillars" +MAP_RESOURCES = [ + "api/map", ] -class MapGenerationTest(BasicTest): +class MapGenerationTest(PerformanceTest): - def __init__(self, name, island_client, config_parser, analyzers, - timeout, log_handler, break_on_timeout=False): - self.name = name + TEST_NAME = "Map generation performance test" + + def __init__(self, island_client, config_parser, analyzers, + timeout, log_handler, break_on_timeout): self.island_client = island_client self.config_parser = config_parser - self.exploitation_test = ExploitationTest(name, island_client, config_parser, analyzers, timeout, log_handler) - self.break_on_timeout = break_on_timeout + exploitation_test = ExploitationTest(MapGenerationTest.TEST_NAME, island_client, + config_parser, analyzers, timeout, log_handler) + performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, + max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, + endpoints_to_test=MAP_RESOURCES, + break_on_timeout=break_on_timeout) + self.performance_test_workflow = PerformanceTestWorkflow(MapGenerationTest.TEST_NAME, + exploitation_test, + performance_config) def run(self): - self.island_client.import_config(self.config_parser.config_raw) - self.exploitation_test.print_test_starting_info() - try: - self.island_client.run_monkey_local() - self.exploitation_test.test_until_timeout() - finally: - self.island_client.kill_all_monkeys() - self.exploitation_test.wait_until_monkeys_die() - self.exploitation_test.wait_for_monkey_process_to_finish() - performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, - max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, - endpoints_to_test=REPORT_RESOURCES, - break_on_timeout=self.break_on_timeout) - performance_test = PerformanceTest("Report generation test", performance_config, self.exploitation_test) - performance_test.run() - self.exploitation_test.parse_logs() - self.island_client.reset_env() + self.performance_test_workflow.run() diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test.py index 294626986..b26c20f93 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test.py @@ -1,42 +1,16 @@ -import logging -from datetime import timedelta +from abc import ABCMeta, abstractmethod from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest -from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig -from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer -LOGGER = logging.getLogger(__name__) +class PerformanceTest(BasicTest, metaclass=ABCMeta): + @abstractmethod + def __init__(self, island_client, config_parser, analyzers, + timeout, log_handler, break_on_timeout): + pass -class PerformanceTest(BasicTest): - - def __init__(self, name, test_config: PerformanceTestConfig, exploitation_test: ExploitationTest): - self.name = name - self.test_config = test_config - self.exploitation_test = exploitation_test - - def run(self) -> bool: - if not self.exploitation_test.island_client.is_all_monkeys_dead(): - raise RuntimeError("Can't test report times since not all Monkeys have died.") - - # Collect timings for all pages - self.exploitation_test.island_client.clear_caches() - endpoint_timings = {} - for endpoint in self.test_config.endpoints_to_test: - endpoint_timings[endpoint] = self.exploitation_test.island_client.get_elapsed_for_get_request(endpoint) - - analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings) - - return analyzer.analyze_test_results() - - def get_elapsed_for_get_request(self, url): - response = self.exploitation_test.island_client.requests.get(url) - if response.ok: - LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") - return response.elapsed - else: - LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") - # instead of raising for status, mark failed responses as maxtime - return timedelta.max() + @property + @abstractmethod + def TEST_NAME(self): + pass diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py new file mode 100644 index 000000000..e496c09a5 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py @@ -0,0 +1,29 @@ +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.endpoint_performance_test import EndpointPerformanceTest + + +class PerformanceTestWorkflow(BasicTest): + + def __init__(self, name, exploitation_test: ExploitationTest, performance_config: PerformanceTestConfig): + self.name = name + self.exploitation_test = exploitation_test + self.island_client = exploitation_test.island_client + self.config_parser = exploitation_test.config_parser + self.performance_config = performance_config + + def run(self): + self.island_client.import_config(self.config_parser.config_raw) + self.exploitation_test.print_test_starting_info() + try: + self.island_client.run_monkey_local() + self.exploitation_test.test_until_timeout() + finally: + self.island_client.kill_all_monkeys() + self.exploitation_test.wait_until_monkeys_die() + self.exploitation_test.wait_for_monkey_process_to_finish() + performance_test = EndpointPerformanceTest(self.name, self.performance_config, self.island_client) + performance_test.run() + self.exploitation_test.parse_logs() + self.island_client.reset_env() diff --git a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py index 8c98a8289..52fe76288 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py @@ -1,43 +1,38 @@ from datetime import timedelta -from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) -MAP_RESOURCES = [ - "api/report/security" +REPORT_RESOURCES = [ + "api/report/security", + "api/attack/report", + "api/report/zero_trust/findings", + "api/report/zero_trust/principles", + "api/report/zero_trust/pillars" ] -class ReportGenerationTest(BasicTest): +class ReportGenerationTest(PerformanceTest): + TEST_NAME = "Report generation performance test" - def __init__(self, name, island_client, config_parser, analyzers, - timeout, log_handler, break_on_timeout=False): - self.name = name + def __init__(self, island_client, config_parser, analyzers, + timeout, log_handler, break_on_timeout): self.island_client = island_client self.config_parser = config_parser - self.exploitation_test = ExploitationTest(name, island_client, config_parser, analyzers, timeout, log_handler) - self.break_on_timeout = break_on_timeout + exploitation_test = ExploitationTest(ReportGenerationTest.TEST_NAME, island_client, + config_parser, analyzers, timeout, log_handler) + performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, + max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, + endpoints_to_test=REPORT_RESOURCES, + break_on_timeout=break_on_timeout) + self.performance_test_workflow = PerformanceTestWorkflow(ReportGenerationTest.TEST_NAME, + exploitation_test, + performance_config) def run(self): - self.island_client.import_config(self.config_parser.config_raw) - self.exploitation_test.print_test_starting_info() - try: - self.island_client.run_monkey_local() - self.exploitation_test.test_until_timeout() - finally: - self.island_client.kill_all_monkeys() - self.exploitation_test.wait_until_monkeys_die() - self.exploitation_test.wait_for_monkey_process_to_finish() - performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, - max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, - endpoints_to_test=REPORT_RESOURCES, - break_on_timeout=self.break_on_timeout) - performance_test = PerformanceTest("Report generation test", performance_config, self.exploitation_test) - performance_test.run() - self.exploitation_test.parse_logs() - self.island_client.reset_env() + self.performance_test_workflow.run() From 5fa27c14e8a24695955334c5f7b1313ba7907d42 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 15 Apr 2020 09:38:08 +0300 Subject: [PATCH 4/4] Bugfixes and debugging comments removed --- envs/monkey_zoo/blackbox/test_blackbox.py | 94 ++++++++++--------- .../performance/endpoint_performance_test.py | 2 +- .../tests/performance/map_generation.py | 2 +- .../performance/performance_test_workflow.py | 6 +- 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 82cd0f0ad..04e510c55 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -7,6 +7,7 @@ from time import sleep from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser +from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest @@ -24,7 +25,7 @@ LOGGER = logging.getLogger(__name__) @pytest.fixture(autouse=True, scope='session') def GCPHandler(request): GCPHandler = gcp_machine_handlers.GCPHandler() - #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) wait_machine_bootup() def fin(): @@ -74,13 +75,13 @@ class TestMonkeyBlackbox(object): log_handler = TestLogsHandler(performance_test_class.TEST_NAME, island_client, TestMonkeyBlackbox.get_log_dir_path()) - analyzer = CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets()) - performance_test_class.__init__(island_client=island_client, - config_parser=config_parser, - analyzers=analyzer, - timeout=timeout_in_seconds, - log_handler=log_handler, - break_on_timeout=break_on_timeout).run() + analyzers = [CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets())] + performance_test_class(island_client=island_client, + config_parser=config_parser, + analyzers=analyzers, + timeout=timeout_in_seconds, + log_handler=log_handler, + break_on_timeout=break_on_timeout).run() @staticmethod def get_log_dir_path(): @@ -89,41 +90,41 @@ class TestMonkeyBlackbox(object): def test_server_online(self, island_client): assert island_client.get_api_status() is not None - #def test_ssh_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") -# - #def test_hadoop_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) -# - #def test_mssql_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") -# - #def test_smb_and_mimikatz_exploiters(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") -# - #def test_smb_pth(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") -# - #def test_elastic_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") -# - #def test_struts_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") -# - #def test_weblogic_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") -# - #def test_shellshock_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") -# - #def test_tunneling(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) -# - #def test_wmi_and_mimikatz_exploiters(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") -# - #def test_wmi_pth(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") + def test_ssh_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") + + def test_hadoop_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) + + def test_mssql_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") + + def test_smb_and_mimikatz_exploiters(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") + + def test_smb_pth(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") + + def test_elastic_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") + + def test_struts_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") + + def test_weblogic_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") + + def test_shellshock_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") + + def test_tunneling(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) + + def test_wmi_and_mimikatz_exploiters(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") + + def test_wmi_pth(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") def test_report_generation_performance(self, island_client): """ @@ -137,3 +138,10 @@ class TestMonkeyBlackbox(object): island_client, "PERFORMANCE.conf", timeout_in_seconds=10*60) + + def test_map_generation_performance(self, island_client): + TestMonkeyBlackbox.run_performance_test(MapGenerationTest, + island_client, + "PERFORMANCE.conf", + timeout_in_seconds=10*60) + diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py index dc2b6b0c7..76a389efd 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -39,4 +39,4 @@ class EndpointPerformanceTest(BasicTest): else: LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") # instead of raising for status, mark failed responses as maxtime - return timedelta.max() + return timedelta.max diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py index 782bf82fe..c597907f4 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py @@ -9,7 +9,7 @@ MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) MAP_RESOURCES = [ - "api/map", + "api/netmap", ] diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py index e496c09a5..3157140a9 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py @@ -23,7 +23,9 @@ class PerformanceTestWorkflow(BasicTest): self.island_client.kill_all_monkeys() self.exploitation_test.wait_until_monkeys_die() self.exploitation_test.wait_for_monkey_process_to_finish() - performance_test = EndpointPerformanceTest(self.name, self.performance_config, self.island_client) - performance_test.run() + performance_test = EndpointPerformanceTest(self.name, self.performance_config, self.island_client) + try: + assert performance_test.run() + finally: self.exploitation_test.parse_logs() self.island_client.reset_env()