diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index b31fbdcab..dbdd54b41 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -10,7 +10,13 @@ In order to execute the entire test suite, you must know the external IP of the this information in the GCP Console `Compute Engine/VM Instances` under _External IP_. #### Running in command line -Run the following command: +Blackbox tests have following parameters: +- `--island=IP` Sets island's IP +- `--no-gcp` (Optional) Use for no interaction with the cloud (local test). +- `--quick-performance-tests` (Optional) If enabled performance tests won't reset island and won't send telemetries, +instead will just test performance of endpoints in already present island state. + +Example run command: `monkey\envs\monkey_zoo\blackbox>python -m pytest -s --island=35.207.152.72:5000 test_blackbox.py` diff --git a/envs/monkey_zoo/blackbox/conftest.py b/envs/monkey_zoo/blackbox/conftest.py index 13aabf5b6..0444a4101 100644 --- a/envs/monkey_zoo/blackbox/conftest.py +++ b/envs/monkey_zoo/blackbox/conftest.py @@ -4,8 +4,24 @@ import pytest def pytest_addoption(parser): parser.addoption("--island", action="store", default="", help="Specify the Monkey Island address (host+port).") + parser.addoption("--no-gcp", action="store_true", default=False, + help="Use for no interaction with the cloud.") + parser.addoption("--quick-performance-tests", action="store_true", default=False, + help="If enabled performance tests won't reset island and won't send telemetries, " + "instead will just test performance of already present island state.") -@pytest.fixture(scope='module') +@pytest.fixture(scope='session') def island(request): return request.config.getoption("--island") + + +@pytest.fixture(scope='session') +def no_gcp(request): + return request.config.getoption("--no-gcp") + + +@pytest.fixture(scope='session') +def quick_performance_tests(request): + return request.config.getoption("--quick-performance-tests") + diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 2408d79be..870b81868 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -27,15 +27,16 @@ LOGGER = logging.getLogger(__name__) @pytest.fixture(autouse=True, scope='session') -def GCPHandler(request): - GCPHandler = gcp_machine_handlers.GCPHandler() - GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) - wait_machine_bootup() +def GCPHandler(request, no_gcp): + if not no_gcp: + GCPHandler = gcp_machine_handlers.GCPHandler() + GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + wait_machine_bootup() - def fin(): - GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + def fin(): + GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) - request.addfinalizer(fin) + request.addfinalizer(fin) @pytest.fixture(autouse=True, scope='session') @@ -49,9 +50,10 @@ def wait_machine_bootup(): @pytest.fixture(scope='class') -def island_client(island): +def island_client(island, quick_performance_tests): island_client_object = MonkeyIslandClient(island) - island_client_object.reset_env() + if not quick_performance_tests: + island_client_object.reset_env() yield island_client_object @@ -130,7 +132,7 @@ class TestMonkeyBlackbox(object): def test_wmi_pth(self, island_client): TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") - def test_report_generation_performance(self, island_client): + def test_report_generation_performance(self, island_client, quick_performance_tests): """ This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test for a total of 8 machines including the Monkey Island. @@ -138,22 +140,31 @@ class TestMonkeyBlackbox(object): Is has 2 analyzers - the regular one which checks all the Monkeys and the Timing one which checks how long the report took to execute """ - TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, - island_client, - "PERFORMANCE.conf", - timeout_in_seconds=10*60) + if not quick_performance_tests: + TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, + island_client, + "PERFORMANCE.conf", + timeout_in_seconds=10*60) + else: + LOGGER.error("This test doesn't support 'quick_performance_tests' option.") + assert False - def test_map_generation_performance(self, island_client): - TestMonkeyBlackbox.run_performance_test(MapGenerationTest, - island_client, - "PERFORMANCE.conf", - timeout_in_seconds=10*60) + def test_map_generation_performance(self, island_client, quick_performance_tests): + if not quick_performance_tests: + TestMonkeyBlackbox.run_performance_test(MapGenerationTest, + island_client, + "PERFORMANCE.conf", + timeout_in_seconds=10*60) + else: + LOGGER.error("This test doesn't support 'quick_performance_tests' option.") + assert False - def test_report_generation_from_fake_telemetries(self, island_client): - ReportGenerationFromTelemetryTest(island_client).run() + def test_report_generation_from_fake_telemetries(self, island_client, quick_performance_tests): + ReportGenerationFromTelemetryTest(island_client, quick_performance_tests).run() - def test_map_generation_from_fake_telemetries(self, island_client): - MapGenerationFromTelemetryTest(island_client).run() + def test_map_generation_from_fake_telemetries(self, island_client, quick_performance_tests): + MapGenerationFromTelemetryTest(island_client, quick_performance_tests).run() - def test_telem_performance(self, island_client): - TelemetryPerformanceTest(island_client).test_telemetry_performance() + def test_telem_performance(self, island_client, quick_performance_tests): + if not quick_performance_tests: + TelemetryPerformanceTest(island_client).test_telemetry_performance() diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py index c5344d8f7..1b31a8962 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation_from_telemetries.py @@ -17,7 +17,7 @@ class MapGenerationFromTelemetryTest(PerformanceTest): TEST_NAME = "Map generation from fake telemetries test" - def __init__(self, island_client, break_on_timeout=False): + def __init__(self, island_client, quick_performance_test: bool, break_on_timeout=False): self.island_client = island_client performance_config = PerformanceTestConfig(max_allowed_single_page_time=MAX_ALLOWED_SINGLE_PAGE_TIME, max_allowed_total_time=MAX_ALLOWED_TOTAL_TIME, @@ -25,7 +25,8 @@ class MapGenerationFromTelemetryTest(PerformanceTest): break_on_timeout=break_on_timeout) self.performance_test_workflow = TelemetryPerformanceTestWorkflow(MapGenerationFromTelemetryTest.TEST_NAME, self.island_client, - performance_config) + performance_config, + quick_performance_test) def run(self): self.performance_test_workflow.run() diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py index b5acf4a9e..e8bef33d8 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test_workflow.py @@ -6,15 +6,18 @@ from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test impor class TelemetryPerformanceTestWorkflow(BasicTest): - def __init__(self, name, island_client, performance_config: PerformanceTestConfig): + def __init__(self, name, island_client, performance_config: PerformanceTestConfig, quick_performance_test): self.name = name self.island_client = island_client self.performance_config = performance_config + self.quick_performance_test = quick_performance_test def run(self): try: - TelemetryPerformanceTest(island_client=self.island_client).test_telemetry_performance() + if not self.quick_performance_test: + TelemetryPerformanceTest(island_client=self.island_client).test_telemetry_performance() performance_test = EndpointPerformanceTest(self.name, self.performance_config, self.island_client) assert performance_test.run() finally: - self.island_client.reset_env() + if not self.quick_performance_test: + self.island_client.reset_env()