From 568a97e5a53008d4bf157c10ed30ad5db1d1f414 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 30 Jul 2021 16:54:15 +0300 Subject: [PATCH] BB performance tests: update the outdated README.md and other small improvements --- .gitignore | 2 +- envs/monkey_zoo/.gitignore | 2 +- envs/monkey_zoo/blackbox/README.md | 17 +++++++++-------- .../island_client/monkey_island_client.py | 4 ++++ envs/monkey_zoo/blackbox/test_blackbox.py | 2 ++ .../telem_sample_parsing/sample_file_parser.py | 2 +- .../sample_multiplier/sample_multiplier.py | 1 - .../performance/telemetry_performance_test.py | 13 ++++++------- 8 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 82df0f640..c6a702e86 100644 --- a/.gitignore +++ b/.gitignore @@ -85,7 +85,7 @@ MonkeyZoo/* monkey/logs # Exported monkey telemetries -/monkey/telem_sample/ +/envs/monkey_zoo/blackbox/tests/performance/telemetry_sample/ # Profiling logs profiler_logs/ diff --git a/envs/monkey_zoo/.gitignore b/envs/monkey_zoo/.gitignore index be22d3037..e7549cdec 100644 --- a/envs/monkey_zoo/.gitignore +++ b/envs/monkey_zoo/.gitignore @@ -1,2 +1,2 @@ logs/ -/blackbox/tests/performance/telem_sample +/blackbox/tests/performance/telemetry_sample diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index b6a12333f..a1cb95962 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -32,19 +32,20 @@ directory `monkey\envs\monkey_zoo\blackbox`. **Before running performance test make sure browser is not sending requests to island!** To run telemetry performance test follow these steps: -0. Set `server_config.json` to "standard" (no password protection) setting. +0. Set no password protection on the island. +Make sure the island parameter is an IP address(not localhost) as the name resolution will increase the time for requests. 1. Gather monkey telemetries. 1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have exported telemetries already. 2. Run monkey and wait until infection is done. - 3. All telemetries are gathered in `monkey/telem_sample` + 3. All telemetries are gathered in `monkey/telem_sample`. If not, restart the island process. 2. Run telemetry performance test. - 1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems` - 2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply + 1. Move directory `monkey/telem_sample` to `envs/monkey_zoo/blackbox/tests/performance/telemetry_sample` + 2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py` to multiply telemetries gathered. - 1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox` + 1. Run `sample_multiplier.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox` 2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate telemetries 4 times. - 3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion. - 3. Performance test will run as part of BlackBox tests or you can run it separately by adding - `-k 'test_telem_performance'` option. + 3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuration. + 3. Add a `--run-performance-tests` flag to blackbox scripts to run performance tests as part of BlackBox tests. + You can run a single test separately by adding `-k 'test_telem_performance'` option. diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index 9ec54a56f..5c5b57e09 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -62,6 +62,10 @@ class MonkeyIslandClient(object): LOGGER.error("Failed to reset the environment.") assert False + @avoid_race_condition + def set_scenario(self, scenario): + self.requests.post_json("api/island-mode", {"mode": scenario}) + def find_monkeys_in_db(self, query): if query is None: raise TypeError diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 5ee5f63c7..26cfa646f 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -39,6 +39,7 @@ from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test impor TelemetryPerformanceTest, ) from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers +from monkey_island.cc.services.mode.mode_enum import IslandModeEnum DEFAULT_TIMEOUT_SECONDS = 5 * 60 MACHINE_BOOTUP_WAIT_SECONDS = 30 @@ -109,6 +110,7 @@ def island_client(island, quick_performance_tests): pytest.exit("BB tests couldn't establish communication to the island.") if not quick_performance_tests: island_client_object.reset_env() + island_client_object.set_scenario(IslandModeEnum.ADVANCED.value) yield island_client_object diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py index 42a851405..c0eeafd5c 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py @@ -5,7 +5,7 @@ from typing import Dict, List from tqdm import tqdm -TELEM_DIR_PATH = "./tests/performance/telem_sample" +TELEM_DIR_PATH = "../envs/monkey_zoo/blackbox/tests/performance/telemetry_sample" MAX_SAME_TYPE_TELEM_FILES = 10000 LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py index 221c3a87a..8ec9bb346 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py @@ -16,7 +16,6 @@ from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_mult FakeMonkey, ) -TELEM_DIR_PATH = "./tests/performance/telemetry_sample" LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index b76c1b68d..31179d713 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -2,8 +2,6 @@ import json import logging from datetime import timedelta -from tqdm import tqdm - from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod @@ -35,11 +33,12 @@ class TelemetryPerformanceTest: LOGGER.info("Telemetries imported successfully.") all_telemetries.sort(key=lambda telem: telem["time"]["$date"]) telemetry_parse_times = {} - for telemetry in tqdm( - all_telemetries, total=len(all_telemetries), ascii=True, desc="Telemetries sent" - ): - telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint(telemetry) - telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(telemetry) + for i in range(len(all_telemetries)): + telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint( + all_telemetries[i] + ) + telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(all_telemetries[i]) + LOGGER.info(f"Telemetry Nr.{i} sent out of {len(all_telemetries)} total.") test_config = PerformanceTestConfig( MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME, MAX_ALLOWED_TOTAL_TIME )