diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index d9067eeee..92d8a68eb 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -14,11 +14,11 @@ class PerformanceAnalyzer(Analyzer): self.endpoint_timings = endpoint_timings def analyze_test_results(self): - # Calculate total time and check each page + # Calculate total time and check each endpoint single_page_time_less_then_max = True total_time = timedelta() - for page, elapsed in self.endpoint_timings.items(): - LOGGER.info(f"page {page} took {str(elapsed)}") + for endpoint, elapsed in self.endpoint_timings.items(): + LOGGER.info(f"Endpoint {endpoint} took {str(elapsed)}") total_time += elapsed if elapsed > self.performance_test_config.max_allowed_single_page_time: single_page_time_less_then_max = False diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index 07785509d..b56f51612 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -1,8 +1,12 @@ from typing import Dict +from datetime import timedelta + import requests import functools +from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod + # SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' import logging @@ -16,6 +20,26 @@ class MonkeyIslandRequests(object): def __init__(self, server_address): self.addr = "https://{IP}/".format(IP=server_address) self.token = self.try_get_jwt_from_server() + self.supported_request_methods = {SupportedRequestMethod.GET: self.get, + SupportedRequestMethod.POST: self.post, + SupportedRequestMethod.PATCH: self.patch, + SupportedRequestMethod.DELETE: self.delete} + + def get_request_time(self, url, method: SupportedRequestMethod, data=None): + response = self.send_request_by_method(url, method, data) + if response.ok: + LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") + return response.elapsed + else: + LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") + # instead of raising for status, mark failed responses as maxtime + return timedelta.max + + def send_request_by_method(self, url, method=SupportedRequestMethod.GET, data=None): + if data: + return self.supported_request_methods[method](url, data) + else: + return self.supported_request_methods[method](url) def try_get_jwt_from_server(self): try: @@ -64,9 +88,9 @@ class MonkeyIslandRequests(object): verify=False) @_Decorators.refresh_jwt_token - def patch_json(self, url, data: Dict): + def patch(self, url, data: Dict): return requests.patch(self.addr + url, # noqa: DUO123 - json=data, + data=data, headers=self.get_jwt_header(), verify=False) diff --git a/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py b/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py new file mode 100644 index 000000000..60cb0877a --- /dev/null +++ b/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class SupportedRequestMethod(Enum): + GET = "GET" + POST = "POST" + PATCH = "PATCH" + DELETE = "DELETE" diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py index 76a389efd..5a83f252c 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -1,10 +1,10 @@ import logging -from datetime import timedelta from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer +from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod LOGGER = logging.getLogger(__name__) @@ -25,18 +25,8 @@ class EndpointPerformanceTest(BasicTest): self.island_client.clear_caches() endpoint_timings = {} for endpoint in self.test_config.endpoints_to_test: - endpoint_timings[endpoint] = self.get_elapsed_for_get_request(endpoint) - + endpoint_timings[endpoint] = self.island_client.requests.get_request_time(endpoint, + SupportedRequestMethod.GET) analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings) return analyzer.analyze_test_results() - - def get_elapsed_for_get_request(self, url): - response = self.island_client.requests.get(url) - if response.ok: - LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") - return response.elapsed - else: - LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") - # instead of raising for status, mark failed responses as maxtime - return timedelta.max diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py index 8ed2b5a62..ad7be5967 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py @@ -5,7 +5,7 @@ from typing import List class PerformanceTestConfig: def __init__(self, max_allowed_single_page_time: timedelta, max_allowed_total_time: timedelta, - endpoints_to_test: List[str], break_on_timeout=False): + endpoints_to_test: List[str] = None, break_on_timeout=False): self.max_allowed_single_page_time = max_allowed_single_page_time self.max_allowed_total_time = max_allowed_total_time self.endpoints_to_test = endpoints_to_test diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 047d07406..0950be4ff 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -1,7 +1,17 @@ - +import logging +from datetime import timedelta +import json from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser +from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod + +LOGGER = logging.getLogger(__name__) + +MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=60) class TelemetryPerformanceTest: @@ -10,24 +20,32 @@ class TelemetryPerformanceTest: self.island_client = island_client def test_telemetry_performance(self): - all_telemetries = TelemParser.get_all_telemetries() + LOGGER.info("Starting telemetry performance test.") + try: + all_telemetries = TelemParser.get_all_telemetries() + except FileNotFoundError: + LOGGER.error("Telemetries to send not found. Refer to readme to figure out how to generate telemetries " + "and where to put them.") + return False + LOGGER.info("Telemetries imported successfully.") all_telemetries.sort(key=lambda telem: telem['time']['$date']) + telemetry_parse_times = {} for telemetry in all_telemetries: - self.send_telemetry(telemetry) + telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint(telemetry) + telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(telemetry) + test_config = PerformanceTestConfig(MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME, MAX_ALLOWED_TOTAL_TIME) + PerformanceAnalyzer(test_config, telemetry_parse_times).analyze_test_results() - def send_telemetry(self, telemetry): + def get_telemetry_time(self, telemetry): content = telemetry['content'] url = telemetry['endpoint'] - method = telemetry['method'] + method = SupportedRequestMethod.__getattr__(telemetry['method']) - if method == 'POST': - result = self.island_client.requests.post(url=url, data=content) - elif method == 'GET': - result = self.island_client.requests.get(url=url) - elif method == 'PATCH': - result = self.island_client.requests.patch_json(url=url, data=content) - elif method == 'DELETE': - result = self.island_client.requests.delete(url=url) - else: - raise Exception - return result + return self.island_client.requests.get_request_time(url=url, method=method, data=content) + + @staticmethod + def get_verbose_telemetry_endpoint(telemetry): + telem_category = "" + if "telem_category" in telemetry['content']: + telem_category = "_" + json.loads(telemetry['content'])['telem_category'] + return telemetry['endpoint'] + telem_category