diff --git a/.gitignore b/.gitignore index 772829801..a10127767 100644 --- a/.gitignore +++ b/.gitignore @@ -82,5 +82,8 @@ MonkeyZoo/* !MonkeyZoo/config.tf !MonkeyZoo/MonkeyZooDocs.pdf +# Exported monkey telemetries +/monkey/telem_sample/ + # vim swap files *.swp diff --git a/envs/monkey_zoo/.gitignore b/envs/monkey_zoo/.gitignore index 333c1e910..be22d3037 100644 --- a/envs/monkey_zoo/.gitignore +++ b/envs/monkey_zoo/.gitignore @@ -1 +1,2 @@ logs/ +/blackbox/tests/performance/telem_sample diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index f1b66de91..b31fbdcab 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -12,8 +12,26 @@ this information in the GCP Console `Compute Engine/VM Instances` under _Externa #### Running in command line Run the following command: -`monkey\envs\monkey_zoo\blackbox>python -m pytest --island=35.207.152.72:5000 test_blackbox.py` +`monkey\envs\monkey_zoo\blackbox>python -m pytest -s --island=35.207.152.72:5000 test_blackbox.py` #### Running in PyCharm -Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the +Configure a PyTest configuration with the additional arguments `-s --island=35.207.152.72` on the `monkey\envs\monkey_zoo\blackbox`. + +### Running telemetry performance test +To run telemetry performance test follow these steps: +1. Gather monkey telemetries. + 1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have + exported telemetries already. + 2. Run monkey and wait until infection is done. + 3. All telemetries are gathered in `monkey/telem_sample` +2. Run telemetry performance test. + 1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems` + 2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply + telemetries gathered. + 1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox` + 2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate + telemetries 4 times. + 3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion. + 3. Performance test will run as part of BlackBox tests or you can run it separately by adding + `-k 'test_telem_performance'` option. diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index d9067eeee..4a43ab6a5 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -4,6 +4,7 @@ from typing import Dict from envs.monkey_zoo.blackbox.analyzers.analyzer import Analyzer from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig + LOGGER = logging.getLogger(__name__) @@ -14,18 +15,18 @@ class PerformanceAnalyzer(Analyzer): self.endpoint_timings = endpoint_timings def analyze_test_results(self): - # Calculate total time and check each page + # Calculate total time and check each endpoint single_page_time_less_then_max = True total_time = timedelta() - for page, elapsed in self.endpoint_timings.items(): - LOGGER.info(f"page {page} took {str(elapsed)}") + for endpoint, elapsed in self.endpoint_timings.items(): total_time += elapsed if elapsed > self.performance_test_config.max_allowed_single_page_time: single_page_time_less_then_max = False total_time_less_then_max = total_time < self.performance_test_config.max_allowed_total_time - LOGGER.info(f"total time is {str(total_time)}") + PerformanceAnalyzer.log_slowest_endpoints(self.endpoint_timings) + LOGGER.info(f"Total time is {str(total_time)}") performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max @@ -37,3 +38,11 @@ class PerformanceAnalyzer(Analyzer): breakpoint() return performance_is_good_enough + + @staticmethod + def log_slowest_endpoints(endpoint_timings, max_endpoints_to_display=100): + slow_endpoint_list = list(endpoint_timings.items()) + slow_endpoint_list.sort(key=lambda x: x[1], reverse=True) + slow_endpoint_list = slow_endpoint_list[:max_endpoints_to_display] + for endpoint in slow_endpoint_list: + LOGGER.info(f"{endpoint[0]} took {str(endpoint[1])}") diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index f7d6f552c..b2370a345 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -1,7 +1,8 @@ -from time import sleep import json import logging +from time import sleep + from bson import json_util from envs.monkey_zoo.blackbox.island_client.monkey_island_requests import MonkeyIslandRequests @@ -30,7 +31,7 @@ class MonkeyIslandClient(object): @avoid_race_condition def run_monkey_local(self): - response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"}) + response = self.requests.post_json("api/local-monkey", data={"action": "run"}) if MonkeyIslandClient.monkey_ran_successfully(response): LOGGER.info("Running the monkey.") else: diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index 23f259a9c..babc9c7a0 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -1,9 +1,15 @@ +from typing import Dict +from datetime import timedelta + + import requests import functools -# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' +from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod + import logging +# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \ '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557' LOGGER = logging.getLogger(__name__) @@ -14,6 +20,26 @@ class MonkeyIslandRequests(object): def __init__(self, server_address): self.addr = "https://{IP}/".format(IP=server_address) self.token = self.try_get_jwt_from_server() + self.supported_request_methods = {SupportedRequestMethod.GET: self.get, + SupportedRequestMethod.POST: self.post, + SupportedRequestMethod.PATCH: self.patch, + SupportedRequestMethod.DELETE: self.delete} + + def get_request_time(self, url, method: SupportedRequestMethod, data=None): + response = self.send_request_by_method(url, method, data) + if response.ok: + LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") + return response.elapsed + else: + LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") + # instead of raising for status, mark failed responses as maxtime + return timedelta.max + + def send_request_by_method(self, url, method=SupportedRequestMethod.GET, data=None): + if data: + return self.supported_request_methods[method](url, data) + else: + return self.supported_request_methods[method](url) def try_get_jwt_from_server(self): try: @@ -55,9 +81,16 @@ class MonkeyIslandRequests(object): verify=False) @_Decorators.refresh_jwt_token - def post_json(self, url, dict_data): + def post_json(self, url, data: Dict): return requests.post(self.addr + url, # noqa: DUO123 - json=dict_data, + json=data, + headers=self.get_jwt_header(), + verify=False) + + @_Decorators.refresh_jwt_token + def patch(self, url, data: Dict): + return requests.patch(self.addr + url, # noqa: DUO123 + data=data, headers=self.get_jwt_header(), verify=False) diff --git a/envs/monkey_zoo/blackbox/island_client/supported_request_method.py b/envs/monkey_zoo/blackbox/island_client/supported_request_method.py new file mode 100644 index 000000000..60cb0877a --- /dev/null +++ b/envs/monkey_zoo/blackbox/island_client/supported_request_method.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class SupportedRequestMethod(Enum): + GET = "GET" + POST = "POST" + PATCH = "PATCH" + DELETE = "DELETE" diff --git a/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py b/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py index 091be570a..b7f424a69 100644 --- a/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py +++ b/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py @@ -1,6 +1,6 @@ +import logging import os -import logging from bson import ObjectId LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py b/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py index b54f773e6..bae6a9adc 100644 --- a/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py +++ b/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py @@ -1,8 +1,7 @@ +import logging import os import shutil -import logging - from envs.monkey_zoo.blackbox.log_handlers.monkey_log_parser import MonkeyLogParser from envs.monkey_zoo.blackbox.log_handlers.monkey_logs_downloader import MonkeyLogsDownloader diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 5109c7652..9e3211691 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -4,14 +4,15 @@ import logging import pytest from time import sleep -from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers -from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler DEFAULT_TIMEOUT_SECONDS = 5*60 MACHINE_BOOTUP_WAIT_SECONDS = 30 @@ -144,3 +145,6 @@ class TestMonkeyBlackbox(object): island_client, "PERFORMANCE.conf", timeout_in_seconds=10*60) + + def test_telem_performance(self, island_client): + TelemetryPerformanceTest(island_client).test_telemetry_performance() diff --git a/envs/monkey_zoo/blackbox/tests/exploitation.py b/envs/monkey_zoo/blackbox/tests/exploitation.py index e731d8f90..2d55f2294 100644 --- a/envs/monkey_zoo/blackbox/tests/exploitation.py +++ b/envs/monkey_zoo/blackbox/tests/exploitation.py @@ -1,9 +1,8 @@ +import logging from time import sleep -import logging - -from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer MAX_TIME_FOR_MONKEYS_TO_DIE = 5 * 60 WAIT_TIME_BETWEEN_REQUESTS = 10 diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py index 76a389efd..798f490af 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -1,11 +1,10 @@ import logging -from datetime import timedelta -from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest -from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer - +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig LOGGER = logging.getLogger(__name__) @@ -25,18 +24,8 @@ class EndpointPerformanceTest(BasicTest): self.island_client.clear_caches() endpoint_timings = {} for endpoint in self.test_config.endpoints_to_test: - endpoint_timings[endpoint] = self.get_elapsed_for_get_request(endpoint) - + endpoint_timings[endpoint] = self.island_client.requests.get_request_time(endpoint, + SupportedRequestMethod.GET) analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings) return analyzer.analyze_test_results() - - def get_elapsed_for_get_request(self, url): - response = self.island_client.requests.get(url) - if response.ok: - LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") - return response.elapsed - else: - LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") - # instead of raising for status, mark failed responses as maxtime - return timedelta.max diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py index c597907f4..eb95fdc6a 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py @@ -1,8 +1,8 @@ from datetime import timedelta from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py index 8ed2b5a62..ad7be5967 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py @@ -5,7 +5,7 @@ from typing import List class PerformanceTestConfig: def __init__(self, max_allowed_single_page_time: timedelta, max_allowed_total_time: timedelta, - endpoints_to_test: List[str], break_on_timeout=False): + endpoints_to_test: List[str] = None, break_on_timeout=False): self.max_allowed_single_page_time = max_allowed_single_page_time self.max_allowed_total_time = max_allowed_total_time self.endpoints_to_test = endpoints_to_test diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py index 3157140a9..f6cd1dada 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py @@ -1,7 +1,7 @@ from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.endpoint_performance_test import EndpointPerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig class PerformanceTestWorkflow(BasicTest): diff --git a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py index 52fe76288..e204cc29f 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py @@ -1,9 +1,9 @@ from datetime import timedelta from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow -from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py new file mode 100644 index 000000000..70e25d8e7 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py @@ -0,0 +1,47 @@ +import json +import logging +from os import listdir, path +from typing import List, Dict + +from tqdm import tqdm + +TELEM_DIR_PATH = './tests/performance/telem_sample' +MAX_SAME_TYPE_TELEM_FILES = 10000 +LOGGER = logging.getLogger(__name__) + + +class SampleFileParser: + + @staticmethod + def save_teletries_to_files(telems: List[Dict]): + for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)): + SampleFileParser.save_telemetry_to_file(telem) + + @staticmethod + def save_telemetry_to_file(telem: Dict): + telem_filename = telem['name'] + telem['method'] + for i in range(MAX_SAME_TYPE_TELEM_FILES): + if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): + telem_filename = str(i) + telem_filename + break + with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: + file.write(json.dumps(telem)) + + @staticmethod + def read_telem_files() -> List[str]: + telems = [] + try: + file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) + if path.isfile(path.join(TELEM_DIR_PATH, f))] + except FileNotFoundError: + raise FileNotFoundError("Telemetries to send not found. " + "Refer to readme to figure out how to generate telemetries and where to put them.") + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telem_string = "".join(telem_file.readlines()).replace("\n", "") + telems.append(telem_string) + return telems + + @staticmethod + def get_all_telemetries() -> List[Dict]: + return [json.loads(t) for t in SampleFileParser.read_telem_files()] diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py new file mode 100644 index 000000000..90422f9a0 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py @@ -0,0 +1,25 @@ +from typing import List + + +class FakeIpGenerator: + def __init__(self): + self.fake_ip_parts = [1, 1, 1, 1] + + def generate_fake_ips_for_real_ips(self, real_ips: List[str]) -> List[str]: + fake_ips = [] + for i in range(len(real_ips)): + fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts)) + self.increment_ip() + return fake_ips + + def increment_ip(self): + self.fake_ip_parts[3] += 1 + self.try_fix_ip_range() + + def try_fix_ip_range(self): + for i in range(len(self.fake_ip_parts)): + if self.fake_ip_parts[i] > 256: + if i-1 < 0: + raise Exception("Fake IP's out of range.") + self.fake_ip_parts[i-1] += 1 + self.fake_ip_parts[i] = 1 diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py new file mode 100644 index 000000000..89cdf5cad --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py @@ -0,0 +1,18 @@ +import random + +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator + + +class FakeMonkey: + def __init__(self, ips, guid, fake_ip_generator: FakeIpGenerator, on_island=False): + self.original_ips = ips + self.original_guid = guid + self.fake_ip_generator = fake_ip_generator + self.on_island = on_island + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) + self.fake_ips = fake_ip_generator.generate_fake_ips_for_real_ips(ips) + + def change_fake_data(self): + self.fake_ips = self.fake_ip_generator.generate_fake_ips_for_real_ips(self.original_ips) + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py new file mode 100644 index 000000000..da3c22b05 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py @@ -0,0 +1,89 @@ +import copy +import json +import logging +import sys +from typing import List, Dict + +from tqdm import tqdm + +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_multiplier.fake_monkey import FakeMonkey + +TELEM_DIR_PATH = './tests/performance/telemetry_sample' +LOGGER = logging.getLogger(__name__) + + +class SampleMultiplier: + + def __init__(self, multiplier: int): + self.multiplier = multiplier + self.fake_ip_generator = FakeIpGenerator() + + def multiply_telems(self): + telems = SampleFileParser.get_all_telemetries() + telem_contents = [json.loads(telem['content']) for telem in telems] + monkeys = self.get_monkeys_from_telems(telem_contents) + for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries", position=1): + for monkey in monkeys: + monkey.change_fake_data() + fake_telem_batch = copy.deepcopy(telems) + SampleMultiplier.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) + SampleMultiplier.offset_telem_times(iteration=i, telems=fake_telem_batch) + SampleFileParser.save_teletries_to_files(fake_telem_batch) + LOGGER.info("") + + @staticmethod + def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]): + for telem in tqdm(telems, desc="Telemetries fabricated", position=2): + for monkey in monkeys: + if monkey.on_island: + continue + if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) \ + and not monkey.on_island: + telem['content'] = telem['content'].replace(monkey.original_guid, monkey.fake_guid) + telem['endpoint'] = telem['endpoint'].replace(monkey.original_guid, monkey.fake_guid) + for i in range(len(monkey.original_ips)): + telem['content'] = telem['content'].replace(monkey.original_ips[i], monkey.fake_ips[i]) + + @staticmethod + def offset_telem_times(iteration: int, telems: List[Dict]): + for telem in telems: + telem['time']['$date'] += iteration * 1000 + + def get_monkeys_from_telems(self, telems: List[Dict]): + island_ips = SampleMultiplier.get_island_ips_from_telems(telems) + monkeys = [] + for telem in [telem for telem in telems + if 'telem_category' in telem and telem['telem_category'] == 'system_info']: + if 'network_info' not in telem['data']: + continue + guid = telem['monkey_guid'] + monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid] + if not monkey_present: + ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']] + if set(island_ips).intersection(ips): + on_island = True + else: + on_island = False + + monkeys.append(FakeMonkey(ips=ips, + guid=guid, + fake_ip_generator=self.fake_ip_generator, + on_island=on_island)) + return monkeys + + @staticmethod + def get_island_ips_from_telems(telems: List[Dict]) -> List[str]: + island_ips = [] + for telem in telems: + if 'config' in telem: + island_ips = telem['config']['command_servers'] + for i in range(len(island_ips)): + island_ips[i] = island_ips[i].replace(":5000", "") + return island_ips + + +if __name__ == "__main__": + SampleMultiplier(multiplier=int(sys.argv[1])).multiply_telems() diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py new file mode 100644 index 000000000..d8adef827 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py @@ -0,0 +1,19 @@ +from unittest import TestCase + +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator + + +class TestFakeIpGenerator(TestCase): + + def test_fake_ip_generation(self): + fake_ip_gen = FakeIpGenerator() + self.assertListEqual([1, 1, 1, 1], fake_ip_gen.fake_ip_parts) + for i in range(256): + fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1']) + self.assertListEqual(['1.1.2.1'], fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1'])) + fake_ip_gen.fake_ip_parts = [256, 256, 255, 256] + self.assertListEqual(['256.256.255.256', '256.256.256.1'], + fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1', '1.1.1.2'])) + fake_ip_gen.fake_ip_parts = [256, 256, 256, 256] + self.assertRaises(Exception, fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1'])) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py new file mode 100644 index 000000000..4de77e41a --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -0,0 +1,52 @@ +import json +import logging +from datetime import timedelta + +from tqdm import tqdm + +from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser + +LOGGER = logging.getLogger(__name__) + +MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=60) + + +class TelemetryPerformanceTest: + + def __init__(self, island_client: MonkeyIslandClient): + self.island_client = island_client + + def test_telemetry_performance(self): + LOGGER.info("Starting telemetry performance test.") + try: + all_telemetries = SampleFileParser.get_all_telemetries() + except FileNotFoundError: + raise FileNotFoundError("Telemetries to send not found. " + "Refer to readme to figure out how to generate telemetries and where to put them.") + LOGGER.info("Telemetries imported successfully.") + all_telemetries.sort(key=lambda telem: telem['time']['$date']) + telemetry_parse_times = {} + for telemetry in tqdm(all_telemetries, total=len(all_telemetries), ascii=True, desc="Telemetries sent"): + telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint(telemetry) + telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(telemetry) + test_config = PerformanceTestConfig(MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME, MAX_ALLOWED_TOTAL_TIME) + PerformanceAnalyzer(test_config, telemetry_parse_times).analyze_test_results() + + def get_telemetry_time(self, telemetry): + content = telemetry['content'] + url = telemetry['endpoint'] + method = SupportedRequestMethod.__getattr__(telemetry['method']) + + return self.island_client.requests.get_request_time(url=url, method=method, data=content) + + @staticmethod + def get_verbose_telemetry_endpoint(telemetry): + telem_category = "" + if "telem_category" in telemetry['content']: + telem_category = "_" + json.loads(telemetry['content'])['telem_category'] + "_" + telemetry['_id']['$oid'] + return telemetry['endpoint'] + telem_category diff --git a/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py b/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py index 3cb2ad6af..633f406a5 100644 --- a/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py +++ b/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py @@ -1,6 +1,5 @@ -import subprocess - import logging +import subprocess LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/utils/json_encoder.py b/envs/monkey_zoo/blackbox/utils/json_encoder.py index 77be9211a..1642ea222 100644 --- a/envs/monkey_zoo/blackbox/utils/json_encoder.py +++ b/envs/monkey_zoo/blackbox/utils/json_encoder.py @@ -1,4 +1,5 @@ import json + from bson import ObjectId diff --git a/monkey/monkey_island/cc/models/test_telem.py b/monkey/monkey_island/cc/models/test_telem.py new file mode 100644 index 000000000..97855d4ed --- /dev/null +++ b/monkey/monkey_island/cc/models/test_telem.py @@ -0,0 +1,13 @@ +""" +Define a Document Schema for the TestTelem document. +""" +from mongoengine import Document, StringField, DateTimeField + + +class TestTelem(Document): + # SCHEMA + name = StringField(required=True) + time = DateTimeField(required=True) + method = StringField(required=True) + endpoint = StringField(required=True) + content = StringField(required=True) diff --git a/monkey/monkey_island/cc/resources/log.py b/monkey/monkey_island/cc/resources/log.py index cbce92a37..920890648 100644 --- a/monkey/monkey_island/cc/resources/log.py +++ b/monkey/monkey_island/cc/resources/log.py @@ -6,6 +6,7 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from monkey_island.cc.services.log import LogService from monkey_island.cc.services.node import NodeService @@ -23,6 +24,7 @@ class Log(flask_restful.Resource): return LogService.log_exists(ObjectId(exists_monkey_id)) # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self): telemetry_json = json.loads(request.data) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 3e3ef40c0..dcdc5bc12 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -3,6 +3,7 @@ from datetime import datetime import dateutil.parser import flask_restful +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from flask import request from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS @@ -33,6 +34,7 @@ class Monkey(flask_restful.Resource): return {} # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def patch(self, guid): monkey_json = json.loads(request.data) update = {"$set": {'modifytime': datetime.now()}} @@ -56,6 +58,7 @@ class Monkey(flask_restful.Resource): return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False) # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self, **kw): monkey_json = json.loads(request.data) monkey_json['creds'] = [] diff --git a/monkey/monkey_island/cc/resources/root.py b/monkey/monkey_island/cc/resources/root.py index 59a8fbe7c..216329905 100644 --- a/monkey/monkey_island/cc/resources/root.py +++ b/monkey/monkey_island/cc/resources/root.py @@ -1,6 +1,5 @@ import logging import threading -from datetime import datetime import flask_restful from flask import request, make_response, jsonify @@ -8,10 +7,7 @@ from flask import request, make_response, jsonify from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo from monkey_island.cc.services.database import Database -from monkey_island.cc.services.node import NodeService -from monkey_island.cc.services.reporting.report import ReportService -from monkey_island.cc.services.reporting.report_generation_synchronisation import is_report_being_generated, \ - safe_generate_reports +from monkey_island.cc.services.infection_lifecycle import InfectionLifecycle from monkey_island.cc.utils import local_ip_addresses __author__ = 'Barak' @@ -32,7 +28,7 @@ class Root(flask_restful.Resource): elif action == "reset": return jwt_required()(Database.reset_db)() elif action == "killall": - return Root.kill_all() + return jwt_required()(InfectionLifecycle.kill_all)() elif action == "is-up": return {'is-up': True} else: @@ -43,33 +39,6 @@ class Root(flask_restful.Resource): return jsonify( ip_addresses=local_ip_addresses(), mongo=str(mongo.db), - completed_steps=self.get_completed_steps()) + completed_steps=InfectionLifecycle.get_completed_steps()) - @staticmethod - @jwt_required() - def kill_all(): - mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}}, - upsert=False, - multi=True) - logger.info('Kill all monkeys was called') - return jsonify(status='OK') - @jwt_required() - def get_completed_steps(self): - is_any_exists = NodeService.is_any_monkey_exists() - infection_done = NodeService.is_monkey_finished_running() - - if infection_done: - # Checking is_report_being_generated here, because we don't want to wait to generate a report; rather, - # we want to skip and reply. - if not is_report_being_generated() and not ReportService.is_latest_report_exists(): - safe_generate_reports() - report_done = ReportService.is_report_generated() - else: # Infection is not done - report_done = False - - return dict( - run_server=True, - run_monkey=is_any_exists, - infection_done=infection_done, - report_done=report_done) diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index dc6a7d512..f6e833eeb 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -8,6 +8,7 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.processing import process_telemetry from monkey_island.cc.models.monkey import Monkey @@ -40,6 +41,7 @@ class Telemetry(flask_restful.Resource): return result # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self): telemetry_json = json.loads(request.data) telemetry_json['timestamp'] = datetime.now() diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py new file mode 100644 index 000000000..18ebfd244 --- /dev/null +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -0,0 +1,68 @@ +import logging +from functools import wraps +from os import mkdir, path +import shutil +from datetime import datetime + +from flask import request + +from monkey_island.cc.models.test_telem import TestTelem +from monkey_island.cc.services.config import ConfigService + +TELEM_SAMPLE_DIR = "./telem_sample" +MAX_SAME_CATEGORY_TELEMS = 10000 + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class TestTelemStore: + + @staticmethod + def store_test_telem(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if ConfigService.is_test_telem_export_enabled(): + time = datetime.now() + method = request.method + content = request.data.decode() + endpoint = request.path + name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') + TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save() + return f(*args, **kwargs) + + return decorated_function + + @staticmethod + def export_test_telems(): + logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}") + try: + mkdir(TELEM_SAMPLE_DIR) + except FileExistsError: + logger.info("Deleting all previous telemetries.") + shutil.rmtree(TELEM_SAMPLE_DIR) + mkdir(TELEM_SAMPLE_DIR) + for test_telem in TestTelem.objects(): + with open(TestTelemStore.get_unique_file_path_for_test_telem(TELEM_SAMPLE_DIR, test_telem), 'w') as file: + file.write(test_telem.to_json(indent=2)) + logger.info("Telemetries exported!") + + @staticmethod + def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem): + telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem) + for i in range(MAX_SAME_CATEGORY_TELEMS): + potential_filepath = path.join(target_dir, (telem_filename + str(i))) + if path.exists(potential_filepath): + continue + return potential_filepath + raise Exception(f"Too many telemetries of the same category. Max amount {MAX_SAME_CATEGORY_TELEMS}") + + @staticmethod + def _get_filename_by_test_telem(test_telem: TestTelem): + endpoint_part = test_telem.name + return endpoint_part + '_' + test_telem.method + + +if __name__ == '__main__': + TestTelemStore.export_test_telems() diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 96c59cad6..e9ed3b0f6 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -307,3 +307,7 @@ class ConfigService: pair['public_key'] = encryptor.dec(pair['public_key']) pair['private_key'] = encryptor.dec(pair['private_key']) return pair + + @staticmethod + def is_test_telem_export_enabled(): + return ConfigService.get_config_value(['internal', 'testing', 'export_monkey_telems']) diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index 85ea7c2ec..afc8591d3 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -737,6 +737,19 @@ SCHEMA = { "description": "List of SSH key pairs to use, when trying to ssh into servers" } } + }, + "testing": { + "title": "Testing", + "type": "object", + "properties": { + "export_monkey_telems": { + "title": "Export monkey telemetries", + "type": "boolean", + "default": False, + "description": "Exports unencrypted telemetries that can be used for tests in development." + " Do not turn on!" + } + } } } }, diff --git a/monkey/monkey_island/cc/services/infection_lifecycle.py b/monkey/monkey_island/cc/services/infection_lifecycle.py new file mode 100644 index 000000000..e79cfe947 --- /dev/null +++ b/monkey/monkey_island/cc/services/infection_lifecycle.py @@ -0,0 +1,51 @@ +import logging +from datetime import datetime + +from flask import jsonify + +from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore +from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.node import NodeService +from monkey_island.cc.services.reporting.report import ReportService +from monkey_island.cc.services.reporting.report_generation_synchronisation import is_report_being_generated, \ + safe_generate_reports + +logger = logging.getLogger(__name__) + + +class InfectionLifecycle: + + @staticmethod + def kill_all(): + mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}}, + upsert=False, + multi=True) + logger.info('Kill all monkeys was called') + return jsonify(status='OK') + + @staticmethod + def get_completed_steps(): + is_any_exists = NodeService.is_any_monkey_exists() + infection_done = NodeService.is_monkey_finished_running() + + if infection_done: + InfectionLifecycle._on_finished_infection() + report_done = ReportService.is_report_generated() + else: # Infection is not done + report_done = False + + return dict( + run_server=True, + run_monkey=is_any_exists, + infection_done=infection_done, + report_done=report_done) + + @staticmethod + def _on_finished_infection(): + # Checking is_report_being_generated here, because we don't want to wait to generate a report; rather, + # we want to skip and reply. + if not is_report_being_generated() and not ReportService.is_latest_report_exists(): + safe_generate_reports() + if ConfigService.is_test_telem_export_enabled(): + TestTelemStore.export_test_telems() diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index b5baed7f4..aa90c1916 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -23,3 +23,4 @@ requests dpath ring stix2 +tqdm