diff --git a/.gitignore b/.gitignore index 97e618d43..a10127767 100644 --- a/.gitignore +++ b/.gitignore @@ -83,7 +83,7 @@ MonkeyZoo/* !MonkeyZoo/MonkeyZooDocs.pdf # Exported monkey telemetries -/monkey/test_telems/ +/monkey/telem_sample/ # vim swap files *.swp diff --git a/envs/monkey_zoo/.gitignore b/envs/monkey_zoo/.gitignore index 04310b6fd..be22d3037 100644 --- a/envs/monkey_zoo/.gitignore +++ b/envs/monkey_zoo/.gitignore @@ -1,2 +1,2 @@ logs/ -/blackbox/tests/performance/test_telems/* +/blackbox/tests/performance/telem_sample diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index e800537de..b31fbdcab 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -24,13 +24,14 @@ To run telemetry performance test follow these steps: 1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have exported telemetries already. 2. Run monkey and wait until infection is done. - 3. All telemetries are gathered in `monkey/test_telems` + 3. All telemetries are gathered in `monkey/telem_sample` 2. Run telemetry performance test. 1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems` 2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply telemetries gathered. - 1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox` + 1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox` 2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate telemetries 4 times. 3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion. - 3. Run blackbox tests, telemetry performance test will run as part of it. + 3. Performance test will run as part of BlackBox tests or you can run it separately by adding + `-k 'test_telem_performance'` option. diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py new file mode 100644 index 000000000..f12704d2d --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py @@ -0,0 +1,43 @@ +import json +import logging +from os import listdir, path +from typing import List, Dict + +from tqdm import tqdm + +TELEM_DIR_PATH = './tests/performance/telem_sample' +MAX_SAME_TYPE_TELEM_FILES = 10000 +LOGGER = logging.getLogger(__name__) + + +class SampleFileParser: + + @staticmethod + def save_teletries_to_files(telems: List[Dict]): + for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)): + SampleFileParser.save_telemetry_to_file(telem) + + @staticmethod + def save_telemetry_to_file(telem: Dict): + telem_filename = telem['name'] + telem['method'] + for i in range(MAX_SAME_TYPE_TELEM_FILES): + if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): + telem_filename = str(i) + telem_filename + break + with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: + file.write(json.dumps(telem)) + + @staticmethod + def read_telem_files() -> List[str]: + telems = [] + file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) + if path.isfile(path.join(TELEM_DIR_PATH, f))] + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telem_string = "".join(telem_file.readlines()).replace("\n", "") + telems.append(telem_string) + return telems + + @staticmethod + def get_all_telemetries() -> List[Dict]: + return [json.loads(t) for t in SampleFileParser.read_telem_files()] diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py similarity index 100% rename from envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py similarity index 82% rename from envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py index dab84a1e6..89cdf5cad 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py @@ -1,6 +1,7 @@ import random -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator class FakeMonkey: diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py similarity index 57% rename from envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py index df7e9f5be..da3c22b05 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py @@ -2,35 +2,37 @@ import copy import json import logging import sys -from os import listdir, path from typing import List, Dict from tqdm import tqdm -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_multiplier.fake_monkey import FakeMonkey -TELEM_DIR_PATH = './tests/performance/test_telems' +TELEM_DIR_PATH = './tests/performance/telemetry_sample' LOGGER = logging.getLogger(__name__) -class TelemParser: +class SampleMultiplier: def __init__(self, multiplier: int): self.multiplier = multiplier self.fake_ip_generator = FakeIpGenerator() def multiply_telems(self): - telems = TelemParser.get_all_telemetries() + telems = SampleFileParser.get_all_telemetries() telem_contents = [json.loads(telem['content']) for telem in telems] monkeys = self.get_monkeys_from_telems(telem_contents) for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries", position=1): for monkey in monkeys: monkey.change_fake_data() fake_telem_batch = copy.deepcopy(telems) - TelemParser.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) - TelemParser.offset_telem_times(iteration=i, telems=fake_telem_batch) - TelemParser.save_teletries_to_files(fake_telem_batch) + SampleMultiplier.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) + SampleMultiplier.offset_telem_times(iteration=i, telems=fake_telem_batch) + SampleFileParser.save_teletries_to_files(fake_telem_batch) + LOGGER.info("") @staticmethod def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]): @@ -38,7 +40,8 @@ class TelemParser: for monkey in monkeys: if monkey.on_island: continue - if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) and not monkey.on_island: + if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) \ + and not monkey.on_island: telem['content'] = telem['content'].replace(monkey.original_guid, monkey.fake_guid) telem['endpoint'] = telem['endpoint'].replace(monkey.original_guid, monkey.fake_guid) for i in range(len(monkey.original_ips)): @@ -49,39 +52,11 @@ class TelemParser: for telem in telems: telem['time']['$date'] += iteration * 1000 - @staticmethod - def save_teletries_to_files(telems: List[Dict]): - for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)): - TelemParser.save_telemetry_to_file(telem) - - @staticmethod - def save_telemetry_to_file(telem: Dict): - telem_filename = telem['name'] + telem['method'] - for i in range(10000): - if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): - telem_filename = str(i) + telem_filename - break - with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: - file.write(json.dumps(telem)) - - @staticmethod - def read_telem_files() -> List[str]: - telems = [] - file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) - if path.isfile(path.join(TELEM_DIR_PATH, f))] - for file_path in file_paths: - with open(file_path, 'r') as telem_file: - telems.append(telem_file.readline()) - return telems - - @staticmethod - def get_all_telemetries() -> List[Dict]: - return [json.loads(t) for t in TelemParser.read_telem_files()] - def get_monkeys_from_telems(self, telems: List[Dict]): - island_ips = TelemParser.get_island_ips_from_telems(telems) + island_ips = SampleMultiplier.get_island_ips_from_telems(telems) monkeys = [] - for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']: + for telem in [telem for telem in telems + if 'telem_category' in telem and telem['telem_category'] == 'system_info']: if 'network_info' not in telem['data']: continue guid = telem['monkey_guid'] @@ -111,4 +86,4 @@ class TelemParser: if __name__ == "__main__": - TelemParser(multiplier=int(sys.argv[1])).multiply_telems() + SampleMultiplier(multiplier=int(sys.argv[1])).multiply_telems() diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py similarity index 85% rename from envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py index 96609a8a9..d8adef827 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py @@ -1,6 +1,7 @@ from unittest import TestCase -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator class TestFakeIpGenerator(TestCase): diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 4086a234e..5fc97d222 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -8,7 +8,7 @@ from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceA from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig -from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser LOGGER = logging.getLogger(__name__) @@ -24,7 +24,7 @@ class TelemetryPerformanceTest: def test_telemetry_performance(self): LOGGER.info("Starting telemetry performance test.") try: - all_telemetries = TelemParser.get_all_telemetries() + all_telemetries = SampleFileParser.get_all_telemetries() except FileNotFoundError: LOGGER.error("Telemetries to send not found. Refer to readme to figure out how to generate telemetries " "and where to put them.") diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index 031a0d02a..18ebfd244 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -9,7 +9,7 @@ from flask import request from monkey_island.cc.models.test_telem import TestTelem from monkey_island.cc.services.config import ConfigService -TEST_TELEM_DIR = "./test_telems" +TELEM_SAMPLE_DIR = "./telem_sample" MAX_SAME_CATEGORY_TELEMS = 10000 @@ -36,16 +36,16 @@ class TestTelemStore: @staticmethod def export_test_telems(): - logger.info(f"Exporting all telemetries to {TEST_TELEM_DIR}") + logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}") try: - mkdir(TEST_TELEM_DIR) + mkdir(TELEM_SAMPLE_DIR) except FileExistsError: logger.info("Deleting all previous telemetries.") - shutil.rmtree(TEST_TELEM_DIR) - mkdir(TEST_TELEM_DIR) + shutil.rmtree(TELEM_SAMPLE_DIR) + mkdir(TELEM_SAMPLE_DIR) for test_telem in TestTelem.objects(): - with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file: - file.write(test_telem.to_json()) + with open(TestTelemStore.get_unique_file_path_for_test_telem(TELEM_SAMPLE_DIR, test_telem), 'w') as file: + file.write(test_telem.to_json(indent=2)) logger.info("Telemetries exported!") @staticmethod