Improved the dir structure of telem parsing, other minor CR comments fixed

.gitignore

@@ -83,7 +83,7 @@ MonkeyZoo/*
 !MonkeyZoo/MonkeyZooDocs.pdf
 
 # Exported monkey telemetries
-/monkey/test_telems/
+/monkey/telem_sample/
 
 # vim swap files
 *.swp

envs/monkey_zoo/.gitignore

@@ -1,2 +1,2 @@
 logs/
-/blackbox/tests/performance/test_telems/*
+/blackbox/tests/performance/telem_sample

envs/monkey_zoo/blackbox/README.md

@@ -24,13 +24,14 @@ To run telemetry performance test follow these steps:
     1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have 
     exported telemetries already.
     2. Run monkey and wait until infection is done.
-    3. All telemetries are gathered in `monkey/test_telems`
+    3. All telemetries are gathered in `monkey/telem_sample`
 2. Run telemetry performance test.
     1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems`
     2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply 
     telemetries gathered.
-        1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox`
+        1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox`
         2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate
         telemetries 4 times.
         3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion.
-    3. Run blackbox tests, telemetry performance test will run as part of it.
+    3. Performance test will run as part of BlackBox tests or you can run it separately by adding 
+    `-k 'test_telem_performance'` option.

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py

@@ -0,0 +1,43 @@
+import json
+import logging
+from os import listdir, path
+from typing import List, Dict
+
+from tqdm import tqdm
+
+TELEM_DIR_PATH = './tests/performance/telem_sample'
+MAX_SAME_TYPE_TELEM_FILES = 10000
+LOGGER = logging.getLogger(__name__)
+
+
+class SampleFileParser:
+
+    @staticmethod
+    def save_teletries_to_files(telems: List[Dict]):
+        for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)):
+            SampleFileParser.save_telemetry_to_file(telem)
+
+    @staticmethod
+    def save_telemetry_to_file(telem: Dict):
+        telem_filename = telem['name'] + telem['method']
+        for i in range(MAX_SAME_TYPE_TELEM_FILES):
+            if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))):
+                telem_filename = str(i) + telem_filename
+                break
+        with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file:
+            file.write(json.dumps(telem))
+
+    @staticmethod
+    def read_telem_files() -> List[str]:
+        telems = []
+        file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH)
+                      if path.isfile(path.join(TELEM_DIR_PATH, f))]
+        for file_path in file_paths:
+            with open(file_path, 'r') as telem_file:
+                telem_string = "".join(telem_file.readlines()).replace("\n", "")
+                telems.append(telem_string)
+        return telems
+
+    @staticmethod
+    def get_all_telemetries() -> List[Dict]:
+        return [json.loads(t) for t in SampleFileParser.read_telem_files()]

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py

@@ -1,6 +1,7 @@
 import random
 
-from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator
+from envs.monkey_zoo.blackbox.tests.performance.\
+    telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator
 
 
 class FakeMonkey:

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py

@@ -2,35 +2,37 @@ import copy
 import json
 import logging
 import sys
-from os import listdir, path
 from typing import List, Dict
 
 from tqdm import tqdm
 
-from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator
+from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser
-from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey
+from envs.monkey_zoo.blackbox.tests.performance.\
+    telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator
+from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_multiplier.fake_monkey import FakeMonkey
 
-TELEM_DIR_PATH = './tests/performance/test_telems'
+TELEM_DIR_PATH = './tests/performance/telemetry_sample'
 LOGGER = logging.getLogger(__name__)
 
 
-class TelemParser:
+class SampleMultiplier:
 
     def __init__(self, multiplier: int):
         self.multiplier = multiplier
         self.fake_ip_generator = FakeIpGenerator()
 
     def multiply_telems(self):
-        telems = TelemParser.get_all_telemetries()
+        telems = SampleFileParser.get_all_telemetries()
         telem_contents = [json.loads(telem['content']) for telem in telems]
         monkeys = self.get_monkeys_from_telems(telem_contents)
         for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries", position=1):
             for monkey in monkeys:
                 monkey.change_fake_data()
             fake_telem_batch = copy.deepcopy(telems)
-            TelemParser.fabricate_monkeys_in_telems(fake_telem_batch, monkeys)
+            SampleMultiplier.fabricate_monkeys_in_telems(fake_telem_batch, monkeys)
-            TelemParser.offset_telem_times(iteration=i, telems=fake_telem_batch)
+            SampleMultiplier.offset_telem_times(iteration=i, telems=fake_telem_batch)
-            TelemParser.save_teletries_to_files(fake_telem_batch)
+            SampleFileParser.save_teletries_to_files(fake_telem_batch)
+            LOGGER.info("")
 
     @staticmethod
     def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]):
@@ -38,7 +40,8 @@ class TelemParser:
             for monkey in monkeys:
                 if monkey.on_island:
                     continue
-                if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) and not monkey.on_island:
+                if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) \
+                        and not monkey.on_island:
                     telem['content'] = telem['content'].replace(monkey.original_guid, monkey.fake_guid)
                     telem['endpoint'] = telem['endpoint'].replace(monkey.original_guid, monkey.fake_guid)
                 for i in range(len(monkey.original_ips)):
@@ -49,39 +52,11 @@ class TelemParser:
         for telem in telems:
             telem['time']['$date'] += iteration * 1000
 
-    @staticmethod
-    def save_teletries_to_files(telems: List[Dict]):
-        for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)):
-            TelemParser.save_telemetry_to_file(telem)
-
-    @staticmethod
-    def save_telemetry_to_file(telem: Dict):
-        telem_filename = telem['name'] + telem['method']
-        for i in range(10000):
-            if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))):
-                telem_filename = str(i) + telem_filename
-                break
-        with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file:
-            file.write(json.dumps(telem))
-
-    @staticmethod
-    def read_telem_files() -> List[str]:
-        telems = []
-        file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH)
-                      if path.isfile(path.join(TELEM_DIR_PATH, f))]
-        for file_path in file_paths:
-            with open(file_path, 'r') as telem_file:
-                telems.append(telem_file.readline())
-        return telems
-
-    @staticmethod
-    def get_all_telemetries() -> List[Dict]:
-        return [json.loads(t) for t in TelemParser.read_telem_files()]
-
     def get_monkeys_from_telems(self, telems: List[Dict]):
-        island_ips = TelemParser.get_island_ips_from_telems(telems)
+        island_ips = SampleMultiplier.get_island_ips_from_telems(telems)
         monkeys = []
-        for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']:
+        for telem in [telem for telem in telems
+                      if 'telem_category' in telem and telem['telem_category'] == 'system_info']:
             if 'network_info' not in telem['data']:
                 continue
             guid = telem['monkey_guid']
@@ -111,4 +86,4 @@ class TelemParser:
 
 
 if __name__ == "__main__":
-    TelemParser(multiplier=int(sys.argv[1])).multiply_telems()
+    SampleMultiplier(multiplier=int(sys.argv[1])).multiply_telems()

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py

@@ -1,6 +1,7 @@
 from unittest import TestCase
 
-from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator
+from envs.monkey_zoo.blackbox.tests.performance.\
+    telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator
 
 
 class TestFakeIpGenerator(TestCase):

envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py

@@ -8,7 +8,7 @@ from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceA
 from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
 from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod
 from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
-from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser
+from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser
 
 LOGGER = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ class TelemetryPerformanceTest:
     def test_telemetry_performance(self):
         LOGGER.info("Starting telemetry performance test.")
         try:
-            all_telemetries = TelemParser.get_all_telemetries()
+            all_telemetries = SampleFileParser.get_all_telemetries()
         except FileNotFoundError:
             LOGGER.error("Telemetries to send not found. Refer to readme to figure out how to generate telemetries "
                          "and where to put them.")

monkey/monkey_island/cc/resources/test/utils/telem_store.py

@@ -9,7 +9,7 @@ from flask import request
 from monkey_island.cc.models.test_telem import TestTelem
 from monkey_island.cc.services.config import ConfigService
 
-TEST_TELEM_DIR = "./test_telems"
+TELEM_SAMPLE_DIR = "./telem_sample"
 MAX_SAME_CATEGORY_TELEMS = 10000
 
 
@@ -36,16 +36,16 @@ class TestTelemStore:
 
     @staticmethod
     def export_test_telems():
-        logger.info(f"Exporting all telemetries to {TEST_TELEM_DIR}")
+        logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}")
         try:
-            mkdir(TEST_TELEM_DIR)
+            mkdir(TELEM_SAMPLE_DIR)
         except FileExistsError:
             logger.info("Deleting all previous telemetries.")
-            shutil.rmtree(TEST_TELEM_DIR)
+            shutil.rmtree(TELEM_SAMPLE_DIR)
-            mkdir(TEST_TELEM_DIR)
+            mkdir(TELEM_SAMPLE_DIR)
         for test_telem in TestTelem.objects():
-            with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file:
+            with open(TestTelemStore.get_unique_file_path_for_test_telem(TELEM_SAMPLE_DIR, test_telem), 'w') as file:
-                file.write(test_telem.to_json())
+                file.write(test_telem.to_json(indent=2))
         logger.info("Telemetries exported!")
 
     @staticmethod