Merge pull request #632 from VakarisZ/monkey_telemetry_fabrication

Monkey telemetry fabrication and tests

.gitignore

@@ -82,5 +82,8 @@ MonkeyZoo/*
 !MonkeyZoo/config.tf
 !MonkeyZoo/MonkeyZooDocs.pdf
 
+# Exported monkey telemetries
+/monkey/telem_sample/
+
 # vim swap files
 *.swp

envs/monkey_zoo/.gitignore

@@ -1 +1,2 @@
 logs/
+/blackbox/tests/performance/telem_sample

envs/monkey_zoo/blackbox/README.md

@@ -12,8 +12,26 @@ this information in the GCP Console `Compute Engine/VM Instances` under _Externa
 #### Running in command line
 Run the following command:
 
-`monkey\envs\monkey_zoo\blackbox>python -m pytest --island=35.207.152.72:5000 test_blackbox.py`
+`monkey\envs\monkey_zoo\blackbox>python -m pytest -s --island=35.207.152.72:5000 test_blackbox.py`
 
 #### Running in PyCharm
-Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the 
+Configure a PyTest configuration with the additional arguments `-s --island=35.207.152.72` on the 
 `monkey\envs\monkey_zoo\blackbox`.
+
+### Running telemetry performance test
+To run telemetry performance test follow these steps:
+1. Gather monkey telemetries.
+    1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have 
+    exported telemetries already.
+    2. Run monkey and wait until infection is done.
+    3. All telemetries are gathered in `monkey/telem_sample`
+2. Run telemetry performance test.
+    1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems`
+    2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply 
+    telemetries gathered.
+        1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox`
+        2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate
+        telemetries 4 times.
+        3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion.
+    3. Performance test will run as part of BlackBox tests or you can run it separately by adding 
+    `-k 'test_telem_performance'` option.

envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py

@@ -4,6 +4,7 @@ from typing import Dict
 
 from envs.monkey_zoo.blackbox.analyzers.analyzer import Analyzer
 from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -14,18 +15,18 @@ class PerformanceAnalyzer(Analyzer):
         self.endpoint_timings = endpoint_timings
 
     def analyze_test_results(self):
-        # Calculate total time and check each page
+        # Calculate total time and check each endpoint
         single_page_time_less_then_max = True
         total_time = timedelta()
-        for page, elapsed in self.endpoint_timings.items():
+        for endpoint, elapsed in self.endpoint_timings.items():
-            LOGGER.info(f"page {page} took {str(elapsed)}")
             total_time += elapsed
             if elapsed > self.performance_test_config.max_allowed_single_page_time:
                 single_page_time_less_then_max = False
 
         total_time_less_then_max = total_time < self.performance_test_config.max_allowed_total_time
 
-        LOGGER.info(f"total time is {str(total_time)}")
+        PerformanceAnalyzer.log_slowest_endpoints(self.endpoint_timings)
+        LOGGER.info(f"Total time is {str(total_time)}")
 
         performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max
 
@@ -37,3 +38,11 @@ class PerformanceAnalyzer(Analyzer):
             breakpoint()
 
         return performance_is_good_enough
+
+    @staticmethod
+    def log_slowest_endpoints(endpoint_timings, max_endpoints_to_display=100):
+        slow_endpoint_list = list(endpoint_timings.items())
+        slow_endpoint_list.sort(key=lambda x: x[1], reverse=True)
+        slow_endpoint_list = slow_endpoint_list[:max_endpoints_to_display]
+        for endpoint in slow_endpoint_list:
+            LOGGER.info(f"{endpoint[0]} took {str(endpoint[1])}")

envs/monkey_zoo/blackbox/island_client/monkey_island_client.py

@@ -1,7 +1,8 @@
-from time import sleep
 import json
 
 import logging
+from time import sleep
+
 from bson import json_util
 
 from envs.monkey_zoo.blackbox.island_client.monkey_island_requests import MonkeyIslandRequests
@@ -30,7 +31,7 @@ class MonkeyIslandClient(object):
 
     @avoid_race_condition
     def run_monkey_local(self):
-        response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"})
+        response = self.requests.post_json("api/local-monkey", data={"action": "run"})
         if MonkeyIslandClient.monkey_ran_successfully(response):
             LOGGER.info("Running the monkey.")
         else:

envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py

@@ -1,9 +1,15 @@
+from typing import Dict
+from datetime import timedelta
+
+
 import requests
 import functools
 
-# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
+from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod
+
 import logging
 
+# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
 NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \
                 '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557'
 LOGGER = logging.getLogger(__name__)
@@ -14,6 +20,26 @@ class MonkeyIslandRequests(object):
     def __init__(self, server_address):
         self.addr = "https://{IP}/".format(IP=server_address)
         self.token = self.try_get_jwt_from_server()
+        self.supported_request_methods = {SupportedRequestMethod.GET: self.get,
+                                          SupportedRequestMethod.POST: self.post,
+                                          SupportedRequestMethod.PATCH: self.patch,
+                                          SupportedRequestMethod.DELETE: self.delete}
+
+    def get_request_time(self, url, method: SupportedRequestMethod, data=None):
+        response = self.send_request_by_method(url, method, data)
+        if response.ok:
+            LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}")
+            return response.elapsed
+        else:
+            LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}")
+            # instead of raising for status, mark failed responses as maxtime
+            return timedelta.max
+
+    def send_request_by_method(self, url, method=SupportedRequestMethod.GET, data=None):
+        if data:
+            return self.supported_request_methods[method](url, data)
+        else:
+            return self.supported_request_methods[method](url)
 
     def try_get_jwt_from_server(self):
         try:
@@ -55,9 +81,16 @@ class MonkeyIslandRequests(object):
                              verify=False)
 
     @_Decorators.refresh_jwt_token
-    def post_json(self, url, dict_data):
+    def post_json(self, url, data: Dict):
         return requests.post(self.addr + url,  # noqa: DUO123
-                             json=dict_data,
+                             json=data,
+                             headers=self.get_jwt_header(),
+                             verify=False)
+
+    @_Decorators.refresh_jwt_token
+    def patch(self, url, data: Dict):
+        return requests.patch(self.addr + url,  # noqa: DUO123
+                             data=data,
                              headers=self.get_jwt_header(),
                              verify=False)
 

envs/monkey_zoo/blackbox/island_client/supported_request_method.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class SupportedRequestMethod(Enum):
+    GET = "GET"
+    POST = "POST"
+    PATCH = "PATCH"
+    DELETE = "DELETE"

envs/monkey_zoo/blackbox/log_handlers/monkey_log.py

@@ -1,6 +1,6 @@
+import logging
 import os
 
-import logging
 from bson import ObjectId
 
 LOGGER = logging.getLogger(__name__)

envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py

@@ -1,8 +1,7 @@
+import logging
 import os
 import shutil
 
-import logging
-
 from envs.monkey_zoo.blackbox.log_handlers.monkey_log_parser import MonkeyLogParser
 from envs.monkey_zoo.blackbox.log_handlers.monkey_logs_downloader import MonkeyLogsDownloader
 

envs/monkey_zoo/blackbox/test_blackbox.py

@@ -4,14 +4,15 @@ import logging
 import pytest
 from time import sleep
 
-from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
 from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer
 from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser
+from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
+from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler
+from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest
 from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest
 from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest
+from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest
 from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers
-from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest
-from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler
 
 DEFAULT_TIMEOUT_SECONDS = 5*60
 MACHINE_BOOTUP_WAIT_SECONDS = 30
@@ -144,3 +145,6 @@ class TestMonkeyBlackbox(object):
                                                 island_client,
                                                 "PERFORMANCE.conf",
                                                 timeout_in_seconds=10*60)
+
+    def test_telem_performance(self, island_client):
+        TelemetryPerformanceTest(island_client).test_telemetry_performance()

envs/monkey_zoo/blackbox/tests/exploitation.py

@@ -1,9 +1,8 @@
+import logging
 from time import sleep
 
-import logging
-
-from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer
 from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest
+from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer
 
 MAX_TIME_FOR_MONKEYS_TO_DIE = 5 * 60
 WAIT_TIME_BETWEEN_REQUESTS = 10

envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py

@@ -1,11 +1,10 @@
 import logging
-from datetime import timedelta
 
-from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest
-from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
-from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer
-
+from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
+from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod
+from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest
+from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 
 LOGGER = logging.getLogger(__name__)
 
@@ -25,18 +24,8 @@ class EndpointPerformanceTest(BasicTest):
         self.island_client.clear_caches()
         endpoint_timings = {}
         for endpoint in self.test_config.endpoints_to_test:
-            endpoint_timings[endpoint] = self.get_elapsed_for_get_request(endpoint)
+            endpoint_timings[endpoint] = self.island_client.requests.get_request_time(endpoint,
-
+                                                                                      SupportedRequestMethod.GET)
         analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings)
 
         return analyzer.analyze_test_results()
-
-    def get_elapsed_for_get_request(self, url):
-        response = self.island_client.requests.get(url)
-        if response.ok:
-            LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}")
-            return response.elapsed
-        else:
-            LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}")
-            # instead of raising for status, mark failed responses as maxtime
-            return timedelta.max

envs/monkey_zoo/blackbox/tests/performance/map_generation.py

@@ -1,8 +1,8 @@
 from datetime import timedelta
 
 from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest
-from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest
+from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow
 
 MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2)

envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py

@@ -5,7 +5,7 @@ from typing import List
 class PerformanceTestConfig:
 
     def __init__(self, max_allowed_single_page_time: timedelta, max_allowed_total_time: timedelta,
-                 endpoints_to_test: List[str], break_on_timeout=False):
+                 endpoints_to_test: List[str] = None, break_on_timeout=False):
         self.max_allowed_single_page_time = max_allowed_single_page_time
         self.max_allowed_total_time = max_allowed_total_time
         self.endpoints_to_test = endpoints_to_test

envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py

@@ -1,7 +1,7 @@
 from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest
 from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest
-from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 from envs.monkey_zoo.blackbox.tests.performance.endpoint_performance_test import EndpointPerformanceTest
+from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 
 
 class PerformanceTestWorkflow(BasicTest):

envs/monkey_zoo/blackbox/tests/performance/report_generation.py

@@ -1,9 +1,9 @@
 from datetime import timedelta
 
 from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest
+from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest
 from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
 from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow
-from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest
 
 MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2)
 MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5)

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py

@@ -0,0 +1,47 @@
+import json
+import logging
+from os import listdir, path
+from typing import List, Dict
+
+from tqdm import tqdm
+
+TELEM_DIR_PATH = './tests/performance/telem_sample'
+MAX_SAME_TYPE_TELEM_FILES = 10000
+LOGGER = logging.getLogger(__name__)
+
+
+class SampleFileParser:
+
+    @staticmethod
+    def save_teletries_to_files(telems: List[Dict]):
+        for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)):
+            SampleFileParser.save_telemetry_to_file(telem)
+
+    @staticmethod
+    def save_telemetry_to_file(telem: Dict):
+        telem_filename = telem['name'] + telem['method']
+        for i in range(MAX_SAME_TYPE_TELEM_FILES):
+            if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))):
+                telem_filename = str(i) + telem_filename
+                break
+        with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file:
+            file.write(json.dumps(telem))
+
+    @staticmethod
+    def read_telem_files() -> List[str]:
+        telems = []
+        try:
+            file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH)
+                          if path.isfile(path.join(TELEM_DIR_PATH, f))]
+        except FileNotFoundError:
+            raise FileNotFoundError("Telemetries to send not found. "
+                                    "Refer to readme to figure out how to generate telemetries and where to put them.")
+        for file_path in file_paths:
+            with open(file_path, 'r') as telem_file:
+                telem_string = "".join(telem_file.readlines()).replace("\n", "")
+                telems.append(telem_string)
+        return telems
+
+    @staticmethod
+    def get_all_telemetries() -> List[Dict]:
+        return [json.loads(t) for t in SampleFileParser.read_telem_files()]

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py

@@ -0,0 +1,25 @@
+from typing import List
+
+
+class FakeIpGenerator:
+    def __init__(self):
+        self.fake_ip_parts = [1, 1, 1, 1]
+
+    def generate_fake_ips_for_real_ips(self, real_ips: List[str]) -> List[str]:
+        fake_ips = []
+        for i in range(len(real_ips)):
+            fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts))
+            self.increment_ip()
+        return fake_ips
+
+    def increment_ip(self):
+        self.fake_ip_parts[3] += 1
+        self.try_fix_ip_range()
+
+    def try_fix_ip_range(self):
+        for i in range(len(self.fake_ip_parts)):
+            if self.fake_ip_parts[i] > 256:
+                if i-1 < 0:
+                    raise Exception("Fake IP's out of range.")
+                self.fake_ip_parts[i-1] += 1
+                self.fake_ip_parts[i] = 1

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py

@@ -0,0 +1,18 @@
+import random
+
+from envs.monkey_zoo.blackbox.tests.performance.\
+    telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator
+
+
+class FakeMonkey:
+    def __init__(self, ips, guid, fake_ip_generator: FakeIpGenerator, on_island=False):
+        self.original_ips = ips
+        self.original_guid = guid
+        self.fake_ip_generator = fake_ip_generator
+        self.on_island = on_island
+        self.fake_guid = str(random.randint(1000000000000, 9999999999999))
+        self.fake_ips = fake_ip_generator.generate_fake_ips_for_real_ips(ips)
+
+    def change_fake_data(self):
+        self.fake_ips = self.fake_ip_generator.generate_fake_ips_for_real_ips(self.original_ips)
+        self.fake_guid = str(random.randint(1000000000000, 9999999999999))

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py

@@ -0,0 +1,89 @@
+import copy
+import json
+import logging
+import sys
+from typing import List, Dict
+
+from tqdm import tqdm
+
+from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser
+from envs.monkey_zoo.blackbox.tests.performance.\
+    telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator
+from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_multiplier.fake_monkey import FakeMonkey
+
+TELEM_DIR_PATH = './tests/performance/telemetry_sample'
+LOGGER = logging.getLogger(__name__)
+
+
+class SampleMultiplier:
+
+    def __init__(self, multiplier: int):
+        self.multiplier = multiplier
+        self.fake_ip_generator = FakeIpGenerator()
+
+    def multiply_telems(self):
+        telems = SampleFileParser.get_all_telemetries()
+        telem_contents = [json.loads(telem['content']) for telem in telems]
+        monkeys = self.get_monkeys_from_telems(telem_contents)
+        for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries", position=1):
+            for monkey in monkeys:
+                monkey.change_fake_data()
+            fake_telem_batch = copy.deepcopy(telems)
+            SampleMultiplier.fabricate_monkeys_in_telems(fake_telem_batch, monkeys)
+            SampleMultiplier.offset_telem_times(iteration=i, telems=fake_telem_batch)
+            SampleFileParser.save_teletries_to_files(fake_telem_batch)
+            LOGGER.info("")
+
+    @staticmethod
+    def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]):
+        for telem in tqdm(telems, desc="Telemetries fabricated", position=2):
+            for monkey in monkeys:
+                if monkey.on_island:
+                    continue
+                if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) \
+                        and not monkey.on_island:
+                    telem['content'] = telem['content'].replace(monkey.original_guid, monkey.fake_guid)
+                    telem['endpoint'] = telem['endpoint'].replace(monkey.original_guid, monkey.fake_guid)
+                for i in range(len(monkey.original_ips)):
+                    telem['content'] = telem['content'].replace(monkey.original_ips[i], monkey.fake_ips[i])
+
+    @staticmethod
+    def offset_telem_times(iteration: int, telems: List[Dict]):
+        for telem in telems:
+            telem['time']['$date'] += iteration * 1000
+
+    def get_monkeys_from_telems(self, telems: List[Dict]):
+        island_ips = SampleMultiplier.get_island_ips_from_telems(telems)
+        monkeys = []
+        for telem in [telem for telem in telems
+                      if 'telem_category' in telem and telem['telem_category'] == 'system_info']:
+            if 'network_info' not in telem['data']:
+                continue
+            guid = telem['monkey_guid']
+            monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid]
+            if not monkey_present:
+                ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']]
+                if set(island_ips).intersection(ips):
+                    on_island = True
+                else:
+                    on_island = False
+
+                monkeys.append(FakeMonkey(ips=ips,
+                                          guid=guid,
+                                          fake_ip_generator=self.fake_ip_generator,
+                                          on_island=on_island))
+        return monkeys
+
+    @staticmethod
+    def get_island_ips_from_telems(telems: List[Dict]) -> List[str]:
+        island_ips = []
+        for telem in telems:
+            if 'config' in telem:
+                island_ips = telem['config']['command_servers']
+                for i in range(len(island_ips)):
+                    island_ips[i] = island_ips[i].replace(":5000", "")
+        return island_ips
+
+
+if __name__ == "__main__":
+    SampleMultiplier(multiplier=int(sys.argv[1])).multiply_telems()

envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py

@@ -0,0 +1,19 @@
+from unittest import TestCase
+
+from envs.monkey_zoo.blackbox.tests.performance.\
+    telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator
+
+
+class TestFakeIpGenerator(TestCase):
+
+    def test_fake_ip_generation(self):
+        fake_ip_gen = FakeIpGenerator()
+        self.assertListEqual([1, 1, 1, 1], fake_ip_gen.fake_ip_parts)
+        for i in range(256):
+            fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1'])
+        self.assertListEqual(['1.1.2.1'], fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1']))
+        fake_ip_gen.fake_ip_parts = [256, 256, 255, 256]
+        self.assertListEqual(['256.256.255.256', '256.256.256.1'],
+                             fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1', '1.1.1.2']))
+        fake_ip_gen.fake_ip_parts = [256, 256, 256, 256]
+        self.assertRaises(Exception, fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1']))

envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py

@@ -0,0 +1,52 @@
+import json
+import logging
+from datetime import timedelta
+
+from tqdm import tqdm
+
+from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer
+from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
+from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod
+from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig
+from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser
+
+LOGGER = logging.getLogger(__name__)
+
+MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME = timedelta(seconds=2)
+MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=60)
+
+
+class TelemetryPerformanceTest:
+
+    def __init__(self, island_client: MonkeyIslandClient):
+        self.island_client = island_client
+
+    def test_telemetry_performance(self):
+        LOGGER.info("Starting telemetry performance test.")
+        try:
+            all_telemetries = SampleFileParser.get_all_telemetries()
+        except FileNotFoundError:
+            raise FileNotFoundError("Telemetries to send not found. "
+                                    "Refer to readme to figure out how to generate telemetries and where to put them.")
+        LOGGER.info("Telemetries imported successfully.")
+        all_telemetries.sort(key=lambda telem: telem['time']['$date'])
+        telemetry_parse_times = {}
+        for telemetry in tqdm(all_telemetries, total=len(all_telemetries), ascii=True, desc="Telemetries sent"):
+            telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint(telemetry)
+            telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(telemetry)
+        test_config = PerformanceTestConfig(MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME, MAX_ALLOWED_TOTAL_TIME)
+        PerformanceAnalyzer(test_config, telemetry_parse_times).analyze_test_results()
+
+    def get_telemetry_time(self, telemetry):
+        content = telemetry['content']
+        url = telemetry['endpoint']
+        method = SupportedRequestMethod.__getattr__(telemetry['method'])
+
+        return self.island_client.requests.get_request_time(url=url, method=method, data=content)
+
+    @staticmethod
+    def get_verbose_telemetry_endpoint(telemetry):
+        telem_category = ""
+        if "telem_category" in telemetry['content']:
+            telem_category = "_" + json.loads(telemetry['content'])['telem_category'] + "_" + telemetry['_id']['$oid']
+        return telemetry['endpoint'] + telem_category

envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py

@@ -1,6 +1,5 @@
-import subprocess
-
 import logging
+import subprocess
 
 LOGGER = logging.getLogger(__name__)
 

envs/monkey_zoo/blackbox/utils/json_encoder.py

@@ -1,4 +1,5 @@
 import json
+
 from bson import ObjectId
 
 

monkey/monkey_island/cc/models/test_telem.py

@@ -0,0 +1,13 @@
+"""
+Define a Document Schema for the TestTelem document.
+"""
+from mongoengine import Document, StringField, DateTimeField
+
+
+class TestTelem(Document):
+    # SCHEMA
+    name = StringField(required=True)
+    time = DateTimeField(required=True)
+    method = StringField(required=True)
+    endpoint = StringField(required=True)
+    content = StringField(required=True)

monkey/monkey_island/cc/resources/log.py

@@ -6,6 +6,7 @@ from flask import request
 
 from monkey_island.cc.auth import jwt_required
 from monkey_island.cc.database import mongo
+from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore
 from monkey_island.cc.services.log import LogService
 from monkey_island.cc.services.node import NodeService
 
@@ -23,6 +24,7 @@ class Log(flask_restful.Resource):
             return LogService.log_exists(ObjectId(exists_monkey_id))
 
     # Used by monkey. can't secure.
+    @TestTelemStore.store_test_telem
     def post(self):
         telemetry_json = json.loads(request.data)
 

monkey/monkey_island/cc/resources/monkey.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 import dateutil.parser
 import flask_restful
+from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore
 from flask import request
 
 from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS
@@ -33,6 +34,7 @@ class Monkey(flask_restful.Resource):
         return {}
 
     # Used by monkey. can't secure.
+    @TestTelemStore.store_test_telem
     def patch(self, guid):
         monkey_json = json.loads(request.data)
         update = {"$set": {'modifytime': datetime.now()}}
@@ -56,6 +58,7 @@ class Monkey(flask_restful.Resource):
         return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False)
 
     # Used by monkey. can't secure.
+    @TestTelemStore.store_test_telem
     def post(self, **kw):
         monkey_json = json.loads(request.data)
         monkey_json['creds'] = []

monkey/monkey_island/cc/resources/root.py

@@ -1,6 +1,5 @@
 import logging
 import threading
-from datetime import datetime
 
 import flask_restful
 from flask import request, make_response, jsonify
@@ -8,10 +7,7 @@ from flask import request, make_response, jsonify
 from monkey_island.cc.auth import jwt_required
 from monkey_island.cc.database import mongo
 from monkey_island.cc.services.database import Database
-from monkey_island.cc.services.node import NodeService
+from monkey_island.cc.services.infection_lifecycle import InfectionLifecycle
-from monkey_island.cc.services.reporting.report import ReportService
-from monkey_island.cc.services.reporting.report_generation_synchronisation import is_report_being_generated, \
-    safe_generate_reports
 from monkey_island.cc.utils import local_ip_addresses
 
 __author__ = 'Barak'
@@ -32,7 +28,7 @@ class Root(flask_restful.Resource):
         elif action == "reset":
             return jwt_required()(Database.reset_db)()
         elif action == "killall":
-            return Root.kill_all()
+            return jwt_required()(InfectionLifecycle.kill_all)()
         elif action == "is-up":
             return {'is-up': True}
         else:
@@ -43,33 +39,6 @@ class Root(flask_restful.Resource):
         return jsonify(
             ip_addresses=local_ip_addresses(),
             mongo=str(mongo.db),
-            completed_steps=self.get_completed_steps())
+            completed_steps=InfectionLifecycle.get_completed_steps())
 
-    @staticmethod
-    @jwt_required()
-    def kill_all():
-        mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}},
-                               upsert=False,
-                               multi=True)
-        logger.info('Kill all monkeys was called')
-        return jsonify(status='OK')
 
-    @jwt_required()
-    def get_completed_steps(self):
-        is_any_exists = NodeService.is_any_monkey_exists()
-        infection_done = NodeService.is_monkey_finished_running()
-
-        if infection_done:
-            # Checking is_report_being_generated here, because we don't want to wait to generate a report; rather,
-            # we want to skip and reply.
-            if not is_report_being_generated() and not ReportService.is_latest_report_exists():
-                safe_generate_reports()
-            report_done = ReportService.is_report_generated()
-        else:  # Infection is not done
-            report_done = False
-
-        return dict(
-            run_server=True,
-            run_monkey=is_any_exists,
-            infection_done=infection_done,
-            report_done=report_done)

monkey/monkey_island/cc/resources/telemetry.py

@@ -8,6 +8,7 @@ from flask import request
 
 from monkey_island.cc.auth import jwt_required
 from monkey_island.cc.database import mongo
+from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore
 from monkey_island.cc.services.node import NodeService
 from monkey_island.cc.services.telemetry.processing.processing import process_telemetry
 from monkey_island.cc.models.monkey import Monkey
@@ -40,6 +41,7 @@ class Telemetry(flask_restful.Resource):
         return result
 
     # Used by monkey. can't secure.
+    @TestTelemStore.store_test_telem
     def post(self):
         telemetry_json = json.loads(request.data)
         telemetry_json['timestamp'] = datetime.now()

monkey/monkey_island/cc/resources/test/utils/telem_store.py

@@ -0,0 +1,68 @@
+import logging
+from functools import wraps
+from os import mkdir, path
+import shutil
+from datetime import datetime
+
+from flask import request
+
+from monkey_island.cc.models.test_telem import TestTelem
+from monkey_island.cc.services.config import ConfigService
+
+TELEM_SAMPLE_DIR = "./telem_sample"
+MAX_SAME_CATEGORY_TELEMS = 10000
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class TestTelemStore:
+
+    @staticmethod
+    def store_test_telem(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            if ConfigService.is_test_telem_export_enabled():
+                time = datetime.now()
+                method = request.method
+                content = request.data.decode()
+                endpoint = request.path
+                name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_')
+                TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save()
+            return f(*args, **kwargs)
+
+        return decorated_function
+
+    @staticmethod
+    def export_test_telems():
+        logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}")
+        try:
+            mkdir(TELEM_SAMPLE_DIR)
+        except FileExistsError:
+            logger.info("Deleting all previous telemetries.")
+            shutil.rmtree(TELEM_SAMPLE_DIR)
+            mkdir(TELEM_SAMPLE_DIR)
+        for test_telem in TestTelem.objects():
+            with open(TestTelemStore.get_unique_file_path_for_test_telem(TELEM_SAMPLE_DIR, test_telem), 'w') as file:
+                file.write(test_telem.to_json(indent=2))
+        logger.info("Telemetries exported!")
+
+    @staticmethod
+    def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem):
+        telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem)
+        for i in range(MAX_SAME_CATEGORY_TELEMS):
+            potential_filepath = path.join(target_dir, (telem_filename + str(i)))
+            if path.exists(potential_filepath):
+                continue
+            return potential_filepath
+        raise Exception(f"Too many telemetries of the same category. Max amount {MAX_SAME_CATEGORY_TELEMS}")
+
+    @staticmethod
+    def _get_filename_by_test_telem(test_telem: TestTelem):
+        endpoint_part = test_telem.name
+        return endpoint_part + '_' + test_telem.method
+
+
+if __name__ == '__main__':
+    TestTelemStore.export_test_telems()

monkey/monkey_island/cc/services/config.py

@@ -307,3 +307,7 @@ class ConfigService:
             pair['public_key'] = encryptor.dec(pair['public_key'])
             pair['private_key'] = encryptor.dec(pair['private_key'])
         return pair
+
+    @staticmethod
+    def is_test_telem_export_enabled():
+        return ConfigService.get_config_value(['internal', 'testing', 'export_monkey_telems'])

monkey/monkey_island/cc/services/config_schema.py

@@ -737,6 +737,19 @@ SCHEMA = {
                             "description": "List of SSH key pairs to use, when trying to ssh into servers"
                         }
                     }
+                },
+                "testing": {
+                    "title": "Testing",
+                    "type": "object",
+                    "properties": {
+                        "export_monkey_telems": {
+                            "title": "Export monkey telemetries",
+                            "type": "boolean",
+                            "default": False,
+                            "description": "Exports unencrypted telemetries that can be used for tests in development."
+                                           " Do not turn on!"
+                        }
+                    }
                 }
             }
         },

monkey/monkey_island/cc/services/infection_lifecycle.py

@@ -0,0 +1,51 @@
+import logging
+from datetime import datetime
+
+from flask import jsonify
+
+from monkey_island.cc.database import mongo
+from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore
+from monkey_island.cc.services.config import ConfigService
+from monkey_island.cc.services.node import NodeService
+from monkey_island.cc.services.reporting.report import ReportService
+from monkey_island.cc.services.reporting.report_generation_synchronisation import is_report_being_generated, \
+    safe_generate_reports
+
+logger = logging.getLogger(__name__)
+
+
+class InfectionLifecycle:
+
+    @staticmethod
+    def kill_all():
+        mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}},
+                               upsert=False,
+                               multi=True)
+        logger.info('Kill all monkeys was called')
+        return jsonify(status='OK')
+
+    @staticmethod
+    def get_completed_steps():
+        is_any_exists = NodeService.is_any_monkey_exists()
+        infection_done = NodeService.is_monkey_finished_running()
+
+        if infection_done:
+            InfectionLifecycle._on_finished_infection()
+            report_done = ReportService.is_report_generated()
+        else:  # Infection is not done
+            report_done = False
+
+        return dict(
+            run_server=True,
+            run_monkey=is_any_exists,
+            infection_done=infection_done,
+            report_done=report_done)
+
+    @staticmethod
+    def _on_finished_infection():
+        # Checking is_report_being_generated here, because we don't want to wait to generate a report; rather,
+        # we want to skip and reply.
+        if not is_report_being_generated() and not ReportService.is_latest_report_exists():
+            safe_generate_reports()
+        if ConfigService.is_test_telem_export_enabled():
+            TestTelemStore.export_test_telems()

monkey/monkey_island/requirements.txt

@@ -23,3 +23,4 @@ requests
 dpath
 ring
 stix2
+tqdm