Finished telemetry fabrication

.gitignore

@@ -82,5 +82,9 @@ MonkeyZoo/*
 !MonkeyZoo/config.tf
 !MonkeyZoo/MonkeyZooDocs.pdf
 
+# Exported monkey telemetries
+/monkey/test_telems/
+
+
 # vim swap files
 *.swp

envs/monkey_zoo/blackbox/README.md

@@ -17,3 +17,19 @@ Run the following command:
 #### Running in PyCharm
 Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the 
 `monkey\envs\monkey_zoo\blackbox`.
+
+### Running telemetry performance test
+To run telemetry performance test follow these steps:
+1. Gather monkey telemetries.
+    1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have 
+    exported telemetries already.
+    2. Run monkey and wait until infection is done.
+    3. All telemetries are gathered in `monkey/test_telems`
+2. Run telemetry performance test.
+    1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems`
+    2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply 
+    telemetries gathered.
+        1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox`
+        2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate
+        telemetries 4 times.
+    3. Run blackbox tests, telemetry performance test will run as part of it.

envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py

@@ -2,6 +2,7 @@ from typing import List, Dict
 from os import listdir, path
 import copy
 import json
+import sys
 
 from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator
 from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey
@@ -11,10 +12,10 @@ TELEM_DIR_PATH = './tests/performance/test_telems'
 
 
 class TelemParser:
-    def __init__(self, island_ip: str, multiplier: int):
+
+    def __init__(self, multiplier: int):
         self.multiplier = multiplier
         self.fake_ip_generator = FakeIpGenerator()
-        self.island_ip = island_ip
 
     def multiply_telems(self):
         telems = TelemParser.get_all_telemetries()
@@ -51,7 +52,7 @@ class TelemParser:
             TelemParser.save_telemetry_to_file(telem)
 
     @staticmethod
-    def save_telemetry_to_file(telem):
+    def save_telemetry_to_file(telem: Dict):
         telem_filename = telem['name'] + telem['method']
         for i in range(10000):
             if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))):
@@ -71,10 +72,11 @@ class TelemParser:
         return telems
 
     @staticmethod
-    def get_all_telemetries():
+    def get_all_telemetries() -> List[Dict]:
         return [json.loads(t) for t in TelemParser.read_telem_files()]
 
     def get_monkeys_from_telems(self, telems: List[Dict]):
+        island_ips = TelemParser.get_island_ips_from_telems(telems)
         monkeys = []
         for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']:
             if 'network_info' not in telem['data']:
@@ -83,7 +85,7 @@ class TelemParser:
             monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid]
             if not monkey_present:
                 ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']]
-                if self.island_ip in ips:
+                if set(island_ips).intersection(ips):
                     on_island = True
                 else:
                     on_island = False
@@ -94,6 +96,16 @@ class TelemParser:
                                           on_island=on_island))
         return monkeys
 
+    @staticmethod
+    def get_island_ips_from_telems(telems: List[Dict]) -> List[str]:
+        island_ips = []
+        for telem in telems:
+            if 'config' in telem:
+                island_ips = telem['config']['command_servers']
+                for i in range(len(island_ips)):
+                    island_ips[i] = island_ips[i].replace(":5000", "")
+        return island_ips
+
 
 if __name__ == "__main__":
-    TelemParser(island_ip='192.168.56.1', multiplier=100).multiply_telems()
+    TelemParser(multiplier=int(sys.argv[1])).multiply_telems()

monkey/monkey_island/cc/models/test_telem.py

@@ -1,5 +1,5 @@
 """
-Define a Document Schema for the Monkey document.
+Define a Document Schema for the TestTelem document.
 """
 from mongoengine import Document, StringField, DateTimeField
 

monkey/monkey_island/cc/resources/test/utils/telem_store.py

@@ -1,12 +1,14 @@
 from functools import wraps
 from os import mkdir, path
+import shutil
 from datetime import datetime
 
 from flask import request
 
 from monkey_island.cc.models.test_telem import TestTelem
+from monkey_island.cc.services.config import ConfigService
 
-MONKEY_TELEM_COLLECTION_NAME = "monkey_telems_for_tests"
+TEST_TELEM_DIR = "./test_telems"
 
 
 class TestTelemStore:
@@ -15,28 +17,28 @@ class TestTelemStore:
     def store_test_telem(f):
         @wraps(f)
         def decorated_function(*args, **kwargs):
-            time = datetime.now()
-            method = request.method
-            content = request.data.decode()
-            endpoint = request.path
-            name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_')
-            TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save()
+            if ConfigService.is_test_telem_export_enabled():
+                time = datetime.now()
+                method = request.method
+                content = request.data.decode()
+                endpoint = request.path
+                name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_')
+                TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save()
             return f(*args, **kwargs)
 
         return decorated_function
 
     @staticmethod
     def export_test_telems():
-        telem_dir = "./test_telems"
         try:
-            mkdir(telem_dir)
+            mkdir(TEST_TELEM_DIR)
         except FileExistsError:
-            pass
+            shutil.rmtree(TEST_TELEM_DIR)
+            mkdir(TEST_TELEM_DIR)
         for test_telem in TestTelem.objects():
-            with open(TestTelemStore.get_unique_file_path_for_test_telem(telem_dir, test_telem), 'w') as file:
+            with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file:
                 file.write(test_telem.to_json())
 
-
     @staticmethod
     def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem):
         telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem)

monkey/monkey_island/cc/services/config.py

@@ -307,3 +307,7 @@ class ConfigService:
             pair['public_key'] = encryptor.dec(pair['public_key'])
             pair['private_key'] = encryptor.dec(pair['private_key'])
         return pair
+
+    @staticmethod
+    def is_test_telem_export_enabled():
+        return ConfigService.get_config_value(['internal', 'testing', 'export_monkey_telems'])

monkey/monkey_island/cc/services/config_schema.py

@@ -736,6 +736,19 @@ SCHEMA = {
                             "description": "List of SSH key pairs to use, when trying to ssh into servers"
                         }
                     }
+                },
+                "testing": {
+                    "title": "Testing",
+                    "type": "object",
+                    "properties": {
+                        "export_monkey_telems": {
+                            "title": "Export monkey telemetries",
+                            "type": "boolean",
+                            "default": False,
+                            "description": "Exports unencrypted telemetries that can be used for tests in development."
+                                           " Do not turn on!"
+                        }
+                    }
                 }
             }
         },