From 7d397da24538cdfaa90764d7993c8c8cd3d6ec2c Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 15 Apr 2020 15:42:37 +0300 Subject: [PATCH 01/16] Added telemetry collection --- monkey/monkey_island/cc/models/test_telem.py | 18 ++++++++++++++++++ monkey/monkey_island/cc/resources/monkey.py | 3 +++ .../cc/resources/test/utils/telem_store.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 monkey/monkey_island/cc/models/test_telem.py create mode 100644 monkey/monkey_island/cc/resources/test/utils/telem_store.py diff --git a/monkey/monkey_island/cc/models/test_telem.py b/monkey/monkey_island/cc/models/test_telem.py new file mode 100644 index 000000000..98bb23ebc --- /dev/null +++ b/monkey/monkey_island/cc/models/test_telem.py @@ -0,0 +1,18 @@ +""" +Define a Document Schema for the Monkey document. +""" +from mongoengine import Document, StringField + + +class TestTelem(Document): + # SCHEMA + method = StringField(required=True) + endpoint = StringField(required=True) + content = StringField(required=True) + + @staticmethod + def try_drop_collection(): + try: + TestTelem.drop_collection() + except Exception: + pass diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 3e3ef40c0..5d4be9645 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -3,6 +3,7 @@ from datetime import datetime import dateutil.parser import flask_restful +from monkey_island.cc.resources.test.utils.telem_store import store_test_telem from flask import request from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS @@ -33,6 +34,7 @@ class Monkey(flask_restful.Resource): return {} # Used by monkey. can't secure. + @store_test_telem def patch(self, guid): monkey_json = json.loads(request.data) update = {"$set": {'modifytime': datetime.now()}} @@ -56,6 +58,7 @@ class Monkey(flask_restful.Resource): return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False) # Used by monkey. can't secure. + @store_test_telem def post(self, **kw): monkey_json = json.loads(request.data) monkey_json['creds'] = [] diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py new file mode 100644 index 000000000..55351a804 --- /dev/null +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -0,0 +1,19 @@ +from functools import wraps + +from flask import request + +from monkey_island.cc.models.test_telem import TestTelem + +MONKEY_TELEM_COLLECTION_NAME = "monkey_telems_for_tests" + + +def store_test_telem(f): + @wraps(f) + def decorated_function(*args, **kwargs): + method = request.method + content = request.data.decode() + endpoint = str(request.url_rule) + TestTelem(method=method, endpoint=endpoint, content=content).save() + return f(*args, **kwargs) + + return decorated_function From a400da276a6841aa4c290db38dc0b4d7dc114c07 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 15 Apr 2020 17:17:03 +0300 Subject: [PATCH 02/16] Added ability to export test telemetries to directory --- monkey/monkey_island/cc/models/test_telem.py | 8 +-- monkey/monkey_island/cc/resources/monkey.py | 6 +-- .../cc/resources/test/utils/telem_store.py | 53 +++++++++++++++---- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/monkey/monkey_island/cc/models/test_telem.py b/monkey/monkey_island/cc/models/test_telem.py index 98bb23ebc..777b71fc2 100644 --- a/monkey/monkey_island/cc/models/test_telem.py +++ b/monkey/monkey_island/cc/models/test_telem.py @@ -6,13 +6,7 @@ from mongoengine import Document, StringField class TestTelem(Document): # SCHEMA + name = StringField(required=True) method = StringField(required=True) endpoint = StringField(required=True) content = StringField(required=True) - - @staticmethod - def try_drop_collection(): - try: - TestTelem.drop_collection() - except Exception: - pass diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 5d4be9645..dcdc5bc12 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -3,7 +3,7 @@ from datetime import datetime import dateutil.parser import flask_restful -from monkey_island.cc.resources.test.utils.telem_store import store_test_telem +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from flask import request from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS @@ -34,7 +34,7 @@ class Monkey(flask_restful.Resource): return {} # Used by monkey. can't secure. - @store_test_telem + @TestTelemStore.store_test_telem def patch(self, guid): monkey_json = json.loads(request.data) update = {"$set": {'modifytime': datetime.now()}} @@ -58,7 +58,7 @@ class Monkey(flask_restful.Resource): return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False) # Used by monkey. can't secure. - @store_test_telem + @TestTelemStore.store_test_telem def post(self, **kw): monkey_json = json.loads(request.data) monkey_json['creds'] = [] diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index 55351a804..dcceeb9d6 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -1,4 +1,5 @@ from functools import wraps +from os import mkdir, path from flask import request @@ -7,13 +8,47 @@ from monkey_island.cc.models.test_telem import TestTelem MONKEY_TELEM_COLLECTION_NAME = "monkey_telems_for_tests" -def store_test_telem(f): - @wraps(f) - def decorated_function(*args, **kwargs): - method = request.method - content = request.data.decode() - endpoint = str(request.url_rule) - TestTelem(method=method, endpoint=endpoint, content=content).save() - return f(*args, **kwargs) +class TestTelemStore: - return decorated_function + @staticmethod + def store_test_telem(f): + @wraps(f) + def decorated_function(*args, **kwargs): + method = request.method + content = request.data.decode() + endpoint = request.path + name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') + TestTelem(name=name, method=method, endpoint=endpoint, content=content).save() + return f(*args, **kwargs) + + return decorated_function + + @staticmethod + def export_test_telems(): + telem_dir = "./test_telems" + try: + mkdir(telem_dir) + except FileExistsError: + pass + for test_telem in TestTelem.objects(): + with open(TestTelemStore.get_unique_file_path_for_test_telem(telem_dir, test_telem), 'w') as file: + file.write(test_telem.to_json()) + + + @staticmethod + def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem): + telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem) + for i in range(100): + potential_filepath = path.join(target_dir, (telem_filename + str(i))) + if path.exists(potential_filepath): + continue + return potential_filepath + + @staticmethod + def _get_filename_by_test_telem(test_telem: TestTelem): + endpoint_part = test_telem.name + return endpoint_part + '_' + test_telem.method + + +if __name__ == '__main__': + TestTelemStore.export_test_telems() From 1ccbb3d98953eca71ce6eedcd6cbe7e5f7d09db4 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 16 Apr 2020 16:39:10 +0300 Subject: [PATCH 03/16] Finished fabricated monkey telemetry sending --- .../island_client/monkey_island_client.py | 2 +- .../island_client/monkey_island_requests.py | 13 +- envs/monkey_zoo/blackbox/test_blackbox.py | 118 +++++++++--------- .../performance/telemetry_performance_test.py | 47 +++++++ monkey/monkey_island/cc/models/test_telem.py | 3 +- .../cc/resources/test/utils/telem_store.py | 4 +- 6 files changed, 125 insertions(+), 62 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index 93780bf3b..22fa57ac1 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -31,7 +31,7 @@ class MonkeyIslandClient(object): @avoid_race_condition def run_monkey_local(self): - response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"}) + response = self.requests.post_json("api/local-monkey", data={"action": "run"}) if MonkeyIslandClient.monkey_ran_successfully(response): LOGGER.info("Running the monkey.") else: diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index 23f259a9c..07785509d 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -1,3 +1,5 @@ +from typing import Dict + import requests import functools @@ -55,9 +57,16 @@ class MonkeyIslandRequests(object): verify=False) @_Decorators.refresh_jwt_token - def post_json(self, url, dict_data): + def post_json(self, url, data: Dict): return requests.post(self.addr + url, # noqa: DUO123 - json=dict_data, + json=data, + headers=self.get_jwt_header(), + verify=False) + + @_Decorators.refresh_jwt_token + def patch_json(self, url, data: Dict): + return requests.patch(self.addr + url, # noqa: DUO123 + json=data, headers=self.get_jwt_header(), verify=False) diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 04e510c55..631229e36 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -9,6 +9,7 @@ from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import Communicat from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler @@ -25,11 +26,12 @@ LOGGER = logging.getLogger(__name__) @pytest.fixture(autouse=True, scope='session') def GCPHandler(request): GCPHandler = gcp_machine_handlers.GCPHandler() - GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) - wait_machine_bootup() + #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #wait_machine_bootup() def fin(): - GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + pass request.addfinalizer(fin) @@ -90,58 +92,60 @@ class TestMonkeyBlackbox(object): def test_server_online(self, island_client): assert island_client.get_api_status() is not None - def test_ssh_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") - - def test_hadoop_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) - - def test_mssql_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") - - def test_smb_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") - - def test_smb_pth(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") - - def test_elastic_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") - - def test_struts_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") - - def test_weblogic_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") - - def test_shellshock_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") - - def test_tunneling(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) - - def test_wmi_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") - - def test_wmi_pth(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") - - def test_report_generation_performance(self, island_client): - """ - This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test - for a total of 8 machines including the Monkey Island. - - Is has 2 analyzers - the regular one which checks all the Monkeys - and the Timing one which checks how long the report took to execute - """ - TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, - island_client, - "PERFORMANCE.conf", - timeout_in_seconds=10*60) - - def test_map_generation_performance(self, island_client): - TestMonkeyBlackbox.run_performance_test(MapGenerationTest, - island_client, - "PERFORMANCE.conf", - timeout_in_seconds=10*60) + #def test_ssh_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") +# + #def test_hadoop_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) +# + #def test_mssql_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") +# + #def test_smb_and_mimikatz_exploiters(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") +# + #def test_smb_pth(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") +# + #def test_elastic_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") +# + #def test_struts_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") +# + #def test_weblogic_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") +# + #def test_shellshock_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") +# + #def test_tunneling(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) +# + #def test_wmi_and_mimikatz_exploiters(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") +# + #def test_wmi_pth(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") +# + #def test_report_generation_performance(self, island_client): + # """ + # This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test + # for a total of 8 machines including the Monkey Island. +# + # Is has 2 analyzers - the regular one which checks all the Monkeys + # and the Timing one which checks how long the report took to execute + # """ + # TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, + # island_client, + # "PERFORMANCE.conf", + # timeout_in_seconds=10*60) +# + #def test_map_generation_performance(self, island_client): + # TestMonkeyBlackbox.run_performance_test(MapGenerationTest, + # island_client, + # "PERFORMANCE.conf", + # timeout_in_seconds=10*60) + def test_telem_performance(self, island_client): + TelemetryPerformanceTest(island_client).test_telemetry_performance() diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py new file mode 100644 index 000000000..20d95474e --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -0,0 +1,47 @@ +import json +from os import listdir +from os.path import isfile, join + +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient + +TELEM_DIR_PATH = './tests/performance/test_telems' +TELEM_TEST_ENDPOINT = '/api/test/telemetry/performance' + + +class TelemetryPerformanceTest: + + def __init__(self, island_client: MonkeyIslandClient): + self.island_client = island_client + + def test_telemetry_performance(self): + all_telemetries = TelemetryPerformanceTest.get_all_telemetries() + all_telemetries.sort(key=lambda telem: telem['time']['$date']) + for telemetry in all_telemetries: + self.send_telemetry(telemetry) + + @staticmethod + def get_all_telemetries(): + telemetries = [] + file_paths = [join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) if isfile(join(TELEM_DIR_PATH, f))] + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telem_contents = json.loads(telem_file.readline()) + telemetries.append(telem_contents) + return telemetries + + def send_telemetry(self, telemetry): + content = telemetry['content'] + url = telemetry['endpoint'] + method = telemetry['method'] + + if method == 'POST': + result = self.island_client.requests.post(url=url, data=content) + elif method == 'GET': + result = self.island_client.requests.get(url=url) + elif method == 'PATCH': + result = self.island_client.requests.patch_json(url=url, data=content) + elif method == 'DELETE': + result = self.island_client.requests.delete(url=url) + else: + raise Exception + return result diff --git a/monkey/monkey_island/cc/models/test_telem.py b/monkey/monkey_island/cc/models/test_telem.py index 777b71fc2..cb0697a33 100644 --- a/monkey/monkey_island/cc/models/test_telem.py +++ b/monkey/monkey_island/cc/models/test_telem.py @@ -1,12 +1,13 @@ """ Define a Document Schema for the Monkey document. """ -from mongoengine import Document, StringField +from mongoengine import Document, StringField, DateTimeField class TestTelem(Document): # SCHEMA name = StringField(required=True) + time = DateTimeField(required=True) method = StringField(required=True) endpoint = StringField(required=True) content = StringField(required=True) diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index dcceeb9d6..a6c569436 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -1,5 +1,6 @@ from functools import wraps from os import mkdir, path +from datetime import datetime from flask import request @@ -14,11 +15,12 @@ class TestTelemStore: def store_test_telem(f): @wraps(f) def decorated_function(*args, **kwargs): + time = datetime.now() method = request.method content = request.data.decode() endpoint = request.path name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') - TestTelem(name=name, method=method, endpoint=endpoint, content=content).save() + TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save() return f(*args, **kwargs) return decorated_function From 636c8f6d3f82ebd33ce12e0234d303d8809466b5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 20 Apr 2020 17:49:24 +0300 Subject: [PATCH 04/16] Added the ability to forge more intricate setup telemetries + small bugfixes, refactors and improvements --- envs/monkey_zoo/.gitignore | 1 + .../performance/telemetry_performance_test.py | 20 +-- .../tests/performance/utils/telem_parser.py | 124 ++++++++++++++++++ monkey/monkey_island/cc/resources/log.py | 2 + .../monkey_island/cc/resources/telemetry.py | 2 + 5 files changed, 132 insertions(+), 17 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py diff --git a/envs/monkey_zoo/.gitignore b/envs/monkey_zoo/.gitignore index 333c1e910..04310b6fd 100644 --- a/envs/monkey_zoo/.gitignore +++ b/envs/monkey_zoo/.gitignore @@ -1 +1,2 @@ logs/ +/blackbox/tests/performance/test_telems/* diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 20d95474e..047d07406 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -1,11 +1,7 @@ -import json -from os import listdir -from os.path import isfile, join + from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient - -TELEM_DIR_PATH = './tests/performance/test_telems' -TELEM_TEST_ENDPOINT = '/api/test/telemetry/performance' +from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser class TelemetryPerformanceTest: @@ -14,21 +10,11 @@ class TelemetryPerformanceTest: self.island_client = island_client def test_telemetry_performance(self): - all_telemetries = TelemetryPerformanceTest.get_all_telemetries() + all_telemetries = TelemParser.get_all_telemetries() all_telemetries.sort(key=lambda telem: telem['time']['$date']) for telemetry in all_telemetries: self.send_telemetry(telemetry) - @staticmethod - def get_all_telemetries(): - telemetries = [] - file_paths = [join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) if isfile(join(TELEM_DIR_PATH, f))] - for file_path in file_paths: - with open(file_path, 'r') as telem_file: - telem_contents = json.loads(telem_file.readline()) - telemetries.append(telem_contents) - return telemetries - def send_telemetry(self, telemetry): content = telemetry['content'] url = telemetry['endpoint'] diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py new file mode 100644 index 000000000..c6ee7095f --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py @@ -0,0 +1,124 @@ +from typing import List, Dict +from os import listdir, path +import copy +import random +import json + + +TELEM_DIR_PATH = './tests/performance/test_telems' + + +class FakeIpGenerator: + def __init__(self): + self.fake_ip_parts = [1, 1, 1, 1] + + def generate_fake_ips_for_real_ips(self, real_ips): + self.fake_ip_parts[2] += 1 + fake_ips = [] + for i in range(len(real_ips)): + fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts)) + self.fake_ip_parts[3] += 1 + return fake_ips + + +class Monkey: + def __init__(self, ips, guid, fake_ip_generator: FakeIpGenerator, on_island=False): + self.ips = ips + self.guid = guid + self.fake_ip_generator = fake_ip_generator + self.on_island = on_island + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) + self.fake_ips = fake_ip_generator.generate_fake_ips_for_real_ips(ips) + + def change_fake_data(self): + self.fake_ips = self.fake_ip_generator.generate_fake_ips_for_real_ips(self.ips) + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) + + +class TelemParser: + def __init__(self, island_ip: str, multiplier: int): + self.multiplier = multiplier + self.fake_ip_generator = FakeIpGenerator() + self.island_ip = island_ip + + def multiply_telems(self): + telems = TelemParser.get_all_telemetries() + telem_contents = [json.loads(telem['content']) for telem in telems] + monkeys = self.get_monkeys_from_telems(telem_contents) + for i in range(self.multiplier): + for monkey in monkeys: + monkey.change_fake_data() + fake_telem_batch = copy.deepcopy(telems) + TelemParser.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) + TelemParser.offset_telem_times(iteration=i, telems=fake_telem_batch) + TelemParser.save_teletries_to_files(fake_telem_batch) + + @staticmethod + def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[Monkey]): + for telem in telems: + for monkey in monkeys: + if monkey.on_island: + continue + if (monkey.guid in telem['content'] or monkey.guid in telem['endpoint']) and not monkey.on_island: + telem['content'] = telem['content'].replace(monkey.guid, monkey.fake_guid) + telem['endpoint'] = telem['endpoint'].replace(monkey.guid, monkey.fake_guid) + for i in range(len(monkey.ips)): + telem['content'] = telem['content'].replace(monkey.ips[i], monkey.fake_ips[i]) + + @staticmethod + def offset_telem_times(iteration: int, telems: List[Dict]): + for telem in telems: + telem['time']['$date'] += iteration * 1000 + + @staticmethod + def save_teletries_to_files(telems: List[Dict]): + for telem in telems: + TelemParser.save_telemetry_to_file(telem) + + @staticmethod + def save_telemetry_to_file(telem): + telem_filename = telem['name'] + telem['method'] + for i in range(10000): + if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): + telem_filename = str(i) + telem_filename + break + with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: + file.write(json.dumps(telem)) + + @staticmethod + def read_telem_files() -> List[str]: + telems = [] + file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) + if path.isfile(path.join(TELEM_DIR_PATH, f))] + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telems.append(telem_file.readline()) + return telems + + @staticmethod + def get_all_telemetries(): + return [json.loads(t) for t in TelemParser.read_telem_files()] + + def get_monkeys_from_telems(self, telems: List[Dict]): + monkeys = [] + for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']: + if 'network_info' not in telem['data']: + continue + guid = telem['monkey_guid'] + monkey_present = [monkey for monkey in monkeys if monkey.guid == guid] + if not monkey_present: + ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']] + if self.island_ip in ips: + on_island = True + else: + on_island = False + + monkeys.append(Monkey(ips=ips, + guid=guid, + fake_ip_generator=self.fake_ip_generator, + on_island=on_island)) + return monkeys + + +if __name__ == "__main__": + TelemParser(island_ip='192.168.56.1', multiplier=100).multiply_telems() diff --git a/monkey/monkey_island/cc/resources/log.py b/monkey/monkey_island/cc/resources/log.py index cbce92a37..920890648 100644 --- a/monkey/monkey_island/cc/resources/log.py +++ b/monkey/monkey_island/cc/resources/log.py @@ -6,6 +6,7 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from monkey_island.cc.services.log import LogService from monkey_island.cc.services.node import NodeService @@ -23,6 +24,7 @@ class Log(flask_restful.Resource): return LogService.log_exists(ObjectId(exists_monkey_id)) # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self): telemetry_json = json.loads(request.data) diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index dc6a7d512..f6e833eeb 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -8,6 +8,7 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.processing import process_telemetry from monkey_island.cc.models.monkey import Monkey @@ -40,6 +41,7 @@ class Telemetry(flask_restful.Resource): return result # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self): telemetry_json = json.loads(request.data) telemetry_json['timestamp'] = datetime.now() From b6e0e30d5876d2e3a51ac201577ed10a499c8c2b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 20 Apr 2020 18:23:25 +0300 Subject: [PATCH 05/16] Splitting classes into multiple files --- .../performance/utils/fake_ip_generator.py | 11 ++++ .../tests/performance/utils/fake_monkey.py | 17 ++++++ .../tests/performance/utils/telem_parser.py | 53 +++++-------------- 3 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py new file mode 100644 index 000000000..8abe53453 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py @@ -0,0 +1,11 @@ +class FakeIpGenerator: + def __init__(self): + self.fake_ip_parts = [1, 1, 1, 1] + + def generate_fake_ips_for_real_ips(self, real_ips): + self.fake_ip_parts[2] += 1 + fake_ips = [] + for i in range(len(real_ips)): + fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts)) + self.fake_ip_parts[3] += 1 + return fake_ips diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py b/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py new file mode 100644 index 000000000..dab84a1e6 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py @@ -0,0 +1,17 @@ +import random + +from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator + + +class FakeMonkey: + def __init__(self, ips, guid, fake_ip_generator: FakeIpGenerator, on_island=False): + self.original_ips = ips + self.original_guid = guid + self.fake_ip_generator = fake_ip_generator + self.on_island = on_island + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) + self.fake_ips = fake_ip_generator.generate_fake_ips_for_real_ips(ips) + + def change_fake_data(self): + self.fake_ips = self.fake_ip_generator.generate_fake_ips_for_real_ips(self.original_ips) + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py index c6ee7095f..fb93c4132 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py @@ -1,40 +1,15 @@ from typing import List, Dict from os import listdir, path import copy -import random import json +from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey + TELEM_DIR_PATH = './tests/performance/test_telems' -class FakeIpGenerator: - def __init__(self): - self.fake_ip_parts = [1, 1, 1, 1] - - def generate_fake_ips_for_real_ips(self, real_ips): - self.fake_ip_parts[2] += 1 - fake_ips = [] - for i in range(len(real_ips)): - fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts)) - self.fake_ip_parts[3] += 1 - return fake_ips - - -class Monkey: - def __init__(self, ips, guid, fake_ip_generator: FakeIpGenerator, on_island=False): - self.ips = ips - self.guid = guid - self.fake_ip_generator = fake_ip_generator - self.on_island = on_island - self.fake_guid = str(random.randint(1000000000000, 9999999999999)) - self.fake_ips = fake_ip_generator.generate_fake_ips_for_real_ips(ips) - - def change_fake_data(self): - self.fake_ips = self.fake_ip_generator.generate_fake_ips_for_real_ips(self.ips) - self.fake_guid = str(random.randint(1000000000000, 9999999999999)) - - class TelemParser: def __init__(self, island_ip: str, multiplier: int): self.multiplier = multiplier @@ -54,16 +29,16 @@ class TelemParser: TelemParser.save_teletries_to_files(fake_telem_batch) @staticmethod - def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[Monkey]): + def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]): for telem in telems: for monkey in monkeys: if monkey.on_island: continue - if (monkey.guid in telem['content'] or monkey.guid in telem['endpoint']) and not monkey.on_island: - telem['content'] = telem['content'].replace(monkey.guid, monkey.fake_guid) - telem['endpoint'] = telem['endpoint'].replace(monkey.guid, monkey.fake_guid) - for i in range(len(monkey.ips)): - telem['content'] = telem['content'].replace(monkey.ips[i], monkey.fake_ips[i]) + if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) and not monkey.on_island: + telem['content'] = telem['content'].replace(monkey.original_guid, monkey.fake_guid) + telem['endpoint'] = telem['endpoint'].replace(monkey.original_guid, monkey.fake_guid) + for i in range(len(monkey.original_ips)): + telem['content'] = telem['content'].replace(monkey.original_ips[i], monkey.fake_ips[i]) @staticmethod def offset_telem_times(iteration: int, telems: List[Dict]): @@ -105,7 +80,7 @@ class TelemParser: if 'network_info' not in telem['data']: continue guid = telem['monkey_guid'] - monkey_present = [monkey for monkey in monkeys if monkey.guid == guid] + monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid] if not monkey_present: ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']] if self.island_ip in ips: @@ -113,10 +88,10 @@ class TelemParser: else: on_island = False - monkeys.append(Monkey(ips=ips, - guid=guid, - fake_ip_generator=self.fake_ip_generator, - on_island=on_island)) + monkeys.append(FakeMonkey(ips=ips, + guid=guid, + fake_ip_generator=self.fake_ip_generator, + on_island=on_island)) return monkeys From 78b01866134aa29bafe2487588bbcfa2775d186d Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 23 Apr 2020 19:44:24 +0300 Subject: [PATCH 06/16] Finished telemetry fabrication --- .gitignore | 4 +++ envs/monkey_zoo/blackbox/README.md | 16 ++++++++++++ .../tests/performance/utils/telem_parser.py | 24 ++++++++++++----- monkey/monkey_island/cc/models/test_telem.py | 2 +- .../cc/resources/test/utils/telem_store.py | 26 ++++++++++--------- monkey/monkey_island/cc/services/config.py | 4 +++ .../cc/services/config_schema.py | 13 ++++++++++ 7 files changed, 70 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 772829801..71aa824de 100644 --- a/.gitignore +++ b/.gitignore @@ -82,5 +82,9 @@ MonkeyZoo/* !MonkeyZoo/config.tf !MonkeyZoo/MonkeyZooDocs.pdf +# Exported monkey telemetries +/monkey/test_telems/ + + # vim swap files *.swp diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index f1b66de91..278ca65ef 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -17,3 +17,19 @@ Run the following command: #### Running in PyCharm Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the `monkey\envs\monkey_zoo\blackbox`. + +### Running telemetry performance test +To run telemetry performance test follow these steps: +1. Gather monkey telemetries. + 1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have + exported telemetries already. + 2. Run monkey and wait until infection is done. + 3. All telemetries are gathered in `monkey/test_telems` +2. Run telemetry performance test. + 1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems` + 2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply + telemetries gathered. + 1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox` + 2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate + telemetries 4 times. + 3. Run blackbox tests, telemetry performance test will run as part of it. diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py index fb93c4132..06846d17f 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py @@ -2,6 +2,7 @@ from typing import List, Dict from os import listdir, path import copy import json +import sys from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey @@ -11,10 +12,10 @@ TELEM_DIR_PATH = './tests/performance/test_telems' class TelemParser: - def __init__(self, island_ip: str, multiplier: int): + + def __init__(self, multiplier: int): self.multiplier = multiplier self.fake_ip_generator = FakeIpGenerator() - self.island_ip = island_ip def multiply_telems(self): telems = TelemParser.get_all_telemetries() @@ -51,7 +52,7 @@ class TelemParser: TelemParser.save_telemetry_to_file(telem) @staticmethod - def save_telemetry_to_file(telem): + def save_telemetry_to_file(telem: Dict): telem_filename = telem['name'] + telem['method'] for i in range(10000): if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): @@ -71,10 +72,11 @@ class TelemParser: return telems @staticmethod - def get_all_telemetries(): + def get_all_telemetries() -> List[Dict]: return [json.loads(t) for t in TelemParser.read_telem_files()] def get_monkeys_from_telems(self, telems: List[Dict]): + island_ips = TelemParser.get_island_ips_from_telems(telems) monkeys = [] for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']: if 'network_info' not in telem['data']: @@ -83,7 +85,7 @@ class TelemParser: monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid] if not monkey_present: ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']] - if self.island_ip in ips: + if set(island_ips).intersection(ips): on_island = True else: on_island = False @@ -94,6 +96,16 @@ class TelemParser: on_island=on_island)) return monkeys + @staticmethod + def get_island_ips_from_telems(telems: List[Dict]) -> List[str]: + island_ips = [] + for telem in telems: + if 'config' in telem: + island_ips = telem['config']['command_servers'] + for i in range(len(island_ips)): + island_ips[i] = island_ips[i].replace(":5000", "") + return island_ips + if __name__ == "__main__": - TelemParser(island_ip='192.168.56.1', multiplier=100).multiply_telems() + TelemParser(multiplier=int(sys.argv[1])).multiply_telems() diff --git a/monkey/monkey_island/cc/models/test_telem.py b/monkey/monkey_island/cc/models/test_telem.py index cb0697a33..97855d4ed 100644 --- a/monkey/monkey_island/cc/models/test_telem.py +++ b/monkey/monkey_island/cc/models/test_telem.py @@ -1,5 +1,5 @@ """ -Define a Document Schema for the Monkey document. +Define a Document Schema for the TestTelem document. """ from mongoengine import Document, StringField, DateTimeField diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index a6c569436..495a0ff88 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -1,12 +1,14 @@ from functools import wraps from os import mkdir, path +import shutil from datetime import datetime from flask import request from monkey_island.cc.models.test_telem import TestTelem +from monkey_island.cc.services.config import ConfigService -MONKEY_TELEM_COLLECTION_NAME = "monkey_telems_for_tests" +TEST_TELEM_DIR = "./test_telems" class TestTelemStore: @@ -15,28 +17,28 @@ class TestTelemStore: def store_test_telem(f): @wraps(f) def decorated_function(*args, **kwargs): - time = datetime.now() - method = request.method - content = request.data.decode() - endpoint = request.path - name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') - TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save() + if ConfigService.is_test_telem_export_enabled(): + time = datetime.now() + method = request.method + content = request.data.decode() + endpoint = request.path + name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') + TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save() return f(*args, **kwargs) return decorated_function @staticmethod def export_test_telems(): - telem_dir = "./test_telems" try: - mkdir(telem_dir) + mkdir(TEST_TELEM_DIR) except FileExistsError: - pass + shutil.rmtree(TEST_TELEM_DIR) + mkdir(TEST_TELEM_DIR) for test_telem in TestTelem.objects(): - with open(TestTelemStore.get_unique_file_path_for_test_telem(telem_dir, test_telem), 'w') as file: + with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file: file.write(test_telem.to_json()) - @staticmethod def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem): telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 96c59cad6..e9ed3b0f6 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -307,3 +307,7 @@ class ConfigService: pair['public_key'] = encryptor.dec(pair['public_key']) pair['private_key'] = encryptor.dec(pair['private_key']) return pair + + @staticmethod + def is_test_telem_export_enabled(): + return ConfigService.get_config_value(['internal', 'testing', 'export_monkey_telems']) diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index 3d0220ee2..72ab9ac42 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -736,6 +736,19 @@ SCHEMA = { "description": "List of SSH key pairs to use, when trying to ssh into servers" } } + }, + "testing": { + "title": "Testing", + "type": "object", + "properties": { + "export_monkey_telems": { + "title": "Export monkey telemetries", + "type": "boolean", + "default": False, + "description": "Exports unencrypted telemetries that can be used for tests in development." + " Do not turn on!" + } + } } } }, From 91d68113f9bf13fc2e3f31cab4235c6e1c321bcb Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 23 Apr 2020 19:46:05 +0300 Subject: [PATCH 07/16] Refactored root endpoint and added service hook for things that should happen after exploitation is done --- monkey/monkey_island/cc/resources/root.py | 37 ++------------ .../cc/services/infection_lifecycle.py | 51 +++++++++++++++++++ 2 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 monkey/monkey_island/cc/services/infection_lifecycle.py diff --git a/monkey/monkey_island/cc/resources/root.py b/monkey/monkey_island/cc/resources/root.py index 59a8fbe7c..216329905 100644 --- a/monkey/monkey_island/cc/resources/root.py +++ b/monkey/monkey_island/cc/resources/root.py @@ -1,6 +1,5 @@ import logging import threading -from datetime import datetime import flask_restful from flask import request, make_response, jsonify @@ -8,10 +7,7 @@ from flask import request, make_response, jsonify from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo from monkey_island.cc.services.database import Database -from monkey_island.cc.services.node import NodeService -from monkey_island.cc.services.reporting.report import ReportService -from monkey_island.cc.services.reporting.report_generation_synchronisation import is_report_being_generated, \ - safe_generate_reports +from monkey_island.cc.services.infection_lifecycle import InfectionLifecycle from monkey_island.cc.utils import local_ip_addresses __author__ = 'Barak' @@ -32,7 +28,7 @@ class Root(flask_restful.Resource): elif action == "reset": return jwt_required()(Database.reset_db)() elif action == "killall": - return Root.kill_all() + return jwt_required()(InfectionLifecycle.kill_all)() elif action == "is-up": return {'is-up': True} else: @@ -43,33 +39,6 @@ class Root(flask_restful.Resource): return jsonify( ip_addresses=local_ip_addresses(), mongo=str(mongo.db), - completed_steps=self.get_completed_steps()) + completed_steps=InfectionLifecycle.get_completed_steps()) - @staticmethod - @jwt_required() - def kill_all(): - mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}}, - upsert=False, - multi=True) - logger.info('Kill all monkeys was called') - return jsonify(status='OK') - @jwt_required() - def get_completed_steps(self): - is_any_exists = NodeService.is_any_monkey_exists() - infection_done = NodeService.is_monkey_finished_running() - - if infection_done: - # Checking is_report_being_generated here, because we don't want to wait to generate a report; rather, - # we want to skip and reply. - if not is_report_being_generated() and not ReportService.is_latest_report_exists(): - safe_generate_reports() - report_done = ReportService.is_report_generated() - else: # Infection is not done - report_done = False - - return dict( - run_server=True, - run_monkey=is_any_exists, - infection_done=infection_done, - report_done=report_done) diff --git a/monkey/monkey_island/cc/services/infection_lifecycle.py b/monkey/monkey_island/cc/services/infection_lifecycle.py new file mode 100644 index 000000000..e79cfe947 --- /dev/null +++ b/monkey/monkey_island/cc/services/infection_lifecycle.py @@ -0,0 +1,51 @@ +import logging +from datetime import datetime + +from flask import jsonify + +from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore +from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.node import NodeService +from monkey_island.cc.services.reporting.report import ReportService +from monkey_island.cc.services.reporting.report_generation_synchronisation import is_report_being_generated, \ + safe_generate_reports + +logger = logging.getLogger(__name__) + + +class InfectionLifecycle: + + @staticmethod + def kill_all(): + mongo.db.monkey.update({'dead': False}, {'$set': {'config.alive': False, 'modifytime': datetime.now()}}, + upsert=False, + multi=True) + logger.info('Kill all monkeys was called') + return jsonify(status='OK') + + @staticmethod + def get_completed_steps(): + is_any_exists = NodeService.is_any_monkey_exists() + infection_done = NodeService.is_monkey_finished_running() + + if infection_done: + InfectionLifecycle._on_finished_infection() + report_done = ReportService.is_report_generated() + else: # Infection is not done + report_done = False + + return dict( + run_server=True, + run_monkey=is_any_exists, + infection_done=infection_done, + report_done=report_done) + + @staticmethod + def _on_finished_infection(): + # Checking is_report_being_generated here, because we don't want to wait to generate a report; rather, + # we want to skip and reply. + if not is_report_being_generated() and not ReportService.is_latest_report_exists(): + safe_generate_reports() + if ConfigService.is_test_telem_export_enabled(): + TestTelemStore.export_test_telems() From 986e52716f00138b694441f8e2a44871310dde59 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 23 Apr 2020 19:46:58 +0300 Subject: [PATCH 08/16] Finished telemetry performance test (time measurements, refactoring) --- .../analyzers/performance_analyzer.py | 6 +-- .../island_client/monkey_island_requests.py | 28 ++++++++++- .../island_client/supported_reuqest_method.py | 8 +++ .../performance/endpoint_performance_test.py | 16 ++---- .../performance/performance_test_config.py | 2 +- .../performance/telemetry_performance_test.py | 50 +++++++++++++------ 6 files changed, 75 insertions(+), 35 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index d9067eeee..92d8a68eb 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -14,11 +14,11 @@ class PerformanceAnalyzer(Analyzer): self.endpoint_timings = endpoint_timings def analyze_test_results(self): - # Calculate total time and check each page + # Calculate total time and check each endpoint single_page_time_less_then_max = True total_time = timedelta() - for page, elapsed in self.endpoint_timings.items(): - LOGGER.info(f"page {page} took {str(elapsed)}") + for endpoint, elapsed in self.endpoint_timings.items(): + LOGGER.info(f"Endpoint {endpoint} took {str(elapsed)}") total_time += elapsed if elapsed > self.performance_test_config.max_allowed_single_page_time: single_page_time_less_then_max = False diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index 07785509d..b56f51612 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -1,8 +1,12 @@ from typing import Dict +from datetime import timedelta + import requests import functools +from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod + # SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' import logging @@ -16,6 +20,26 @@ class MonkeyIslandRequests(object): def __init__(self, server_address): self.addr = "https://{IP}/".format(IP=server_address) self.token = self.try_get_jwt_from_server() + self.supported_request_methods = {SupportedRequestMethod.GET: self.get, + SupportedRequestMethod.POST: self.post, + SupportedRequestMethod.PATCH: self.patch, + SupportedRequestMethod.DELETE: self.delete} + + def get_request_time(self, url, method: SupportedRequestMethod, data=None): + response = self.send_request_by_method(url, method, data) + if response.ok: + LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") + return response.elapsed + else: + LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") + # instead of raising for status, mark failed responses as maxtime + return timedelta.max + + def send_request_by_method(self, url, method=SupportedRequestMethod.GET, data=None): + if data: + return self.supported_request_methods[method](url, data) + else: + return self.supported_request_methods[method](url) def try_get_jwt_from_server(self): try: @@ -64,9 +88,9 @@ class MonkeyIslandRequests(object): verify=False) @_Decorators.refresh_jwt_token - def patch_json(self, url, data: Dict): + def patch(self, url, data: Dict): return requests.patch(self.addr + url, # noqa: DUO123 - json=data, + data=data, headers=self.get_jwt_header(), verify=False) diff --git a/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py b/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py new file mode 100644 index 000000000..60cb0877a --- /dev/null +++ b/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class SupportedRequestMethod(Enum): + GET = "GET" + POST = "POST" + PATCH = "PATCH" + DELETE = "DELETE" diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py index 76a389efd..5a83f252c 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -1,10 +1,10 @@ import logging -from datetime import timedelta from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer +from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod LOGGER = logging.getLogger(__name__) @@ -25,18 +25,8 @@ class EndpointPerformanceTest(BasicTest): self.island_client.clear_caches() endpoint_timings = {} for endpoint in self.test_config.endpoints_to_test: - endpoint_timings[endpoint] = self.get_elapsed_for_get_request(endpoint) - + endpoint_timings[endpoint] = self.island_client.requests.get_request_time(endpoint, + SupportedRequestMethod.GET) analyzer = PerformanceAnalyzer(self.test_config, endpoint_timings) return analyzer.analyze_test_results() - - def get_elapsed_for_get_request(self, url): - response = self.island_client.requests.get(url) - if response.ok: - LOGGER.debug(f"Got ok for {url} content peek:\n{response.content[:120].strip()}") - return response.elapsed - else: - LOGGER.error(f"Trying to get {url} but got unexpected {str(response)}") - # instead of raising for status, mark failed responses as maxtime - return timedelta.max diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py index 8ed2b5a62..ad7be5967 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_config.py @@ -5,7 +5,7 @@ from typing import List class PerformanceTestConfig: def __init__(self, max_allowed_single_page_time: timedelta, max_allowed_total_time: timedelta, - endpoints_to_test: List[str], break_on_timeout=False): + endpoints_to_test: List[str] = None, break_on_timeout=False): self.max_allowed_single_page_time = max_allowed_single_page_time self.max_allowed_total_time = max_allowed_total_time self.endpoints_to_test = endpoints_to_test diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 047d07406..0950be4ff 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -1,7 +1,17 @@ - +import logging +from datetime import timedelta +import json from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser +from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod + +LOGGER = logging.getLogger(__name__) + +MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME = timedelta(seconds=2) +MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=60) class TelemetryPerformanceTest: @@ -10,24 +20,32 @@ class TelemetryPerformanceTest: self.island_client = island_client def test_telemetry_performance(self): - all_telemetries = TelemParser.get_all_telemetries() + LOGGER.info("Starting telemetry performance test.") + try: + all_telemetries = TelemParser.get_all_telemetries() + except FileNotFoundError: + LOGGER.error("Telemetries to send not found. Refer to readme to figure out how to generate telemetries " + "and where to put them.") + return False + LOGGER.info("Telemetries imported successfully.") all_telemetries.sort(key=lambda telem: telem['time']['$date']) + telemetry_parse_times = {} for telemetry in all_telemetries: - self.send_telemetry(telemetry) + telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint(telemetry) + telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(telemetry) + test_config = PerformanceTestConfig(MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME, MAX_ALLOWED_TOTAL_TIME) + PerformanceAnalyzer(test_config, telemetry_parse_times).analyze_test_results() - def send_telemetry(self, telemetry): + def get_telemetry_time(self, telemetry): content = telemetry['content'] url = telemetry['endpoint'] - method = telemetry['method'] + method = SupportedRequestMethod.__getattr__(telemetry['method']) - if method == 'POST': - result = self.island_client.requests.post(url=url, data=content) - elif method == 'GET': - result = self.island_client.requests.get(url=url) - elif method == 'PATCH': - result = self.island_client.requests.patch_json(url=url, data=content) - elif method == 'DELETE': - result = self.island_client.requests.delete(url=url) - else: - raise Exception - return result + return self.island_client.requests.get_request_time(url=url, method=method, data=content) + + @staticmethod + def get_verbose_telemetry_endpoint(telemetry): + telem_category = "" + if "telem_category" in telemetry['content']: + telem_category = "_" + json.loads(telemetry['content'])['telem_category'] + return telemetry['endpoint'] + telem_category From baa1598a1b64ba19d9cd5fde1a8f8c392464cd40 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 24 Apr 2020 13:19:07 +0300 Subject: [PATCH 09/16] Added progress bars, optimized imports, fixed bug that happened on big numbers of telems --- .gitignore | 1 - envs/monkey_zoo/blackbox/README.md | 4 ++-- .../blackbox/analyzers/performance_analyzer.py | 1 + .../blackbox/island_client/monkey_island_client.py | 5 ++--- .../island_client/monkey_island_requests.py | 2 +- .../monkey_zoo/blackbox/log_handlers/monkey_log.py | 2 +- .../blackbox/log_handlers/test_logs_handler.py | 3 +-- envs/monkey_zoo/blackbox/test_blackbox.py | 14 ++++++-------- envs/monkey_zoo/blackbox/tests/exploitation.py | 5 ++--- .../tests/performance/endpoint_performance_test.py | 7 +++---- .../blackbox/tests/performance/map_generation.py | 2 +- .../tests/performance/performance_test_workflow.py | 2 +- .../tests/performance/report_generation.py | 2 +- .../performance/telemetry_performance_test.py | 12 +++++++----- .../tests/performance/utils/telem_parser.py | 13 ++++++++----- .../blackbox/utils/gcp_machine_handlers.py | 3 +-- envs/monkey_zoo/blackbox/utils/json_encoder.py | 1 + .../cc/resources/test/utils/telem_store.py | 4 +++- monkey/monkey_island/requirements.txt | 1 + 19 files changed, 43 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 71aa824de..97e618d43 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,5 @@ MonkeyZoo/* # Exported monkey telemetries /monkey/test_telems/ - # vim swap files *.swp diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index 278ca65ef..334dbf3ac 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -12,10 +12,10 @@ this information in the GCP Console `Compute Engine/VM Instances` under _Externa #### Running in command line Run the following command: -`monkey\envs\monkey_zoo\blackbox>python -m pytest --island=35.207.152.72:5000 test_blackbox.py` +`monkey\envs\monkey_zoo\blackbox>python -m pytest -s --island=35.207.152.72:5000 test_blackbox.py` #### Running in PyCharm -Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the +Configure a PyTest configuration with the additional arguments `-s --island=35.207.152.72` on the `monkey\envs\monkey_zoo\blackbox`. ### Running telemetry performance test diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index 92d8a68eb..665ec18af 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -4,6 +4,7 @@ from typing import Dict from envs.monkey_zoo.blackbox.analyzers.analyzer import Analyzer from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig + LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index 22fa57ac1..e3ecb6eb8 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -1,8 +1,7 @@ -from datetime import timedelta -from time import sleep import json - import logging +from time import sleep + from bson import json_util from envs.monkey_zoo.blackbox.island_client.monkey_island_requests import MonkeyIslandRequests diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index b56f51612..edcffe674 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -7,9 +7,9 @@ import functools from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod -# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' import logging +# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \ '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557' LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py b/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py index 091be570a..b7f424a69 100644 --- a/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py +++ b/envs/monkey_zoo/blackbox/log_handlers/monkey_log.py @@ -1,6 +1,6 @@ +import logging import os -import logging from bson import ObjectId LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py b/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py index b54f773e6..bae6a9adc 100644 --- a/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py +++ b/envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py @@ -1,8 +1,7 @@ +import logging import os import shutil -import logging - from envs.monkey_zoo.blackbox.log_handlers.monkey_log_parser import MonkeyLogParser from envs.monkey_zoo.blackbox.log_handlers.monkey_logs_downloader import MonkeyLogsDownloader diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 631229e36..3c4b5e3df 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -1,18 +1,16 @@ -import os import logging - -import pytest +import os from time import sleep -from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +import pytest + from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser -from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest -from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient +from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler +from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers -from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler DEFAULT_TIMEOUT_SECONDS = 5*60 MACHINE_BOOTUP_WAIT_SECONDS = 30 diff --git a/envs/monkey_zoo/blackbox/tests/exploitation.py b/envs/monkey_zoo/blackbox/tests/exploitation.py index e731d8f90..2d55f2294 100644 --- a/envs/monkey_zoo/blackbox/tests/exploitation.py +++ b/envs/monkey_zoo/blackbox/tests/exploitation.py @@ -1,9 +1,8 @@ +import logging from time import sleep -import logging - -from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer MAX_TIME_FOR_MONKEYS_TO_DIE = 5 * 60 WAIT_TIME_BETWEEN_REQUESTS = 10 diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py index 5a83f252c..5b8a43227 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -1,11 +1,10 @@ import logging -from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest -from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod - +from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py index c597907f4..eb95fdc6a 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/map_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/map_generation.py @@ -1,8 +1,8 @@ from datetime import timedelta from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) diff --git a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py index 3157140a9..f6cd1dada 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py +++ b/envs/monkey_zoo/blackbox/tests/performance/performance_test_workflow.py @@ -1,7 +1,7 @@ from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.endpoint_performance_test import EndpointPerformanceTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig class PerformanceTestWorkflow(BasicTest): diff --git a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py index 52fe76288..e204cc29f 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/report_generation.py +++ b/envs/monkey_zoo/blackbox/tests/performance/report_generation.py @@ -1,9 +1,9 @@ from datetime import timedelta from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.performance_test_workflow import PerformanceTestWorkflow -from envs.monkey_zoo.blackbox.tests.performance.performance_test import PerformanceTest MAX_ALLOWED_SINGLE_PAGE_TIME = timedelta(seconds=2) MAX_ALLOWED_TOTAL_TIME = timedelta(seconds=5) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 0950be4ff..236625288 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -1,12 +1,14 @@ +import json import logging from datetime import timedelta -import json -from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient -from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser +from tqdm import tqdm + from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer -from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod +from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig +from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser LOGGER = logging.getLogger(__name__) @@ -30,7 +32,7 @@ class TelemetryPerformanceTest: LOGGER.info("Telemetries imported successfully.") all_telemetries.sort(key=lambda telem: telem['time']['$date']) telemetry_parse_times = {} - for telemetry in all_telemetries: + for telemetry in tqdm(all_telemetries, total=len(all_telemetries), ascii=True, desc="Telemetries sent"): telemetry_endpoint = TelemetryPerformanceTest.get_verbose_telemetry_endpoint(telemetry) telemetry_parse_times[telemetry_endpoint] = self.get_telemetry_time(telemetry) test_config = PerformanceTestConfig(MAX_ALLOWED_SINGLE_TELEM_PARSE_TIME, MAX_ALLOWED_TOTAL_TIME) diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py index 06846d17f..049454a32 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py @@ -1,14 +1,17 @@ -from typing import List, Dict -from os import listdir, path import copy import json +import logging import sys +from os import listdir, path +from typing import List, Dict + +from tqdm import tqdm from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey - TELEM_DIR_PATH = './tests/performance/test_telems' +LOGGER = logging.getLogger(__name__) class TelemParser: @@ -21,7 +24,7 @@ class TelemParser: telems = TelemParser.get_all_telemetries() telem_contents = [json.loads(telem['content']) for telem in telems] monkeys = self.get_monkeys_from_telems(telem_contents) - for i in range(self.multiplier): + for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries"): for monkey in monkeys: monkey.change_fake_data() fake_telem_batch = copy.deepcopy(telems) @@ -31,7 +34,7 @@ class TelemParser: @staticmethod def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]): - for telem in telems: + for telem in tqdm(telems, desc="Teletry"): for monkey in monkeys: if monkey.on_island: continue diff --git a/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py b/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py index 3cb2ad6af..633f406a5 100644 --- a/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py +++ b/envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py @@ -1,6 +1,5 @@ -import subprocess - import logging +import subprocess LOGGER = logging.getLogger(__name__) diff --git a/envs/monkey_zoo/blackbox/utils/json_encoder.py b/envs/monkey_zoo/blackbox/utils/json_encoder.py index 77be9211a..1642ea222 100644 --- a/envs/monkey_zoo/blackbox/utils/json_encoder.py +++ b/envs/monkey_zoo/blackbox/utils/json_encoder.py @@ -1,4 +1,5 @@ import json + from bson import ObjectId diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index 495a0ff88..49d7ea27b 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -9,6 +9,7 @@ from monkey_island.cc.models.test_telem import TestTelem from monkey_island.cc.services.config import ConfigService TEST_TELEM_DIR = "./test_telems" +MAX_SAME_CATEGORY_TELEMS = 10000 class TestTelemStore: @@ -42,11 +43,12 @@ class TestTelemStore: @staticmethod def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem): telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem) - for i in range(100): + for i in range(MAX_SAME_CATEGORY_TELEMS): potential_filepath = path.join(target_dir, (telem_filename + str(i))) if path.exists(potential_filepath): continue return potential_filepath + raise Exception(f"Too many telemetries of the same category. Max amount {MAX_SAME_CATEGORY_TELEMS}") @staticmethod def _get_filename_by_test_telem(test_telem: TestTelem): diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index cad53d1c8..39364cbb5 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -24,3 +24,4 @@ requests dpath ring stix2 +tcdm From 41ae125980192f7e4eb89edd2fbfaec29832fbaf Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 24 Apr 2020 16:41:00 +0300 Subject: [PATCH 10/16] Fixed telemetry performance test logging bugs and improved logging to display only N longest telems instead of all --- .../blackbox/analyzers/performance_analyzer.py | 12 ++++++++++-- .../tests/performance/telemetry_performance_test.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py index 665ec18af..4a43ab6a5 100644 --- a/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/performance_analyzer.py @@ -19,14 +19,14 @@ class PerformanceAnalyzer(Analyzer): single_page_time_less_then_max = True total_time = timedelta() for endpoint, elapsed in self.endpoint_timings.items(): - LOGGER.info(f"Endpoint {endpoint} took {str(elapsed)}") total_time += elapsed if elapsed > self.performance_test_config.max_allowed_single_page_time: single_page_time_less_then_max = False total_time_less_then_max = total_time < self.performance_test_config.max_allowed_total_time - LOGGER.info(f"total time is {str(total_time)}") + PerformanceAnalyzer.log_slowest_endpoints(self.endpoint_timings) + LOGGER.info(f"Total time is {str(total_time)}") performance_is_good_enough = total_time_less_then_max and single_page_time_less_then_max @@ -38,3 +38,11 @@ class PerformanceAnalyzer(Analyzer): breakpoint() return performance_is_good_enough + + @staticmethod + def log_slowest_endpoints(endpoint_timings, max_endpoints_to_display=100): + slow_endpoint_list = list(endpoint_timings.items()) + slow_endpoint_list.sort(key=lambda x: x[1], reverse=True) + slow_endpoint_list = slow_endpoint_list[:max_endpoints_to_display] + for endpoint in slow_endpoint_list: + LOGGER.info(f"{endpoint[0]} took {str(endpoint[1])}") diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 236625288..719ec5806 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -49,5 +49,5 @@ class TelemetryPerformanceTest: def get_verbose_telemetry_endpoint(telemetry): telem_category = "" if "telem_category" in telemetry['content']: - telem_category = "_" + json.loads(telemetry['content'])['telem_category'] + telem_category = "_" + json.loads(telemetry['content'])['telem_category'] + "_" + telemetry['_id']['$oid'] return telemetry['endpoint'] + telem_category From 55d3fd5bd8b16e49bc9073c161aef3ac89096cdb Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 28 Apr 2020 17:54:12 +0300 Subject: [PATCH 11/16] More improvement on telemetry fabrication logging --- envs/monkey_zoo/blackbox/README.md | 1 + .../blackbox/tests/performance/utils/telem_parser.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index 334dbf3ac..e800537de 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -32,4 +32,5 @@ To run telemetry performance test follow these steps: 1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox` 2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate telemetries 4 times. + 3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion. 3. Run blackbox tests, telemetry performance test will run as part of it. diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py index 049454a32..df7e9f5be 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py @@ -24,7 +24,7 @@ class TelemParser: telems = TelemParser.get_all_telemetries() telem_contents = [json.loads(telem['content']) for telem in telems] monkeys = self.get_monkeys_from_telems(telem_contents) - for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries"): + for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries", position=1): for monkey in monkeys: monkey.change_fake_data() fake_telem_batch = copy.deepcopy(telems) @@ -34,7 +34,7 @@ class TelemParser: @staticmethod def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]): - for telem in tqdm(telems, desc="Teletry"): + for telem in tqdm(telems, desc="Telemetries fabricated", position=2): for monkey in monkeys: if monkey.on_island: continue @@ -51,7 +51,7 @@ class TelemParser: @staticmethod def save_teletries_to_files(telems: List[Dict]): - for telem in telems: + for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)): TelemParser.save_telemetry_to_file(telem) @staticmethod From 1352747c4eff1b0bc3c69da0b12abbdab3be45e2 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 29 Apr 2020 10:39:10 +0300 Subject: [PATCH 12/16] Removed debugging comments --- envs/monkey_zoo/blackbox/test_blackbox.py | 116 +++++++++++----------- 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 3c4b5e3df..c0a7caf3d 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -9,6 +9,8 @@ from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandCo from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest +from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers @@ -24,11 +26,11 @@ LOGGER = logging.getLogger(__name__) @pytest.fixture(autouse=True, scope='session') def GCPHandler(request): GCPHandler = gcp_machine_handlers.GCPHandler() - #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) - #wait_machine_bootup() + GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + wait_machine_bootup() def fin(): - #GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) pass request.addfinalizer(fin) @@ -90,60 +92,60 @@ class TestMonkeyBlackbox(object): def test_server_online(self, island_client): assert island_client.get_api_status() is not None - #def test_ssh_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") -# - #def test_hadoop_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) -# - #def test_mssql_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") -# - #def test_smb_and_mimikatz_exploiters(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") -# - #def test_smb_pth(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") -# - #def test_elastic_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") -# - #def test_struts_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") -# - #def test_weblogic_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") -# - #def test_shellshock_exploiter(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") -# - #def test_tunneling(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) -# - #def test_wmi_and_mimikatz_exploiters(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") -# - #def test_wmi_pth(self, island_client): - # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") -# - #def test_report_generation_performance(self, island_client): - # """ - # This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test - # for a total of 8 machines including the Monkey Island. -# - # Is has 2 analyzers - the regular one which checks all the Monkeys - # and the Timing one which checks how long the report took to execute - # """ - # TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, - # island_client, - # "PERFORMANCE.conf", - # timeout_in_seconds=10*60) -# - #def test_map_generation_performance(self, island_client): - # TestMonkeyBlackbox.run_performance_test(MapGenerationTest, - # island_client, - # "PERFORMANCE.conf", - # timeout_in_seconds=10*60) + def test_ssh_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") + + def test_hadoop_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) + + def test_mssql_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") + + def test_smb_and_mimikatz_exploiters(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") + + def test_smb_pth(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") + + def test_elastic_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") + + def test_struts_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") + + def test_weblogic_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") + + def test_shellshock_exploiter(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") + + def test_tunneling(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) + + def test_wmi_and_mimikatz_exploiters(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") + + def test_wmi_pth(self, island_client): + TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") + + def test_report_generation_performance(self, island_client): + """ + This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test + for a total of 8 machines including the Monkey Island. + + Is has 2 analyzers - the regular one which checks all the Monkeys + and the Timing one which checks how long the report took to execute + """ + TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, + island_client, + "PERFORMANCE.conf", + timeout_in_seconds=10*60) + + def test_map_generation_performance(self, island_client): + TestMonkeyBlackbox.run_performance_test(MapGenerationTest, + island_client, + "PERFORMANCE.conf", + timeout_in_seconds=10*60) def test_telem_performance(self, island_client): TelemetryPerformanceTest(island_client).test_telemetry_performance() From 1a784791fdcc1a79f0cb816f67b7dff73c1919e8 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 29 Apr 2020 11:17:08 +0300 Subject: [PATCH 13/16] Fixed a typo in supported_request_method.py filename --- .../monkey_zoo/blackbox/island_client/monkey_island_requests.py | 2 +- ...{supported_reuqest_method.py => supported_request_method.py} | 0 .../blackbox/tests/performance/endpoint_performance_test.py | 2 +- .../blackbox/tests/performance/telemetry_performance_test.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename envs/monkey_zoo/blackbox/island_client/{supported_reuqest_method.py => supported_request_method.py} (100%) diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index edcffe674..babc9c7a0 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -5,7 +5,7 @@ from datetime import timedelta import requests import functools -from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod +from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod import logging diff --git a/envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py b/envs/monkey_zoo/blackbox/island_client/supported_request_method.py similarity index 100% rename from envs/monkey_zoo/blackbox/island_client/supported_reuqest_method.py rename to envs/monkey_zoo/blackbox/island_client/supported_request_method.py diff --git a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py index 5b8a43227..798f490af 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/endpoint_performance_test.py @@ -2,7 +2,7 @@ import logging from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient -from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod +from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 719ec5806..4086a234e 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -6,7 +6,7 @@ from tqdm import tqdm from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceAnalyzer from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient -from envs.monkey_zoo.blackbox.island_client.supported_reuqest_method import SupportedRequestMethod +from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser From 9aff40d974524e0502da1528a8906591729f9514 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 7 May 2020 11:20:08 +0300 Subject: [PATCH 14/16] Fake IP's don't get out of range, fixed typo in requirements and added logging to telem parsing --- .../performance/utils/fake_ip_generator.py | 20 ++++++++++++++++--- .../utils/test_fake_ip_generator.py | 18 +++++++++++++++++ .../cc/resources/test/utils/telem_store.py | 8 ++++++++ monkey/monkey_island/requirements.txt | 2 +- 4 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py index 8abe53453..90422f9a0 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py @@ -1,11 +1,25 @@ +from typing import List + + class FakeIpGenerator: def __init__(self): self.fake_ip_parts = [1, 1, 1, 1] - def generate_fake_ips_for_real_ips(self, real_ips): - self.fake_ip_parts[2] += 1 + def generate_fake_ips_for_real_ips(self, real_ips: List[str]) -> List[str]: fake_ips = [] for i in range(len(real_ips)): fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts)) - self.fake_ip_parts[3] += 1 + self.increment_ip() return fake_ips + + def increment_ip(self): + self.fake_ip_parts[3] += 1 + self.try_fix_ip_range() + + def try_fix_ip_range(self): + for i in range(len(self.fake_ip_parts)): + if self.fake_ip_parts[i] > 256: + if i-1 < 0: + raise Exception("Fake IP's out of range.") + self.fake_ip_parts[i-1] += 1 + self.fake_ip_parts[i] = 1 diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py new file mode 100644 index 000000000..96609a8a9 --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py @@ -0,0 +1,18 @@ +from unittest import TestCase + +from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator + + +class TestFakeIpGenerator(TestCase): + + def test_fake_ip_generation(self): + fake_ip_gen = FakeIpGenerator() + self.assertListEqual([1, 1, 1, 1], fake_ip_gen.fake_ip_parts) + for i in range(256): + fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1']) + self.assertListEqual(['1.1.2.1'], fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1'])) + fake_ip_gen.fake_ip_parts = [256, 256, 255, 256] + self.assertListEqual(['256.256.255.256', '256.256.256.1'], + fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1', '1.1.1.2'])) + fake_ip_gen.fake_ip_parts = [256, 256, 256, 256] + self.assertRaises(Exception, fake_ip_gen.generate_fake_ips_for_real_ips(['1.1.1.1'])) diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index 49d7ea27b..031a0d02a 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -1,3 +1,4 @@ +import logging from functools import wraps from os import mkdir, path import shutil @@ -12,6 +13,10 @@ TEST_TELEM_DIR = "./test_telems" MAX_SAME_CATEGORY_TELEMS = 10000 +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + class TestTelemStore: @staticmethod @@ -31,14 +36,17 @@ class TestTelemStore: @staticmethod def export_test_telems(): + logger.info(f"Exporting all telemetries to {TEST_TELEM_DIR}") try: mkdir(TEST_TELEM_DIR) except FileExistsError: + logger.info("Deleting all previous telemetries.") shutil.rmtree(TEST_TELEM_DIR) mkdir(TEST_TELEM_DIR) for test_telem in TestTelem.objects(): with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file: file.write(test_telem.to_json()) + logger.info("Telemetries exported!") @staticmethod def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem): diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index 39364cbb5..ce376ab4c 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -24,4 +24,4 @@ requests dpath ring stix2 -tcdm +tqdm From e189e9625962f88526b47e16da36471d0c790f6e Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 11 May 2020 16:32:18 +0300 Subject: [PATCH 15/16] Improved the dir structure of telem parsing, other minor CR comments fixed --- .gitignore | 2 +- envs/monkey_zoo/.gitignore | 2 +- envs/monkey_zoo/blackbox/README.md | 7 ++- .../telem_sample_parsing/__init__.py | 0 .../sample_file_parser.py | 43 ++++++++++++++ .../sample_multiplier/__init__.py | 0 .../sample_multiplier}/fake_ip_generator.py | 0 .../sample_multiplier}/fake_monkey.py | 3 +- .../sample_multiplier/sample_multiplier.py} | 59 ++++++------------- .../test_fake_ip_generator.py | 3 +- .../performance/telemetry_performance_test.py | 4 +- .../cc/resources/test/utils/telem_store.py | 14 ++--- 12 files changed, 79 insertions(+), 58 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py create mode 100644 envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py rename envs/monkey_zoo/blackbox/tests/performance/{utils => telem_sample_parsing/sample_multiplier}/fake_ip_generator.py (100%) rename envs/monkey_zoo/blackbox/tests/performance/{utils => telem_sample_parsing/sample_multiplier}/fake_monkey.py (82%) rename envs/monkey_zoo/blackbox/tests/performance/{utils/telem_parser.py => telem_sample_parsing/sample_multiplier/sample_multiplier.py} (57%) rename envs/monkey_zoo/blackbox/tests/performance/{utils => telem_sample_parsing/sample_multiplier}/test_fake_ip_generator.py (85%) diff --git a/.gitignore b/.gitignore index 97e618d43..a10127767 100644 --- a/.gitignore +++ b/.gitignore @@ -83,7 +83,7 @@ MonkeyZoo/* !MonkeyZoo/MonkeyZooDocs.pdf # Exported monkey telemetries -/monkey/test_telems/ +/monkey/telem_sample/ # vim swap files *.swp diff --git a/envs/monkey_zoo/.gitignore b/envs/monkey_zoo/.gitignore index 04310b6fd..be22d3037 100644 --- a/envs/monkey_zoo/.gitignore +++ b/envs/monkey_zoo/.gitignore @@ -1,2 +1,2 @@ logs/ -/blackbox/tests/performance/test_telems/* +/blackbox/tests/performance/telem_sample diff --git a/envs/monkey_zoo/blackbox/README.md b/envs/monkey_zoo/blackbox/README.md index e800537de..b31fbdcab 100644 --- a/envs/monkey_zoo/blackbox/README.md +++ b/envs/monkey_zoo/blackbox/README.md @@ -24,13 +24,14 @@ To run telemetry performance test follow these steps: 1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have exported telemetries already. 2. Run monkey and wait until infection is done. - 3. All telemetries are gathered in `monkey/test_telems` + 3. All telemetries are gathered in `monkey/telem_sample` 2. Run telemetry performance test. 1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems` 2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply telemetries gathered. - 1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox` + 1. Run `telem_parser.py` script with working directory set to `monkey\envs\monkey_zoo\blackbox` 2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate telemetries 4 times. 3. If you're using pycharm check "Emulate terminal in output console" on debug/run configuraion. - 3. Run blackbox tests, telemetry performance test will run as part of it. + 3. Performance test will run as part of BlackBox tests or you can run it separately by adding + `-k 'test_telem_performance'` option. diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py new file mode 100644 index 000000000..f12704d2d --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py @@ -0,0 +1,43 @@ +import json +import logging +from os import listdir, path +from typing import List, Dict + +from tqdm import tqdm + +TELEM_DIR_PATH = './tests/performance/telem_sample' +MAX_SAME_TYPE_TELEM_FILES = 10000 +LOGGER = logging.getLogger(__name__) + + +class SampleFileParser: + + @staticmethod + def save_teletries_to_files(telems: List[Dict]): + for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)): + SampleFileParser.save_telemetry_to_file(telem) + + @staticmethod + def save_telemetry_to_file(telem: Dict): + telem_filename = telem['name'] + telem['method'] + for i in range(MAX_SAME_TYPE_TELEM_FILES): + if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): + telem_filename = str(i) + telem_filename + break + with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: + file.write(json.dumps(telem)) + + @staticmethod + def read_telem_files() -> List[str]: + telems = [] + file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) + if path.isfile(path.join(TELEM_DIR_PATH, f))] + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telem_string = "".join(telem_file.readlines()).replace("\n", "") + telems.append(telem_string) + return telems + + @staticmethod + def get_all_telemetries() -> List[Dict]: + return [json.loads(t) for t in SampleFileParser.read_telem_files()] diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py similarity index 100% rename from envs/monkey_zoo/blackbox/tests/performance/utils/fake_ip_generator.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_ip_generator.py diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py similarity index 82% rename from envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py index dab84a1e6..89cdf5cad 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/fake_monkey.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/fake_monkey.py @@ -1,6 +1,7 @@ import random -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator class FakeMonkey: diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py similarity index 57% rename from envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py index df7e9f5be..da3c22b05 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/sample_multiplier.py @@ -2,35 +2,37 @@ import copy import json import logging import sys -from os import listdir, path from typing import List, Dict from tqdm import tqdm -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_multiplier.fake_monkey import FakeMonkey -TELEM_DIR_PATH = './tests/performance/test_telems' +TELEM_DIR_PATH = './tests/performance/telemetry_sample' LOGGER = logging.getLogger(__name__) -class TelemParser: +class SampleMultiplier: def __init__(self, multiplier: int): self.multiplier = multiplier self.fake_ip_generator = FakeIpGenerator() def multiply_telems(self): - telems = TelemParser.get_all_telemetries() + telems = SampleFileParser.get_all_telemetries() telem_contents = [json.loads(telem['content']) for telem in telems] monkeys = self.get_monkeys_from_telems(telem_contents) for i in tqdm(range(self.multiplier), desc="Batch of fabricated telemetries", position=1): for monkey in monkeys: monkey.change_fake_data() fake_telem_batch = copy.deepcopy(telems) - TelemParser.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) - TelemParser.offset_telem_times(iteration=i, telems=fake_telem_batch) - TelemParser.save_teletries_to_files(fake_telem_batch) + SampleMultiplier.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) + SampleMultiplier.offset_telem_times(iteration=i, telems=fake_telem_batch) + SampleFileParser.save_teletries_to_files(fake_telem_batch) + LOGGER.info("") @staticmethod def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[FakeMonkey]): @@ -38,7 +40,8 @@ class TelemParser: for monkey in monkeys: if monkey.on_island: continue - if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) and not monkey.on_island: + if (monkey.original_guid in telem['content'] or monkey.original_guid in telem['endpoint']) \ + and not monkey.on_island: telem['content'] = telem['content'].replace(monkey.original_guid, monkey.fake_guid) telem['endpoint'] = telem['endpoint'].replace(monkey.original_guid, monkey.fake_guid) for i in range(len(monkey.original_ips)): @@ -49,39 +52,11 @@ class TelemParser: for telem in telems: telem['time']['$date'] += iteration * 1000 - @staticmethod - def save_teletries_to_files(telems: List[Dict]): - for telem in (tqdm(telems, desc="Telemetries saved to files", position=3)): - TelemParser.save_telemetry_to_file(telem) - - @staticmethod - def save_telemetry_to_file(telem: Dict): - telem_filename = telem['name'] + telem['method'] - for i in range(10000): - if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): - telem_filename = str(i) + telem_filename - break - with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: - file.write(json.dumps(telem)) - - @staticmethod - def read_telem_files() -> List[str]: - telems = [] - file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) - if path.isfile(path.join(TELEM_DIR_PATH, f))] - for file_path in file_paths: - with open(file_path, 'r') as telem_file: - telems.append(telem_file.readline()) - return telems - - @staticmethod - def get_all_telemetries() -> List[Dict]: - return [json.loads(t) for t in TelemParser.read_telem_files()] - def get_monkeys_from_telems(self, telems: List[Dict]): - island_ips = TelemParser.get_island_ips_from_telems(telems) + island_ips = SampleMultiplier.get_island_ips_from_telems(telems) monkeys = [] - for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']: + for telem in [telem for telem in telems + if 'telem_category' in telem and telem['telem_category'] == 'system_info']: if 'network_info' not in telem['data']: continue guid = telem['monkey_guid'] @@ -111,4 +86,4 @@ class TelemParser: if __name__ == "__main__": - TelemParser(multiplier=int(sys.argv[1])).multiply_telems() + SampleMultiplier(multiplier=int(sys.argv[1])).multiply_telems() diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py similarity index 85% rename from envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py rename to envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py index 96609a8a9..d8adef827 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/utils/test_fake_ip_generator.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_multiplier/test_fake_ip_generator.py @@ -1,6 +1,7 @@ from unittest import TestCase -from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator +from envs.monkey_zoo.blackbox.tests.performance.\ + telem_sample_parsing.sample_multiplier.fake_ip_generator import FakeIpGenerator class TestFakeIpGenerator(TestCase): diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 4086a234e..5fc97d222 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -8,7 +8,7 @@ from envs.monkey_zoo.blackbox.analyzers.performance_analyzer import PerformanceA from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.island_client.supported_request_method import SupportedRequestMethod from envs.monkey_zoo.blackbox.tests.performance.performance_test_config import PerformanceTestConfig -from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser +from envs.monkey_zoo.blackbox.tests.performance.telem_sample_parsing.sample_file_parser import SampleFileParser LOGGER = logging.getLogger(__name__) @@ -24,7 +24,7 @@ class TelemetryPerformanceTest: def test_telemetry_performance(self): LOGGER.info("Starting telemetry performance test.") try: - all_telemetries = TelemParser.get_all_telemetries() + all_telemetries = SampleFileParser.get_all_telemetries() except FileNotFoundError: LOGGER.error("Telemetries to send not found. Refer to readme to figure out how to generate telemetries " "and where to put them.") diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index 031a0d02a..18ebfd244 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -9,7 +9,7 @@ from flask import request from monkey_island.cc.models.test_telem import TestTelem from monkey_island.cc.services.config import ConfigService -TEST_TELEM_DIR = "./test_telems" +TELEM_SAMPLE_DIR = "./telem_sample" MAX_SAME_CATEGORY_TELEMS = 10000 @@ -36,16 +36,16 @@ class TestTelemStore: @staticmethod def export_test_telems(): - logger.info(f"Exporting all telemetries to {TEST_TELEM_DIR}") + logger.info(f"Exporting all telemetries to {TELEM_SAMPLE_DIR}") try: - mkdir(TEST_TELEM_DIR) + mkdir(TELEM_SAMPLE_DIR) except FileExistsError: logger.info("Deleting all previous telemetries.") - shutil.rmtree(TEST_TELEM_DIR) - mkdir(TEST_TELEM_DIR) + shutil.rmtree(TELEM_SAMPLE_DIR) + mkdir(TELEM_SAMPLE_DIR) for test_telem in TestTelem.objects(): - with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file: - file.write(test_telem.to_json()) + with open(TestTelemStore.get_unique_file_path_for_test_telem(TELEM_SAMPLE_DIR, test_telem), 'w') as file: + file.write(test_telem.to_json(indent=2)) logger.info("Telemetries exported!") @staticmethod From 67b7d0b769902232fc25382f5fac238ded888f52 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 11 May 2020 16:56:05 +0300 Subject: [PATCH 16/16] Improved telemetry fabrication logging and failing if telemetry folders don't exist --- .../telem_sample_parsing/sample_file_parser.py | 8 ++++++-- .../tests/performance/telemetry_performance_test.py | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py index f12704d2d..70e25d8e7 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telem_sample_parsing/sample_file_parser.py @@ -30,8 +30,12 @@ class SampleFileParser: @staticmethod def read_telem_files() -> List[str]: telems = [] - file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) - if path.isfile(path.join(TELEM_DIR_PATH, f))] + try: + file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) + if path.isfile(path.join(TELEM_DIR_PATH, f))] + except FileNotFoundError: + raise FileNotFoundError("Telemetries to send not found. " + "Refer to readme to figure out how to generate telemetries and where to put them.") for file_path in file_paths: with open(file_path, 'r') as telem_file: telem_string = "".join(telem_file.readlines()).replace("\n", "") diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 5fc97d222..4de77e41a 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -26,9 +26,8 @@ class TelemetryPerformanceTest: try: all_telemetries = SampleFileParser.get_all_telemetries() except FileNotFoundError: - LOGGER.error("Telemetries to send not found. Refer to readme to figure out how to generate telemetries " - "and where to put them.") - return False + raise FileNotFoundError("Telemetries to send not found. " + "Refer to readme to figure out how to generate telemetries and where to put them.") LOGGER.info("Telemetries imported successfully.") all_telemetries.sort(key=lambda telem: telem['time']['$date']) telemetry_parse_times = {}