From 636c8f6d3f82ebd33ce12e0234d303d8809466b5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 20 Apr 2020 17:49:24 +0300 Subject: [PATCH] Added the ability to forge more intricate setup telemetries + small bugfixes, refactors and improvements --- envs/monkey_zoo/.gitignore | 1 + .../performance/telemetry_performance_test.py | 20 +-- .../tests/performance/utils/telem_parser.py | 124 ++++++++++++++++++ monkey/monkey_island/cc/resources/log.py | 2 + .../monkey_island/cc/resources/telemetry.py | 2 + 5 files changed, 132 insertions(+), 17 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py diff --git a/envs/monkey_zoo/.gitignore b/envs/monkey_zoo/.gitignore index 333c1e910..04310b6fd 100644 --- a/envs/monkey_zoo/.gitignore +++ b/envs/monkey_zoo/.gitignore @@ -1 +1,2 @@ logs/ +/blackbox/tests/performance/test_telems/* diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py index 20d95474e..047d07406 100644 --- a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -1,11 +1,7 @@ -import json -from os import listdir -from os.path import isfile, join + from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient - -TELEM_DIR_PATH = './tests/performance/test_telems' -TELEM_TEST_ENDPOINT = '/api/test/telemetry/performance' +from envs.monkey_zoo.blackbox.tests.performance.utils.telem_parser import TelemParser class TelemetryPerformanceTest: @@ -14,21 +10,11 @@ class TelemetryPerformanceTest: self.island_client = island_client def test_telemetry_performance(self): - all_telemetries = TelemetryPerformanceTest.get_all_telemetries() + all_telemetries = TelemParser.get_all_telemetries() all_telemetries.sort(key=lambda telem: telem['time']['$date']) for telemetry in all_telemetries: self.send_telemetry(telemetry) - @staticmethod - def get_all_telemetries(): - telemetries = [] - file_paths = [join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) if isfile(join(TELEM_DIR_PATH, f))] - for file_path in file_paths: - with open(file_path, 'r') as telem_file: - telem_contents = json.loads(telem_file.readline()) - telemetries.append(telem_contents) - return telemetries - def send_telemetry(self, telemetry): content = telemetry['content'] url = telemetry['endpoint'] diff --git a/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py new file mode 100644 index 000000000..c6ee7095f --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py @@ -0,0 +1,124 @@ +from typing import List, Dict +from os import listdir, path +import copy +import random +import json + + +TELEM_DIR_PATH = './tests/performance/test_telems' + + +class FakeIpGenerator: + def __init__(self): + self.fake_ip_parts = [1, 1, 1, 1] + + def generate_fake_ips_for_real_ips(self, real_ips): + self.fake_ip_parts[2] += 1 + fake_ips = [] + for i in range(len(real_ips)): + fake_ips.append('.'.join(str(part) for part in self.fake_ip_parts)) + self.fake_ip_parts[3] += 1 + return fake_ips + + +class Monkey: + def __init__(self, ips, guid, fake_ip_generator: FakeIpGenerator, on_island=False): + self.ips = ips + self.guid = guid + self.fake_ip_generator = fake_ip_generator + self.on_island = on_island + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) + self.fake_ips = fake_ip_generator.generate_fake_ips_for_real_ips(ips) + + def change_fake_data(self): + self.fake_ips = self.fake_ip_generator.generate_fake_ips_for_real_ips(self.ips) + self.fake_guid = str(random.randint(1000000000000, 9999999999999)) + + +class TelemParser: + def __init__(self, island_ip: str, multiplier: int): + self.multiplier = multiplier + self.fake_ip_generator = FakeIpGenerator() + self.island_ip = island_ip + + def multiply_telems(self): + telems = TelemParser.get_all_telemetries() + telem_contents = [json.loads(telem['content']) for telem in telems] + monkeys = self.get_monkeys_from_telems(telem_contents) + for i in range(self.multiplier): + for monkey in monkeys: + monkey.change_fake_data() + fake_telem_batch = copy.deepcopy(telems) + TelemParser.fabricate_monkeys_in_telems(fake_telem_batch, monkeys) + TelemParser.offset_telem_times(iteration=i, telems=fake_telem_batch) + TelemParser.save_teletries_to_files(fake_telem_batch) + + @staticmethod + def fabricate_monkeys_in_telems(telems: List[Dict], monkeys: List[Monkey]): + for telem in telems: + for monkey in monkeys: + if monkey.on_island: + continue + if (monkey.guid in telem['content'] or monkey.guid in telem['endpoint']) and not monkey.on_island: + telem['content'] = telem['content'].replace(monkey.guid, monkey.fake_guid) + telem['endpoint'] = telem['endpoint'].replace(monkey.guid, monkey.fake_guid) + for i in range(len(monkey.ips)): + telem['content'] = telem['content'].replace(monkey.ips[i], monkey.fake_ips[i]) + + @staticmethod + def offset_telem_times(iteration: int, telems: List[Dict]): + for telem in telems: + telem['time']['$date'] += iteration * 1000 + + @staticmethod + def save_teletries_to_files(telems: List[Dict]): + for telem in telems: + TelemParser.save_telemetry_to_file(telem) + + @staticmethod + def save_telemetry_to_file(telem): + telem_filename = telem['name'] + telem['method'] + for i in range(10000): + if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): + telem_filename = str(i) + telem_filename + break + with open(path.join(TELEM_DIR_PATH, telem_filename), 'w') as file: + file.write(json.dumps(telem)) + + @staticmethod + def read_telem_files() -> List[str]: + telems = [] + file_paths = [path.join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) + if path.isfile(path.join(TELEM_DIR_PATH, f))] + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telems.append(telem_file.readline()) + return telems + + @staticmethod + def get_all_telemetries(): + return [json.loads(t) for t in TelemParser.read_telem_files()] + + def get_monkeys_from_telems(self, telems: List[Dict]): + monkeys = [] + for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']: + if 'network_info' not in telem['data']: + continue + guid = telem['monkey_guid'] + monkey_present = [monkey for monkey in monkeys if monkey.guid == guid] + if not monkey_present: + ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']] + if self.island_ip in ips: + on_island = True + else: + on_island = False + + monkeys.append(Monkey(ips=ips, + guid=guid, + fake_ip_generator=self.fake_ip_generator, + on_island=on_island)) + return monkeys + + +if __name__ == "__main__": + TelemParser(island_ip='192.168.56.1', multiplier=100).multiply_telems() diff --git a/monkey/monkey_island/cc/resources/log.py b/monkey/monkey_island/cc/resources/log.py index cbce92a37..920890648 100644 --- a/monkey/monkey_island/cc/resources/log.py +++ b/monkey/monkey_island/cc/resources/log.py @@ -6,6 +6,7 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from monkey_island.cc.services.log import LogService from monkey_island.cc.services.node import NodeService @@ -23,6 +24,7 @@ class Log(flask_restful.Resource): return LogService.log_exists(ObjectId(exists_monkey_id)) # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self): telemetry_json = json.loads(request.data) diff --git a/monkey/monkey_island/cc/resources/telemetry.py b/monkey/monkey_island/cc/resources/telemetry.py index dc6a7d512..f6e833eeb 100644 --- a/monkey/monkey_island/cc/resources/telemetry.py +++ b/monkey/monkey_island/cc/resources/telemetry.py @@ -8,6 +8,7 @@ from flask import request from monkey_island.cc.auth import jwt_required from monkey_island.cc.database import mongo +from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.processing import process_telemetry from monkey_island.cc.models.monkey import Monkey @@ -40,6 +41,7 @@ class Telemetry(flask_restful.Resource): return result # Used by monkey. can't secure. + @TestTelemStore.store_test_telem def post(self): telemetry_json = json.loads(request.data) telemetry_json['timestamp'] = datetime.now()