Finished telemetry fabrication

This commit is contained in:
VakarisZ 2020-04-23 19:44:24 +03:00
parent b6e0e30d58
commit 78b0186613
7 changed files with 70 additions and 19 deletions

4
.gitignore vendored
View File

@ -82,5 +82,9 @@ MonkeyZoo/*
!MonkeyZoo/config.tf !MonkeyZoo/config.tf
!MonkeyZoo/MonkeyZooDocs.pdf !MonkeyZoo/MonkeyZooDocs.pdf
# Exported monkey telemetries
/monkey/test_telems/
# vim swap files # vim swap files
*.swp *.swp

View File

@ -17,3 +17,19 @@ Run the following command:
#### Running in PyCharm #### Running in PyCharm
Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the
`monkey\envs\monkey_zoo\blackbox`. `monkey\envs\monkey_zoo\blackbox`.
### Running telemetry performance test
To run telemetry performance test follow these steps:
1. Gather monkey telemetries.
1. Enable "Export monkey telemetries" in Configuration -> Internal -> Tests if you don't have
exported telemetries already.
2. Run monkey and wait until infection is done.
3. All telemetries are gathered in `monkey/test_telems`
2. Run telemetry performance test.
1. Move directory `monkey/test_telems` to `envs/monkey_zoo/blackbox/tests/performance/test_telems`
2. (Optional) Use `envs/monkey_zoo/blackbox/tests/performance/utils/telem_parser.py` to multiply
telemetries gathered.
1. Run `telem_parser.py` scrip with working directory set to `monkey\envs\monkey_zoo\blackbox`
2. Pass integer to indicate the multiplier. For example running `telem_parser.py 4` will replicate
telemetries 4 times.
3. Run blackbox tests, telemetry performance test will run as part of it.

View File

@ -2,6 +2,7 @@ from typing import List, Dict
from os import listdir, path from os import listdir, path
import copy import copy
import json import json
import sys
from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator from envs.monkey_zoo.blackbox.tests.performance.utils.fake_ip_generator import FakeIpGenerator
from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey from envs.monkey_zoo.blackbox.tests.performance.utils.fake_monkey import FakeMonkey
@ -11,10 +12,10 @@ TELEM_DIR_PATH = './tests/performance/test_telems'
class TelemParser: class TelemParser:
def __init__(self, island_ip: str, multiplier: int):
def __init__(self, multiplier: int):
self.multiplier = multiplier self.multiplier = multiplier
self.fake_ip_generator = FakeIpGenerator() self.fake_ip_generator = FakeIpGenerator()
self.island_ip = island_ip
def multiply_telems(self): def multiply_telems(self):
telems = TelemParser.get_all_telemetries() telems = TelemParser.get_all_telemetries()
@ -51,7 +52,7 @@ class TelemParser:
TelemParser.save_telemetry_to_file(telem) TelemParser.save_telemetry_to_file(telem)
@staticmethod @staticmethod
def save_telemetry_to_file(telem): def save_telemetry_to_file(telem: Dict):
telem_filename = telem['name'] + telem['method'] telem_filename = telem['name'] + telem['method']
for i in range(10000): for i in range(10000):
if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))): if not path.exists(path.join(TELEM_DIR_PATH, (str(i) + telem_filename))):
@ -71,10 +72,11 @@ class TelemParser:
return telems return telems
@staticmethod @staticmethod
def get_all_telemetries(): def get_all_telemetries() -> List[Dict]:
return [json.loads(t) for t in TelemParser.read_telem_files()] return [json.loads(t) for t in TelemParser.read_telem_files()]
def get_monkeys_from_telems(self, telems: List[Dict]): def get_monkeys_from_telems(self, telems: List[Dict]):
island_ips = TelemParser.get_island_ips_from_telems(telems)
monkeys = [] monkeys = []
for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']: for telem in [telem for telem in telems if 'telem_category' in telem and telem['telem_category'] == 'system_info']:
if 'network_info' not in telem['data']: if 'network_info' not in telem['data']:
@ -83,7 +85,7 @@ class TelemParser:
monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid] monkey_present = [monkey for monkey in monkeys if monkey.original_guid == guid]
if not monkey_present: if not monkey_present:
ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']] ips = [net_info['addr'] for net_info in telem['data']['network_info']['networks']]
if self.island_ip in ips: if set(island_ips).intersection(ips):
on_island = True on_island = True
else: else:
on_island = False on_island = False
@ -94,6 +96,16 @@ class TelemParser:
on_island=on_island)) on_island=on_island))
return monkeys return monkeys
@staticmethod
def get_island_ips_from_telems(telems: List[Dict]) -> List[str]:
island_ips = []
for telem in telems:
if 'config' in telem:
island_ips = telem['config']['command_servers']
for i in range(len(island_ips)):
island_ips[i] = island_ips[i].replace(":5000", "")
return island_ips
if __name__ == "__main__": if __name__ == "__main__":
TelemParser(island_ip='192.168.56.1', multiplier=100).multiply_telems() TelemParser(multiplier=int(sys.argv[1])).multiply_telems()

View File

@ -1,5 +1,5 @@
""" """
Define a Document Schema for the Monkey document. Define a Document Schema for the TestTelem document.
""" """
from mongoengine import Document, StringField, DateTimeField from mongoengine import Document, StringField, DateTimeField

View File

@ -1,12 +1,14 @@
from functools import wraps from functools import wraps
from os import mkdir, path from os import mkdir, path
import shutil
from datetime import datetime from datetime import datetime
from flask import request from flask import request
from monkey_island.cc.models.test_telem import TestTelem from monkey_island.cc.models.test_telem import TestTelem
from monkey_island.cc.services.config import ConfigService
MONKEY_TELEM_COLLECTION_NAME = "monkey_telems_for_tests" TEST_TELEM_DIR = "./test_telems"
class TestTelemStore: class TestTelemStore:
@ -15,28 +17,28 @@ class TestTelemStore:
def store_test_telem(f): def store_test_telem(f):
@wraps(f) @wraps(f)
def decorated_function(*args, **kwargs): def decorated_function(*args, **kwargs):
time = datetime.now() if ConfigService.is_test_telem_export_enabled():
method = request.method time = datetime.now()
content = request.data.decode() method = request.method
endpoint = request.path content = request.data.decode()
name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') endpoint = request.path
TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save() name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_')
TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save()
return f(*args, **kwargs) return f(*args, **kwargs)
return decorated_function return decorated_function
@staticmethod @staticmethod
def export_test_telems(): def export_test_telems():
telem_dir = "./test_telems"
try: try:
mkdir(telem_dir) mkdir(TEST_TELEM_DIR)
except FileExistsError: except FileExistsError:
pass shutil.rmtree(TEST_TELEM_DIR)
mkdir(TEST_TELEM_DIR)
for test_telem in TestTelem.objects(): for test_telem in TestTelem.objects():
with open(TestTelemStore.get_unique_file_path_for_test_telem(telem_dir, test_telem), 'w') as file: with open(TestTelemStore.get_unique_file_path_for_test_telem(TEST_TELEM_DIR, test_telem), 'w') as file:
file.write(test_telem.to_json()) file.write(test_telem.to_json())
@staticmethod @staticmethod
def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem): def get_unique_file_path_for_test_telem(target_dir: str, test_telem: TestTelem):
telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem) telem_filename = TestTelemStore._get_filename_by_test_telem(test_telem)

View File

@ -307,3 +307,7 @@ class ConfigService:
pair['public_key'] = encryptor.dec(pair['public_key']) pair['public_key'] = encryptor.dec(pair['public_key'])
pair['private_key'] = encryptor.dec(pair['private_key']) pair['private_key'] = encryptor.dec(pair['private_key'])
return pair return pair
@staticmethod
def is_test_telem_export_enabled():
return ConfigService.get_config_value(['internal', 'testing', 'export_monkey_telems'])

View File

@ -736,6 +736,19 @@ SCHEMA = {
"description": "List of SSH key pairs to use, when trying to ssh into servers" "description": "List of SSH key pairs to use, when trying to ssh into servers"
} }
} }
},
"testing": {
"title": "Testing",
"type": "object",
"properties": {
"export_monkey_telems": {
"title": "Export monkey telemetries",
"type": "boolean",
"default": False,
"description": "Exports unencrypted telemetries that can be used for tests in development."
" Do not turn on!"
}
}
} }
} }
}, },