From 1ccbb3d98953eca71ce6eedcd6cbe7e5f7d09db4 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 16 Apr 2020 16:39:10 +0300 Subject: [PATCH] Finished fabricated monkey telemetry sending --- .../island_client/monkey_island_client.py | 2 +- .../island_client/monkey_island_requests.py | 13 +- envs/monkey_zoo/blackbox/test_blackbox.py | 118 +++++++++--------- .../performance/telemetry_performance_test.py | 47 +++++++ monkey/monkey_island/cc/models/test_telem.py | 3 +- .../cc/resources/test/utils/telem_store.py | 4 +- 6 files changed, 125 insertions(+), 62 deletions(-) create mode 100644 envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index 93780bf3b..22fa57ac1 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -31,7 +31,7 @@ class MonkeyIslandClient(object): @avoid_race_condition def run_monkey_local(self): - response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"}) + response = self.requests.post_json("api/local-monkey", data={"action": "run"}) if MonkeyIslandClient.monkey_ran_successfully(response): LOGGER.info("Running the monkey.") else: diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py index 23f259a9c..07785509d 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py @@ -1,3 +1,5 @@ +from typing import Dict + import requests import functools @@ -55,9 +57,16 @@ class MonkeyIslandRequests(object): verify=False) @_Decorators.refresh_jwt_token - def post_json(self, url, dict_data): + def post_json(self, url, data: Dict): return requests.post(self.addr + url, # noqa: DUO123 - json=dict_data, + json=data, + headers=self.get_jwt_header(), + verify=False) + + @_Decorators.refresh_jwt_token + def patch_json(self, url, data: Dict): + return requests.patch(self.addr + url, # noqa: DUO123 + json=data, headers=self.get_jwt_header(), verify=False) diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 04e510c55..631229e36 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -9,6 +9,7 @@ from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import Communicat from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser from envs.monkey_zoo.blackbox.tests.performance.map_generation import MapGenerationTest from envs.monkey_zoo.blackbox.tests.performance.report_generation import ReportGenerationTest +from envs.monkey_zoo.blackbox.tests.performance.telemetry_performance_test import TelemetryPerformanceTest from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers from envs.monkey_zoo.blackbox.tests.exploitation import ExploitationTest from envs.monkey_zoo.blackbox.log_handlers.test_logs_handler import TestLogsHandler @@ -25,11 +26,12 @@ LOGGER = logging.getLogger(__name__) @pytest.fixture(autouse=True, scope='session') def GCPHandler(request): GCPHandler = gcp_machine_handlers.GCPHandler() - GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) - wait_machine_bootup() + #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #wait_machine_bootup() def fin(): - GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + #GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST)) + pass request.addfinalizer(fin) @@ -90,58 +92,60 @@ class TestMonkeyBlackbox(object): def test_server_online(self, island_client): assert island_client.get_api_status() is not None - def test_ssh_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") - - def test_hadoop_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) - - def test_mssql_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") - - def test_smb_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") - - def test_smb_pth(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") - - def test_elastic_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") - - def test_struts_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") - - def test_weblogic_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") - - def test_shellshock_exploiter(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") - - def test_tunneling(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) - - def test_wmi_and_mimikatz_exploiters(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") - - def test_wmi_pth(self, island_client): - TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") - - def test_report_generation_performance(self, island_client): - """ - This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test - for a total of 8 machines including the Monkey Island. - - Is has 2 analyzers - the regular one which checks all the Monkeys - and the Timing one which checks how long the report took to execute - """ - TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, - island_client, - "PERFORMANCE.conf", - timeout_in_seconds=10*60) - - def test_map_generation_performance(self, island_client): - TestMonkeyBlackbox.run_performance_test(MapGenerationTest, - island_client, - "PERFORMANCE.conf", - timeout_in_seconds=10*60) + #def test_ssh_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SSH.conf", "SSH_exploiter_and_keys") +# + #def test_hadoop_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6 * 60) +# + #def test_mssql_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "MSSQL.conf", "MSSQL_exploiter") +# + #def test_smb_and_mimikatz_exploiters(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz") +# + #def test_smb_pth(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SMB_PTH.conf", "SMB_PTH") +# + #def test_elastic_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "ELASTIC.conf", "Elastic_exploiter") +# + #def test_struts_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter") +# + #def test_weblogic_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter") +# + #def test_shellshock_exploiter(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter") +# + #def test_tunneling(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 15 * 60) +# + #def test_wmi_and_mimikatz_exploiters(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz") +# + #def test_wmi_pth(self, island_client): + # TestMonkeyBlackbox.run_exploitation_test(island_client, "WMI_PTH.conf", "WMI_PTH") +# + #def test_report_generation_performance(self, island_client): + # """ + # This test includes the SSH + Elastic + Hadoop + MSSQL machines all in one test + # for a total of 8 machines including the Monkey Island. +# + # Is has 2 analyzers - the regular one which checks all the Monkeys + # and the Timing one which checks how long the report took to execute + # """ + # TestMonkeyBlackbox.run_performance_test(ReportGenerationTest, + # island_client, + # "PERFORMANCE.conf", + # timeout_in_seconds=10*60) +# + #def test_map_generation_performance(self, island_client): + # TestMonkeyBlackbox.run_performance_test(MapGenerationTest, + # island_client, + # "PERFORMANCE.conf", + # timeout_in_seconds=10*60) + def test_telem_performance(self, island_client): + TelemetryPerformanceTest(island_client).test_telemetry_performance() diff --git a/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py new file mode 100644 index 000000000..20d95474e --- /dev/null +++ b/envs/monkey_zoo/blackbox/tests/performance/telemetry_performance_test.py @@ -0,0 +1,47 @@ +import json +from os import listdir +from os.path import isfile, join + +from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient + +TELEM_DIR_PATH = './tests/performance/test_telems' +TELEM_TEST_ENDPOINT = '/api/test/telemetry/performance' + + +class TelemetryPerformanceTest: + + def __init__(self, island_client: MonkeyIslandClient): + self.island_client = island_client + + def test_telemetry_performance(self): + all_telemetries = TelemetryPerformanceTest.get_all_telemetries() + all_telemetries.sort(key=lambda telem: telem['time']['$date']) + for telemetry in all_telemetries: + self.send_telemetry(telemetry) + + @staticmethod + def get_all_telemetries(): + telemetries = [] + file_paths = [join(TELEM_DIR_PATH, f) for f in listdir(TELEM_DIR_PATH) if isfile(join(TELEM_DIR_PATH, f))] + for file_path in file_paths: + with open(file_path, 'r') as telem_file: + telem_contents = json.loads(telem_file.readline()) + telemetries.append(telem_contents) + return telemetries + + def send_telemetry(self, telemetry): + content = telemetry['content'] + url = telemetry['endpoint'] + method = telemetry['method'] + + if method == 'POST': + result = self.island_client.requests.post(url=url, data=content) + elif method == 'GET': + result = self.island_client.requests.get(url=url) + elif method == 'PATCH': + result = self.island_client.requests.patch_json(url=url, data=content) + elif method == 'DELETE': + result = self.island_client.requests.delete(url=url) + else: + raise Exception + return result diff --git a/monkey/monkey_island/cc/models/test_telem.py b/monkey/monkey_island/cc/models/test_telem.py index 777b71fc2..cb0697a33 100644 --- a/monkey/monkey_island/cc/models/test_telem.py +++ b/monkey/monkey_island/cc/models/test_telem.py @@ -1,12 +1,13 @@ """ Define a Document Schema for the Monkey document. """ -from mongoengine import Document, StringField +from mongoengine import Document, StringField, DateTimeField class TestTelem(Document): # SCHEMA name = StringField(required=True) + time = DateTimeField(required=True) method = StringField(required=True) endpoint = StringField(required=True) content = StringField(required=True) diff --git a/monkey/monkey_island/cc/resources/test/utils/telem_store.py b/monkey/monkey_island/cc/resources/test/utils/telem_store.py index dcceeb9d6..a6c569436 100644 --- a/monkey/monkey_island/cc/resources/test/utils/telem_store.py +++ b/monkey/monkey_island/cc/resources/test/utils/telem_store.py @@ -1,5 +1,6 @@ from functools import wraps from os import mkdir, path +from datetime import datetime from flask import request @@ -14,11 +15,12 @@ class TestTelemStore: def store_test_telem(f): @wraps(f) def decorated_function(*args, **kwargs): + time = datetime.now() method = request.method content = request.data.decode() endpoint = request.path name = str(request.url_rule).replace('/', '_').replace('<', '_').replace('>', '_').replace(':', '_') - TestTelem(name=name, method=method, endpoint=endpoint, content=content).save() + TestTelem(name=name, method=method, endpoint=endpoint, content=content, time=time).save() return f(*args, **kwargs) return decorated_function