diff --git a/envs/monkey_zoo/blackbox/analyzers/communication_analyzer.py b/envs/monkey_zoo/blackbox/analyzers/communication_analyzer.py index e4638233f..a91850057 100644 --- a/envs/monkey_zoo/blackbox/analyzers/communication_analyzer.py +++ b/envs/monkey_zoo/blackbox/analyzers/communication_analyzer.py @@ -1,22 +1,41 @@ import json +LOG_INIT_MESSAGE = "Analysis didn't run." + class CommunicationAnalyzer(object): def __init__(self, island_client, machine_ips): self.island_client = island_client self.machine_ips = machine_ips + self.log = AnalyzerLog(self.__class__.__name__) def analyze_test_results(self): + self.log.clear() for machine_ip in self.machine_ips: if not self.did_monkey_communicate_back(machine_ip): + self.log.add_entry("Monkey from {} didn't communicate back".format(machine_ip)) return False - print("Monkey from {} communicated back".format(machine_ip)) + self.log.add_entry("Monkey from {} communicated back".format(machine_ip)) return True def did_monkey_communicate_back(self, machine_ip): query = json.dumps({'ip_addresses': {'$elemMatch': {'$eq': machine_ip}}}) - response = self.island_client.send_get_request("api/test/monkey", {'find_query': query}) + response = self.island_client.request_get("api/test/monkey", {'find_query': query}) return len(json.loads(response.content)['results']) > 0 +class AnalyzerLog(object): + + def __init__(self, analyzer_name): + self.contents = LOG_INIT_MESSAGE + self.name = analyzer_name + + def clear(self): + self.contents = "" + + def add_entry(self, message): + self.contents = "{}\n{}".format(self.contents, message) + + def get_contents(self): + return "{}: {}\n".format(self.name, self.contents) diff --git a/envs/monkey_zoo/blackbox/gcp_machine_handlers.py b/envs/monkey_zoo/blackbox/gcp_machine_handlers.py index 4937191ed..3d6e75bf4 100644 --- a/envs/monkey_zoo/blackbox/gcp_machine_handlers.py +++ b/envs/monkey_zoo/blackbox/gcp_machine_handlers.py @@ -22,6 +22,7 @@ class GCPHandler(object): print("GCP Handler failed to initialize: %s." % e) def start_machines(self, machine_list): + print("Setting up all GCP machines...") try: subprocess.call((GCPHandler.MACHINE_STARTING_COMMAND % (machine_list, self.zone)), shell=True) print("GCP machines successfully started.") diff --git a/envs/monkey_zoo/blackbox/monkey_island_client.py b/envs/monkey_zoo/blackbox/monkey_island_client.py index 662ad71b8..a003f4d90 100644 --- a/envs/monkey_zoo/blackbox/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/monkey_island_client.py @@ -10,35 +10,31 @@ NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d206 class MonkeyIslandClient(object): def __init__(self, server_address): self.addr = "https://{IP}/".format(IP=server_address) - self.token = self.get_jwt_token_from_server() + self.token = self.get_jwt_from_server() - def get_jwt_token_from_server(self): - resp = requests.post(self.addr + "api/auth", json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS}, + def get_jwt_from_server(self): + resp = requests.post(self.addr + "api/auth", + json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS}, verify=False) return resp.json()["access_token"] - def request_get(self, url): - return requests.get( - self.addr + url, - headers={"Authorization": "JWT " + self.token}, - verify=False - ) + def request_get(self, url, data=None): + return requests.get(self.addr + url, + headers={"Authorization": "JWT " + self.token}, + params=data, + verify=False) def request_post(self, url, data): - return requests.post( - self.addr + url, - data=data, - headers={"Authorization": "JWT " + self.token}, - verify=False - ) + return requests.post(self.addr + url, + data=data, + headers={"Authorization": "JWT " + self.token}, + verify=False) - def request_json(self, url, dict_data): - return requests.post( - self.addr + url, - json=dict_data, - headers={"Authorization": "JWT " + self.token}, - verify=False - ) + def request_post_json(self, url, dict_data): + return requests.post(self.addr + url, + json=dict_data, + headers={"Authorization": "JWT " + self.token}, + verify=False) def get_api_status(self): return self.request_get("api") @@ -47,12 +43,15 @@ class MonkeyIslandClient(object): _ = self.request_post("api/configuration/island", data=config_contents) def run_monkey_local(self): - resp = self.request_json("api/local-monkey", dict_data={"action": "run"}) - print(resp.text) + if self.request_post_json("api/local-monkey", dict_data={"action": "run"}).ok: + print("Running the monkey.") + else: + print("Failed to run the monkey.") + assert False - def send_get_request(self, endpoint, data): - resp = requests.get(self.addr + endpoint, - headers={"Authorization": "JWT " + self.token}, - params=data, - verify=False) - return resp + def reset_env(self): + if self.request_get("api", {"action": "reset"}).ok: + print("Resetting environment after the test.") + else: + print("Failed to reset the environment.") + assert False diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 5ef7e16f5..a8b31f4c5 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -1,41 +1,106 @@ import unittest +from time import sleep, time import pytest from envs.monkey_zoo.blackbox.monkey_island_client import MonkeyIslandClient from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer from envs.monkey_zoo.blackbox.island_config_parser import IslandConfigParser +from envs.monkey_zoo.blackbox.gcp_machine_handlers import GCPHandler -def generic_blackbox_test_case(client, raw_config, analyzers): - client.import_config(raw_config) - # client.run_monkey_local() - for analyzer in analyzers: - assert analyzer.analyze_test_results() +MACHINE_BOOT_TIME_SECONDS = 20 +TEST_TIME_SECONDS = 70 +DELAY_BETWEEN_TESTS = 1 + + +class BlackBoxTest(object): + + def __init__(self, name, island_client, island_config, analyzers, timeout=TEST_TIME_SECONDS): + self.name = name + self.island_client = island_client + self.island_config = island_config + self.analyzers = analyzers + self.timeout = timeout + + def run(self): + self.island_client.import_config(self.island_config) + self.island_client.run_monkey_local() + self.test_until_timeout() + self.island_client.reset_env() + + def test_until_timeout(self): + timer = TestTimer(self.timeout) + while not timer.timed_out(): + if self.analyzers_pass(): + self.log_success(timer) + return + sleep(DELAY_BETWEEN_TESTS) + self.log_failure(timer) + assert False + + def log_success(self, timer): + print(self.get_analyzer_logs()) + print("{} test passed, time taken: {:.1f} seconds.".format(self.name, timer.get_time_taken())) + + def log_failure(self, timer): + print(self.get_analyzer_logs()) + print("{} test failed because of timeout. Time taken: {:.1f} seconds.".format(self.name, + timer.get_time_taken())) + + def analyzers_pass(self): + for analyzer in self.analyzers: + if not analyzer.analyze_test_results(): + return False + return True + + def get_analyzer_logs(self): + log = "" + for analyzer in self.analyzers: + log += "\n"+analyzer.log.get_contents() + return log + + +class TestTimer(object): + def __init__(self, timeout): + self.timeout_time = TestTimer.get_timeout_time(timeout) + self.start_time = time() + + def timed_out(self): + return time() > self.timeout_time + + def get_time_taken(self): + return time() - self.start_time + + @staticmethod + def get_timeout_time(timeout): + return time() + timeout @pytest.mark.usefixtures("island") # noinspection PyUnresolvedReferences class TestMonkeyBlackbox(unittest.TestCase): - @classmethod - def setUpClass(cls): - # GCPHandler().start_machines("elastic-4") - print("Setting up all GCP machines...") - @classmethod - def tearDownClass(cls): - # GCPHandler().stop_machines("elastic-4") + def setUp(self): + self.GCPHandler = GCPHandler() + self.island_client = MonkeyIslandClient(self.island) + self.GCPHandler.start_machines("sshkeys-11 sshkeys-12") + TestMonkeyBlackbox.wait_for_machine_boot() + + def tearDown(self): + self.GCPHandler.stop_machines("sshkeys-11 sshkeys-12") print("Killing all GCP machines...") def test_server_online(self): - client = MonkeyIslandClient(self.island) - assert client.get_api_status() is not None + assert self.island_client.get_api_status() is not None def test_ssh_exec(self): conf_file_name = 'SSH.conf' - client = MonkeyIslandClient(self.island) config_parser = IslandConfigParser(conf_file_name) - analyzer = CommunicationAnalyzer(client, config_parser.get_ips_of_targets()) - generic_blackbox_test_case(client, config_parser.config_raw, [analyzer]) - + analyzer = CommunicationAnalyzer(self.island_client, config_parser.get_ips_of_targets()) + BlackBoxTest("SSH test", self.island_client, config_parser.config_raw, [analyzer]).run() + @staticmethod + def wait_for_machine_boot(time=MACHINE_BOOT_TIME_SECONDS): + print("Waiting for machines to fully boot up({:.0f} seconds).".format(time)) + sleep(time)