Added timeout to tests, added logs

envs/monkey_zoo/blackbox/analyzers/communication_analyzer.py

@@ -1,22 +1,41 @@
 import json
 
+LOG_INIT_MESSAGE = "Analysis didn't run."
+
 
 class CommunicationAnalyzer(object):
 
     def __init__(self, island_client, machine_ips):
         self.island_client = island_client
         self.machine_ips = machine_ips
+        self.log = AnalyzerLog(self.__class__.__name__)
 
     def analyze_test_results(self):
+        self.log.clear()
         for machine_ip in self.machine_ips:
             if not self.did_monkey_communicate_back(machine_ip):
+                self.log.add_entry("Monkey from {} didn't communicate back".format(machine_ip))
                 return False
-            print("Monkey from {} communicated back".format(machine_ip))
+            self.log.add_entry("Monkey from {} communicated back".format(machine_ip))
         return True
 
     def did_monkey_communicate_back(self, machine_ip):
         query = json.dumps({'ip_addresses': {'$elemMatch': {'$eq': machine_ip}}})
-        response = self.island_client.send_get_request("api/test/monkey", {'find_query': query})
+        response = self.island_client.request_get("api/test/monkey", {'find_query': query})
         return len(json.loads(response.content)['results']) > 0
 
 
+class AnalyzerLog(object):
+
+    def __init__(self, analyzer_name):
+        self.contents = LOG_INIT_MESSAGE
+        self.name = analyzer_name
+
+    def clear(self):
+        self.contents = ""
+
+    def add_entry(self, message):
+        self.contents = "{}\n{}".format(self.contents, message)
+
+    def get_contents(self):
+        return "{}: {}\n".format(self.name, self.contents)

envs/monkey_zoo/blackbox/gcp_machine_handlers.py

@@ -22,6 +22,7 @@ class GCPHandler(object):
             print("GCP Handler failed to initialize: %s." % e)
 
     def start_machines(self, machine_list):
+        print("Setting up all GCP machines...")
         try:
             subprocess.call((GCPHandler.MACHINE_STARTING_COMMAND % (machine_list, self.zone)), shell=True)
             print("GCP machines successfully started.")

envs/monkey_zoo/blackbox/monkey_island_client.py

@@ -10,35 +10,31 @@ NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d206
 class MonkeyIslandClient(object):
     def __init__(self, server_address):
         self.addr = "https://{IP}/".format(IP=server_address)
-        self.token = self.get_jwt_token_from_server()
+        self.token = self.get_jwt_from_server()
 
-    def get_jwt_token_from_server(self):
+    def get_jwt_from_server(self):
-        resp = requests.post(self.addr + "api/auth", json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
+        resp = requests.post(self.addr + "api/auth",
+                             json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
                              verify=False)
         return resp.json()["access_token"]
 
-    def request_get(self, url):
+    def request_get(self, url, data=None):
-        return requests.get(
+        return requests.get(self.addr + url,
-            self.addr + url,
                             headers={"Authorization": "JWT " + self.token},
-            verify=False
+                            params=data,
-        )
+                            verify=False)
 
     def request_post(self, url, data):
-        return requests.post(
+        return requests.post(self.addr + url,
-            self.addr + url,
                              data=data,
                              headers={"Authorization": "JWT " + self.token},
-            verify=False
+                             verify=False)
-        )
 
-    def request_json(self, url, dict_data):
+    def request_post_json(self, url, dict_data):
-        return requests.post(
+        return requests.post(self.addr + url,
-            self.addr + url,
                              json=dict_data,
                              headers={"Authorization": "JWT " + self.token},
-            verify=False
+                             verify=False)
-        )
 
     def get_api_status(self):
         return self.request_get("api")
@@ -47,12 +43,15 @@ class MonkeyIslandClient(object):
         _ = self.request_post("api/configuration/island", data=config_contents)
 
     def run_monkey_local(self):
-        resp = self.request_json("api/local-monkey", dict_data={"action": "run"})
+        if self.request_post_json("api/local-monkey", dict_data={"action": "run"}).ok:
-        print(resp.text)
+            print("Running the monkey.")
+        else:
+            print("Failed to run the monkey.")
+            assert False
 
-    def send_get_request(self, endpoint, data):
+    def reset_env(self):
-        resp = requests.get(self.addr + endpoint,
+        if self.request_get("api", {"action": "reset"}).ok:
-                            headers={"Authorization": "JWT " + self.token},
+            print("Resetting environment after the test.")
-                            params=data,
+        else:
-                            verify=False)
+            print("Failed to reset the environment.")
-        return resp
+            assert False

envs/monkey_zoo/blackbox/test_blackbox.py

@@ -1,41 +1,106 @@
 import unittest
+from time import sleep, time
 
 import pytest
 
 from envs.monkey_zoo.blackbox.monkey_island_client import MonkeyIslandClient
 from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer
 from envs.monkey_zoo.blackbox.island_config_parser import IslandConfigParser
+from envs.monkey_zoo.blackbox.gcp_machine_handlers import GCPHandler
 
 
-def generic_blackbox_test_case(client, raw_config, analyzers):
+MACHINE_BOOT_TIME_SECONDS = 20
-    client.import_config(raw_config)
+TEST_TIME_SECONDS = 70
-    # client.run_monkey_local()
+DELAY_BETWEEN_TESTS = 1
-    for analyzer in analyzers:
+
-        assert analyzer.analyze_test_results()
+
+class BlackBoxTest(object):
+
+    def __init__(self, name, island_client, island_config, analyzers, timeout=TEST_TIME_SECONDS):
+        self.name = name
+        self.island_client = island_client
+        self.island_config = island_config
+        self.analyzers = analyzers
+        self.timeout = timeout
+
+    def run(self):
+        self.island_client.import_config(self.island_config)
+        self.island_client.run_monkey_local()
+        self.test_until_timeout()
+        self.island_client.reset_env()
+
+    def test_until_timeout(self):
+        timer = TestTimer(self.timeout)
+        while not timer.timed_out():
+            if self.analyzers_pass():
+                self.log_success(timer)
+                return
+            sleep(DELAY_BETWEEN_TESTS)
+        self.log_failure(timer)
+        assert False
+
+    def log_success(self, timer):
+        print(self.get_analyzer_logs())
+        print("{} test passed, time taken: {:.1f} seconds.".format(self.name, timer.get_time_taken()))
+
+    def log_failure(self, timer):
+        print(self.get_analyzer_logs())
+        print("{} test failed because of timeout. Time taken: {:.1f} seconds.".format(self.name,
+                                                                                      timer.get_time_taken()))
+
+    def analyzers_pass(self):
+        for analyzer in self.analyzers:
+            if not analyzer.analyze_test_results():
+                return False
+        return True
+
+    def get_analyzer_logs(self):
+        log = ""
+        for analyzer in self.analyzers:
+            log += "\n"+analyzer.log.get_contents()
+        return log
+
+
+class TestTimer(object):
+    def __init__(self, timeout):
+        self.timeout_time = TestTimer.get_timeout_time(timeout)
+        self.start_time = time()
+
+    def timed_out(self):
+        return time() > self.timeout_time
+
+    def get_time_taken(self):
+        return time() - self.start_time
+
+    @staticmethod
+    def get_timeout_time(timeout):
+        return time() + timeout
 
 
 @pytest.mark.usefixtures("island")
 # noinspection PyUnresolvedReferences
 class TestMonkeyBlackbox(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # GCPHandler().start_machines("elastic-4")
-        print("Setting up all GCP machines...")
 
-    @classmethod
+    def setUp(self):
-    def tearDownClass(cls):
+        self.GCPHandler = GCPHandler()
-        # GCPHandler().stop_machines("elastic-4")
+        self.island_client = MonkeyIslandClient(self.island)
+        self.GCPHandler.start_machines("sshkeys-11 sshkeys-12")
+        TestMonkeyBlackbox.wait_for_machine_boot()
+
+    def tearDown(self):
+        self.GCPHandler.stop_machines("sshkeys-11 sshkeys-12")
         print("Killing all GCP machines...")
 
     def test_server_online(self):
-        client = MonkeyIslandClient(self.island)
+        assert self.island_client.get_api_status() is not None
-        assert client.get_api_status() is not None
 
     def test_ssh_exec(self):
         conf_file_name = 'SSH.conf'
-        client = MonkeyIslandClient(self.island)
         config_parser = IslandConfigParser(conf_file_name)
-        analyzer = CommunicationAnalyzer(client, config_parser.get_ips_of_targets())
+        analyzer = CommunicationAnalyzer(self.island_client, config_parser.get_ips_of_targets())
-        generic_blackbox_test_case(client, config_parser.config_raw, [analyzer])
+        BlackBoxTest("SSH test", self.island_client, config_parser.config_raw, [analyzer]).run()
-
 
+    @staticmethod
+    def wait_for_machine_boot(time=MACHINE_BOOT_TIME_SECONDS):
+        print("Waiting for machines to fully boot up({:.0f} seconds).".format(time))
+        sleep(time)