Implemented logs

envs/monkey_zoo/.gitignore

@@ -1 +1,2 @@
 gcp_keys/
+logs/

envs/monkey_zoo/blackbox/analyzers/communication_analyzer.py

@@ -1,5 +1,3 @@
-import json
-
 LOG_INIT_MESSAGE = "Analysis didn't run."
 
 
@@ -20,9 +18,8 @@ class CommunicationAnalyzer(object):
         return True
 
     def did_monkey_communicate_back(self, machine_ip):
-        query = json.dumps({'ip_addresses': {'$elemMatch': {'$eq': machine_ip}}})
+        query = {'ip_addresses': {'$elemMatch': {'$eq': machine_ip}}}
-        response = self.island_client.request_get("api/test/monkey", {'find_query': query})
+        return len(self.island_client.find_monkeys_in_db(query)) > 0
-        return len(json.loads(response.content)['results']) > 0
 
 
 class AnalyzerLog(object):

envs/monkey_zoo/blackbox/island_client/monkey_island_client.py

@@ -0,0 +1,86 @@
+from time import sleep
+import json
+
+from bson import json_util
+
+from envs.monkey_zoo.blackbox.island_client.monkey_island_requests import MonkeyIslandRequests
+
+SLEEP_BETWEEN_REQUESTS_SECONDS = 0.5
+MONKEY_TEST_ENDPOINT = 'api/test/monkey'
+LOG_TEST_ENDPOINT = 'api/test/log'
+
+
+def avoid_race_condition(func):
+    sleep(SLEEP_BETWEEN_REQUESTS_SECONDS)
+    return func
+
+
+class MonkeyIslandClient(object):
+    def __init__(self, server_address):
+        self.requests = MonkeyIslandRequests(server_address)
+
+    def get_api_status(self):
+        return self.requests.get("api")
+
+    @avoid_race_condition
+    def import_config(self, config_contents):
+        _ = self.requests.post("api/configuration/island", data=config_contents)
+
+    @avoid_race_condition
+    def run_monkey_local(self):
+        response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"})
+        if MonkeyIslandClient.monkey_ran_successfully(response):
+            print("Running the monkey.")
+        else:
+            print("Failed to run the monkey.")
+            assert False
+
+    @staticmethod
+    def monkey_ran_successfully(response):
+        return response.ok and json.loads(response.content)['is_running']
+
+    @avoid_race_condition
+    def kill_all_monkeys(self):
+        if self.requests.get("api", {"action": "killall"}).ok:
+            print("Killing all monkeys after the test.")
+        else:
+            print("Failed to kill all monkeys.")
+            assert False
+
+    @avoid_race_condition
+    def reset_env(self):
+        if self.requests.get("api", {"action": "reset"}).ok:
+            print("Resetting environment after the test.")
+        else:
+            print("Failed to reset the environment.")
+            assert False
+
+    def find_monkeys_in_db(self, query):
+        response = self.requests.get(MONKEY_TEST_ENDPOINT,
+                                     MonkeyIslandClient.form_find_query_for_request(query))
+        try:
+            return MonkeyIslandClient.get_test_query_results(response)
+        except Exception:
+            print("Ran into trouble parsing response for monkey query")
+            raise
+
+    def find_log_in_db(self, query):
+        response = self.requests.get(LOG_TEST_ENDPOINT,
+                                     MonkeyIslandClient.form_find_query_for_request(query))
+        try:
+            return MonkeyIslandClient.get_test_query_results(response)
+        except Exception:
+            print("Ran into trouble parsing response for log query")
+            raise
+
+    @staticmethod
+    def form_find_query_for_request(query):
+        return {'find_query': json_util.dumps(query)}
+
+    @staticmethod
+    def get_test_query_results(response):
+        return json.loads(response.content)['results']
+
+    def is_all_monkeys_dead(self):
+        query = {'dead': 'false'}
+        return len(self.find_monkeys_in_db(query)) == 0

envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py

@@ -0,0 +1,45 @@
+import requests
+
+# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
+NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \
+                '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557'
+
+
+class MonkeyIslandRequests(object):
+    def __init__(self, server_address):
+        self.addr = "https://{IP}/".format(IP=server_address)
+        self.token = self.try_get_jwt_from_server()
+
+    def try_get_jwt_from_server(self):
+        try:
+            return self.get_jwt_from_server()
+        except requests.ConnectionError:
+            print("Unable to connect to island, aborting!")
+            assert False
+
+    def get_jwt_from_server(self):
+        resp = requests.post(self.addr + "api/auth",
+                             json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
+                             verify=False)
+        return resp.json()["access_token"]
+
+    def get(self, url, data=None):
+        return requests.get(self.addr + url,
+                            headers=self.get_jwt_header(),
+                            params=data,
+                            verify=False)
+
+    def post(self, url, data):
+        return requests.post(self.addr + url,
+                             data=data,
+                             headers=self.get_jwt_header(),
+                             verify=False)
+
+    def post_json(self, url, dict_data):
+        return requests.post(self.addr + url,
+                             json=dict_data,
+                             headers=self.get_jwt_header(),
+                             verify=False)
+
+    def get_jwt_header(self):
+        return {"Authorization": "JWT " + self.token}

envs/monkey_zoo/blackbox/island_configs/HADOOP.conf

@@ -46,16 +46,7 @@
   "exploits": {
     "general": {
       "exploiter_classes": [
-        "SmbExploiter",
+        "HadoopExploiter"
-        "WmiExploiter",
-        "SSHExploiter",
-        "ShellShockExploiter",
-        "SambaCryExploiter",
-        "ElasticGroovyExploiter",
-        "Struts2Exploiter",
-        "WebLogicExploiter",
-        "HadoopExploiter",
-        "VSFTPDExploiter"
       ],
       "skip_exploit_if_file_exist": false
     },

envs/monkey_zoo/blackbox/log_handlers/monkey_log.py

@@ -0,0 +1,33 @@
+import os
+
+from bson import ObjectId
+
+
+class MonkeyLog(object):
+    def __init__(self, monkey, log_dir_path):
+        self.monkey = monkey
+        self.log_dir_path = log_dir_path
+
+    def download_log(self, island_client):
+        log = island_client.find_log_in_db({'monkey_id': ObjectId(self.monkey['id'])})
+        if not log:
+            print("Log for monkey {} not found".format(self.monkey['ip_addresses'][0]))
+        else:
+            self.write_log_to_file(log)
+
+    def write_log_to_file(self, log):
+        with open(self.get_log_path_for_monkey(self.monkey), 'w') as log_file:
+            log_file.write(MonkeyLog.parse_log(log))
+
+    @staticmethod
+    def parse_log(log):
+        log = log.strip('"')
+        log = log.replace("\\n", "\n ")
+        return log
+
+    @staticmethod
+    def get_filename_for_monkey_log(monkey):
+        return "{}.txt".format(monkey['ip_addresses'][0])
+
+    def get_log_path_for_monkey(self, monkey):
+        return os.path.join(self.log_dir_path, MonkeyLog.get_filename_for_monkey_log(monkey))

envs/monkey_zoo/blackbox/log_handlers/test_logs.py

@@ -0,0 +1,35 @@
+import os
+import shutil
+
+from envs.monkey_zoo.blackbox.log_handlers.monkey_log import MonkeyLog
+
+LOG_DIR_NAME = 'logs'
+
+
+class TestLogsHandler(object):
+    def __init__(self, test_name, island_client):
+        self.test_name = test_name
+        self.island_client = island_client
+        self.log_dir_path = os.path.join(TestLogsHandler.get_log_dir_path(), self.test_name)
+
+    def download_logs(self):
+        self.try_create_log_dir_for_test()
+        print("Downloading logs")
+        all_monkeys = self.island_client.find_monkeys_in_db(None)
+        for monkey in all_monkeys:
+            MonkeyLog(monkey, self.log_dir_path).download_log(self.island_client)
+
+    def try_create_log_dir_for_test(self):
+        try:
+            os.mkdir(self.log_dir_path)
+        except Exception as e:
+            print("Can't create a dir for test logs: {}".format(e))
+
+    @staticmethod
+    def get_log_dir_path():
+        return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), LOG_DIR_NAME)
+
+    @staticmethod
+    def delete_log_folder_contents():
+        shutil.rmtree(TestLogsHandler.get_log_dir_path(), ignore_errors=True)
+        os.mkdir(TestLogsHandler.get_log_dir_path())

envs/monkey_zoo/blackbox/test_blackbox.py

@@ -1,15 +1,15 @@
+import pytest
 from time import sleep
 
-import pytest
+from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
-
-from envs.monkey_zoo.blackbox.utils.monkey_island_client import MonkeyIslandClient
 from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer
-from envs.monkey_zoo.blackbox.utils.island_config_parser import IslandConfigParser
+from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser
 from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers
 from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest
+from envs.monkey_zoo.blackbox.log_handlers.test_logs import TestLogsHandler
 
-DEFAULT_TIMEOUT_SECONDS = 3 * 60
+DEFAULT_TIMEOUT_SECONDS = 4*60
-DELAY_BETWEEN_TESTS = 30
+MACHINE_BOOTUP_WAIT_SECONDS = 30
 GCP_TEST_MACHINE_LIST = ['sshkeys-11', 'sshkeys-12', 'elastic-4', 'elastic-5', 'haddop-2-v3', 'hadoop-3', 'mssql-16',
                          'mimikatz-14', 'mimikatz-15', 'final-test-struts2-23', 'final-test-struts2-24',
                          'tunneling-9', 'tunneling-10', 'tunneling-11', 'weblogic-18', 'weblogic-19', 'shellshock-8']
@@ -18,15 +18,25 @@ GCP_TEST_MACHINE_LIST = ['sshkeys-11', 'sshkeys-12', 'elastic-4', 'elastic-5', '
 @pytest.fixture(autouse=True, scope='session')
 def GCPHandler(request):
     GCPHandler = gcp_machine_handlers.GCPHandler()
-    #GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST))
+    GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST))
+    wait_machine_bootup()
 
     def fin():
-        pass
+        GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST))
-        # GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST))
 
     request.addfinalizer(fin)
 
 
+@pytest.fixture(autouse=True, scope='session')
+def delete_logs():
+    print("Deleting monkey logs before new tests.")
+    TestLogsHandler.delete_log_folder_contents()
+
+
+def wait_machine_bootup():
+    sleep(MACHINE_BOOTUP_WAIT_SECONDS)
+
+
 @pytest.fixture(scope='class')
 def island_client(island):
     island_client_object = MonkeyIslandClient(island)
@@ -37,8 +47,8 @@ def island_client(island):
 # noinspection PyUnresolvedReferences
 class TestMonkeyBlackbox(object):
 
-    def run_basic_test(self, island_client, conf_filename, test_name, timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS):
+    @staticmethod
-        TestMonkeyBlackbox.wait_between_tests()
+    def run_basic_test(island_client, conf_filename, test_name, timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS):
         config_parser = IslandConfigParser(conf_filename)
         analyzer = CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets())
         BasicTest(test_name,
@@ -47,44 +57,39 @@ class TestMonkeyBlackbox(object):
                   [analyzer],
                   timeout_in_seconds).run()
 
-    @staticmethod
-    def wait_between_tests():
-        print("Waiting for ({:.0f} seconds) for old monkey's to die or GCP machines to boot up.".format(DELAY_BETWEEN_TESTS))
-        sleep(DELAY_BETWEEN_TESTS)
-
     def test_server_online(self, island_client):
         assert island_client.get_api_status() is not None
 
     def test_ssh_exploiter(self, island_client):
-        self.run_basic_test(island_client, "SSH.conf", "SSH exploiter and keys")
+        TestMonkeyBlackbox.run_basic_test(island_client, "SSH.conf", "SSH_exploiter_and_keys")
 
     def test_hadoop_exploiter(self, island_client):
-        self.run_basic_test(island_client, "HADOOP.conf", "Hadoop exploiter")
+        TestMonkeyBlackbox.run_basic_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6*60)
 
     """
     def test_mssql_exploiter(self, island_client):
-        self.run_basic_test(island_client, "MSSQL.conf", "MSSQL exploiter") 
+        TestMonkeyBlackbox.run_basic_test(island_client, "MSSQL.conf", "MSSQL_exploiter") 
     """
 
     def test_smb_and_mimikatz_exploiters(self, island_client):
-        self.run_basic_test(island_client, "SMB_MIMIKATZ.conf", "SMB exploiter, mimikatz")
+        TestMonkeyBlackbox.run_basic_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz")
 
     """
     def test_elastic_exploiter(self, island_client):
-        self.run_basic_test(island_client, "ELASTIC.conf", "Elastic exploiter")
+        TestMonkeyBlackbox.run_basic_test(island_client, "ELASTIC.conf", "Elastic_exploiter")
     """
 
     def test_struts_exploiter(self, island_client):
-        self.run_basic_test(island_client, "STRUTS2.conf", "Strtuts2 exploiter")
+        TestMonkeyBlackbox.run_basic_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter")
 
     def test_weblogic_exploiter(self, island_client):
-        self.run_basic_test(island_client, "WEBLOGIC.conf", "Weblogic exploiter")
+        TestMonkeyBlackbox.run_basic_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter")
 
     def test_shellshock_exploiter(self, island_client):
-        self.run_basic_test(island_client, "SHELLSHOCK.conf", "Shellschock exploiter")
+        TestMonkeyBlackbox.run_basic_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter")
 
     def test_tunneling(self, island_client):
-        self.run_basic_test(island_client, "TUNNELING.conf", "Tunneling exploiter")
+        TestMonkeyBlackbox.run_basic_test(island_client, "TUNNELING.conf", "Tunneling_exploiter")
 
     def test_wmi_exploiter(self, island_client):
-        self.run_basic_test(island_client, "WMI_MIMIKATZ.conf", "WMI exploiter, mimikatz")
+        TestMonkeyBlackbox.run_basic_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz")

envs/monkey_zoo/blackbox/tests/basic_test.py

@@ -1,8 +1,10 @@
 from time import sleep
 
 from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer
+from envs.monkey_zoo.blackbox.log_handlers.test_logs import TestLogsHandler
 
-DELAY_BETWEEN_ANALYSIS = 1
+MAX_TIME_FOR_MONKEYS_TO_DIE = 5*60
+WAIT_TIME_BETWEEN_REQUESTS = 10
 
 
 class BasicTest(object):
@@ -19,8 +21,13 @@ class BasicTest(object):
         try:
             self.island_client.run_monkey_local()
             self.test_until_timeout()
+        except AssertionError:
+            print("Test {} failed. Downloading logs of all monkeys.".format(self.name))
+            TestLogsHandler(self.name, self.island_client).download_logs()
+            raise
         finally:
             self.island_client.kill_all_monkeys()
+            self.wait_until_monkeys_die()
             self.island_client.reset_env()
 
     def test_until_timeout(self):
@@ -52,3 +59,12 @@ class BasicTest(object):
         for analyzer in self.analyzers:
             log += "\n"+analyzer.log.get_contents()
         return log
+
+    def wait_until_monkeys_die(self):
+        time_passed = 0
+        while not self.island_client.is_all_monkeys_dead() and time_passed < MAX_TIME_FOR_MONKEYS_TO_DIE:
+            sleep(WAIT_TIME_BETWEEN_REQUESTS)
+            time_passed += WAIT_TIME_BETWEEN_REQUESTS
+        if time_passed > MAX_TIME_FOR_MONKEYS_TO_DIE:
+            print("Some monkeys didn't die after the test, passing")
+            assert False

envs/monkey_zoo/blackbox/utils/json_encoder.py

@@ -0,0 +1,9 @@
+import json
+from bson import ObjectId
+
+
+class MongoQueryJSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, ObjectId):
+            return str(o)
+        return json.JSONEncoder.default(self, o)

envs/monkey_zoo/blackbox/utils/monkey_island_client.py

@@ -1,87 +0,0 @@
-import json
-from time import sleep
-
-import requests
-
-# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
-NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \
-                '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557'
-SLEEP_BETWEEN_REQUESTS_SECONDS = 0.5
-
-
-def avoid_race_condition(func):
-    sleep(SLEEP_BETWEEN_REQUESTS_SECONDS)
-    return func
-
-
-class MonkeyIslandClient(object):
-    def __init__(self, server_address):
-        self.addr = "https://{IP}/".format(IP=server_address)
-        self.token = self.try_get_jwt_from_server()
-
-    def try_get_jwt_from_server(self):
-        try:
-            return self.get_jwt_from_server()
-        except requests.ConnectionError:
-            print("Unable to connect to island, aborting!")
-            assert False
-
-    def get_jwt_from_server(self):
-        resp = requests.post(self.addr + "api/auth",
-                             json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
-                             verify=False)
-        return resp.json()["access_token"]
-
-    def request_get(self, url, data=None):
-        return requests.get(self.addr + url,
-                            headers={"Authorization": "JWT " + self.token},
-                            params=data,
-                            verify=False)
-
-    def request_post(self, url, data):
-        return requests.post(self.addr + url,
-                             data=data,
-                             headers={"Authorization": "JWT " + self.token},
-                             verify=False)
-
-    def request_post_json(self, url, dict_data):
-        return requests.post(self.addr + url,
-                             json=dict_data,
-                             headers={"Authorization": "JWT " + self.token},
-                             verify=False)
-
-    def get_api_status(self):
-        return self.request_get("api")
-
-    @avoid_race_condition
-    def import_config(self, config_contents):
-        _ = self.request_post("api/configuration/island", data=config_contents)
-
-    @avoid_race_condition
-    def run_monkey_local(self):
-        response = self.request_post_json("api/local-monkey", dict_data={"action": "run"})
-        if MonkeyIslandClient.monkey_ran_successfully(response):
-            print("Running the monkey.")
-        else:
-            print("Failed to run the monkey.")
-            assert False
-
-    @staticmethod
-    def monkey_ran_successfully(response):
-        return response.ok and json.loads(response.content)['is_running']
-
-    @avoid_race_condition
-    def kill_all_monkeys(self):
-        if self.request_get("api", {"action": "killall"}).ok:
-            print("Killing all monkeys after the test.")
-        else:
-            print("Failed to kill all monkeys.")
-            assert False
-
-    @avoid_race_condition
-    def reset_env(self):
-        if self.request_get("api", {"action": "reset"}).ok:
-            print("Resetting environment after the test.")
-        else:
-            print("Failed to reset the environment.")
-            assert False

monkey/infection_monkey/model/__init__.py

@@ -38,4 +38,4 @@ HADOOP_LINUX_COMMAND = "! [ -f %(monkey_path)s ] " \
                     "; chmod +x %(monkey_path)s " \
                     "&&  %(monkey_path)s %(monkey_type)s %(parameters)s"
 
-DOWNLOAD_TIMEOUT = 300
+DOWNLOAD_TIMEOUT = 180

monkey/monkey_island/cc/app.py

@@ -35,7 +35,9 @@ from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService
 from monkey_island.cc.resources.pba_file_upload import FileUpload
 from monkey_island.cc.resources.attack.attack_config import AttackConfiguration
 from monkey_island.cc.resources.attack.attack_report import AttackReport
+
 from monkey_island.cc.resources.test.monkey_test import MonkeyTest
+from monkey_island.cc.resources.test.log_test import LogTest
 
 __author__ = 'Barak'
 
@@ -135,7 +137,9 @@ def init_api_resources(api):
     api.add_resource(AttackConfiguration, '/api/attack')
     api.add_resource(AttackReport, '/api/attack/report')
     api.add_resource(VersionUpdate, '/api/version-update', '/api/version-update/')
+
     api.add_resource(MonkeyTest, '/api/test/monkey')
+    api.add_resource(LogTest, '/api/test/log')
 
 
 def init_app(mongo_url):

monkey/monkey_island/cc/resources/test/log_test.py

@@ -0,0 +1,18 @@
+from bson import json_util
+import flask_restful
+from flask import request
+
+
+from monkey_island.cc.auth import jwt_required
+from monkey_island.cc.database import mongo, database
+
+
+class LogTest(flask_restful.Resource):
+    @jwt_required()
+    def get(self):
+        find_query = json_util.loads(request.args.get('find_query'))
+        log = mongo.db.log.find_one(find_query)
+        if not log:
+            return {'results': None}
+        log_file = database.gridfs.get(log['file_id'])
+        return {'results': log_file.read()}

monkey/monkey_island/cc/resources/test/monkey_test.py

@@ -1,5 +1,4 @@
-import json
+from bson import json_util
-
 import flask_restful
 from flask import request
 
@@ -10,5 +9,5 @@ from monkey_island.cc.database import mongo
 class MonkeyTest(flask_restful.Resource):
     @jwt_required()
     def get(self, **kw):
-        find_query = json.loads(request.args.get('find_query'))
+        find_query = json_util.loads(request.args.get('find_query'))
         return {'results': list(mongo.db.monkey.find(find_query))}