Implemented logs

This commit is contained in:
VakarisZ 2019-09-13 16:12:58 +03:00
parent a17305a324
commit 9938ba9574
18 changed files with 285 additions and 133 deletions

View File

@ -1 +1,2 @@
gcp_keys/
logs/

View File

@ -1,5 +1,3 @@
import json
LOG_INIT_MESSAGE = "Analysis didn't run."
@ -20,9 +18,8 @@ class CommunicationAnalyzer(object):
return True
def did_monkey_communicate_back(self, machine_ip):
query = json.dumps({'ip_addresses': {'$elemMatch': {'$eq': machine_ip}}})
response = self.island_client.request_get("api/test/monkey", {'find_query': query})
return len(json.loads(response.content)['results']) > 0
query = {'ip_addresses': {'$elemMatch': {'$eq': machine_ip}}}
return len(self.island_client.find_monkeys_in_db(query)) > 0
class AnalyzerLog(object):

View File

@ -0,0 +1,86 @@
from time import sleep
import json
from bson import json_util
from envs.monkey_zoo.blackbox.island_client.monkey_island_requests import MonkeyIslandRequests
SLEEP_BETWEEN_REQUESTS_SECONDS = 0.5
MONKEY_TEST_ENDPOINT = 'api/test/monkey'
LOG_TEST_ENDPOINT = 'api/test/log'
def avoid_race_condition(func):
sleep(SLEEP_BETWEEN_REQUESTS_SECONDS)
return func
class MonkeyIslandClient(object):
def __init__(self, server_address):
self.requests = MonkeyIslandRequests(server_address)
def get_api_status(self):
return self.requests.get("api")
@avoid_race_condition
def import_config(self, config_contents):
_ = self.requests.post("api/configuration/island", data=config_contents)
@avoid_race_condition
def run_monkey_local(self):
response = self.requests.post_json("api/local-monkey", dict_data={"action": "run"})
if MonkeyIslandClient.monkey_ran_successfully(response):
print("Running the monkey.")
else:
print("Failed to run the monkey.")
assert False
@staticmethod
def monkey_ran_successfully(response):
return response.ok and json.loads(response.content)['is_running']
@avoid_race_condition
def kill_all_monkeys(self):
if self.requests.get("api", {"action": "killall"}).ok:
print("Killing all monkeys after the test.")
else:
print("Failed to kill all monkeys.")
assert False
@avoid_race_condition
def reset_env(self):
if self.requests.get("api", {"action": "reset"}).ok:
print("Resetting environment after the test.")
else:
print("Failed to reset the environment.")
assert False
def find_monkeys_in_db(self, query):
response = self.requests.get(MONKEY_TEST_ENDPOINT,
MonkeyIslandClient.form_find_query_for_request(query))
try:
return MonkeyIslandClient.get_test_query_results(response)
except Exception:
print("Ran into trouble parsing response for monkey query")
raise
def find_log_in_db(self, query):
response = self.requests.get(LOG_TEST_ENDPOINT,
MonkeyIslandClient.form_find_query_for_request(query))
try:
return MonkeyIslandClient.get_test_query_results(response)
except Exception:
print("Ran into trouble parsing response for log query")
raise
@staticmethod
def form_find_query_for_request(query):
return {'find_query': json_util.dumps(query)}
@staticmethod
def get_test_query_results(response):
return json.loads(response.content)['results']
def is_all_monkeys_dead(self):
query = {'dead': 'false'}
return len(self.find_monkeys_in_db(query)) == 0

View File

@ -0,0 +1,45 @@
import requests
# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \
'8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557'
class MonkeyIslandRequests(object):
def __init__(self, server_address):
self.addr = "https://{IP}/".format(IP=server_address)
self.token = self.try_get_jwt_from_server()
def try_get_jwt_from_server(self):
try:
return self.get_jwt_from_server()
except requests.ConnectionError:
print("Unable to connect to island, aborting!")
assert False
def get_jwt_from_server(self):
resp = requests.post(self.addr + "api/auth",
json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
verify=False)
return resp.json()["access_token"]
def get(self, url, data=None):
return requests.get(self.addr + url,
headers=self.get_jwt_header(),
params=data,
verify=False)
def post(self, url, data):
return requests.post(self.addr + url,
data=data,
headers=self.get_jwt_header(),
verify=False)
def post_json(self, url, dict_data):
return requests.post(self.addr + url,
json=dict_data,
headers=self.get_jwt_header(),
verify=False)
def get_jwt_header(self):
return {"Authorization": "JWT " + self.token}

View File

@ -46,16 +46,7 @@
"exploits": {
"general": {
"exploiter_classes": [
"SmbExploiter",
"WmiExploiter",
"SSHExploiter",
"ShellShockExploiter",
"SambaCryExploiter",
"ElasticGroovyExploiter",
"Struts2Exploiter",
"WebLogicExploiter",
"HadoopExploiter",
"VSFTPDExploiter"
"HadoopExploiter"
],
"skip_exploit_if_file_exist": false
},

View File

@ -0,0 +1,33 @@
import os
from bson import ObjectId
class MonkeyLog(object):
def __init__(self, monkey, log_dir_path):
self.monkey = monkey
self.log_dir_path = log_dir_path
def download_log(self, island_client):
log = island_client.find_log_in_db({'monkey_id': ObjectId(self.monkey['id'])})
if not log:
print("Log for monkey {} not found".format(self.monkey['ip_addresses'][0]))
else:
self.write_log_to_file(log)
def write_log_to_file(self, log):
with open(self.get_log_path_for_monkey(self.monkey), 'w') as log_file:
log_file.write(MonkeyLog.parse_log(log))
@staticmethod
def parse_log(log):
log = log.strip('"')
log = log.replace("\\n", "\n ")
return log
@staticmethod
def get_filename_for_monkey_log(monkey):
return "{}.txt".format(monkey['ip_addresses'][0])
def get_log_path_for_monkey(self, monkey):
return os.path.join(self.log_dir_path, MonkeyLog.get_filename_for_monkey_log(monkey))

View File

@ -0,0 +1,35 @@
import os
import shutil
from envs.monkey_zoo.blackbox.log_handlers.monkey_log import MonkeyLog
LOG_DIR_NAME = 'logs'
class TestLogsHandler(object):
def __init__(self, test_name, island_client):
self.test_name = test_name
self.island_client = island_client
self.log_dir_path = os.path.join(TestLogsHandler.get_log_dir_path(), self.test_name)
def download_logs(self):
self.try_create_log_dir_for_test()
print("Downloading logs")
all_monkeys = self.island_client.find_monkeys_in_db(None)
for monkey in all_monkeys:
MonkeyLog(monkey, self.log_dir_path).download_log(self.island_client)
def try_create_log_dir_for_test(self):
try:
os.mkdir(self.log_dir_path)
except Exception as e:
print("Can't create a dir for test logs: {}".format(e))
@staticmethod
def get_log_dir_path():
return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), LOG_DIR_NAME)
@staticmethod
def delete_log_folder_contents():
shutil.rmtree(TestLogsHandler.get_log_dir_path(), ignore_errors=True)
os.mkdir(TestLogsHandler.get_log_dir_path())

View File

@ -1,15 +1,15 @@
import pytest
from time import sleep
import pytest
from envs.monkey_zoo.blackbox.utils.monkey_island_client import MonkeyIslandClient
from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
from envs.monkey_zoo.blackbox.analyzers.communication_analyzer import CommunicationAnalyzer
from envs.monkey_zoo.blackbox.utils.island_config_parser import IslandConfigParser
from envs.monkey_zoo.blackbox.island_client.island_config_parser import IslandConfigParser
from envs.monkey_zoo.blackbox.utils import gcp_machine_handlers
from envs.monkey_zoo.blackbox.tests.basic_test import BasicTest
from envs.monkey_zoo.blackbox.log_handlers.test_logs import TestLogsHandler
DEFAULT_TIMEOUT_SECONDS = 3 * 60
DELAY_BETWEEN_TESTS = 30
DEFAULT_TIMEOUT_SECONDS = 4*60
MACHINE_BOOTUP_WAIT_SECONDS = 30
GCP_TEST_MACHINE_LIST = ['sshkeys-11', 'sshkeys-12', 'elastic-4', 'elastic-5', 'haddop-2-v3', 'hadoop-3', 'mssql-16',
'mimikatz-14', 'mimikatz-15', 'final-test-struts2-23', 'final-test-struts2-24',
'tunneling-9', 'tunneling-10', 'tunneling-11', 'weblogic-18', 'weblogic-19', 'shellshock-8']
@ -18,15 +18,25 @@ GCP_TEST_MACHINE_LIST = ['sshkeys-11', 'sshkeys-12', 'elastic-4', 'elastic-5', '
@pytest.fixture(autouse=True, scope='session')
def GCPHandler(request):
GCPHandler = gcp_machine_handlers.GCPHandler()
#GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST))
GCPHandler.start_machines(" ".join(GCP_TEST_MACHINE_LIST))
wait_machine_bootup()
def fin():
pass
# GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST))
GCPHandler.stop_machines(" ".join(GCP_TEST_MACHINE_LIST))
request.addfinalizer(fin)
@pytest.fixture(autouse=True, scope='session')
def delete_logs():
print("Deleting monkey logs before new tests.")
TestLogsHandler.delete_log_folder_contents()
def wait_machine_bootup():
sleep(MACHINE_BOOTUP_WAIT_SECONDS)
@pytest.fixture(scope='class')
def island_client(island):
island_client_object = MonkeyIslandClient(island)
@ -37,8 +47,8 @@ def island_client(island):
# noinspection PyUnresolvedReferences
class TestMonkeyBlackbox(object):
def run_basic_test(self, island_client, conf_filename, test_name, timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS):
TestMonkeyBlackbox.wait_between_tests()
@staticmethod
def run_basic_test(island_client, conf_filename, test_name, timeout_in_seconds=DEFAULT_TIMEOUT_SECONDS):
config_parser = IslandConfigParser(conf_filename)
analyzer = CommunicationAnalyzer(island_client, config_parser.get_ips_of_targets())
BasicTest(test_name,
@ -47,44 +57,39 @@ class TestMonkeyBlackbox(object):
[analyzer],
timeout_in_seconds).run()
@staticmethod
def wait_between_tests():
print("Waiting for ({:.0f} seconds) for old monkey's to die or GCP machines to boot up.".format(DELAY_BETWEEN_TESTS))
sleep(DELAY_BETWEEN_TESTS)
def test_server_online(self, island_client):
assert island_client.get_api_status() is not None
def test_ssh_exploiter(self, island_client):
self.run_basic_test(island_client, "SSH.conf", "SSH exploiter and keys")
TestMonkeyBlackbox.run_basic_test(island_client, "SSH.conf", "SSH_exploiter_and_keys")
def test_hadoop_exploiter(self, island_client):
self.run_basic_test(island_client, "HADOOP.conf", "Hadoop exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "HADOOP.conf", "Hadoop_exploiter", 6*60)
"""
def test_mssql_exploiter(self, island_client):
self.run_basic_test(island_client, "MSSQL.conf", "MSSQL exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "MSSQL.conf", "MSSQL_exploiter")
"""
def test_smb_and_mimikatz_exploiters(self, island_client):
self.run_basic_test(island_client, "SMB_MIMIKATZ.conf", "SMB exploiter, mimikatz")
TestMonkeyBlackbox.run_basic_test(island_client, "SMB_MIMIKATZ.conf", "SMB_exploiter_mimikatz")
"""
def test_elastic_exploiter(self, island_client):
self.run_basic_test(island_client, "ELASTIC.conf", "Elastic exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "ELASTIC.conf", "Elastic_exploiter")
"""
def test_struts_exploiter(self, island_client):
self.run_basic_test(island_client, "STRUTS2.conf", "Strtuts2 exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "STRUTS2.conf", "Strtuts2_exploiter")
def test_weblogic_exploiter(self, island_client):
self.run_basic_test(island_client, "WEBLOGIC.conf", "Weblogic exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "WEBLOGIC.conf", "Weblogic_exploiter")
def test_shellshock_exploiter(self, island_client):
self.run_basic_test(island_client, "SHELLSHOCK.conf", "Shellschock exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter")
def test_tunneling(self, island_client):
self.run_basic_test(island_client, "TUNNELING.conf", "Tunneling exploiter")
TestMonkeyBlackbox.run_basic_test(island_client, "TUNNELING.conf", "Tunneling_exploiter")
def test_wmi_exploiter(self, island_client):
self.run_basic_test(island_client, "WMI_MIMIKATZ.conf", "WMI exploiter, mimikatz")
TestMonkeyBlackbox.run_basic_test(island_client, "WMI_MIMIKATZ.conf", "WMI_exploiter,_mimikatz")

View File

@ -1,8 +1,10 @@
from time import sleep
from envs.monkey_zoo.blackbox.utils.test_timer import TestTimer
from envs.monkey_zoo.blackbox.log_handlers.test_logs import TestLogsHandler
DELAY_BETWEEN_ANALYSIS = 1
MAX_TIME_FOR_MONKEYS_TO_DIE = 5*60
WAIT_TIME_BETWEEN_REQUESTS = 10
class BasicTest(object):
@ -19,8 +21,13 @@ class BasicTest(object):
try:
self.island_client.run_monkey_local()
self.test_until_timeout()
except AssertionError:
print("Test {} failed. Downloading logs of all monkeys.".format(self.name))
TestLogsHandler(self.name, self.island_client).download_logs()
raise
finally:
self.island_client.kill_all_monkeys()
self.wait_until_monkeys_die()
self.island_client.reset_env()
def test_until_timeout(self):
@ -52,3 +59,12 @@ class BasicTest(object):
for analyzer in self.analyzers:
log += "\n"+analyzer.log.get_contents()
return log
def wait_until_monkeys_die(self):
time_passed = 0
while not self.island_client.is_all_monkeys_dead() and time_passed < MAX_TIME_FOR_MONKEYS_TO_DIE:
sleep(WAIT_TIME_BETWEEN_REQUESTS)
time_passed += WAIT_TIME_BETWEEN_REQUESTS
if time_passed > MAX_TIME_FOR_MONKEYS_TO_DIE:
print("Some monkeys didn't die after the test, passing")
assert False

View File

@ -0,0 +1,9 @@
import json
from bson import ObjectId
class MongoQueryJSONEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, ObjectId):
return str(o)
return json.JSONEncoder.default(self, o)

View File

@ -1,87 +0,0 @@
import json
from time import sleep
import requests
# SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()'
NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \
'8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557'
SLEEP_BETWEEN_REQUESTS_SECONDS = 0.5
def avoid_race_condition(func):
sleep(SLEEP_BETWEEN_REQUESTS_SECONDS)
return func
class MonkeyIslandClient(object):
def __init__(self, server_address):
self.addr = "https://{IP}/".format(IP=server_address)
self.token = self.try_get_jwt_from_server()
def try_get_jwt_from_server(self):
try:
return self.get_jwt_from_server()
except requests.ConnectionError:
print("Unable to connect to island, aborting!")
assert False
def get_jwt_from_server(self):
resp = requests.post(self.addr + "api/auth",
json={"username": NO_AUTH_CREDS, "password": NO_AUTH_CREDS},
verify=False)
return resp.json()["access_token"]
def request_get(self, url, data=None):
return requests.get(self.addr + url,
headers={"Authorization": "JWT " + self.token},
params=data,
verify=False)
def request_post(self, url, data):
return requests.post(self.addr + url,
data=data,
headers={"Authorization": "JWT " + self.token},
verify=False)
def request_post_json(self, url, dict_data):
return requests.post(self.addr + url,
json=dict_data,
headers={"Authorization": "JWT " + self.token},
verify=False)
def get_api_status(self):
return self.request_get("api")
@avoid_race_condition
def import_config(self, config_contents):
_ = self.request_post("api/configuration/island", data=config_contents)
@avoid_race_condition
def run_monkey_local(self):
response = self.request_post_json("api/local-monkey", dict_data={"action": "run"})
if MonkeyIslandClient.monkey_ran_successfully(response):
print("Running the monkey.")
else:
print("Failed to run the monkey.")
assert False
@staticmethod
def monkey_ran_successfully(response):
return response.ok and json.loads(response.content)['is_running']
@avoid_race_condition
def kill_all_monkeys(self):
if self.request_get("api", {"action": "killall"}).ok:
print("Killing all monkeys after the test.")
else:
print("Failed to kill all monkeys.")
assert False
@avoid_race_condition
def reset_env(self):
if self.request_get("api", {"action": "reset"}).ok:
print("Resetting environment after the test.")
else:
print("Failed to reset the environment.")
assert False

View File

@ -38,4 +38,4 @@ HADOOP_LINUX_COMMAND = "! [ -f %(monkey_path)s ] " \
"; chmod +x %(monkey_path)s " \
"&& %(monkey_path)s %(monkey_type)s %(parameters)s"
DOWNLOAD_TIMEOUT = 300
DOWNLOAD_TIMEOUT = 180

View File

@ -35,7 +35,9 @@ from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService
from monkey_island.cc.resources.pba_file_upload import FileUpload
from monkey_island.cc.resources.attack.attack_config import AttackConfiguration
from monkey_island.cc.resources.attack.attack_report import AttackReport
from monkey_island.cc.resources.test.monkey_test import MonkeyTest
from monkey_island.cc.resources.test.log_test import LogTest
__author__ = 'Barak'
@ -135,7 +137,9 @@ def init_api_resources(api):
api.add_resource(AttackConfiguration, '/api/attack')
api.add_resource(AttackReport, '/api/attack/report')
api.add_resource(VersionUpdate, '/api/version-update', '/api/version-update/')
api.add_resource(MonkeyTest, '/api/test/monkey')
api.add_resource(LogTest, '/api/test/log')
def init_app(mongo_url):

View File

@ -0,0 +1,18 @@
from bson import json_util
import flask_restful
from flask import request
from monkey_island.cc.auth import jwt_required
from monkey_island.cc.database import mongo, database
class LogTest(flask_restful.Resource):
@jwt_required()
def get(self):
find_query = json_util.loads(request.args.get('find_query'))
log = mongo.db.log.find_one(find_query)
if not log:
return {'results': None}
log_file = database.gridfs.get(log['file_id'])
return {'results': log_file.read()}

View File

@ -1,5 +1,4 @@
import json
from bson import json_util
import flask_restful
from flask import request
@ -10,5 +9,5 @@ from monkey_island.cc.database import mongo
class MonkeyTest(flask_restful.Resource):
@jwt_required()
def get(self, **kw):
find_query = json.loads(request.args.get('find_query'))
find_query = json_util.loads(request.args.get('find_query'))
return {'results': list(mongo.db.monkey.find(find_query))}