p15670423
/
monkey
1
0
Fork 5
Code Issues Pull Requests 3 Projects Releases Wiki Activity

Release/1.7.0 -> develop (#457) - added some logs and blackbox improvments. 

Release/1.7.0 -> develop
This commit is contained in:
Shay Nehmad 2019-10-07 15:59:46 +03:00 committed by GitHub
parent 34f73bd0a2 3b7b0f0998
commit 68e2a83aac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 89 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
envs/monkey_zoo/blackbox/README.md
View File

@ -3,3 +3,17 @@
1. Download google sdk: https://cloud.google.com/sdk/docs/
2. Download service account key for MonkeyZoo project (if you deployed MonkeyZoo via terraform scripts then you already have it).
GCP console -> IAM -> service accounts(you can use the same key used to authenticate terraform scripts)
3. Deploy the relevant branch + complied executables to the Island machine on GCP.
### Running the tests
In order to execute the entire test suite, you must know the external IP of the Island machine on GCP. You can find
this information in the GCP Console `Compute Engine/VM Instances` under _External IP_.
#### Running in command line
Run the following command:
`monkey\envs\monkey_zoo\blackbox>python -m pytest --island=35.207.152.72:5000 test_blackbox.py`
#### Running in PyCharm
Configure a PyTest configuration with the additional argument `--island=35.207.152.72` on the
`monkey\envs\monkey_zoo\blackbox`.

5
envs/monkey_zoo/blackbox/island_client/monkey_island_requests.py
View File

@ -16,8 +16,9 @@ class MonkeyIslandRequests(object):
def try_get_jwt_from_server(self):
try:
return self.get_jwt_from_server()
except requests.ConnectionError:
LOGGER.error("Unable to connect to island, aborting!")
except requests.ConnectionError as err:
LOGGER.error(
"Unable to connect to island, aborting! Error information: {}. Server: {}".format(err, self.addr))
assert False
def get_jwt_from_server(self):

23
envs/monkey_zoo/blackbox/log_handlers/monkey_log_parser.py
View File

@ -1,5 +1,8 @@
import logging
import re
LOGGER = logging.getLogger(__name__)
class MonkeyLogParser(object):
@ -12,9 +15,13 @@ class MonkeyLogParser(object):
return log.read()
def print_errors(self):
print("Errors:")
for error_line in MonkeyLogParser.get_errors(self.log_contents):
print(error_line)
errors = MonkeyLogParser.get_errors(self.log_contents)
if len(errors) > 0:
LOGGER.info("Found {} errors:".format(len(errors)))
for index, error_line in enumerate(errors):
LOGGER.info("Err #{}: {}".format(index, error_line))
else:
LOGGER.info("No errors!")
@staticmethod
def get_errors(log_contents):
@ -22,9 +29,13 @@ class MonkeyLogParser(object):
return searcher.findall(log_contents)
def print_warnings(self):
print("Warnings:")
for warning_line in MonkeyLogParser.get_warnings(self.log_contents):
print(warning_line)
warnings = MonkeyLogParser.get_warnings(self.log_contents)
if len(warnings) > 0:
LOGGER.info("Found {} warnings:".format(len(warnings)))
for index, warning_line in enumerate(warnings):
LOGGER.info("Warn #{}: {}".format(index, warning_line))
else:
LOGGER.info("No warnings!")
@staticmethod
def get_warnings(log_contents):

2
envs/monkey_zoo/blackbox/log_handlers/test_logs_handler.py
View File

@ -44,7 +44,7 @@ class TestLogsHandler(object):
@staticmethod
def parse_logs(log_paths):
for log_path in log_paths:
print("Info from log at {}".format(log_path))
LOGGER.info("Info from log at {}".format(log_path))
log_parser = MonkeyLogParser(log_path)
log_parser.print_errors()
log_parser.print_warnings()

2
envs/monkey_zoo/blackbox/pytest.ini
View File

@ -1,5 +1,5 @@
[pytest]
log_cli = 1
log_cli_level = INFO
log_cli_format = %(asctime)s [%(levelname)s] %(message)s
log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s
log_cli_date_format=%H:%M:%S

1
envs/monkey_zoo/blackbox/test_blackbox.py
View File

@ -99,6 +99,7 @@ class TestMonkeyBlackbox(object):
def test_shellshock_exploiter(self, island_client):
TestMonkeyBlackbox.run_basic_test(island_client, "SHELLSHOCK.conf", "Shellschock_exploiter")
@pytest.mark.xfail(reason="Test fails randomly - still investigating.")
def test_tunneling(self, island_client):
TestMonkeyBlackbox.run_basic_test(island_client, "TUNNELING.conf", "Tunneling_exploiter", 10*60)

6
envs/monkey_zoo/blackbox/tests/basic_test.py
View File

@ -1,3 +1,4 @@
import json
from time import sleep
import logging
@ -22,6 +23,7 @@ class BasicTest(object):
self.log_handler = log_handler
def run(self):
LOGGER.info("Uploading configuration:\n{}".format(json.dumps(self.config_parser.config_json, indent=2)))
self.island_client.import_config(self.config_parser.config_raw)
self.print_test_starting_info()
try:
@ -47,6 +49,7 @@ class BasicTest(object):
self.log_success(timer)
return
sleep(DELAY_BETWEEN_ANALYSIS)
LOGGER.debug("Waiting until all analyzers passed. Time passed: {}".format(timer.get_time_taken()))
self.log_failure(timer)
assert False
@ -76,12 +79,13 @@ class BasicTest(object):
while not self.island_client.is_all_monkeys_dead() and time_passed < MAX_TIME_FOR_MONKEYS_TO_DIE:
sleep(WAIT_TIME_BETWEEN_REQUESTS)
time_passed += WAIT_TIME_BETWEEN_REQUESTS
LOGGER.debug("Waiting for all monkeys to die. Time passed: {}".format(time_passed))
if time_passed > MAX_TIME_FOR_MONKEYS_TO_DIE:
LOGGER.error("Some monkeys didn't die after the test, failing")
assert False
def parse_logs(self):
LOGGER.info("\nParsing test logs:")
LOGGER.info("Parsing test logs:")
self.log_handler.parse_test_logs()
@staticmethod

5
envs/monkey_zoo/blackbox/utils/gcp_machine_handlers.py
View File

@ -26,6 +26,11 @@ class GCPHandler(object):
LOGGER.error("GCP Handler failed to initialize: %s." % e)
def start_machines(self, machine_list):
"""
Start all the machines in the list.
:param machine_list: A space-separated string with all the machine names. Example:
start_machines(`" ".join(["elastic-3", "mssql-16"])`)
"""
LOGGER.info("Setting up all GCP machines...")
try:
subprocess.call((GCPHandler.MACHINE_STARTING_COMMAND % (machine_list, self.zone)), shell=True)

4
monkey/common/cloud/aws_instance.py
View File

@ -30,14 +30,14 @@ class AwsInstance(object):
self.region = self._parse_region(
urllib2.urlopen(AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/placement/availability-zone').read())
except urllib2.URLError as e:
logger.warning("Failed init of AwsInstance while getting metadata: {}".format(e.message))
logger.debug("Failed init of AwsInstance while getting metadata: {}".format(e.message))
try:
self.account_id = self._extract_account_id(
urllib2.urlopen(
AWS_LATEST_METADATA_URI_PREFIX + 'dynamic/instance-identity/document', timeout=2).read())
except urllib2.URLError as e:
logger.warning("Failed init of AwsInstance while getting dynamic instance data: {}".format(e.message))
logger.debug("Failed init of AwsInstance while getting dynamic instance data: {}".format(e.message))
@staticmethod
def _parse_region(region_url_response):

6
monkey/monkey_island/cc/environment/__init__.py
View File

@ -13,7 +13,8 @@ class Environment(object):
_MONGO_DB_NAME = "monkeyisland"
_MONGO_DB_HOST = "localhost"
_MONGO_DB_PORT = 27017
_MONGO_URL = os.environ.get("MONKEY_MONGO_URL", "mongodb://{0}:{1}/{2}".format(_MONGO_DB_HOST, _MONGO_DB_PORT, str(_MONGO_DB_NAME)))
_MONGO_URL = os.environ.get("MONKEY_MONGO_URL",
"mongodb://{0}:{1}/{2}".format(_MONGO_DB_HOST, _MONGO_DB_PORT, str(_MONGO_DB_NAME)))
_DEBUG_SERVER = False
_AUTH_EXPIRATION_TIME = timedelta(hours=1)
@ -27,8 +28,7 @@ class Environment(object):
def testing(self, value):
self._testing = value
_MONKEY_VERSION = "1.6.3"
_MONKEY_VERSION = "1.7.0"
def __init__(self):
self.config = None

14
monkey/monkey_island/cc/main.py
View File

@ -25,6 +25,7 @@ from monkey_island.cc.services.reporting.exporter_init import populate_exporter_
from monkey_island.cc.utils import local_ip_addresses
from monkey_island.cc.environment.environment import env
from monkey_island.cc.database import is_db_server_up, get_db_version
from monkey_island.cc.resources.monkey_download import MonkeyDownload
def main():
@ -49,12 +50,19 @@ def main():
ssl_options={'certfile': os.environ.get('SERVER_CRT', crt_path),
'keyfile': os.environ.get('SERVER_KEY', key_path)})
http_server.listen(env.get_island_port())
logger.info(
'Monkey Island Server is running on https://{}:{}'.format(local_ip_addresses()[0], env.get_island_port()))
log_init_info()
IOLoop.instance().start()
def log_init_info():
logger.info(
'Monkey Island Server is running. Listening on the following URLs: {}'.format(
", ".join(["https://{}:{}".format(x, env.get_island_port()) for x in local_ip_addresses()])
)
)
MonkeyDownload.log_executable_hashes()
def wait_for_mongo_db_server(mongo_url):
while not is_db_server_up(mongo_url):
logger.info('Waiting for MongoDB server on {0}'.format(mongo_url))

27
monkey/monkey_island/cc/resources/monkey_download.py
View File

@ -1,3 +1,4 @@
import hashlib
import json
import logging
import os
@ -83,9 +84,33 @@ class MonkeyDownload(flask_restful.Resource):
if result:
# change resulting from new base path
real_path = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", 'binaries', result['filename'])
executable_filename = result['filename']
real_path = MonkeyDownload.get_executable_full_path(executable_filename)
if os.path.isfile(real_path):
result['size'] = os.path.getsize(real_path)
return result
return {}
@staticmethod
def get_executable_full_path(executable_filename):
real_path = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", 'binaries', executable_filename)
return real_path
@staticmethod
def log_executable_hashes():
"""
Logs all the hashes of the monkey executables for debugging ease (can check what Monkey version you have etc.).
"""
filenames = set([x['filename'] for x in MONKEY_DOWNLOADS])
for filename in filenames:
filepath = MonkeyDownload.get_executable_full_path(filename)
if os.path.isfile(filepath):
with open(filepath, 'rb') as monkey_exec_file:
file_contents = monkey_exec_file.read()
logger.debug("{} hashes:\nSHA-256 {}".format(
filename,
hashlib.sha256(file_contents).hexdigest()
))
else:
logger.debug("No monkey executable for {}.".format(filepath))