From 790c8fd0799a55cdffb8a2ba10244d5bcba615f2 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 24 Apr 2019 14:16:38 +0300 Subject: [PATCH 01/26] Added PyCharm ignore inspection to PyInstaller import and updated gitignore --- .gitignore | 3 ++- monkey/infection_monkey/main.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 63de45992..772829801 100644 --- a/.gitignore +++ b/.gitignore @@ -82,4 +82,5 @@ MonkeyZoo/* !MonkeyZoo/config.tf !MonkeyZoo/MonkeyZooDocs.pdf - +# vim swap files +*.swp diff --git a/monkey/infection_monkey/main.py b/monkey/infection_monkey/main.py index d12414eae..6e06d4aa6 100644 --- a/monkey/infection_monkey/main.py +++ b/monkey/infection_monkey/main.py @@ -13,7 +13,8 @@ from infection_monkey.config import WormConfiguration, EXTERNAL_CONFIG_FILE from infection_monkey.dropper import MonkeyDrops from infection_monkey.model import MONKEY_ARG, DROPPER_ARG from infection_monkey.monkey import InfectionMonkey -import infection_monkey.post_breach # dummy import for pyinstaller +# noinspection PyUnresolvedReferences +import infection_monkey.post_breach # dummy import for pyinstaller __author__ = 'itamar' @@ -23,7 +24,7 @@ LOG_CONFIG = {'version': 1, 'disable_existing_loggers': False, 'formatters': {'standard': { 'format': '%(asctime)s [%(process)d:%(thread)d:%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s'}, - }, + }, 'handlers': {'console': {'class': 'logging.StreamHandler', 'level': 'DEBUG', 'formatter': 'standard'}, @@ -70,7 +71,8 @@ def main(): print("Loaded Configuration: %r" % WormConfiguration.as_dict()) # Make sure we're not in a machine that has the kill file - kill_path = os.path.expandvars(WormConfiguration.kill_file_path_windows) if sys.platform == "win32" else WormConfiguration.kill_file_path_linux + kill_path = os.path.expandvars( + WormConfiguration.kill_file_path_windows) if sys.platform == "win32" else WormConfiguration.kill_file_path_linux if os.path.exists(kill_path): print("Kill path found, finished run") return True From 1a5ebf8dbdf52f18f8abf37994dfb7a2dad40387 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 24 Apr 2019 15:16:28 +0300 Subject: [PATCH 02/26] chmod +x install_mongo --- monkey/monkey_island/linux/install_mongo.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 monkey/monkey_island/linux/install_mongo.sh diff --git a/monkey/monkey_island/linux/install_mongo.sh b/monkey/monkey_island/linux/install_mongo.sh old mode 100644 new mode 100755 From 99bc61f9d965d955700b6887bbe67dea3f8769bf Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Thu, 25 Apr 2019 14:45:37 +0300 Subject: [PATCH 03/26] Added a decorator in which the "timer start" for the timeout can be implemented --- monkey/infection_monkey/dropper.py | 1 - monkey/infection_monkey/main.py | 1 + monkey/monkey_island/cc/resources/monkey.py | 6 +++-- .../cc/services/monkey_timeout.py | 22 +++++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 monkey/monkey_island/cc/services/monkey_timeout.py diff --git a/monkey/infection_monkey/dropper.py b/monkey/infection_monkey/dropper.py index 02bd649c2..cc065a745 100644 --- a/monkey/infection_monkey/dropper.py +++ b/monkey/infection_monkey/dropper.py @@ -51,7 +51,6 @@ class MonkeyDrops(object): LOG.debug("Dropper is running with config:\n%s", pprint.pformat(self._config)) def start(self): - if self._config['destination_path'] is None: LOG.error("No destination path specified") return False diff --git a/monkey/infection_monkey/main.py b/monkey/infection_monkey/main.py index 6e06d4aa6..b8e292243 100644 --- a/monkey/infection_monkey/main.py +++ b/monkey/infection_monkey/main.py @@ -98,6 +98,7 @@ def main(): except OSError: pass LOG_CONFIG['handlers']['file']['filename'] = log_path + # noinspection PyUnresolvedReferences LOG_CONFIG['root']['handlers'].append('file') else: del LOG_CONFIG['handlers']['file'] diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 7eb7ecc69..057ebf149 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -2,11 +2,12 @@ import json from datetime import datetime import dateutil.parser -from flask import request import flask_restful +from flask import request from monkey_island.cc.database import mongo from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.monkey_timeout import start_timer_decorator from monkey_island.cc.services.node import NodeService __author__ = 'Barak' @@ -17,6 +18,7 @@ __author__ = 'Barak' class Monkey(flask_restful.Resource): # Used by monkey. can't secure. + @start_timer_decorator def get(self, guid=None, **kw): NodeService.update_dead_monkeys() # refresh monkeys status if not guid: @@ -88,7 +90,7 @@ class Monkey(flask_restful.Resource): parent_to_add = (exploit_telem[0].get('monkey_guid'), exploit_telem[0].get('data').get('exploiter')) else: parent_to_add = (parent, None) - elif (not parent or parent == monkey_json.get('guid')) and 'ip_addresses' in monkey_json: + elif (not parent or parent == monkey_json.get('guid')) and 'ip_addresses' in monkey_json: exploit_telem = [x for x in mongo.db.telemetry.find({'telem_type': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}})] diff --git a/monkey/monkey_island/cc/services/monkey_timeout.py b/monkey/monkey_island/cc/services/monkey_timeout.py new file mode 100644 index 000000000..2c3bf4637 --- /dev/null +++ b/monkey/monkey_island/cc/services/monkey_timeout.py @@ -0,0 +1,22 @@ +import functools +import pprint + + +def start_timer_decorator(func): + @functools.wraps(func) + def wrapper_decorator(*args, **kwargs): + print("ib4get - start the timer, folks. \nargs:") + pprint.pprint(args) + print("kwargs: ") + pprint.pprint(kwargs) + value = func(*args, **kwargs) + print("after party woohoo") + + try: + print("Starting timer on " + kwargs['guid']) + except KeyError as e: + print("NO GUID AVAILABLE") + + return value + + return wrapper_decorator From 03420aae50ddeb2f1f2dd0cd85a30b889287d260 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 5 May 2019 20:47:36 +0300 Subject: [PATCH 04/26] Update const of timeout --- monkey/infection_monkey/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/control.py b/monkey/infection_monkey/control.py index 98ad55671..d66ab9d69 100644 --- a/monkey/infection_monkey/control.py +++ b/monkey/infection_monkey/control.py @@ -21,7 +21,7 @@ LOG = logging.getLogger(__name__) DOWNLOAD_CHUNK = 1024 # random number greater than 5, # to prevent the monkey from just waiting forever to try and connect to an island before going elsewhere. -TIMEOUT = 15 +TIMEOUT_IN_SECONDS = 15 class ControlClient(object): @@ -76,7 +76,7 @@ class ControlClient(object): requests.get("https://%s/api?action=is-up" % (server,), verify=False, proxies=ControlClient.proxies, - timeout=TIMEOUT) + timeout=TIMEOUT_IN_SECONDS) WormConfiguration.current_server = current_server break From fd2e0887fff22809be8d6debfa1c737f3dc74716 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 5 May 2019 20:48:05 +0300 Subject: [PATCH 05/26] Refactor Environment to enable access to mongo url --- monkey/monkey_island/cc/environment/__init__.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/environment/__init__.py b/monkey/monkey_island/cc/environment/__init__.py index 62b0e9eed..8a3f61b21 100644 --- a/monkey/monkey_island/cc/environment/__init__.py +++ b/monkey/monkey_island/cc/environment/__init__.py @@ -10,7 +10,10 @@ class Environment(object): __metaclass__ = abc.ABCMeta _ISLAND_PORT = 5000 - _MONGO_URL = os.environ.get("MONKEY_MONGO_URL", "mongodb://localhost:27017/monkeyisland") + _MONGO_DB_NAME = "monkeyisland" + _MONGO_DB_HOST = "localhost" + _MONGO_DB_PORT = 27017 + _MONGO_URL = os.environ.get("MONKEY_MONGO_URL", "mongodb://{0}:{1}/{2}".format(_MONGO_DB_HOST, _MONGO_DB_PORT, str(_MONGO_DB_NAME))) _DEBUG_SERVER = False _AUTH_EXPIRATION_TIME = timedelta(hours=1) @@ -40,3 +43,15 @@ class Environment(object): @abc.abstractmethod def get_auth_users(self): return + + @property + def mongo_db_name(self): + return self._MONGO_DB_NAME + + @property + def mongo_db_host(self): + return self._MONGO_DB_HOST + + @property + def mongo_db_port(self): + return self._MONGO_DB_PORT From 1018906602e1a99b55cfae8a5a7a60234448c12f Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 5 May 2019 20:50:11 +0300 Subject: [PATCH 06/26] Added models using mongoengine and started using them in the code, and added TTL field TTL doesn't get expired for some reason, trying to solve in https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents --- monkey/monkey_island/cc/models/__init__.py | 1 + monkey/monkey_island/cc/models/monkey.py | 89 +++++++++++++++++++++ monkey/monkey_island/cc/resources/monkey.py | 10 ++- monkey/monkey_island/cc/services/node.py | 5 +- monkey/monkey_island/requirements.txt | 1 + 5 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 monkey/monkey_island/cc/models/__init__.py create mode 100644 monkey/monkey_island/cc/models/monkey.py diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py new file mode 100644 index 000000000..c02d7f0c1 --- /dev/null +++ b/monkey/monkey_island/cc/models/__init__.py @@ -0,0 +1 @@ +from monkey import Monkey diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py new file mode 100644 index 000000000..d54fe1b43 --- /dev/null +++ b/monkey/monkey_island/cc/models/monkey.py @@ -0,0 +1,89 @@ +""" +Define a Document Schema for the Monkey document. +""" +import mongoengine +from mongoengine import Document, StringField, ListField, BooleanField, EmbeddedDocumentField, DateField, \ + EmbeddedDocument, connect, ReferenceField, DateTimeField + +from monkey_island.cc.environment.environment import env + +connect(db=env.mongo_db_name, host=env.mongo_db_host, port=env.mongo_db_port) + + +class Config(EmbeddedDocument): + """ + No need to define this schema here. It will change often and is already is defined in + monkey_island.cc.services.config_schema. + See https://mongoengine-odm.readthedocs.io/apireference.html#mongoengine.FieldDoesNotExist + """ + meta = {'strict': False} + pass + + +class Creds(EmbeddedDocument): + """ + TODO get an example of this data + """ + meta = {'strict': False} + pass + + +class PbaResults(EmbeddedDocument): + ip = StringField() + hostname = StringField() + command = StringField() + name = StringField() + result = ListField() + + +class Ttl(Document): + meta = { + 'indexes': [ + { + 'name': 'TTL_index', + 'fields': ['expire_at'], + 'expireAfterSeconds': 0 + } + ] + } + + expire_at = DateTimeField() + + +class Monkey(Document): + """ + This class has 2 main section: + * The schema section defines the DB fields in the document. This is the data of the object. + * The logic section defines complex questions we can ask about a single document which are asked multiple + times, somewhat like an API. + """ + guid = StringField(required=True) + config = EmbeddedDocumentField('Config') + creds = ListField(EmbeddedDocumentField('Creds')) + dead = BooleanField() + description = StringField() + hostname = StringField() + internet_access = BooleanField() + ip_addresses = ListField(StringField()) + keepalive = DateField() + modifytime = DateField() + # TODO change this to an embedded document as well - RN it's an unnamed tuple which is confusing. + parent = ListField(ListField(StringField())) + config_error = BooleanField() + critical_services = ListField(StringField()) + pba_results = ListField() + ttl_ref = ReferenceField(Ttl) + + def is_dead(self): + monkey_is_dead = False + if self.dead: + monkey_is_dead = True + else: + try: + if Ttl.objects(id=self.ttl_ref.id).count() == 0: + # No TTLs - monkey has timed out. The monkey is MIA + monkey_is_dead = True + except mongoengine.DoesNotExist: + # Trying to dereference unknown document + monkey_is_dead = True + return monkey_is_dead diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 057ebf149..c6e7af908 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -1,13 +1,13 @@ import json -from datetime import datetime +from datetime import datetime, timedelta import dateutil.parser import flask_restful from flask import request +from monkey_island.cc import models from monkey_island.cc.database import mongo from monkey_island.cc.services.config import ConfigService -from monkey_island.cc.services.monkey_timeout import start_timer_decorator from monkey_island.cc.services.node import NodeService __author__ = 'Barak' @@ -18,7 +18,6 @@ __author__ = 'Barak' class Monkey(flask_restful.Resource): # Used by monkey. can't secure. - @start_timer_decorator def get(self, guid=None, **kw): NodeService.update_dead_monkeys() # refresh monkeys status if not guid: @@ -49,6 +48,11 @@ class Monkey(flask_restful.Resource): tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "") NodeService.set_monkey_tunnel(monkey["_id"], tunnel_host_ip) + current_ttl = models.monkey.Ttl(expire_at=datetime.now() + timedelta(seconds=30)) + current_ttl.save() + + update['$set']['ttl_ref'] = current_ttl.id + return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False) # Used by monkey. can't secure. diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index fa500aab5..83474a2c3 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -7,6 +7,7 @@ from monkey_island.cc.database import mongo from monkey_island.cc.services.edge import EdgeService from monkey_island.cc.utils import local_ip_addresses import socket +from monkey_island.cc import models __author__ = "itay.mizeretz" @@ -141,7 +142,7 @@ class NodeService: "label": label, "group": NodeService.get_monkey_group(monkey), "os": NodeService.get_monkey_os(monkey), - "dead": monkey["dead"], + "dead": models.Monkey.objects(id=monkey["_id"])[0].is_dead(), "domain_name": "", "pba_results": monkey["pba_results"] if "pba_results" in monkey else [] } @@ -293,7 +294,7 @@ class NodeService: @staticmethod def is_any_monkey_alive(): - return mongo.db.monkey.find_one({'dead': False}) is not None + return models.Monkey.objects(dead=False).count() > 0 @staticmethod def is_any_monkey_exists(): diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index 5e0faea19..953cad2cb 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -22,3 +22,4 @@ bson cffi virtualenv wheel +mongoengine From 0602a3bc83f4ed61b9de09d3c902a31901c44b1c Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 11:51:17 +0300 Subject: [PATCH 07/26] Renamed collection from TTL to MonkeyTTL --- monkey/monkey_island/cc/models/monkey.py | 8 ++++---- monkey/monkey_island/cc/resources/monkey.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index d54fe1b43..73f924847 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -22,7 +22,7 @@ class Config(EmbeddedDocument): class Creds(EmbeddedDocument): """ - TODO get an example of this data + TODO get an example of this data, and make it strict """ meta = {'strict': False} pass @@ -36,7 +36,7 @@ class PbaResults(EmbeddedDocument): result = ListField() -class Ttl(Document): +class MonkeyTtl(Document): meta = { 'indexes': [ { @@ -72,7 +72,7 @@ class Monkey(Document): config_error = BooleanField() critical_services = ListField(StringField()) pba_results = ListField() - ttl_ref = ReferenceField(Ttl) + ttl_ref = ReferenceField(MonkeyTtl) def is_dead(self): monkey_is_dead = False @@ -80,7 +80,7 @@ class Monkey(Document): monkey_is_dead = True else: try: - if Ttl.objects(id=self.ttl_ref.id).count() == 0: + if MonkeyTtl.objects(id=self.ttl_ref.id).count() == 0: # No TTLs - monkey has timed out. The monkey is MIA monkey_is_dead = True except mongoengine.DoesNotExist: diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index c6e7af908..ace843c85 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -48,7 +48,7 @@ class Monkey(flask_restful.Resource): tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "") NodeService.set_monkey_tunnel(monkey["_id"], tunnel_host_ip) - current_ttl = models.monkey.Ttl(expire_at=datetime.now() + timedelta(seconds=30)) + current_ttl = models.monkey.MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) current_ttl.save() update['$set']['ttl_ref'] = current_ttl.id From 295525dfed928048330198e92b44288e3bd9a579 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 13:54:17 +0300 Subject: [PATCH 08/26] Extracted models to files, created TestingEnv, and added unit testing for Monkey.is_dead The init of models checks the env and sets up the DB connection. --- .../monkey_island/cc/environment/__init__.py | 10 ++++ .../cc/environment/environment.py | 6 +++ .../monkey_island/cc/environment/testing.py | 17 +++++++ monkey/monkey_island/cc/models/__init__.py | 15 ++++++ monkey/monkey_island/cc/models/config.py | 11 ++++ monkey/monkey_island/cc/models/creds.py | 9 ++++ monkey/monkey_island/cc/models/monkey.py | 50 ++----------------- monkey/monkey_island/cc/models/monkey_ttl.py | 15 ++++++ monkey/monkey_island/cc/models/pba_results.py | 9 ++++ monkey/monkey_island/cc/models/test_monkey.py | 37 ++++++++++++++ monkey/monkey_island/cc/resources/monkey.py | 5 +- monkey/monkey_island/requirements.txt | 1 + 12 files changed, 137 insertions(+), 48 deletions(-) create mode 100644 monkey/monkey_island/cc/environment/testing.py create mode 100644 monkey/monkey_island/cc/models/config.py create mode 100644 monkey/monkey_island/cc/models/creds.py create mode 100644 monkey/monkey_island/cc/models/monkey_ttl.py create mode 100644 monkey/monkey_island/cc/models/pba_results.py create mode 100644 monkey/monkey_island/cc/models/test_monkey.py diff --git a/monkey/monkey_island/cc/environment/__init__.py b/monkey/monkey_island/cc/environment/__init__.py index 8a3f61b21..e30e0eebd 100644 --- a/monkey/monkey_island/cc/environment/__init__.py +++ b/monkey/monkey_island/cc/environment/__init__.py @@ -16,9 +16,19 @@ class Environment(object): _MONGO_URL = os.environ.get("MONKEY_MONGO_URL", "mongodb://{0}:{1}/{2}".format(_MONGO_DB_HOST, _MONGO_DB_PORT, str(_MONGO_DB_NAME))) _DEBUG_SERVER = False _AUTH_EXPIRATION_TIME = timedelta(hours=1) + _testing = False + + @property + def testing(self): + return self._testing + + @testing.setter + def testing(self, value): + self._testing = value def __init__(self): self.config = None + self._testing = False # Assume env is not for unit testing. def set_config(self, config): self.config = config diff --git a/monkey/monkey_island/cc/environment/environment.py b/monkey/monkey_island/cc/environment/environment.py index 3cd6bb587..6115e8dd9 100644 --- a/monkey/monkey_island/cc/environment/environment.py +++ b/monkey/monkey_island/cc/environment/environment.py @@ -2,7 +2,10 @@ import json import logging import os +env = None + from monkey_island.cc.environment import standard +from monkey_island.cc.environment import testing from monkey_island.cc.environment import aws from monkey_island.cc.environment import password from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH @@ -14,11 +17,13 @@ logger = logging.getLogger(__name__) AWS = 'aws' STANDARD = 'standard' PASSWORD = 'password' +TESTING = 'testing' ENV_DICT = { STANDARD: standard.StandardEnvironment, AWS: aws.AwsEnvironment, PASSWORD: password.PasswordEnvironment, + TESTING: testing.TestingEnvironment } @@ -32,6 +37,7 @@ def load_env_from_file(): config_json = load_server_configuration_from_file() return config_json['server_config'] + try: config_json = load_server_configuration_from_file() __env_type = config_json['server_config'] diff --git a/monkey/monkey_island/cc/environment/testing.py b/monkey/monkey_island/cc/environment/testing.py new file mode 100644 index 000000000..286e442dd --- /dev/null +++ b/monkey/monkey_island/cc/environment/testing.py @@ -0,0 +1,17 @@ +from monkey_island.cc.environment import Environment +import monkey_island.cc.auth + + +class TestingEnvironment(Environment): + def __init__(self): + super(TestingEnvironment, self).__init__() + self.testing = True + + # SHA3-512 of '1234567890!@#$%^&*()_nothing_up_my_sleeve_1234567890!@#$%^&*()' + NO_AUTH_CREDS = '55e97c9dcfd22b8079189ddaeea9bce8125887e3237b800c6176c9afa80d2062' \ + '8d2c8d0b1538d2208c1444ac66535b764a3d902b35e751df3faec1e477ed3557' + + def get_auth_users(self): + return [ + monkey_island.cc.auth.User(1, self.NO_AUTH_CREDS, self.NO_AUTH_CREDS) + ] diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index c02d7f0c1..673b6454a 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -1 +1,16 @@ +from mongoengine import connect + +# set up the DB connection. +from monkey_island.cc.environment.environment import env + +if env.testing: + connect('mongoenginetest', host='mongomock://localhost') +else: + connect(db=env.mongo_db_name, host=env.mongo_db_host, port=env.mongo_db_port) + +# Order or importing matters, for registering the embedded and referenced documents before using them. +from config import Config +from creds import Creds +from monkey_ttl import MonkeyTtl +from pba_results import PbaResults from monkey import Monkey diff --git a/monkey/monkey_island/cc/models/config.py b/monkey/monkey_island/cc/models/config.py new file mode 100644 index 000000000..cfe128111 --- /dev/null +++ b/monkey/monkey_island/cc/models/config.py @@ -0,0 +1,11 @@ +from mongoengine import EmbeddedDocument + + +class Config(EmbeddedDocument): + """ + No need to define this schema here. It will change often and is already is defined in + monkey_island.cc.services.config_schema. + See https://mongoengine-odm.readthedocs.io/apireference.html#mongoengine.FieldDoesNotExist + """ + meta = {'strict': False} + pass diff --git a/monkey/monkey_island/cc/models/creds.py b/monkey/monkey_island/cc/models/creds.py new file mode 100644 index 000000000..61322362e --- /dev/null +++ b/monkey/monkey_island/cc/models/creds.py @@ -0,0 +1,9 @@ +from mongoengine import EmbeddedDocument + + +class Creds(EmbeddedDocument): + """ + TODO get an example of this data, and make it strict + """ + meta = {'strict': False} + pass diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index 73f924847..f2a8ae5a4 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -3,51 +3,9 @@ Define a Document Schema for the Monkey document. """ import mongoengine from mongoengine import Document, StringField, ListField, BooleanField, EmbeddedDocumentField, DateField, \ - EmbeddedDocument, connect, ReferenceField, DateTimeField + ReferenceField -from monkey_island.cc.environment.environment import env - -connect(db=env.mongo_db_name, host=env.mongo_db_host, port=env.mongo_db_port) - - -class Config(EmbeddedDocument): - """ - No need to define this schema here. It will change often and is already is defined in - monkey_island.cc.services.config_schema. - See https://mongoengine-odm.readthedocs.io/apireference.html#mongoengine.FieldDoesNotExist - """ - meta = {'strict': False} - pass - - -class Creds(EmbeddedDocument): - """ - TODO get an example of this data, and make it strict - """ - meta = {'strict': False} - pass - - -class PbaResults(EmbeddedDocument): - ip = StringField() - hostname = StringField() - command = StringField() - name = StringField() - result = ListField() - - -class MonkeyTtl(Document): - meta = { - 'indexes': [ - { - 'name': 'TTL_index', - 'fields': ['expire_at'], - 'expireAfterSeconds': 0 - } - ] - } - - expire_at = DateTimeField() +from monkey_island.cc.models.monkey_ttl import MonkeyTtl class Monkey(Document): @@ -81,9 +39,9 @@ class Monkey(Document): else: try: if MonkeyTtl.objects(id=self.ttl_ref.id).count() == 0: - # No TTLs - monkey has timed out. The monkey is MIA + # No TTLs - monkey has timed out. The monkey is MIA. monkey_is_dead = True except mongoengine.DoesNotExist: - # Trying to dereference unknown document + # Trying to dereference unknown document - the monkey is MIA. monkey_is_dead = True return monkey_is_dead diff --git a/monkey/monkey_island/cc/models/monkey_ttl.py b/monkey/monkey_island/cc/models/monkey_ttl.py new file mode 100644 index 000000000..e6d867628 --- /dev/null +++ b/monkey/monkey_island/cc/models/monkey_ttl.py @@ -0,0 +1,15 @@ +from mongoengine import Document, DateTimeField + + +class MonkeyTtl(Document): + meta = { + 'indexes': [ + { + 'name': 'TTL_index', + 'fields': ['expire_at'], + 'expireAfterSeconds': 0 + } + ] + } + + expire_at = DateTimeField() diff --git a/monkey/monkey_island/cc/models/pba_results.py b/monkey/monkey_island/cc/models/pba_results.py new file mode 100644 index 000000000..d2cc48080 --- /dev/null +++ b/monkey/monkey_island/cc/models/pba_results.py @@ -0,0 +1,9 @@ +from mongoengine import EmbeddedDocument, StringField, ListField + + +class PbaResults(EmbeddedDocument): + ip = StringField() + hostname = StringField() + command = StringField() + name = StringField() + result = ListField() diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py new file mode 100644 index 000000000..a79bf9bbb --- /dev/null +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -0,0 +1,37 @@ +import uuid +from datetime import timedelta, datetime +from time import sleep +from unittest import TestCase + +# noinspection PyUnresolvedReferences +import mongomock + +from monkey import Monkey +from monkey_ttl import MonkeyTtl + + +class TestMonkey(TestCase): + def test_is_dead(self): + alive_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) + alive_monkey_ttl.save() + alive_monkey = Monkey( + guid=str(uuid.uuid4()), + dead=False, + ttl_ref=alive_monkey_ttl.id) + alive_monkey.save() + + mia_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) + mia_monkey_ttl.save() + mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl) + mia_monkey.save() + # Emulate timeout + sleep(1) + mia_monkey_ttl.delete() + + dead_monkey = Monkey(guid=str(uuid.uuid4()), dead=True) + dead_monkey.save() + + self.assertTrue(dead_monkey.is_dead()) + self.assertTrue(mia_monkey.is_dead()) + self.assertFalse(alive_monkey.is_dead()) + diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index ace843c85..9a028e71e 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -5,7 +5,7 @@ import dateutil.parser import flask_restful from flask import request -from monkey_island.cc import models +from monkey_island.cc.models.monkey_ttl import MonkeyTtl from monkey_island.cc.database import mongo from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService @@ -48,7 +48,8 @@ class Monkey(flask_restful.Resource): tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "") NodeService.set_monkey_tunnel(monkey["_id"], tunnel_host_ip) - current_ttl = models.monkey.MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) + # The TTL data uses the new `models` module which depends on mongoengine. + current_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) current_ttl.save() update['$set']['ttl_ref'] = current_ttl.id diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index 953cad2cb..6b0b68031 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -23,3 +23,4 @@ cffi virtualenv wheel mongoengine +mongomock From 3c6cda03af2cf80914e7a3973bbe075cc2c1d9ea Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 15:56:27 +0300 Subject: [PATCH 09/26] Added get_single_monkey_by_id API to the monkey model Useful in many cases in the code. Also added unittest for this method which passed. --- monkey/monkey_island/cc/models/errors.py | 2 ++ monkey/monkey_island/cc/models/monkey.py | 10 +++++++++- monkey/monkey_island/cc/models/test_monkey.py | 7 +++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 monkey/monkey_island/cc/models/errors.py diff --git a/monkey/monkey_island/cc/models/errors.py b/monkey/monkey_island/cc/models/errors.py new file mode 100644 index 000000000..ff9eb13ea --- /dev/null +++ b/monkey/monkey_island/cc/models/errors.py @@ -0,0 +1,2 @@ +class MonkeyNotFoundError(Exception): + pass \ No newline at end of file diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index f2a8ae5a4..c0e4e88b2 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -5,6 +5,7 @@ import mongoengine from mongoengine import Document, StringField, ListField, BooleanField, EmbeddedDocumentField, DateField, \ ReferenceField +from monkey_island.cc.models.errors import MonkeyNotFoundError from monkey_island.cc.models.monkey_ttl import MonkeyTtl @@ -32,6 +33,13 @@ class Monkey(Document): pba_results = ListField() ttl_ref = ReferenceField(MonkeyTtl) + @staticmethod + def get_single_monkey_by_id(db_id): + try: + return Monkey.objects(id=db_id)[0] + except IndexError: + raise MonkeyNotFoundError("id: {0}".format(str(db_id))) + def is_dead(self): monkey_is_dead = False if self.dead: @@ -41,7 +49,7 @@ class Monkey(Document): if MonkeyTtl.objects(id=self.ttl_ref.id).count() == 0: # No TTLs - monkey has timed out. The monkey is MIA. monkey_is_dead = True - except mongoengine.DoesNotExist: + except (mongoengine.DoesNotExist, AttributeError): # Trying to dereference unknown document - the monkey is MIA. monkey_is_dead = True return monkey_is_dead diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index a79bf9bbb..31f3ff511 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -7,6 +7,7 @@ from unittest import TestCase import mongomock from monkey import Monkey +from monkey_island.cc.models.errors import MonkeyNotFoundError from monkey_ttl import MonkeyTtl @@ -35,3 +36,9 @@ class TestMonkey(TestCase): self.assertTrue(mia_monkey.is_dead()) self.assertFalse(alive_monkey.is_dead()) + def test_get_single_monkey_by_id(self): + a_monkey = Monkey(guid=str(uuid.uuid4())) + a_monkey.save() + + self.assertIsNotNone(Monkey.get_single_monkey_by_id(a_monkey.id)) + self.assertRaises(MonkeyNotFoundError, Monkey.get_single_monkey_by_id, "abcdefabcdefabcdefabcdef") From 0e4dbfb078567e6594441ff57fd7d1fb797d00df Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 15:57:11 +0300 Subject: [PATCH 10/26] Replacing existing code with the new model API Only for is_dead calls --- monkey/monkey_island/cc/resources/local_run.py | 3 ++- monkey/monkey_island/cc/services/node.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/resources/local_run.py b/monkey/monkey_island/cc/resources/local_run.py index d402a440c..54a16f518 100644 --- a/monkey/monkey_island/cc/resources/local_run.py +++ b/monkey/monkey_island/cc/resources/local_run.py @@ -7,6 +7,7 @@ from flask import request, jsonify, make_response import flask_restful from monkey_island.cc.environment.environment import env +from monkey_island.cc.models import Monkey from monkey_island.cc.resources.monkey_download import get_monkey_executable from monkey_island.cc.services.node import NodeService from monkey_island.cc.utils import local_ip_addresses @@ -57,7 +58,7 @@ class LocalRun(flask_restful.Resource): NodeService.update_dead_monkeys() island_monkey = NodeService.get_monkey_island_monkey() if island_monkey is not None: - is_monkey_running = not island_monkey["dead"] + is_monkey_running = not Monkey.get_single_monkey_by_id(island_monkey["_id"]).is_dead() else: is_monkey_running = False diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index 83474a2c3..a9dde51e2 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -4,6 +4,7 @@ from bson import ObjectId import monkey_island.cc.services.log from monkey_island.cc.database import mongo +from monkey_island.cc.models import Monkey from monkey_island.cc.services.edge import EdgeService from monkey_island.cc.utils import local_ip_addresses import socket @@ -124,7 +125,7 @@ class NodeService: monkey_type = "manual" if NodeService.get_monkey_manual_run(monkey) else "monkey" monkey_os = NodeService.get_monkey_os(monkey) - monkey_running = "" if monkey["dead"] else "_running" + monkey_running = "" if Monkey.get_single_monkey_by_id(monkey["_id"]).is_dead() else "_running" return "%s_%s%s" % (monkey_type, monkey_os, monkey_running) @staticmethod @@ -136,13 +137,14 @@ class NodeService: @staticmethod def monkey_to_net_node(monkey, for_report=False): label = monkey['hostname'] if for_report else NodeService.get_monkey_label(monkey) + is_monkey_dead = Monkey.get_single_monkey_by_id(monkey["_id"]).is_dead() return \ { "id": monkey["_id"], "label": label, "group": NodeService.get_monkey_group(monkey), "os": NodeService.get_monkey_os(monkey), - "dead": models.Monkey.objects(id=monkey["_id"])[0].is_dead(), + "dead": is_monkey_dead, "domain_name": "", "pba_results": monkey["pba_results"] if "pba_results" in monkey else [] } From 00a4ffd028a5a67a6d5dfff2d7a19853744e6ad6 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 15:58:05 +0300 Subject: [PATCH 11/26] The bug was found, TTL now works! should have been utcnow(). see https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents Feature was tested locally and works! --- monkey/monkey_island/cc/resources/monkey.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 9a028e71e..43c73692b 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -10,6 +10,8 @@ from monkey_island.cc.database import mongo from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService +MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 30 + __author__ = 'Barak' # TODO: separate logic from interface @@ -49,7 +51,9 @@ class Monkey(flask_restful.Resource): NodeService.set_monkey_tunnel(monkey["_id"], tunnel_host_ip) # The TTL data uses the new `models` module which depends on mongoengine. - current_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) + # Using UTC to make the mongodb TTL feature work. See + # https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents. + current_ttl = MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)) current_ttl.save() update['$set']['ttl_ref'] = current_ttl.id From dc8f9294cddbd63a5430e9cf97d6a8371e8a5931 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 15:58:44 +0300 Subject: [PATCH 12/26] Added documentation about mongo connection setup --- monkey/monkey_island/cc/models/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 673b6454a..8db8c6f23 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -1,14 +1,17 @@ from mongoengine import connect -# set up the DB connection. from monkey_island.cc.environment.environment import env +# This section sets up the DB connection according to the environment. +# If testing, use mongomock which only emulates mongo. for more information, see +# http://docs.mongoengine.org/guide/mongomock.html . +# Otherwise, use an actual mongod instance with connection parameters supplied by env. if env.testing: connect('mongoenginetest', host='mongomock://localhost') else: connect(db=env.mongo_db_name, host=env.mongo_db_host, port=env.mongo_db_port) -# Order or importing matters, for registering the embedded and referenced documents before using them. +# Order or importing matters here, for registering the embedded and referenced documents before using them. from config import Config from creds import Creds from monkey_ttl import MonkeyTtl From 81e49b45ce9146250fd2f798e549418f1d44afb3 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 15:59:16 +0300 Subject: [PATCH 13/26] Small fix Overshadows local --- monkey/monkey_island/cc/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/auth.py b/monkey/monkey_island/cc/auth.py index 2e7eb69ff..7f15cb45e 100644 --- a/monkey/monkey_island/cc/auth.py +++ b/monkey/monkey_island/cc/auth.py @@ -10,8 +10,8 @@ __author__ = 'itay.mizeretz' class User(object): - def __init__(self, id, username, secret): - self.id = id + def __init__(self, user_id, username, secret): + self.id = user_id self.username = username self.secret = secret From bf3b2df253021b6d6dc6356b70cb48ae2b7a316e Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 17:21:05 +0300 Subject: [PATCH 14/26] Added section markers to monkey model --- monkey/monkey_island/cc/models/monkey.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index c0e4e88b2..68bdf6154 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -16,6 +16,7 @@ class Monkey(Document): * The logic section defines complex questions we can ask about a single document which are asked multiple times, somewhat like an API. """ + # SCHEMA guid = StringField(required=True) config = EmbeddedDocumentField('Config') creds = ListField(EmbeddedDocumentField('Creds')) @@ -33,6 +34,7 @@ class Monkey(Document): pba_results = ListField() ttl_ref = ReferenceField(MonkeyTtl) + # LOGIC @staticmethod def get_single_monkey_by_id(db_id): try: From 29b7bb1adf6e234ce589b3c9d26fccddc77514b1 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 17:31:15 +0300 Subject: [PATCH 15/26] Added documentation --- monkey/monkey_island/cc/models/monkey_ttl.py | 9 +++++++++ monkey/monkey_island/cc/models/test_monkey.py | 9 ++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/models/monkey_ttl.py b/monkey/monkey_island/cc/models/monkey_ttl.py index e6d867628..853edb9b0 100644 --- a/monkey/monkey_island/cc/models/monkey_ttl.py +++ b/monkey/monkey_island/cc/models/monkey_ttl.py @@ -2,6 +2,15 @@ from mongoengine import Document, DateTimeField class MonkeyTtl(Document): + """ + This model represents the monkey's TTL, and is referenced by the main Monkey document. + See https://docs.mongodb.com/manual/tutorial/expire-data/ and + https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents/56021663#56021663 + for more information about how TTL indexing works. + + When initializing this object, do it like so: + t = MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=XXX)) + """ meta = { 'indexes': [ { diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index 31f3ff511..2c4a171df 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -13,6 +13,7 @@ from monkey_ttl import MonkeyTtl class TestMonkey(TestCase): def test_is_dead(self): + # Arrange alive_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) alive_monkey_ttl.save() alive_monkey = Monkey( @@ -21,24 +22,30 @@ class TestMonkey(TestCase): ttl_ref=alive_monkey_ttl.id) alive_monkey.save() + # MIA stands for Missing In Action mia_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) mia_monkey_ttl.save() mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl) mia_monkey.save() - # Emulate timeout + # Emulate timeout - ttl is manually deleted here, since we're using mongomock and not a real mongo instance. sleep(1) mia_monkey_ttl.delete() dead_monkey = Monkey(guid=str(uuid.uuid4()), dead=True) dead_monkey.save() + # act + assert self.assertTrue(dead_monkey.is_dead()) self.assertTrue(mia_monkey.is_dead()) self.assertFalse(alive_monkey.is_dead()) def test_get_single_monkey_by_id(self): + # Arrange a_monkey = Monkey(guid=str(uuid.uuid4())) a_monkey.save() + # Act + assert + # Find the existing one self.assertIsNotNone(Monkey.get_single_monkey_by_id(a_monkey.id)) + # Raise on non-existent monkey self.assertRaises(MonkeyNotFoundError, Monkey.get_single_monkey_by_id, "abcdefabcdefabcdefabcdef") From 26bbf07d7507850c06d571e1bedc4123493fc925 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 17:31:38 +0300 Subject: [PATCH 16/26] Deleted unused file (early implementation idea) --- .../cc/services/monkey_timeout.py | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 monkey/monkey_island/cc/services/monkey_timeout.py diff --git a/monkey/monkey_island/cc/services/monkey_timeout.py b/monkey/monkey_island/cc/services/monkey_timeout.py deleted file mode 100644 index 2c3bf4637..000000000 --- a/monkey/monkey_island/cc/services/monkey_timeout.py +++ /dev/null @@ -1,22 +0,0 @@ -import functools -import pprint - - -def start_timer_decorator(func): - @functools.wraps(func) - def wrapper_decorator(*args, **kwargs): - print("ib4get - start the timer, folks. \nargs:") - pprint.pprint(args) - print("kwargs: ") - pprint.pprint(kwargs) - value = func(*args, **kwargs) - print("after party woohoo") - - try: - print("Starting timer on " + kwargs['guid']) - except KeyError as e: - print("NO GUID AVAILABLE") - - return value - - return wrapper_decorator From 5ec82647e4926deea8c9b076330e9073c0207c7a Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 17:32:03 +0300 Subject: [PATCH 17/26] Increased expiry time to fit production scenarios --- monkey/monkey_island/cc/resources/monkey.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 43c73692b..e1f52f73e 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -10,7 +10,7 @@ from monkey_island.cc.database import mongo from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService -MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 30 +MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 120 __author__ = 'Barak' From 963305db6ed3370dfaaca0e5e70fc3c26aa652b9 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 17:36:22 +0300 Subject: [PATCH 18/26] Added usage documentation to the unit tests --- monkey/monkey_island/cc/models/test_monkey.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index 2c4a171df..23f26addb 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -12,6 +12,13 @@ from monkey_ttl import MonkeyTtl class TestMonkey(TestCase): + """ + Make sure to set server environment to `testing` in server.json! Otherwise this will mess up your mongo instance and + won't work. + + Also, the working directory needs to be the working directory from which you usually run the island so the + server.json file is found and loaded. + """ def test_is_dead(self): # Arrange alive_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) From df049ef842ff53c9d0419634ac72d1f462e2cae6 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 7 May 2019 18:59:07 +0300 Subject: [PATCH 19/26] Fixed report page not using the new API Even though all monkeys have finished, the report page still used the dead=False check instead of is_dead. So even though all monkeys were dead or MIA the report page said that some monkeys are still runnning. --- monkey/monkey_island/cc/services/node.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index a9dde51e2..442fb391a 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -68,7 +68,7 @@ class NodeService: def get_node_label(node): domain_name = "" if node["domain_name"]: - domain_name = " ("+node["domain_name"]+")" + domain_name = " (" + node["domain_name"] + ")" return node["os"]["version"] + " : " + node["ip_addresses"][0] + domain_name @staticmethod @@ -106,7 +106,8 @@ class NodeService: @staticmethod def get_monkey_critical_services(monkey_id): - critical_services = mongo.db.monkey.find_one({'_id': monkey_id}, {'critical_services': 1}).get('critical_services', []) + critical_services = mongo.db.monkey.find_one({'_id': monkey_id}, {'critical_services': 1}).get( + 'critical_services', []) return critical_services @staticmethod @@ -296,7 +297,8 @@ class NodeService: @staticmethod def is_any_monkey_alive(): - return models.Monkey.objects(dead=False).count() > 0 + all_monkeys = models.Monkey.objects() + return any(not monkey.is_dead() for monkey in all_monkeys) @staticmethod def is_any_monkey_exists(): From 62b107884f5910b4e7fe9f406371ffa06a2ac3a2 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Thu, 23 May 2019 11:33:21 +0300 Subject: [PATCH 20/26] Added TTL creation to post as well --- monkey/monkey_island/cc/models/errors.py | 2 +- monkey/monkey_island/cc/resources/monkey.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/models/errors.py b/monkey/monkey_island/cc/models/errors.py index ff9eb13ea..69cb529dd 100644 --- a/monkey/monkey_island/cc/models/errors.py +++ b/monkey/monkey_island/cc/models/errors.py @@ -1,2 +1,2 @@ class MonkeyNotFoundError(Exception): - pass \ No newline at end of file + pass diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index e1f52f73e..e7744f007 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -17,6 +17,16 @@ __author__ = 'Barak' # TODO: separate logic from interface +def create_monkey_ttl(): + # The TTL data uses the new `models` module which depends on mongoengine. + # Using UTC to make the mongodb TTL feature work. See + # https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents. + current_ttl = MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)) + current_ttl.save() + ttlid = current_ttl.id + return ttlid + + class Monkey(flask_restful.Resource): # Used by monkey. can't secure. @@ -50,13 +60,8 @@ class Monkey(flask_restful.Resource): tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "") NodeService.set_monkey_tunnel(monkey["_id"], tunnel_host_ip) - # The TTL data uses the new `models` module which depends on mongoengine. - # Using UTC to make the mongodb TTL feature work. See - # https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents. - current_ttl = MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)) - current_ttl.save() - - update['$set']['ttl_ref'] = current_ttl.id + ttlid = create_monkey_ttl() + update['$set']['ttl_ref'] = ttlid return mongo.db.monkey.update({"_id": monkey["_id"]}, update, upsert=False) @@ -117,6 +122,8 @@ class Monkey(flask_restful.Resource): tunnel_host_ip = monkey_json['tunnel'].split(":")[-2].replace("//", "") monkey_json.pop('tunnel') + monkey_json['ttl_ref'] = create_monkey_ttl() + mongo.db.monkey.update({"guid": monkey_json["guid"]}, {"$set": monkey_json}, upsert=True) From bb40560fc75790ebb1594c3a78a2b809545cfdc8 Mon Sep 17 00:00:00 2001 From: Shay Nehmad <48879847+ShayNehmad@users.noreply.github.com> Date: Mon, 27 May 2019 16:37:10 +0300 Subject: [PATCH 21/26] Update monkey/monkey_island/cc/models/__init__.py Fixed type Co-Authored-By: Itay Mizeretz <30774653+itaymmguardicore@users.noreply.github.com> --- monkey/monkey_island/cc/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 8db8c6f23..9d69a51fc 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -11,7 +11,7 @@ if env.testing: else: connect(db=env.mongo_db_name, host=env.mongo_db_host, port=env.mongo_db_port) -# Order or importing matters here, for registering the embedded and referenced documents before using them. +# Order of importing matters here, for registering the embedded and referenced documents before using them. from config import Config from creds import Creds from monkey_ttl import MonkeyTtl From 2ecc683216bdd0cbfd191a67213f742c8b27a7c3 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 27 May 2019 16:47:30 +0300 Subject: [PATCH 22/26] Moved monketnotfound to monkey.py CR comment --- monkey/monkey_island/cc/models/errors.py | 2 -- monkey/monkey_island/cc/models/monkey.py | 5 ++++- monkey/monkey_island/cc/models/test_monkey.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 monkey/monkey_island/cc/models/errors.py diff --git a/monkey/monkey_island/cc/models/errors.py b/monkey/monkey_island/cc/models/errors.py deleted file mode 100644 index 69cb529dd..000000000 --- a/monkey/monkey_island/cc/models/errors.py +++ /dev/null @@ -1,2 +0,0 @@ -class MonkeyNotFoundError(Exception): - pass diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index 68bdf6154..bb018caa9 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -5,7 +5,6 @@ import mongoengine from mongoengine import Document, StringField, ListField, BooleanField, EmbeddedDocumentField, DateField, \ ReferenceField -from monkey_island.cc.models.errors import MonkeyNotFoundError from monkey_island.cc.models.monkey_ttl import MonkeyTtl @@ -55,3 +54,7 @@ class Monkey(Document): # Trying to dereference unknown document - the monkey is MIA. monkey_is_dead = True return monkey_is_dead + + +class MonkeyNotFoundError(Exception): + pass diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index 23f26addb..ad5fc17de 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -7,7 +7,7 @@ from unittest import TestCase import mongomock from monkey import Monkey -from monkey_island.cc.models.errors import MonkeyNotFoundError +from monkey_island.cc.models.monkey import MonkeyNotFoundError from monkey_ttl import MonkeyTtl From 2e5864067b07b9040656dca15679998d8fbcaa33 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 27 May 2019 17:01:43 +0300 Subject: [PATCH 23/26] Created helper constructor for MonkeyTTL to make it more friendly to use CR comment --- monkey/monkey_island/cc/models/monkey_ttl.py | 20 +++++++++++++++---- monkey/monkey_island/cc/models/test_monkey.py | 4 ++-- monkey/monkey_island/cc/resources/monkey.py | 4 +--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island/cc/models/monkey_ttl.py b/monkey/monkey_island/cc/models/monkey_ttl.py index 853edb9b0..20d3d3eb0 100644 --- a/monkey/monkey_island/cc/models/monkey_ttl.py +++ b/monkey/monkey_island/cc/models/monkey_ttl.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta + from mongoengine import Document, DateTimeField @@ -6,11 +8,21 @@ class MonkeyTtl(Document): This model represents the monkey's TTL, and is referenced by the main Monkey document. See https://docs.mongodb.com/manual/tutorial/expire-data/ and https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents/56021663#56021663 - for more information about how TTL indexing works. - - When initializing this object, do it like so: - t = MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=XXX)) + for more information about how TTL indexing works and why this class is set up the way it is. """ + + def __init__(self, expiry_in_seconds, *args, **values): + """ + Initializes a TTL object which will expire in expire_in_seconds seconds from when created. + Remember to call .save() on the object after creation. + :param expiry_in_seconds: How long should the TTL be in the DB, in seconds. Please take into consideration + that the cleanup thread of mongo might take extra time to delete the TTL from the DB. + """ + # Using UTC to make the mongodb TTL feature work. See + # https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents. + super(MonkeyTtl, self).__init__( + expire_at=datetime.utcnow() + timedelta(seconds=expiry_in_seconds), *args, **values) + meta = { 'indexes': [ { diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index ad5fc17de..8f54635c0 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -21,7 +21,7 @@ class TestMonkey(TestCase): """ def test_is_dead(self): # Arrange - alive_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) + alive_monkey_ttl = MonkeyTtl(30) alive_monkey_ttl.save() alive_monkey = Monkey( guid=str(uuid.uuid4()), @@ -30,7 +30,7 @@ class TestMonkey(TestCase): alive_monkey.save() # MIA stands for Missing In Action - mia_monkey_ttl = MonkeyTtl(expire_at=datetime.now() + timedelta(seconds=30)) + mia_monkey_ttl = MonkeyTtl(30) mia_monkey_ttl.save() mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl) mia_monkey.save() diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index e7744f007..a5544c5ce 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -19,9 +19,7 @@ __author__ = 'Barak' def create_monkey_ttl(): # The TTL data uses the new `models` module which depends on mongoengine. - # Using UTC to make the mongodb TTL feature work. See - # https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents. - current_ttl = MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS)) + current_ttl = MonkeyTtl(MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS) current_ttl.save() ttlid = current_ttl.id return ttlid From 591b75c55515436f73ecfb03de021da876a7065e Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 27 May 2019 17:20:11 +0300 Subject: [PATCH 24/26] Can't override init of Documents. Created factory function instead. --- monkey/monkey_island/cc/models/monkey_ttl.py | 10 +++++++--- monkey/monkey_island/cc/models/test_monkey.py | 5 ++--- monkey/monkey_island/cc/resources/monkey.py | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island/cc/models/monkey_ttl.py b/monkey/monkey_island/cc/models/monkey_ttl.py index 20d3d3eb0..63c1ad4db 100644 --- a/monkey/monkey_island/cc/models/monkey_ttl.py +++ b/monkey/monkey_island/cc/models/monkey_ttl.py @@ -9,9 +9,14 @@ class MonkeyTtl(Document): See https://docs.mongodb.com/manual/tutorial/expire-data/ and https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents/56021663#56021663 for more information about how TTL indexing works and why this class is set up the way it is. + + If you wish to use this class, you can create it using the create_ttl_expire_in(seconds) function. + If you wish to create an instance of this class directly, see the inner implementation of + create_ttl_expire_in(seconds) to see how to do so. """ - def __init__(self, expiry_in_seconds, *args, **values): + @staticmethod + def create_ttl_expire_in(expiry_in_seconds): """ Initializes a TTL object which will expire in expire_in_seconds seconds from when created. Remember to call .save() on the object after creation. @@ -20,8 +25,7 @@ class MonkeyTtl(Document): """ # Using UTC to make the mongodb TTL feature work. See # https://stackoverflow.com/questions/55994379/mongodb-ttl-index-doesnt-delete-expired-documents. - super(MonkeyTtl, self).__init__( - expire_at=datetime.utcnow() + timedelta(seconds=expiry_in_seconds), *args, **values) + return MonkeyTtl(expire_at=datetime.utcnow() + timedelta(seconds=expiry_in_seconds)) meta = { 'indexes': [ diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index 8f54635c0..41920d2a0 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -1,5 +1,4 @@ import uuid -from datetime import timedelta, datetime from time import sleep from unittest import TestCase @@ -21,7 +20,7 @@ class TestMonkey(TestCase): """ def test_is_dead(self): # Arrange - alive_monkey_ttl = MonkeyTtl(30) + alive_monkey_ttl = MonkeyTtl.create_ttl_expire_in(30) alive_monkey_ttl.save() alive_monkey = Monkey( guid=str(uuid.uuid4()), @@ -30,7 +29,7 @@ class TestMonkey(TestCase): alive_monkey.save() # MIA stands for Missing In Action - mia_monkey_ttl = MonkeyTtl(30) + mia_monkey_ttl = MonkeyTtl.create_ttl_expire_in(30) mia_monkey_ttl.save() mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl) mia_monkey.save() diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index a5544c5ce..0207709eb 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -1,12 +1,12 @@ import json -from datetime import datetime, timedelta +from datetime import datetime import dateutil.parser import flask_restful from flask import request -from monkey_island.cc.models.monkey_ttl import MonkeyTtl from monkey_island.cc.database import mongo +from monkey_island.cc.models.monkey_ttl import MonkeyTtl from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService @@ -19,7 +19,7 @@ __author__ = 'Barak' def create_monkey_ttl(): # The TTL data uses the new `models` module which depends on mongoengine. - current_ttl = MonkeyTtl(MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS) + current_ttl = MonkeyTtl.create_ttl_expire_in(MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS) current_ttl.save() ttlid = current_ttl.id return ttlid From 729c0c50b9dfda90775755e4742c7522536d39ba Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 27 May 2019 17:25:09 +0300 Subject: [PATCH 25/26] Deleted mongomock - not required. --- monkey/monkey_island/cc/models/monkey_ttl.py | 2 +- monkey/monkey_island/cc/models/test_monkey.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/models/monkey_ttl.py b/monkey/monkey_island/cc/models/monkey_ttl.py index 63c1ad4db..9ccf77974 100644 --- a/monkey/monkey_island/cc/models/monkey_ttl.py +++ b/monkey/monkey_island/cc/models/monkey_ttl.py @@ -12,7 +12,7 @@ class MonkeyTtl(Document): If you wish to use this class, you can create it using the create_ttl_expire_in(seconds) function. If you wish to create an instance of this class directly, see the inner implementation of - create_ttl_expire_in(seconds) to see how to do so. + create_ttl_expire_in(seconds) to see how to do so. """ @staticmethod diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index 41920d2a0..008fb0ce6 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -2,9 +2,6 @@ import uuid from time import sleep from unittest import TestCase -# noinspection PyUnresolvedReferences -import mongomock - from monkey import Monkey from monkey_island.cc.models.monkey import MonkeyNotFoundError from monkey_ttl import MonkeyTtl From 212ed68dc21f58f68682a84b731ecd807b05cbba Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 28 May 2019 16:44:49 +0300 Subject: [PATCH 26/26] Increased TTL to 5 minutes --- monkey/monkey_island/cc/resources/monkey.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 0207709eb..2f464f068 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -10,7 +10,7 @@ from monkey_island.cc.models.monkey_ttl import MonkeyTtl from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService -MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 120 +MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS = 60 * 5 __author__ = 'Barak'