diff --git a/monkey/monkey_island/cc/resources/bootloader.py b/monkey/monkey_island/cc/resources/bootloader.py index 66eb48ae1..9ef11cc25 100644 --- a/monkey/monkey_island/cc/resources/bootloader.py +++ b/monkey/monkey_island/cc/resources/bootloader.py @@ -18,7 +18,7 @@ class Bootloader(flask_restful.Resource): else: return make_response({"status": "OS_NOT_FOUND"}, 404) - resp = BootloaderService.parse_bootloader_data(data) + resp = BootloaderService.parse_bootloader_telem(data) if resp: return make_response({"status": "RUN"}, 200) diff --git a/monkey/monkey_island/cc/services/bootloader.py b/monkey/monkey_island/cc/services/bootloader.py index 96ecbf640..f84bcf745 100644 --- a/monkey/monkey_island/cc/services/bootloader.py +++ b/monkey/monkey_island/cc/services/bootloader.py @@ -1,7 +1,9 @@ from typing import Dict, List +from bson import ObjectId + from monkey_island.cc.database import mongo -from monkey_island.cc.services.node import NodeService +from monkey_island.cc.services.node import NodeService, NodeNotFoundException from monkey_island.cc.services.utils.node_groups import NodeGroups from monkey_island.cc.services.utils.bootloader_config import SUPPORTED_WINDOWS_VERSIONS, MIN_GLIBC_VERSION @@ -9,18 +11,42 @@ from monkey_island.cc.services.utils.bootloader_config import SUPPORTED_WINDOWS_ class BootloaderService: @staticmethod - def parse_bootloader_data(data: Dict) -> bool: - data['ips'] = BootloaderService.remove_local_ips(data['ips']) - if data['os_version'] == "": - data['os_version'] = "Unknown OS" - mongo.db.bootloader_telems.insert(data) - will_monkey_run = BootloaderService.is_os_compatible(data) - node = NodeService.get_or_create_node_from_bootloader_data(data, will_monkey_run) - group_keywords = [data['system'], 'monkey'] - group_keywords.append('starting') if will_monkey_run else group_keywords.append('old') - NodeService.set_node_group(node['_id'], NodeGroups.get_group_by_keywords(group_keywords)) + def parse_bootloader_telem(telem: Dict) -> bool: + telem['ips'] = BootloaderService.remove_local_ips(telem['ips']) + if telem['os_version'] == "": + telem['os_version'] = "Unknown OS" + + telem_id = BootloaderService.get_mongo_id_for_bootloader_telem(telem) + mongo.db.bootloader_telems.update({'_id': telem_id}, telem, upsert=True) + + will_monkey_run = BootloaderService.is_os_compatible(telem) + try: + node = NodeService.get_or_create_node_from_bootloader_telem(telem, will_monkey_run) + except NodeNotFoundException: + # Didn't find the node, but allow monkey to run anyways + return True + + node_group = BootloaderService.get_next_node_state(node, telem['system'], will_monkey_run) + if 'group' not in node or node['group'] != node_group.value: + NodeService.set_node_group(node['_id'], node_group) return will_monkey_run + @staticmethod + def get_next_node_state(node: Dict, system: str, will_monkey_run: bool) -> NodeGroups: + group_keywords = [system, 'monkey'] + if 'group' in node and node['group'] == 'island': + group_keywords.extend(['island', 'starting']) + else: + group_keywords.append('starting') if will_monkey_run else group_keywords.append('old') + node_group = NodeGroups.get_group_by_keywords(group_keywords) + return node_group + + @staticmethod + def get_mongo_id_for_bootloader_telem(bootloader_telem) -> ObjectId: + ip_hash = hex(hash(str(bootloader_telem['ips'])))[3:15] + hostname_hash = hex(hash(bootloader_telem['hostname']))[3:15] + return ObjectId(ip_hash + hostname_hash) + @staticmethod def is_os_compatible(bootloader_data) -> bool: if bootloader_data['system'] == 'windows': @@ -32,7 +58,6 @@ class BootloaderService: def is_windows_version_supported(windows_version) -> bool: return SUPPORTED_WINDOWS_VERSIONS.get(windows_version) - @staticmethod def is_glibc_supported(glibc_version_string) -> bool: glibc_version_string = glibc_version_string.lower() diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index 901df20ce..1f0723925 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -8,7 +8,7 @@ import monkey_island.cc.services.log from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey from monkey_island.cc.services.edge import EdgeService -from monkey_island.cc.utils import local_ip_addresses +from monkey_island.cc.utils import local_ip_addresses, is_local_ips from monkey_island.cc import models from monkey_island.cc.services.utils.node_groups import NodeGroups @@ -221,39 +221,42 @@ class NodeService: return mongo.db.node.find_one({"_id": new_node_insert_result.inserted_id}) @staticmethod - def create_node_from_bootloader_data(bootloader_data: Dict, will_monkey_run: bool): + def create_node_from_bootloader_telem(bootloader_telem: Dict, will_monkey_run: bool): new_node_insert_result = mongo.db.node.insert_one( { - "ip_addresses": bootloader_data['ips'], - "domain_name": bootloader_data['hostname'], + "ip_addresses": bootloader_telem['ips'], + "domain_name": bootloader_telem['hostname'], "will_monkey_run": will_monkey_run, "exploited": False, "creds": [], "os": { - "type": bootloader_data['system'], - "version": bootloader_data['os_version'] + "type": bootloader_telem['system'], + "version": bootloader_telem['os_version'] } }) return mongo.db.node.find_one({"_id": new_node_insert_result.inserted_id}) @staticmethod - def get_or_create_node_from_bootloader_data(bootloader_data: Dict, will_monkey_run: bool) -> Dict: - new_node = mongo.db.node.find_one({"domain_name": bootloader_data['hostname'], - "ip_addresses": bootloader_data['ips']}) + def get_or_create_node_from_bootloader_telem(bootloader_telem: Dict, will_monkey_run: bool) -> Dict: + if is_local_ips(bootloader_telem['ips']): + raise NodeNotFoundException("Bootloader ran on island, no need to create new node.") + #return NodeService.get_monkey_island_pseudo_net_node() + + new_node = mongo.db.node.find_one({"domain_name": bootloader_telem['hostname'], + "ip_addresses": bootloader_telem['ips']}) if new_node is None: - new_node = NodeService.create_node_from_bootloader_data(bootloader_data, will_monkey_run) - if bootloader_data['tunnel']: - dst_node = NodeService.get_node_or_monkey_by_ip(bootloader_data['tunnel']) + new_node = NodeService.create_node_from_bootloader_telem(bootloader_telem, will_monkey_run) + if bootloader_telem['tunnel']: + dst_node = NodeService.get_node_or_monkey_by_ip(bootloader_telem['tunnel']) else: dst_node = NodeService.get_monkey_island_node() edge = EdgeService.get_or_create_edge(new_node['_id'], dst_node['id']) mongo.db.edge.update({"_id": edge["_id"]}, - {'$set': {'tunnel': bool(bootloader_data['tunnel']), - 'ip_address': bootloader_data['ips'][0], + {'$set': {'tunnel': bool(bootloader_telem['tunnel']), + 'ip_address': bootloader_telem['ips'][0], 'group': NodeGroups.get_group_by_keywords(['island']).value}}, upsert=False) - return new_node @staticmethod @@ -403,3 +406,6 @@ class NodeService: @staticmethod def get_hostname_by_id(node_id): return NodeService.get_node_hostname(mongo.db.monkey.find_one({'_id': node_id}, {'hostname': 1})) + +class NodeNotFoundException(Exception): + pass diff --git a/monkey/monkey_island/cc/services/utils/node_groups.py b/monkey/monkey_island/cc/services/utils/node_groups.py index 751798b18..b5c25cd9e 100644 --- a/monkey/monkey_island/cc/services/utils/node_groups.py +++ b/monkey/monkey_island/cc/services/utils/node_groups.py @@ -5,6 +5,7 @@ from typing import List import collections +# This list must correspond to the one on front end in src/components/map/MapOptions.js class NodeGroups(Enum): CLEAN_UNKNOWN = 'clean_unknown' CLEAN_LINUX = 'clean_linux' @@ -14,8 +15,10 @@ class NodeGroups(Enum): ISLAND = 'island' ISLAND_MONKEY_LINUX = 'island_monkey_linux' ISLAND_MONKEY_LINUX_RUNNING = 'island_monkey_linux_running' + ISLAND_MONKEY_LINUX_STARTING = 'island_monkey_linux_starting' ISLAND_MONKEY_WINDOWS = 'island_monkey_windows' ISLAND_MONKEY_WINDOWS_RUNNING = 'island_monkey_windows_running' + ISLAND_MONKEY_WINDOWS_STARTING = 'island_monkey_windows_starting' MANUAL_LINUX = 'manual_linux' MANUAL_LINUX_RUNNING = 'manual_linux_running' MANUAL_WINDOWS = 'manual_windows' diff --git a/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js b/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js index 08e8afd0d..c1474f5ec 100644 --- a/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js +++ b/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js @@ -1,8 +1,10 @@ +// This list must correspond to the one on back end in cc/services/utils/node_groups.py const groupNames = ['clean_unknown', 'clean_linux', 'clean_windows', 'exploited_linux', 'exploited_windows', 'island', - 'island_monkey_linux', 'island_monkey_linux_running', 'island_monkey_windows', 'island_monkey_windows_running', - 'manual_linux', 'manual_linux_running', 'manual_windows', 'manual_windows_running', 'monkey_linux', - 'monkey_linux_running', 'monkey_windows', 'monkey_windows_running', 'monkey_windows_starting', - 'monkey_linux_starting', 'monkey_windows_old', 'monkey_linux_old' ]; + 'island_monkey_linux', 'island_monkey_linux_running', 'island_monkey_linux_starting', 'island_monkey_windows', + 'island_monkey_windows_running', 'island_monkey_windows_starting', 'manual_linux', 'manual_linux_running', + 'manual_windows', 'manual_windows_running', 'monkey_linux', 'monkey_linux_running', 'monkey_windows', + 'monkey_windows_running', 'monkey_windows_starting', 'monkey_linux_starting', 'monkey_windows_old', + 'monkey_linux_old' ]; let getGroupsOptions = () => { let groupOptions = {}; diff --git a/monkey/monkey_island/cc/ui/src/images/nodes/island_monkey_linux_starting.png b/monkey/monkey_island/cc/ui/src/images/nodes/island_monkey_linux_starting.png new file mode 100644 index 000000000..cd53d7cb7 Binary files /dev/null and b/monkey/monkey_island/cc/ui/src/images/nodes/island_monkey_linux_starting.png differ diff --git a/monkey/monkey_island/cc/ui/src/images/nodes/island_monkey_windows_starting.png b/monkey/monkey_island/cc/ui/src/images/nodes/island_monkey_windows_starting.png new file mode 100644 index 000000000..67129b5cc Binary files /dev/null and b/monkey/monkey_island/cc/ui/src/images/nodes/island_monkey_windows_starting.png differ diff --git a/monkey/monkey_island/cc/utils.py b/monkey/monkey_island/cc/utils.py index 01c69e648..5504c34b6 100644 --- a/monkey/monkey_island/cc/utils.py +++ b/monkey/monkey_island/cc/utils.py @@ -1,5 +1,7 @@ import socket import sys +from typing import List +import collections import array @@ -49,6 +51,11 @@ else: return result +def is_local_ips(ips: List) -> bool: + filtered_local_ips = [ip for ip in local_ip_addresses() if not ip.startswith('169.254')] + return collections.Counter(ips) == collections.Counter(filtered_local_ips) + + # The local IP addresses list should not change often. Therefore, we can cache the result and never call this function # more than once. This stopgap measure is here since this function is called a lot of times during the report # generation.