Fixed bugs related to bootloader on island

monkey/monkey_island/cc/resources/bootloader.py

@@ -18,7 +18,7 @@ class Bootloader(flask_restful.Resource):
         else:
             return make_response({"status": "OS_NOT_FOUND"}, 404)
 
-        resp = BootloaderService.parse_bootloader_data(data)
+        resp = BootloaderService.parse_bootloader_telem(data)
 
         if resp:
             return make_response({"status": "RUN"}, 200)

monkey/monkey_island/cc/services/bootloader.py

@@ -1,7 +1,9 @@
 from typing import Dict, List
 
+from bson import ObjectId
+
 from monkey_island.cc.database import mongo
-from monkey_island.cc.services.node import NodeService
+from monkey_island.cc.services.node import NodeService, NodeNotFoundException
 from monkey_island.cc.services.utils.node_groups import NodeGroups
 from monkey_island.cc.services.utils.bootloader_config import SUPPORTED_WINDOWS_VERSIONS, MIN_GLIBC_VERSION
 
@@ -9,18 +11,42 @@ from monkey_island.cc.services.utils.bootloader_config import SUPPORTED_WINDOWS_
 class BootloaderService:
 
     @staticmethod
-    def parse_bootloader_data(data: Dict) -> bool:
-        data['ips'] = BootloaderService.remove_local_ips(data['ips'])
-        if data['os_version'] == "":
-            data['os_version'] = "Unknown OS"
-        mongo.db.bootloader_telems.insert(data)
-        will_monkey_run = BootloaderService.is_os_compatible(data)
-        node = NodeService.get_or_create_node_from_bootloader_data(data, will_monkey_run)
-        group_keywords = [data['system'], 'monkey']
-        group_keywords.append('starting') if will_monkey_run else group_keywords.append('old')
-        NodeService.set_node_group(node['_id'], NodeGroups.get_group_by_keywords(group_keywords))
+    def parse_bootloader_telem(telem: Dict) -> bool:
+        telem['ips'] = BootloaderService.remove_local_ips(telem['ips'])
+        if telem['os_version'] == "":
+            telem['os_version'] = "Unknown OS"
+
+        telem_id = BootloaderService.get_mongo_id_for_bootloader_telem(telem)
+        mongo.db.bootloader_telems.update({'_id': telem_id}, telem, upsert=True)
+
+        will_monkey_run = BootloaderService.is_os_compatible(telem)
+        try:
+            node = NodeService.get_or_create_node_from_bootloader_telem(telem, will_monkey_run)
+        except NodeNotFoundException:
+            # Didn't find the node, but allow monkey to run anyways
+            return True
+
+        node_group = BootloaderService.get_next_node_state(node, telem['system'], will_monkey_run)
+        if 'group' not in node or node['group'] != node_group.value:
+            NodeService.set_node_group(node['_id'], node_group)
         return will_monkey_run
 
+    @staticmethod
+    def get_next_node_state(node: Dict, system: str, will_monkey_run: bool) -> NodeGroups:
+        group_keywords = [system, 'monkey']
+        if 'group' in node and node['group'] == 'island':
+            group_keywords.extend(['island', 'starting'])
+        else:
+            group_keywords.append('starting') if will_monkey_run else group_keywords.append('old')
+        node_group = NodeGroups.get_group_by_keywords(group_keywords)
+        return node_group
+
+    @staticmethod
+    def get_mongo_id_for_bootloader_telem(bootloader_telem) -> ObjectId:
+        ip_hash = hex(hash(str(bootloader_telem['ips'])))[3:15]
+        hostname_hash = hex(hash(bootloader_telem['hostname']))[3:15]
+        return ObjectId(ip_hash + hostname_hash)
+
     @staticmethod
     def is_os_compatible(bootloader_data) -> bool:
         if bootloader_data['system'] == 'windows':
@@ -32,7 +58,6 @@ class BootloaderService:
     def is_windows_version_supported(windows_version) -> bool:
         return SUPPORTED_WINDOWS_VERSIONS.get(windows_version)
 
-
     @staticmethod
     def is_glibc_supported(glibc_version_string) -> bool:
         glibc_version_string = glibc_version_string.lower()

monkey/monkey_island/cc/services/node.py

@@ -8,7 +8,7 @@ import monkey_island.cc.services.log
 from monkey_island.cc.database import mongo
 from monkey_island.cc.models import Monkey
 from monkey_island.cc.services.edge import EdgeService
-from monkey_island.cc.utils import local_ip_addresses
+from monkey_island.cc.utils import local_ip_addresses, is_local_ips
 from monkey_island.cc import models
 from monkey_island.cc.services.utils.node_groups import NodeGroups
 
@@ -221,39 +221,42 @@ class NodeService:
         return mongo.db.node.find_one({"_id": new_node_insert_result.inserted_id})
 
     @staticmethod
-    def create_node_from_bootloader_data(bootloader_data: Dict, will_monkey_run: bool):
+    def create_node_from_bootloader_telem(bootloader_telem: Dict, will_monkey_run: bool):
         new_node_insert_result = mongo.db.node.insert_one(
             {
-                "ip_addresses": bootloader_data['ips'],
-                "domain_name": bootloader_data['hostname'],
+                "ip_addresses": bootloader_telem['ips'],
+                "domain_name": bootloader_telem['hostname'],
                 "will_monkey_run": will_monkey_run,
                 "exploited": False,
                 "creds": [],
                 "os":
                     {
-                        "type": bootloader_data['system'],
-                        "version": bootloader_data['os_version']
+                        "type": bootloader_telem['system'],
+                        "version": bootloader_telem['os_version']
                     }
             })
         return mongo.db.node.find_one({"_id": new_node_insert_result.inserted_id})
 
     @staticmethod
-    def get_or_create_node_from_bootloader_data(bootloader_data: Dict, will_monkey_run: bool) -> Dict:
-        new_node = mongo.db.node.find_one({"domain_name": bootloader_data['hostname'],
-                                           "ip_addresses": bootloader_data['ips']})
+    def get_or_create_node_from_bootloader_telem(bootloader_telem: Dict, will_monkey_run: bool) -> Dict:
+        if is_local_ips(bootloader_telem['ips']):
+            raise NodeNotFoundException("Bootloader ran on island, no need to create new node.")
+            #return NodeService.get_monkey_island_pseudo_net_node()
+
+        new_node = mongo.db.node.find_one({"domain_name": bootloader_telem['hostname'],
+                                           "ip_addresses": bootloader_telem['ips']})
         if new_node is None:
-            new_node = NodeService.create_node_from_bootloader_data(bootloader_data, will_monkey_run)
-            if bootloader_data['tunnel']:
-                dst_node = NodeService.get_node_or_monkey_by_ip(bootloader_data['tunnel'])
+            new_node = NodeService.create_node_from_bootloader_telem(bootloader_telem, will_monkey_run)
+            if bootloader_telem['tunnel']:
+                dst_node = NodeService.get_node_or_monkey_by_ip(bootloader_telem['tunnel'])
             else:
                 dst_node = NodeService.get_monkey_island_node()
             edge = EdgeService.get_or_create_edge(new_node['_id'], dst_node['id'])
             mongo.db.edge.update({"_id": edge["_id"]},
-                                 {'$set': {'tunnel': bool(bootloader_data['tunnel']),
-                                           'ip_address': bootloader_data['ips'][0],
+                                 {'$set': {'tunnel': bool(bootloader_telem['tunnel']),
+                                           'ip_address': bootloader_telem['ips'][0],
                                            'group': NodeGroups.get_group_by_keywords(['island']).value}},
                                  upsert=False)
-
         return new_node
 
     @staticmethod
@@ -403,3 +406,6 @@ class NodeService:
     @staticmethod
     def get_hostname_by_id(node_id):
         return NodeService.get_node_hostname(mongo.db.monkey.find_one({'_id': node_id}, {'hostname': 1}))
+
+class NodeNotFoundException(Exception):
+    pass

monkey/monkey_island/cc/services/utils/node_groups.py

@@ -5,6 +5,7 @@ from typing import List
 import collections
 
 
+# This list must correspond to the one on front end in src/components/map/MapOptions.js
 class NodeGroups(Enum):
     CLEAN_UNKNOWN = 'clean_unknown'
     CLEAN_LINUX = 'clean_linux'
@@ -14,8 +15,10 @@ class NodeGroups(Enum):
     ISLAND = 'island'
     ISLAND_MONKEY_LINUX = 'island_monkey_linux'
     ISLAND_MONKEY_LINUX_RUNNING = 'island_monkey_linux_running'
+    ISLAND_MONKEY_LINUX_STARTING = 'island_monkey_linux_starting'
     ISLAND_MONKEY_WINDOWS = 'island_monkey_windows'
     ISLAND_MONKEY_WINDOWS_RUNNING = 'island_monkey_windows_running'
+    ISLAND_MONKEY_WINDOWS_STARTING = 'island_monkey_windows_starting'
     MANUAL_LINUX = 'manual_linux'
     MANUAL_LINUX_RUNNING = 'manual_linux_running'
     MANUAL_WINDOWS = 'manual_windows'

monkey/monkey_island/cc/ui/src/components/map/MapOptions.js

@@ -1,8 +1,10 @@
+// This list must correspond to the one on back end in cc/services/utils/node_groups.py
 const groupNames = ['clean_unknown', 'clean_linux', 'clean_windows', 'exploited_linux', 'exploited_windows', 'island',
-  'island_monkey_linux', 'island_monkey_linux_running', 'island_monkey_windows', 'island_monkey_windows_running',
-  'manual_linux', 'manual_linux_running', 'manual_windows', 'manual_windows_running', 'monkey_linux',
-  'monkey_linux_running', 'monkey_windows', 'monkey_windows_running', 'monkey_windows_starting',
-  'monkey_linux_starting', 'monkey_windows_old', 'monkey_linux_old' ];
+  'island_monkey_linux', 'island_monkey_linux_running', 'island_monkey_linux_starting', 'island_monkey_windows',
+  'island_monkey_windows_running', 'island_monkey_windows_starting', 'manual_linux', 'manual_linux_running',
+  'manual_windows', 'manual_windows_running', 'monkey_linux', 'monkey_linux_running', 'monkey_windows',
+  'monkey_windows_running', 'monkey_windows_starting', 'monkey_linux_starting', 'monkey_windows_old',
+  'monkey_linux_old' ];
 
 let getGroupsOptions = () => {
   let groupOptions = {};

monkey/monkey_island/cc/utils.py

@@ -1,5 +1,7 @@
 import socket
 import sys
+from typing import List
+import collections
 
 import array
 
@@ -49,6 +51,11 @@ else:
             return result
 
 
+def is_local_ips(ips: List) -> bool:
+    filtered_local_ips = [ip for ip in local_ip_addresses() if not ip.startswith('169.254')]
+    return collections.Counter(ips) == collections.Counter(filtered_local_ips)
+
+
 # The local IP addresses list should not change often. Therefore, we can cache the result and never call this function
 # more than once. This stopgap measure is here since this function is called a lot of times during the report
 # generation.