From 5ea4a9022354fd141978ccaa71efed8617780b1a Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Tue, 26 May 2020 10:52:41 +0300 Subject: [PATCH 01/21] Bump path version --- monkey/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/common/version.py b/monkey/common/version.py index fd706d909..f8bac8916 100644 --- a/monkey/common/version.py +++ b/monkey/common/version.py @@ -4,7 +4,7 @@ from pathlib import Path MAJOR = "1" MINOR = "8" -PATCH = "1" +PATCH = "2" build_file_path = Path(__file__).parent.joinpath("BUILD") with open(build_file_path, "r") as build_file: BUILD = build_file.read() From e1229baa61ccd4fbd4e6dfe4a027e2aa7fb43b38 Mon Sep 17 00:00:00 2001 From: ophirharpazg Date: Wed, 27 May 2020 17:10:36 +0300 Subject: [PATCH 02/21] The Missing Binaries modal works but in a non-elegant way --- .../ui/src/components/pages/RunMonkeyPage.js | 26 ++++++++++-- .../ui-components/MissingBinariesModal.js | 41 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js diff --git a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js index a741148fb..8ca334e73 100644 --- a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js +++ b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js @@ -13,6 +13,8 @@ import {Link} from 'react-router-dom'; import AuthComponent from '../AuthComponent'; import AwsRunTable from '../run-monkey/AwsRunTable'; +import MissingBinariesModal from '../ui-components/MissingBinariesModal'; + import '../../styles/MonkeyRunPage.scss'; const loading_css_override = css` @@ -40,8 +42,11 @@ class RunMonkeyPageComponent extends AuthComponent { awsMachines: [], isLoadingAws: true, isErrorWhileCollectingAwsMachines: false, - awsMachineCollectionErrorMsg: '' + awsMachineCollectionErrorMsg: '', + showModal: false }; + + this.closeModal = this.closeModal.bind(this); } componentDidMount() { @@ -130,6 +135,12 @@ class RunMonkeyPageComponent extends AuthComponent { runningOnIslandState: 'installing' }); } else { + /* If Monkey binaries are missing, change the state accordingly */ + if (res['error_text'].startsWith('Copy file failed')) { + this.setState({ + showModal: true} + ); + } this.setState({ runningOnIslandState: 'not_running' }); @@ -285,6 +296,12 @@ class RunMonkeyPageComponent extends AuthComponent { ) } + closeModal = () => { + this.setState({ + showModal: false + }) + }; + render() { return ( @@ -296,11 +313,14 @@ class RunMonkeyPageComponent extends AuthComponent {

+ { // TODO: implement button functionality /* diff --git a/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js new file mode 100644 index 000000000..afd1ff0de --- /dev/null +++ b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js @@ -0,0 +1,41 @@ +import {Modal} from 'react-bootstrap'; +import React from 'react'; +import {GridLoader} from 'react-spinners'; + + +class MissingBinariesModal extends React.PureComponent { + + constructor(props) { + super(props); + + this.state = { + showModal: this.props.showModal + }; + } + + render = () => { + return ( + this.props.onClose()}> + +

+
Uh oh...
+

+

+ Some Monkey binaries are not found where they should be...{"\r\n"} + Try downloading them from here, + at the bottommost section titled "Assets". +

+
+ +
+ + + ) + }; + +} + +export default MissingBinariesModal; From a4d4f629e0146189b569539640521adddc863f7f Mon Sep 17 00:00:00 2001 From: ophirharpazg Date: Wed, 27 May 2020 19:32:09 +0300 Subject: [PATCH 03/21] Raise a modal indicating that Monkey binaries are missing. --- .../ui/src/components/pages/RunMonkeyPage.js | 8 +++-- .../ui-components/MissingBinariesModal.js | 36 +++++++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js index 8ca334e73..afb8afafb 100644 --- a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js +++ b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage.js @@ -43,7 +43,8 @@ class RunMonkeyPageComponent extends AuthComponent { isLoadingAws: true, isErrorWhileCollectingAwsMachines: false, awsMachineCollectionErrorMsg: '', - showModal: false + showModal: false, + errorDetails: '' }; this.closeModal = this.closeModal.bind(this); @@ -138,7 +139,8 @@ class RunMonkeyPageComponent extends AuthComponent { /* If Monkey binaries are missing, change the state accordingly */ if (res['error_text'].startsWith('Copy file failed')) { this.setState({ - showModal: true} + showModal: true, + errorDetails: res['error_text']} ); } this.setState({ @@ -320,7 +322,7 @@ class RunMonkeyPageComponent extends AuthComponent { + errorDetails = {this.state.errorDetails}/> { // TODO: implement button functionality /* diff --git a/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js index afd1ff0de..b09774c2e 100644 --- a/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js +++ b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js @@ -9,22 +9,44 @@ class MissingBinariesModal extends React.PureComponent { super(props); this.state = { - showModal: this.props.showModal + showModal: this.props.showModal, + errorDetails: this.props.errorDetails }; } + componentDidUpdate(prevProps) { + if (this.props !== prevProps) { + this.setState({ + showModal: this.props.showModal, + errorDetails: this.props.errorDetails + }) + } + } + render = () => { return ( - this.props.onClose()}> + this.props.onClose()}>

Uh oh...

-

- Some Monkey binaries are not found where they should be...{"\r\n"} - Try downloading them from here, - at the bottommost section titled "Assets". -

+
+

+ + Some Monkey binaries are not found where they should be...
+ Try downloading them from here, + at the bottommost section titled "Assets". +

+
+
+

+ Error Details +

+
+ 
+              {this.state.errorDetails}
+
+
From a07ec9251c928e87eb313d866c00e84d48cbe12c Mon Sep 17 00:00:00 2001 From: ophirharpazg Date: Fri, 29 May 2020 00:30:03 +0300 Subject: [PATCH 05/21] formatting of the binaries path --- .../cc/ui/src/components/ui-components/MissingBinariesModal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js index 19f6c3dbe..c73094fb6 100644 --- a/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js +++ b/monkey/monkey_island/cc/ui/src/components/ui-components/MissingBinariesModal.js @@ -35,7 +35,7 @@ class MissingBinariesModal extends React.PureComponent { Some Monkey binaries are not found where they should be...
You can download the files from here, - at the bottommost section titled "Assets", and place them under the directory monkey/monkey_island/cc/binaries. + at the bottommost section titled "Assets", and place them under the directory monkey/monkey_island/cc/binaries.

From 092482ad8755d1720ba97773c72949545eac66d2 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Thu, 28 May 2020 20:12:10 +0300 Subject: [PATCH 06/21] Refactored edge from mongo queries to DAL --- monkey/monkey_island/cc/models/edge.py | 16 ++ monkey/monkey_island/cc/resources/edge.py | 7 +- monkey/monkey_island/cc/resources/monkey.py | 7 +- monkey/monkey_island/cc/resources/netmap.py | 21 +-- .../cc/services/attack/attack_schema.py | 2 +- monkey/monkey_island/cc/services/edge.py | 173 ------------------ .../cc/services/edge/__init__.py | 0 .../cc/services/edge/displayed_edge.py | 100 ++++++++++ monkey/monkey_island/cc/services/edge/edge.py | 78 ++++++++ .../cc/services/edge/test_displayed_edge.py | 97 ++++++++++ .../cc/services/edge/test_edge.py | 60 ++++++ .../cc/services/netmap/net_edge.py | 68 +++++++ .../cc/services/netmap/net_node.py | 23 +++ monkey/monkey_island/cc/services/node.py | 63 +++++-- .../services/telemetry/processing/exploit.py | 19 +- .../cc/services/telemetry/processing/scan.py | 25 +-- .../telemetry/processing/system_info.py | 1 - .../cc/services/telemetry/processing/utils.py | 7 +- .../cc/testing/IslandTestCase.py | 5 + 19 files changed, 527 insertions(+), 245 deletions(-) create mode 100644 monkey/monkey_island/cc/models/edge.py delete mode 100644 monkey/monkey_island/cc/services/edge.py create mode 100644 monkey/monkey_island/cc/services/edge/__init__.py create mode 100644 monkey/monkey_island/cc/services/edge/displayed_edge.py create mode 100644 monkey/monkey_island/cc/services/edge/edge.py create mode 100644 monkey/monkey_island/cc/services/edge/test_displayed_edge.py create mode 100644 monkey/monkey_island/cc/services/edge/test_edge.py create mode 100644 monkey/monkey_island/cc/services/netmap/net_edge.py create mode 100644 monkey/monkey_island/cc/services/netmap/net_node.py diff --git a/monkey/monkey_island/cc/models/edge.py b/monkey/monkey_island/cc/models/edge.py new file mode 100644 index 000000000..31a51598e --- /dev/null +++ b/monkey/monkey_island/cc/models/edge.py @@ -0,0 +1,16 @@ +from mongoengine import Document, ObjectIdField, ListField, DynamicField, BooleanField, StringField + + +class Edge(Document): + # SCHEMA + src_node_id = ObjectIdField(required=True) + dst_node_id = ObjectIdField(required=True) + scans = ListField(DynamicField(), default=[]) + exploits = ListField(DynamicField(), default=[]) + tunnel = BooleanField(default=False) + exploited = BooleanField(default=False) + src_label = StringField() + dst_label = StringField() + group = StringField() + domain_name = StringField() + ip_address = StringField() diff --git a/monkey/monkey_island/cc/resources/edge.py b/monkey/monkey_island/cc/resources/edge.py index f3ce94ee3..3d284e82c 100644 --- a/monkey/monkey_island/cc/resources/edge.py +++ b/monkey/monkey_island/cc/resources/edge.py @@ -1,7 +1,7 @@ -from flask import request import flask_restful +from flask import request -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService __author__ = 'Barak' @@ -9,7 +9,8 @@ __author__ = 'Barak' class Edge(flask_restful.Resource): def get(self): edge_id = request.args.get('id') + displayed_edge = DisplayedEdgeService.get_displayed_edge_by_id(edge_id) if edge_id: - return {"edge": EdgeService.get_displayed_edge_by_id(edge_id)} + return {"edge": displayed_edge} return {} diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 181ce94b2..5b5e1af4a 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -3,6 +3,8 @@ from datetime import datetime import dateutil.parser import flask_restful + +from monkey_island.cc.models.edge import Edge from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from flask import request @@ -129,8 +131,9 @@ class Monkey(flask_restful.Resource): if existing_node: node_id = existing_node["_id"] - for edge in mongo.db.edge.find({"to": node_id}): - mongo.db.edge.update({"_id": edge["_id"]}, {"$set": {"to": new_monkey_id}}) + for edge in Edge.objects(dst_node_id=node_id): + edge.dst_node_id = new_monkey_id + edge.save() for creds in existing_node['creds']: NodeService.add_credentials_to_monkey(new_monkey_id, creds) mongo.db.node.remove({"_id": node_id}) diff --git a/monkey/monkey_island/cc/resources/netmap.py b/monkey/monkey_island/cc/resources/netmap.py index 3b7e471d8..3b2c4c12e 100644 --- a/monkey/monkey_island/cc/resources/netmap.py +++ b/monkey/monkey_island/cc/resources/netmap.py @@ -1,9 +1,8 @@ import flask_restful from monkey_island.cc.auth import jwt_required -from monkey_island.cc.services.edge import EdgeService -from monkey_island.cc.services.node import NodeService -from monkey_island.cc.database import mongo +from monkey_island.cc.services.netmap.net_edge import NetEdgeService +from monkey_island.cc.services.netmap.net_node import NetNodeService __author__ = 'Barak' @@ -11,19 +10,11 @@ __author__ = 'Barak' class NetMap(flask_restful.Resource): @jwt_required() def get(self, **kw): - monkeys = [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] - nodes = [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})] - edges = [EdgeService.edge_to_net_edge(x) for x in mongo.db.edge.find({})] - - if NodeService.get_monkey_island_monkey() is None: - monkey_island = [NodeService.get_monkey_island_pseudo_net_node()] - edges += EdgeService.get_monkey_island_pseudo_edges() - else: - monkey_island = [] - edges += EdgeService.get_infected_monkey_island_pseudo_edges() + net_nodes = NetNodeService.get_all_net_nodes() + net_edges = NetEdgeService.get_all_net_edges() return \ { - "nodes": monkeys + nodes + monkey_island, - "edges": edges + "nodes": net_nodes, + "edges": net_edges } diff --git a/monkey/monkey_island/cc/services/attack/attack_schema.py b/monkey/monkey_island/cc/services/attack/attack_schema.py index 3c3a451f2..177dc9eaa 100644 --- a/monkey/monkey_island/cc/services/attack/attack_schema.py +++ b/monkey/monkey_island/cc/services/attack/attack_schema.py @@ -78,7 +78,7 @@ SCHEMA = { "necessary": False, "link": "https://attack.mitre.org/techniques/T1136", "description": "Adversaries with a sufficient level of access " - "may create a local system, domain, or cloud tenant account." + "may create a local system, domain, or cloud tenant account." } } }, diff --git a/monkey/monkey_island/cc/services/edge.py b/monkey/monkey_island/cc/services/edge.py deleted file mode 100644 index cd4ef737b..000000000 --- a/monkey/monkey_island/cc/services/edge.py +++ /dev/null @@ -1,173 +0,0 @@ -from bson import ObjectId - -from monkey_island.cc.database import mongo -import monkey_island.cc.services.node -from monkey_island.cc.models.monkey import Monkey, MonkeyNotFoundError - -__author__ = "itay.mizeretz" - - -class EdgeService: - def __init__(self): - pass - - @staticmethod - def get_displayed_edge_by_id(edge_id, for_report=False): - edge = mongo.db.edge.find({"_id": ObjectId(edge_id)})[0] - return EdgeService.edge_to_displayed_edge(edge, for_report) - - @staticmethod - def get_displayed_edges_by_to(to, for_report=False): - edges = mongo.db.edge.find({"to": ObjectId(to)}) - return [EdgeService.edge_to_displayed_edge(edge, for_report) for edge in edges] - - @staticmethod - def edge_to_displayed_edge(edge, for_report=False): - services = [] - os = {} - - if len(edge["scans"]) > 0: - services = EdgeService.services_to_displayed_services(edge["scans"][-1]["data"]["services"], for_report) - os = edge["scans"][-1]["data"]["os"] - - displayed_edge = EdgeService.edge_to_net_edge(edge) - - displayed_edge["ip_address"] = edge["ip_address"] - displayed_edge["services"] = services - displayed_edge["os"] = os - displayed_edge["exploits"] = edge['exploits'] - displayed_edge["_label"] = EdgeService.get_edge_label(displayed_edge) - return displayed_edge - - @staticmethod - def insert_edge(from_id, to_id): - edge_insert_result = mongo.db.edge.insert_one( - { - "from": from_id, - "to": to_id, - "scans": [], - "exploits": [], - "tunnel": False, - "exploited": False, - "src_label": EdgeService.get_label_for_endpoint(from_id), - "dst_label": EdgeService.get_label_for_endpoint(to_id) - }) - return mongo.db.edge.find_one({"_id": edge_insert_result.inserted_id}) - - @staticmethod - def get_or_create_edge(edge_from, edge_to): - tunnel_edge = mongo.db.edge.find_one({"from": edge_from, "to": edge_to}) - if tunnel_edge is None: - tunnel_edge = EdgeService.insert_edge(edge_from, edge_to) - - return tunnel_edge - - @staticmethod - def generate_pseudo_edge(edge_id, edge_from, edge_to): - edge = \ - { - "id": edge_id, - "from": edge_from, - "to": edge_to, - "group": "island", - "src_label": EdgeService.get_label_for_endpoint(edge_from), - "dst_label": EdgeService.get_label_for_endpoint(edge_to) - } - edge["_label"] = EdgeService.get_edge_label(edge) - return edge - - @staticmethod - def get_monkey_island_pseudo_edges(): - edges = [] - monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) if "tunnel" not in x] - # We're using fake ids because the frontend graph module requires unique ids. - # Collision with real id is improbable. - count = 0 - for monkey_id in monkey_ids: - count += 1 - edges.append(EdgeService.generate_pseudo_edge( - ObjectId(hex(count)[2:].zfill(24)), monkey_id, ObjectId("000000000000000000000000"))) - - return edges - - @staticmethod - def get_infected_monkey_island_pseudo_edges(): - monkey = monkey_island.cc.services.node.NodeService.get_monkey_island_monkey() - existing_ids = [x["from"] for x in mongo.db.edge.find({"to": monkey["_id"]})] - monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) - if ("tunnel" not in x) and (x["_id"] not in existing_ids) and (x["_id"] != monkey["_id"])] - edges = [] - - # We're using fake ids because the frontend graph module requires unique ids. - # Collision with real id is improbable. - count = 0 - for monkey_id in monkey_ids: - count += 1 - edges.append(EdgeService.generate_pseudo_edge( - ObjectId(hex(count)[2:].zfill(24)), monkey_id, monkey["_id"])) - - return edges - - @staticmethod - def services_to_displayed_services(services, for_report=False): - if for_report: - return [x for x in services] - else: - return [x + ": " + (services[x]['name'] if 'name' in services[x] else 'unknown') for x in services] - - @staticmethod - def edge_to_net_edge(edge): - return \ - { - "id": edge["_id"], - "from": edge["from"], - "to": edge["to"], - "group": EdgeService.get_edge_group(edge), - "src_label": edge["src_label"], - "dst_label": edge["dst_label"] - } - - @staticmethod - def get_edge_group(edge): - if edge.get("exploited"): - return "exploited" - if edge.get("tunnel"): - return "tunnel" - if (len(edge.get("scans", [])) > 0) or (len(edge.get("exploits", [])) > 0): - return "scan" - return "empty" - - @staticmethod - def set_edge_exploited(edge): - mongo.db.edge.update( - {"_id": edge["_id"]}, - {"$set": {"exploited": True}} - ) - monkey_island.cc.services.node.NodeService.set_node_exploited(edge["to"]) - - @staticmethod - def get_edge_label(edge): - return "%s %s %s" % (edge['src_label'], RIGHT_ARROW, edge['dst_label']) - - @staticmethod - def get_label_for_endpoint(endpoint_id): - node_service = monkey_island.cc.services.node.NodeService - if endpoint_id == ObjectId("000000000000000000000000"): - return 'MonkeyIsland' - if Monkey.is_monkey(endpoint_id): - return Monkey.get_label_by_id(endpoint_id) - else: - return node_service.get_node_label(node_service.get_node_by_id(endpoint_id)) - - @staticmethod - def update_label_by_endpoint(edge, endpoint_id): - label = EdgeService.get_label_for_endpoint(endpoint_id) - if endpoint_id == edge["to"]: - mongo_field = {"dst_label": label} - else: - mongo_field = {"src_label": label} - mongo.db.edge.update({"_id": edge["_id"]}, - {"$set": mongo_field}) - - -RIGHT_ARROW = "\u2192" diff --git a/monkey/monkey_island/cc/services/edge/__init__.py b/monkey/monkey_island/cc/services/edge/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/monkey_island/cc/services/edge/displayed_edge.py b/monkey/monkey_island/cc/services/edge/displayed_edge.py new file mode 100644 index 000000000..761dfa555 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/displayed_edge.py @@ -0,0 +1,100 @@ +from copy import deepcopy + +from bson import ObjectId + +from monkey_island.cc.database import mongo +from monkey_island.cc.models import Monkey +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.edge import EdgeService + +__author__ = "itay.mizeretz" + + +class DisplayedEdgeService: + + @staticmethod + def get_displayed_edges_by_dst(dst_id, for_report=False): + edges = Edge.objects(dst_node_id=ObjectId(dst_id)) + return [DisplayedEdgeService.edge_to_displayed_edge(edge, for_report) for edge in edges] + + @staticmethod + def get_displayed_edge_by_id(edge_id, for_report=False): + edge = Edge.objects.get(id=edge_id) + displayed_edge = DisplayedEdgeService.edge_to_displayed_edge(edge, for_report) + return displayed_edge + + @staticmethod + def edge_to_displayed_edge(edge: Edge, for_report=False): + services = [] + os = {} + + if len(edge.scans) > 0: + services = DisplayedEdgeService.services_to_displayed_services(edge.scans[-1]["data"]["services"], + for_report) + os = edge.scans[-1]["data"]["os"] + + displayed_edge = DisplayedEdgeService.edge_to_net_edge(edge) + + displayed_edge["ip_address"] = edge['ip_address'] + displayed_edge["services"] = services + displayed_edge["os"] = os + # we need to deepcopy all mutable edge properties, because weak-reference link is made otherwise, + # which is destroyed after method is exited and causes an error later. + displayed_edge["exploits"] = deepcopy(edge['exploits']) + displayed_edge["_label"] = EdgeService.get_edge_label(displayed_edge) + return displayed_edge + + @staticmethod + def generate_pseudo_edge(edge_id, src_node_id, dst_node_id, src_label, dst_label): + edge = \ + { + "id": edge_id, + "from": src_node_id, + "to": dst_node_id, + "group": "island", + "src_label": src_label, + "dst_label": dst_label + } + edge["_label"] = EdgeService.get_edge_label(edge) + return edge + + @staticmethod + def get_infected_monkey_island_pseudo_edges(monkey_island_monkey): + existing_ids = [x.src_node_id for x in Edge.objects(dst_node_id=monkey_island_monkey["_id"])] + monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) + if ("tunnel" not in x) and + (x["_id"] not in existing_ids) and + (x["_id"] != monkey_island_monkey["_id"])] + edges = [] + + # We're using fake ids because the frontend graph module requires unique ids. + # Collision with real id is improbable. + count = 0 + for monkey_id in monkey_ids: + count += 1 + edges.append(DisplayedEdgeService.generate_pseudo_edge( + ObjectId(hex(count)[2:].zfill(24)), monkey_id, monkey_island_monkey["_id"])) + + return edges + + @staticmethod + def services_to_displayed_services(services, for_report=False): + if for_report: + return [x for x in services] + else: + return [x + ": " + (services[x]['name'] if 'name' in services[x] else 'unknown') for x in services] + + @staticmethod + def edge_to_net_edge(edge: Edge): + return \ + { + "id": edge.id, + "from": edge.src_node_id, + "to": edge.dst_node_id, + "group": EdgeService.get_edge_group(edge), + "src_label": edge.src_label, + "dst_label": edge.dst_label + } + + +RIGHT_ARROW = "\u2192" diff --git a/monkey/monkey_island/cc/services/edge/edge.py b/monkey/monkey_island/cc/services/edge/edge.py new file mode 100644 index 000000000..e34ce56d1 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/edge.py @@ -0,0 +1,78 @@ +import copy +from typing import Dict + +from bson import ObjectId +from mongoengine import DoesNotExist + +from monkey_island.cc.models.edge import Edge + +RIGHT_ARROW = "\u2192" + + +class EdgeService: + + @staticmethod + def get_or_create_edge(src_node_id, dst_node_id, src_label, dst_label): + edge = False + try: + edge = Edge.objects.get(src_node_id=src_node_id, dst_node_id=dst_node_id) + except DoesNotExist: + edge = Edge(src_node_id=src_node_id, dst_node_id=dst_node_id) + finally: + if edge: + edge.src_label = src_label + edge.dst_label = dst_label + edge.save() + return edge + + @staticmethod + def update_label(edge: Edge, node_id: ObjectId, label: str): + if edge.src_node_id == node_id: + edge.src_label = label + elif edge.dst_node_id == node_id: + edge.dst_label = label + else: + raise DoesNotExist("Node id provided does not match with any endpoint of an edge provided.") + edge.save() + pass + + @staticmethod + def update_based_on_scan_telemetry(edge: Edge, telemetry: Dict): + machine_info = copy.deepcopy(telemetry['data']['machine']) + new_scan = \ + { + "timestamp": telemetry["timestamp"], + "data": machine_info + } + ip_address = machine_info.pop("ip_addr") + domain_name = machine_info.pop("domain_name") + edge.scans.append(new_scan) + edge.ip_address = ip_address + edge.domain_name = domain_name + edge.save() + + @staticmethod + def update_based_on_exploit(edge: Edge, exploit: Dict): + edge.exploits.append(exploit) + edge.save() + if exploit['result']: + EdgeService.set_edge_exploited(edge) + + @staticmethod + def get_edge_group(edge: Edge): + if edge.exploited: + return "exploited" + if edge.tunnel: + return "tunnel" + if edge.scans or edge.exploits: + return "scan" + return "empty" + + @staticmethod + def set_edge_exploited(edge: Edge): + edge.exploited = True + edge.save() + + @staticmethod + def get_edge_label(edge): + return "%s %s %s" % (edge['src_label'], RIGHT_ARROW, edge['dst_label']) diff --git a/monkey/monkey_island/cc/services/edge/test_displayed_edge.py b/monkey/monkey_island/cc/services/edge/test_displayed_edge.py new file mode 100644 index 000000000..340efe771 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/test_displayed_edge.py @@ -0,0 +1,97 @@ +from bson import ObjectId + +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService +from monkey_island.cc.services.edge.edge import EdgeService, RIGHT_ARROW +from monkey_island.cc.testing.IslandTestCase import IslandTestCase + +SCAN_DATA_MOCK = [{ + "timestamp": "2020-05-27T14:59:28.944Z", + "data": { + "os": { + "type": "linux", + "version": "Ubuntu-4ubuntu2.8" + }, + "services": { + "tcp-8088": { + "display_name": "unknown(TCP)", + "port": 8088 + }, + "tcp-22": { + "display_name": "SSH", + "port": 22, + "banner": "SSH-2.0-OpenSSH_7.2p2 Ubuntu-4ubuntu2.8\r\n", + "name": "ssh" + } + }, + "monkey_exe": None, + "default_tunnel": None, + "default_server": None + } +}] + +EXPLOIT_DATA_MOCK = [{ + "result": True, + "exploiter": "ElasticGroovyExploiter", + "info": { + "display_name": "Elastic search", + "started": "2020-05-11T08:59:38.105Z", + "finished": "2020-05-11T08:59:38.106Z", + "vulnerable_urls": [], + "vulnerable_ports": [], + "executed_cmds": [] + }, + "attempts": [], + "timestamp": "2020-05-27T14:59:29.048Z" +}] + + +class TestDisplayedEdgeService(IslandTestCase): + def test_get_displayed_edges_by_to(self): + self.clean_edge_db() + + dst_id = ObjectId() + + src_id = ObjectId() + EdgeService.get_or_create_edge(src_id, dst_id, "Ubuntu-4ubuntu2.8", "Ubuntu-4ubuntu2.8") + + src_id2 = ObjectId() + EdgeService.get_or_create_edge(src_id2, dst_id, "Ubuntu-4ubuntu3.2", "Ubuntu-4ubuntu2.8") + + displayed_edges = DisplayedEdgeService.get_displayed_edges_by_dst(dst_id) + self.assertEqual(len(displayed_edges), 2) + + def test_edge_to_displayed_edge(self): + src_node_id = ObjectId() + dst_node_id = ObjectId() + edge = Edge(src_node_id=src_node_id, + dst_node_id=dst_node_id, + scans=SCAN_DATA_MOCK, + exploits=EXPLOIT_DATA_MOCK, + exploited=True, + domain_name=None, + ip_address="10.2.2.2", + dst_label="Ubuntu-4ubuntu2.8", + src_label="Ubuntu-4ubuntu3.2") + + displayed_edge = DisplayedEdgeService.edge_to_displayed_edge(edge) + + self.assertEqual(displayed_edge['to'], dst_node_id) + self.assertEqual(displayed_edge['from'], src_node_id) + self.assertEqual(displayed_edge['ip_address'], "10.2.2.2") + self.assertListEqual(displayed_edge['services'], ["tcp-8088: unknown", "tcp-22: ssh"]) + self.assertEqual(displayed_edge['os'], {"type": "linux", + "version": "Ubuntu-4ubuntu2.8"}) + self.assertEqual(displayed_edge['exploits'], EXPLOIT_DATA_MOCK) + self.assertEqual(displayed_edge['_label'], "Ubuntu-4ubuntu3.2 " + RIGHT_ARROW + " Ubuntu-4ubuntu2.8") + self.assertEqual(displayed_edge['group'], "exploited") + return displayed_edge + + def test_services_to_displayed_services(self): + services1 = DisplayedEdgeService.services_to_displayed_services(SCAN_DATA_MOCK[-1]["data"]["services"], + True) + self.assertEqual(services1, ["tcp-8088", "tcp-22"]) + + services2 = DisplayedEdgeService.services_to_displayed_services(SCAN_DATA_MOCK[-1]["data"]["services"], + False) + self.assertEqual(services2, ["tcp-8088: unknown", "tcp-22: ssh"]) diff --git a/monkey/monkey_island/cc/services/edge/test_edge.py b/monkey/monkey_island/cc/services/edge/test_edge.py new file mode 100644 index 000000000..8053f45e3 --- /dev/null +++ b/monkey/monkey_island/cc/services/edge/test_edge.py @@ -0,0 +1,60 @@ +import logging + +from mongomock import ObjectId + +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.edge import EdgeService +from monkey_island.cc.testing.IslandTestCase import IslandTestCase + +logger = logging.getLogger(__name__) + + +class TestEdgeService(IslandTestCase): + """ + Make sure to set server environment to `testing` in server_config.json! + Otherwise this will mess up your mongo instance and won't work. + + Also, the working directory needs to be the working directory from which you usually run the island so the + server_config.json file is found and loaded. + """ + + def test_get_or_create_edge(self): + self.fail_if_not_testing_env() + self.clean_edge_db() + + src_id = ObjectId() + dst_id = ObjectId() + + test_edge1 = EdgeService.get_or_create_edge(src_id, dst_id, "Mock label 1", "Mock label 2") + self.assertEqual(test_edge1.src_node_id, src_id) + self.assertEqual(test_edge1.dst_node_id, dst_id) + self.assertFalse(test_edge1.exploited) + self.assertFalse(test_edge1.tunnel) + self.assertListEqual(test_edge1.scans, []) + self.assertListEqual(test_edge1.exploits, []) + self.assertEqual(test_edge1.src_label, "Mock label 1") + self.assertEqual(test_edge1.dst_label, "Mock label 2") + self.assertIsNone(test_edge1.group) + self.assertIsNone(test_edge1.domain_name) + self.assertIsNone(test_edge1.ip_address) + + EdgeService.get_or_create_edge(src_id, dst_id, "Mock label 1", "Mock label 2") + self.assertEqual(len(Edge.objects()), 1) + + def test_get_edge_group(self): + edge = Edge(src_node_id=ObjectId(), + dst_node_id=ObjectId(), + exploited=True) + self.assertEqual("exploited", EdgeService.get_edge_group(edge)) + + edge.exploited = False + edge.tunnel = True + self.assertEqual("tunnel", EdgeService.get_edge_group(edge)) + + edge.tunnel = False + edge.exploits.append(["mock_exploit_data"]) + self.assertEqual("scan", EdgeService.get_edge_group(edge)) + + edge.exploits = [] + edge.scans = [] + self.assertEqual("empty", EdgeService.get_edge_group(edge)) diff --git a/monkey/monkey_island/cc/services/netmap/net_edge.py b/monkey/monkey_island/cc/services/netmap/net_edge.py new file mode 100644 index 000000000..f9d5e1932 --- /dev/null +++ b/monkey/monkey_island/cc/services/netmap/net_edge.py @@ -0,0 +1,68 @@ +from bson import ObjectId + +from monkey_island.cc.models import Monkey +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService +from monkey_island.cc.services.node import NodeService + + +class NetEdgeService: + + @staticmethod + def get_all_net_edges(): + edges = NetEdgeService._get_standard_net_edges() + if NodeService.get_monkey_island_monkey() is None: + edges += NetEdgeService._get_uninfected_island_net_edges() + else: + monkey_island_monkey = NodeService.get_monkey_island_monkey() + edges += NetEdgeService._get_infected_island_net_edges(monkey_island_monkey) + return edges + + @staticmethod + def _get_standard_net_edges(): + return [DisplayedEdgeService.edge_to_net_edge(x) for x in Edge.objects()] + + @staticmethod + def _get_uninfected_island_net_edges(): + edges = [] + monkey_ids = [x.id for x in Monkey.objects() if "tunnel" not in x] + count = 0 + for monkey_id in monkey_ids: + count += 1 + # generating fake ID, because front end requires unique ID's for each edge. Collision improbable + fake_id = ObjectId(hex(count)[2:].zfill(24)) + island_id = ObjectId("000000000000000000000000") + monkey_label = NodeService.get_label_for_endpoint(monkey_id) + island_label = NodeService.get_label_for_endpoint(island_id) + island_pseudo_edge = DisplayedEdgeService.generate_pseudo_edge(edge_id=fake_id, + src_node_id=monkey_id, + dst_node_id=island_id, + src_label=monkey_label, + dst_label=island_label) + edges.append(island_pseudo_edge) + return edges + + @staticmethod + def _get_infected_island_net_edges(monkey_island_monkey): + existing_ids = [x.src_node_id for x in Edge.objects(dst_node_id=monkey_island_monkey["_id"])] + monkey_ids = [x.id for x in Monkey.objects() + if ("tunnel" not in x) and + (x.id not in existing_ids) and + (x.id != monkey_island_monkey["_id"])] + edges = [] + + count = 0 + for monkey_id in monkey_ids: + count += 1 + # generating fake ID, because front end requires unique ID's for each edge. Collision improbable + fake_id = ObjectId(hex(count)[2:].zfill(24)) + src_label = NodeService.get_label_for_endpoint(monkey_id) + dst_label = NodeService.get_label_for_endpoint(monkey_island_monkey["_id"]) + edge = DisplayedEdgeService.generate_pseudo_edge(edge_id=fake_id, + src_node_id=monkey_id, + dst_node_id=monkey_island_monkey["_id"], + src_label=src_label, + dst_label=dst_label) + edges.append(edge) + + return edges diff --git a/monkey/monkey_island/cc/services/netmap/net_node.py b/monkey/monkey_island/cc/services/netmap/net_node.py new file mode 100644 index 000000000..796167cf5 --- /dev/null +++ b/monkey/monkey_island/cc/services/netmap/net_node.py @@ -0,0 +1,23 @@ +from monkey_island.cc.database import mongo +from monkey_island.cc.services.node import NodeService + + +class NetNodeService: + + @staticmethod + def get_all_net_nodes(): + monkeys = NetNodeService._get_monkey_net_nodes() + nodes = NetNodeService._get_standard_net_nodes() + if NodeService.get_monkey_island_monkey() is None: + monkey_island = [NodeService.get_monkey_island_pseudo_net_node()] + else: + monkey_island = [] + return monkeys + nodes + monkey_island + + @staticmethod + def _get_monkey_net_nodes(): + return [NodeService.monkey_to_net_node(x) for x in mongo.db.monkey.find({})] + + @staticmethod + def _get_standard_net_nodes(): + return [NodeService.node_to_net_node(x) for x in mongo.db.node.find({})] diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index 9c0921580..ecce06c85 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -3,13 +3,16 @@ from typing import Dict import socket from bson import ObjectId +from mongoengine import DoesNotExist import monkey_island.cc.services.log from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService from monkey_island.cc.network_utils import local_ip_addresses, is_local_ips from monkey_island.cc import models +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.utils.node_states import NodeStates __author__ = "itay.mizeretz" @@ -48,18 +51,18 @@ class NodeService: accessible_from_nodes_hostnames = [] exploits = [] - edges = EdgeService.get_displayed_edges_by_to(node_id, for_report) + edges = DisplayedEdgeService.get_displayed_edges_by_dst(node_id, for_report) for edge in edges: - from_node_id = edge["from"] + from_node_id = edge['from'] from_node_label = Monkey.get_label_by_id(from_node_id) from_node_hostname = Monkey.get_hostname_by_id(from_node_id) accessible_from_nodes.append(from_node_label) accessible_from_nodes_hostnames.append(from_node_hostname) - for edge_exploit in edge["exploits"]: - edge_exploit["origin"] = from_node_label + for edge_exploit in edge['exploits']: + edge_exploit['origin'] = from_node_label exploits.append(edge_exploit) exploits = sorted(exploits, key=lambda exploit: exploit['timestamp']) @@ -186,23 +189,30 @@ class NodeService: {'$unset': {'tunnel': ''}}, upsert=False) - mongo.db.edge.update( - {"from": monkey_id, 'tunnel': True}, - {'$set': {'tunnel': False}}, - upsert=False) + try: + edge = Edge.objects.get(src_node_id=monkey_id, tunnel=True) + edge.tunnel = False + edge.save() + except DoesNotExist: + pass @staticmethod def set_monkey_tunnel(monkey_id, tunnel_host_ip): tunnel_host_id = NodeService.get_monkey_by_ip(tunnel_host_ip)["_id"] NodeService.unset_all_monkey_tunnels(monkey_id) mongo.db.monkey.update( - {"_id": monkey_id}, + {'_id': monkey_id}, {'$set': {'tunnel': tunnel_host_id}}, upsert=False) - tunnel_edge = EdgeService.get_or_create_edge(monkey_id, tunnel_host_id) - mongo.db.edge.update({"_id": tunnel_edge["_id"]}, - {'$set': {'tunnel': True, 'ip_address': tunnel_host_ip}}, - upsert=False) + monkey_label = NodeService.get_label_for_endpoint(monkey_id) + tunnel_host_label = NodeService.get_label_for_endpoint(tunnel_host_id) + tunnel_edge = EdgeService.get_or_create_edge(src_node_id=monkey_id, + dst_node_id=tunnel_host_id, + src_label=monkey_label, + dst_label=tunnel_host_label) + tunnel_edge.tunnel = True + tunnel_edge.ip_address = tunnel_host_ip + tunnel_edge.save() @staticmethod def insert_node(ip_address, domain_name=''): @@ -255,12 +265,16 @@ class NodeService: dst_node = NodeService.get_node_or_monkey_by_ip(bootloader_telem['tunnel']) else: dst_node = NodeService.get_monkey_island_node() - edge = EdgeService.get_or_create_edge(new_node['_id'], dst_node['id']) - mongo.db.edge.update({"_id": edge["_id"]}, - {'$set': {'tunnel': bool(bootloader_telem['tunnel']), - 'ip_address': bootloader_telem['ips'][0], - 'group': NodeStates.get_by_keywords(['island']).value}}, - upsert=False) + src_label = NodeService.get_label_for_endpoint(new_node['_id']) + dst_label = NodeService.get_label_for_endpoint(dst_node['id']) + edge = EdgeService.get_or_create_edge(src_node_id=new_node['_id'], + dst_node_id=dst_node['id'], + src_label=src_label, + dst_label=dst_label) + edge.tunnel = bool(bootloader_telem['tunnel']) + edge.ip_address = bootloader_telem['ips'][0] + edge.group = NodeStates.get_by_keywords(['island']).value + edge.save() return new_node @staticmethod @@ -411,6 +425,15 @@ class NodeService: def get_hostname_by_id(node_id): return NodeService.get_node_hostname(mongo.db.monkey.find_one({'_id': node_id}, {'hostname': 1})) + @staticmethod + def get_label_for_endpoint(endpoint_id): + if endpoint_id == ObjectId("000000000000000000000000"): + return 'MonkeyIsland' + if Monkey.is_monkey(endpoint_id): + return Monkey.get_label_by_id(endpoint_id) + else: + return NodeService.get_node_label(NodeService.get_node_by_id(endpoint_id)) + class NodeCreationException(Exception): pass diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index 9b25c97ef..8cb5db0d6 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -2,10 +2,10 @@ import copy import dateutil -from monkey_island.cc.database import mongo from monkey_island.cc.encryptor import encryptor from monkey_island.cc.models import Monkey -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.models.edge import Edge +from monkey_island.cc.services.edge.displayed_edge import EdgeService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry from monkey_island.cc.services.telemetry.zero_trust_tests.machine_exploited import test_machine_exploited @@ -14,7 +14,7 @@ from monkey_island.cc.services.telemetry.zero_trust_tests.machine_exploited impo def process_exploit_telemetry(telemetry_json): encrypt_exploit_creds(telemetry_json) edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) - update_edge_info_with_new_exploit(edge, telemetry_json) + update_network_with_exploit(edge, telemetry_json) update_node_credentials_from_successful_attempts(edge, telemetry_json) test_machine_exploited( @@ -25,28 +25,25 @@ def process_exploit_telemetry(telemetry_json): timestamp=telemetry_json['timestamp']) -def update_node_credentials_from_successful_attempts(edge, telemetry_json): +def update_node_credentials_from_successful_attempts(edge: Edge, telemetry_json): for attempt in telemetry_json['data']['attempts']: if attempt['result']: found_creds = {'user': attempt['user']} for field in ['password', 'lm_hash', 'ntlm_hash', 'ssh_key']: if len(attempt[field]) != 0: found_creds[field] = attempt[field] - NodeService.add_credentials_to_node(edge['to'], found_creds) + NodeService.add_credentials_to_node(edge.dst_node_id, found_creds) -def update_edge_info_with_new_exploit(edge, telemetry_json): +def update_network_with_exploit(edge: Edge, telemetry_json): telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started']) telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished']) new_exploit = copy.deepcopy(telemetry_json['data']) new_exploit.pop('machine') new_exploit['timestamp'] = telemetry_json['timestamp'] - mongo.db.edge.update( - {'_id': edge['_id']}, - {'$push': {'exploits': new_exploit}} - ) + EdgeService.update_based_on_exploit(edge, new_exploit) if new_exploit['result']: - EdgeService.set_edge_exploited(edge) + NodeService.set_node_exploited(edge.dst_node_id) def encrypt_exploit_creds(telemetry_json): diff --git a/monkey/monkey_island/cc/services/telemetry/processing/scan.py b/monkey/monkey_island/cc/services/telemetry/processing/scan.py index d57aef7c0..6566b91f2 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/scan.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/scan.py @@ -2,7 +2,8 @@ import copy from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.edge.edge import EdgeService +from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.utils import get_edge_by_scan_or_exploit_telemetry from monkey_island.cc.services.telemetry.zero_trust_tests.data_endpoints import test_open_data_endpoints from monkey_island.cc.services.telemetry.zero_trust_tests.segmentation import test_segmentation_violation @@ -19,22 +20,11 @@ def process_scan_telemetry(telemetry_json): def update_edges_and_nodes_based_on_scan_telemetry(telemetry_json): edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) - data = copy.deepcopy(telemetry_json['data']['machine']) - ip_address = data.pop("ip_addr") - domain_name = data.pop("domain_name") - new_scan = \ - { - "timestamp": telemetry_json["timestamp"], - "data": data - } - mongo.db.edge.update( - {"_id": edge["_id"]}, - {"$push": {"scans": new_scan}, - "$set": {"ip_address": ip_address, 'domain_name': domain_name}} - ) - node = mongo.db.node.find_one({"_id": edge["to"]}) + EdgeService.update_based_on_scan_telemetry(edge, telemetry_json) + + node = mongo.db.node.find_one({"_id": edge.dst_node_id}) if node is not None: - scan_os = new_scan["data"]["os"] + scan_os = telemetry_json['data']['machine']["os"] if "type" in scan_os: mongo.db.node.update({"_id": node["_id"]}, {"$set": {"os.type": scan_os["type"]}}, @@ -43,4 +33,5 @@ def update_edges_and_nodes_based_on_scan_telemetry(telemetry_json): mongo.db.node.update({"_id": node["_id"]}, {"$set": {"os.version": scan_os["version"]}}, upsert=False) - EdgeService.update_label_by_endpoint(edge, node["_id"]) + label = NodeService.get_label_for_endpoint(node["_id"]) + EdgeService.update_label(edge, node["_id"], label) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 844724163..5b842df0b 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,5 +1,4 @@ import logging -from ipaddress import ip_address from monkey_island.cc.encryptor import encryptor from monkey_island.cc.services import mimikatz_utils diff --git a/monkey/monkey_island/cc/services/telemetry/processing/utils.py b/monkey/monkey_island/cc/services/telemetry/processing/utils.py index 466b81bf1..df898945e 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/utils.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/utils.py @@ -1,4 +1,4 @@ -from monkey_island.cc.services.edge import EdgeService +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.node import NodeService @@ -10,7 +10,10 @@ def get_edge_by_scan_or_exploit_telemetry(telemetry_json): if dst_node is None: dst_node = NodeService.get_or_create_node(dst_ip, dst_domain_name) - return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"]) + src_label = NodeService.get_label_for_endpoint(src_monkey["_id"]) + dst_label = NodeService.get_label_for_endpoint(dst_node["_id"]) + + return EdgeService.get_or_create_edge(src_monkey["_id"], dst_node["_id"], src_label, dst_label) def get_tunnel_host_ip_from_proxy_field(telemetry_json): diff --git a/monkey/monkey_island/cc/testing/IslandTestCase.py b/monkey/monkey_island/cc/testing/IslandTestCase.py index 35e568399..e85d2f9ec 100644 --- a/monkey/monkey_island/cc/testing/IslandTestCase.py +++ b/monkey/monkey_island/cc/testing/IslandTestCase.py @@ -1,6 +1,7 @@ import unittest from monkey_island.cc.environment.environment import env from monkey_island.cc.models import Monkey +from monkey_island.cc.models.edge import Edge from monkey_island.cc.models.zero_trust.finding import Finding @@ -12,6 +13,10 @@ class IslandTestCase(unittest.TestCase): def clean_monkey_db(): Monkey.objects().delete() + @staticmethod + def clean_edge_db(): + Edge.objects().delete() + @staticmethod def clean_finding_db(): Finding.objects().delete() From a160e3396b542369860ae6d9e1995d2e4151446b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 29 May 2020 11:42:27 +0300 Subject: [PATCH 07/21] Small PEP improvements all around, unused method deleted on displayed_edge.py --- monkey/monkey_island/cc/bootloader_server.py | 3 ++- monkey/monkey_island/cc/resources/monkey.py | 6 ++++-- .../cc/resources/test/clear_caches.py | 4 ++-- .../attack/technique_reports/T1082.py | 3 ++- .../cc/services/edge/displayed_edge.py | 21 ------------------- monkey/monkey_island/cc/services/edge/edge.py | 10 ++++----- .../cc/services/reporting/pth_report.py | 3 ++- .../cc/services/telemetry/processing/scan.py | 2 -- .../system_info_telemetry_dispatcher.py | 5 +++-- .../test_system_info_telemetry_dispatcher.py | 8 +++++-- 10 files changed, 26 insertions(+), 39 deletions(-) diff --git a/monkey/monkey_island/cc/bootloader_server.py b/monkey/monkey_island/cc/bootloader_server.py index b1f7ec484..30196ff34 100644 --- a/monkey/monkey_island/cc/bootloader_server.py +++ b/monkey/monkey_island/cc/bootloader_server.py @@ -29,7 +29,8 @@ class BootloaderHTTPRequestHandler(BaseHTTPRequestHandler): post_data = self.rfile.read(content_length).decode() island_server_path = BootloaderHTTPRequestHandler.get_bootloader_resource_url(self.request.getsockname()[0]) island_server_path = parse.urljoin(island_server_path, self.path[1:]) - # The island server doesn't always have a correct SSL cert installed (By default it comes with a self signed one), + # The island server doesn't always have a correct SSL cert installed + # (By default it comes with a self signed one), # that's why we're not verifying the cert in this request. r = requests.post(url=island_server_path, data=post_data, verify=False) # noqa: DUO123 diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index 5b5e1af4a..ae34c624d 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -88,7 +88,8 @@ class Monkey(flask_restful.Resource): parent_to_add = (monkey_json.get('guid'), None) # default values in case of manual run if parent and parent != monkey_json.get('guid'): # current parent is known exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, + 'data.result': {'$eq': True}, 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}, 'monkey_guid': {'$eq': parent}})] if 1 == len(exploit_telem): @@ -97,7 +98,8 @@ class Monkey(flask_restful.Resource): parent_to_add = (parent, None) elif (not parent or parent == monkey_json.get('guid')) and 'ip_addresses' in monkey_json: exploit_telem = [x for x in - mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, 'data.result': {'$eq': True}, + mongo.db.telemetry.find({'telem_category': {'$eq': 'exploit'}, + 'data.result': {'$eq': True}, 'data.machine.ip_addr': {'$in': monkey_json['ip_addresses']}})] if 1 == len(exploit_telem): diff --git a/monkey/monkey_island/cc/resources/test/clear_caches.py b/monkey/monkey_island/cc/resources/test/clear_caches.py index f17193821..8d510a8bb 100644 --- a/monkey/monkey_island/cc/resources/test/clear_caches.py +++ b/monkey/monkey_island/cc/resources/test/clear_caches.py @@ -13,8 +13,8 @@ logger = logging.getLogger(__name__) class ClearCaches(flask_restful.Resource): """ - Used for timing tests - we want to get actual execution time of functions in BlackBox without caching - so we use this - to clear the caches. + Used for timing tests - we want to get actual execution time of functions in BlackBox without caching - + so we use this to clear the caches. :note: DO NOT CALL THIS IN PRODUCTION CODE as this will slow down the user experience. """ @jwt_required() diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py b/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py index cdbdb42ec..c80a3d476 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/T1082.py @@ -23,7 +23,8 @@ class T1082(AttackTechnique): 'collections': [ {'used': {'$and': [{'$ifNull': ['$netstat', False]}, {'$gt': ['$aws', {}]}]}, 'name': {'$literal': 'Amazon Web Services info'}}, - {'used': {'$and': [{'$ifNull': ['$process_list', False]}, {'$gt': ['$process_list', {}]}]}, + {'used': {'$and': [{'$ifNull': ['$process_list', False]}, + {'$gt': ['$process_list', {}]}]}, 'name': {'$literal': 'Running process list'}}, {'used': {'$and': [{'$ifNull': ['$netstat', False]}, {'$ne': ['$netstat', []]}]}, 'name': {'$literal': 'Network connections'}}, diff --git a/monkey/monkey_island/cc/services/edge/displayed_edge.py b/monkey/monkey_island/cc/services/edge/displayed_edge.py index 761dfa555..8f94a6ffa 100644 --- a/monkey/monkey_island/cc/services/edge/displayed_edge.py +++ b/monkey/monkey_island/cc/services/edge/displayed_edge.py @@ -2,8 +2,6 @@ from copy import deepcopy from bson import ObjectId -from monkey_island.cc.database import mongo -from monkey_island.cc.models import Monkey from monkey_island.cc.models.edge import Edge from monkey_island.cc.services.edge.edge import EdgeService @@ -58,25 +56,6 @@ class DisplayedEdgeService: edge["_label"] = EdgeService.get_edge_label(edge) return edge - @staticmethod - def get_infected_monkey_island_pseudo_edges(monkey_island_monkey): - existing_ids = [x.src_node_id for x in Edge.objects(dst_node_id=monkey_island_monkey["_id"])] - monkey_ids = [x["_id"] for x in mongo.db.monkey.find({}) - if ("tunnel" not in x) and - (x["_id"] not in existing_ids) and - (x["_id"] != monkey_island_monkey["_id"])] - edges = [] - - # We're using fake ids because the frontend graph module requires unique ids. - # Collision with real id is improbable. - count = 0 - for monkey_id in monkey_ids: - count += 1 - edges.append(DisplayedEdgeService.generate_pseudo_edge( - ObjectId(hex(count)[2:].zfill(24)), monkey_id, monkey_island_monkey["_id"])) - - return edges - @staticmethod def services_to_displayed_services(services, for_report=False): if for_report: diff --git a/monkey/monkey_island/cc/services/edge/edge.py b/monkey/monkey_island/cc/services/edge/edge.py index e34ce56d1..ab4c3c114 100644 --- a/monkey/monkey_island/cc/services/edge/edge.py +++ b/monkey/monkey_island/cc/services/edge/edge.py @@ -58,6 +58,11 @@ class EdgeService: if exploit['result']: EdgeService.set_edge_exploited(edge) + @staticmethod + def set_edge_exploited(edge: Edge): + edge.exploited = True + edge.save() + @staticmethod def get_edge_group(edge: Edge): if edge.exploited: @@ -68,11 +73,6 @@ class EdgeService: return "scan" return "empty" - @staticmethod - def set_edge_exploited(edge: Edge): - edge.exploited = True - edge.save() - @staticmethod def get_edge_label(edge): return "%s %s %s" % (edge['src_label'], RIGHT_ARROW, edge['dst_label']) diff --git a/monkey/monkey_island/cc/services/reporting/pth_report.py b/monkey/monkey_island/cc/services/reporting/pth_report.py index ecb209c69..f6d7b615a 100644 --- a/monkey/monkey_island/cc/services/reporting/pth_report.py +++ b/monkey/monkey_island/cc/services/reporting/pth_report.py @@ -106,7 +106,8 @@ class PTHReportService(object): { 'username': user['name'], 'domain_name': user['domain_name'], - 'hostname': NodeService.get_hostname_by_id(ObjectId(user['machine_id'])) if user['machine_id'] else None + 'hostname': NodeService.get_hostname_by_id(ObjectId(user['machine_id'])) if + user['machine_id'] else None } for user in doc['Docs'] ] users_cred_groups.append({'cred_groups': users_list}) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/scan.py b/monkey/monkey_island/cc/services/telemetry/processing/scan.py index 6566b91f2..48c1f11c3 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/scan.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/scan.py @@ -1,5 +1,3 @@ -import copy - from monkey_island.cc.database import mongo from monkey_island.cc.models import Monkey from monkey_island.cc.services.edge.edge import EdgeService diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index b5f2d24ea..af477ebb4 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -21,8 +21,9 @@ SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS = { class SystemInfoTelemetryDispatcher(object): def __init__(self, collector_to_parsing_functions: typing.Mapping[str, typing.List[typing.Callable]] = None): """ - :param collector_to_parsing_functions: Map between collector names and a list of functions that process the output of - that collector. If `None` is supplied, uses the default one; This should be the normal flow, overriding the + :param collector_to_parsing_functions: Map between collector names and a list of functions + that process the output of that collector. + If `None` is supplied, uses the default one; This should be the normal flow, overriding the collector->functions mapping is useful mostly for testing. """ if collector_to_parsing_functions is None: diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py index c5cc7aca2..f5a72405d 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_system_info_telemetry_dispatcher.py @@ -22,9 +22,13 @@ class SystemInfoTelemetryDispatcherTest(IslandTestCase): bad_empty_telem_json = {} self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_empty_telem_json) bad_no_data_telem_json = {"monkey_guid": "bla"} - self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_no_data_telem_json) + self.assertRaises(KeyError, + dispatcher.dispatch_collector_results_to_relevant_processors, + bad_no_data_telem_json) bad_no_monkey_telem_json = {"data": {"collectors": {"AwsCollector": "Bla"}}} - self.assertRaises(KeyError, dispatcher.dispatch_collector_results_to_relevant_processors, bad_no_monkey_telem_json) + self.assertRaises(KeyError, + dispatcher.dispatch_collector_results_to_relevant_processors, + bad_no_monkey_telem_json) # Telem JSON with no collectors - nothing gets dispatched good_telem_no_collectors = {"monkey_guid": "bla", "data": {"bla": "bla"}} From 90b47a4bb644665ebabc2d4f375f6927d440ace9 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 3 Jun 2020 10:02:31 +0300 Subject: [PATCH 08/21] Migrated to pypykatz on monkey --- monkey/infection_monkey/requirements.txt | 1 + .../system_info/mimikatz_collector.py | 129 ------------------ .../windows_cred_collector/__init__.py | 0 .../pypykatz_handler.py | 72 ++++++++++ .../test_pypykatz_handler.py | 84 ++++++++++++ .../windows_cred_collector.py | 22 +++ .../windows_credential.py | 15 ++ .../system_info/windows_info_collector.py | 25 ++-- 8 files changed, 208 insertions(+), 140 deletions(-) delete mode 100644 monkey/infection_monkey/system_info/mimikatz_collector.py create mode 100644 monkey/infection_monkey/system_info/windows_cred_collector/__init__.py create mode 100644 monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py create mode 100644 monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py create mode 100644 monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py create mode 100644 monkey/infection_monkey/system_info/windows_cred_collector/windows_credential.py diff --git a/monkey/infection_monkey/requirements.txt b/monkey/infection_monkey/requirements.txt index 7cbb5369f..dd4addcbd 100644 --- a/monkey/infection_monkey/requirements.txt +++ b/monkey/infection_monkey/requirements.txt @@ -17,3 +17,4 @@ wmi==1.4.9 ; sys_platform == 'win32' pymssql<3.0 pyftpdlib WinSys-3.x +pypykatz diff --git a/monkey/infection_monkey/system_info/mimikatz_collector.py b/monkey/infection_monkey/system_info/mimikatz_collector.py deleted file mode 100644 index 8b62217cc..000000000 --- a/monkey/infection_monkey/system_info/mimikatz_collector.py +++ /dev/null @@ -1,129 +0,0 @@ -import binascii -import ctypes -import logging -import socket -import zipfile - -import infection_monkey.config -from common.utils.attack_utils import ScanStatus, UsageEnum -from infection_monkey.telemetry.attack.t1129_telem import T1129Telem -from infection_monkey.telemetry.attack.t1106_telem import T1106Telem -from infection_monkey.pyinstaller_utils import get_binary_file_path, get_binaries_dir_path - -__author__ = 'itay.mizeretz' - -LOG = logging.getLogger(__name__) - - -class MimikatzCollector(object): - """ - Password collection module for Windows using Mimikatz. - """ - - # Name of Mimikatz DLL. Must be name of file in Mimikatz zip. - MIMIKATZ_DLL_NAME = 'tmpzipfile123456.dll' - - # Name of ZIP containing Mimikatz. Must be identical to one on monkey.spec - MIMIKATZ_ZIP_NAME = 'tmpzipfile123456.zip' - - # Password to Mimikatz zip file - MIMIKATZ_ZIP_PASSWORD = b'VTQpsJPXgZuXhX6x3V84G' - - def __init__(self): - self._config = infection_monkey.config.WormConfiguration - self._isInit = False - self._dll = None - self._collect = None - self._get = None - self.init_mimikatz() - - def init_mimikatz(self): - try: - with zipfile.ZipFile(get_binary_file_path(MimikatzCollector.MIMIKATZ_ZIP_NAME), 'r') as mimikatz_zip: - mimikatz_zip.extract(self.MIMIKATZ_DLL_NAME, path=get_binaries_dir_path(), - pwd=self.MIMIKATZ_ZIP_PASSWORD) - - self._dll = ctypes.WinDLL(get_binary_file_path(self.MIMIKATZ_DLL_NAME)) - collect_proto = ctypes.WINFUNCTYPE(ctypes.c_int) - get_proto = ctypes.WINFUNCTYPE(MimikatzCollector.LogonData) - get_text_output_proto = ctypes.WINFUNCTYPE(ctypes.c_wchar_p) - self._collect = collect_proto(("collect", self._dll)) - self._get = get_proto(("get", self._dll)) - self._get_text_output_proto = get_text_output_proto(("getTextOutput", self._dll)) - self._isInit = True - status = ScanStatus.USED - except Exception: - LOG.exception("Error initializing mimikatz collector") - status = ScanStatus.SCANNED - T1106Telem(status, UsageEnum.MIMIKATZ_WINAPI).send() - T1129Telem(status, UsageEnum.MIMIKATZ).send() - - def get_logon_info(self): - """ - Gets the logon info from mimikatz. - Returns a dictionary of users with their known credentials. - """ - LOG.info('Getting mimikatz logon information') - if not self._isInit: - return {} - LOG.debug("Running mimikatz collector") - - try: - entry_count = self._collect() - - logon_data_dictionary = {} - hostname = socket.gethostname() - - self.mimikatz_text = self._get_text_output_proto() - - for i in range(entry_count): - entry = self._get() - username = entry.username - - password = entry.password - lm_hash = binascii.hexlify(bytearray(entry.lm_hash)).decode() - ntlm_hash = binascii.hexlify(bytearray(entry.ntlm_hash)).decode() - - if 0 == len(password): - has_password = False - elif (username[-1] == '$') and (hostname.lower() == username[0:-1].lower()): - # Don't save the password of the host domain user (HOSTNAME$) - has_password = False - else: - has_password = True - - has_lm = ("00000000000000000000000000000000" != lm_hash) - has_ntlm = ("00000000000000000000000000000000" != ntlm_hash) - - if username not in logon_data_dictionary: - logon_data_dictionary[username] = {} - if has_password: - logon_data_dictionary[username]["password"] = password - if has_lm: - logon_data_dictionary[username]["lm_hash"] = lm_hash - if has_ntlm: - logon_data_dictionary[username]["ntlm_hash"] = ntlm_hash - - return logon_data_dictionary - except Exception: - LOG.exception("Error getting logon info") - return {} - - def get_mimikatz_text(self): - return self.mimikatz_text - - class LogonData(ctypes.Structure): - """ - Logon data structure returned from mimikatz. - """ - - WINDOWS_MAX_USERNAME_PASS_LENGTH = 257 - LM_NTLM_HASH_LENGTH = 16 - - _fields_ = \ - [ - ("username", ctypes.c_wchar * WINDOWS_MAX_USERNAME_PASS_LENGTH), - ("password", ctypes.c_wchar * WINDOWS_MAX_USERNAME_PASS_LENGTH), - ("lm_hash", ctypes.c_byte * LM_NTLM_HASH_LENGTH), - ("ntlm_hash", ctypes.c_byte * LM_NTLM_HASH_LENGTH) - ] diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/__init__.py b/monkey/infection_monkey/system_info/windows_cred_collector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py b/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py new file mode 100644 index 000000000..3e726a989 --- /dev/null +++ b/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py @@ -0,0 +1,72 @@ +import binascii +from typing import Dict, List + +from pypykatz.pypykatz import pypykatz + +from infection_monkey.system_info.windows_cred_collector.windows_credential import WindowsCredential + +CREDENTIAL_TYPES = ['msv_creds', 'wdigest_creds', 'ssp_creds', 'livessp_creds', 'dpapi_creds', + 'kerberos_creds', 'credman_creds', 'tspkg_creds'] + + +def get_windows_creds(): + pypy_handle = pypykatz.go_live() + logon_data = pypy_handle.to_dict() + windows_creds = _parse_pypykatz_results(logon_data) + return windows_creds + + +def _parse_pypykatz_results(pypykatz_data: Dict) -> List: + windows_creds = [] + for session in pypykatz_data['logon_sessions'].values(): + windows_creds.extend(_get_creds_from_pypykatz_session(session)) + return windows_creds + + +def _get_creds_from_pypykatz_session(pypykatz_session: Dict): + windows_creds = [] + for cred_type_key in CREDENTIAL_TYPES: + pypykatz_creds = pypykatz_session[cred_type_key] + windows_creds.extend(_get_creds_from_pypykatz_creds(pypykatz_creds)) + return windows_creds + + +def _get_creds_from_pypykatz_creds(pypykatz_creds): + creds = _filter_empty_creds(pypykatz_creds) + return [_get_windows_cred(cred) for cred in creds] + + +def _filter_empty_creds(pypykatz_creds: List[Dict]): + return [cred for cred in pypykatz_creds if not _is_cred_empty(cred)] + + +def _is_cred_empty(pypykatz_cred: Dict): + password_empty = 'password' not in pypykatz_cred or not pypykatz_cred['password'] + ntlm_hash_empty = 'NThash' not in pypykatz_cred or not pypykatz_cred['NThash'] + lm_hash_empty = 'LMhash' not in pypykatz_cred or not pypykatz_cred['LMhash'] + return password_empty and ntlm_hash_empty and lm_hash_empty + + +def _get_windows_cred(pypykatz_cred: Dict): + password = '' + ntlm_hash = '' + lm_hash = '' + username = pypykatz_cred['username'] + if 'password' in pypykatz_cred: + password = pypykatz_cred['password'] + if 'NThash' in pypykatz_cred: + ntlm_hash = _hash_to_string(pypykatz_cred['NThash']) + if 'LMhash' in pypykatz_cred: + lm_hash = _hash_to_string(pypykatz_cred['LMhash']) + return WindowsCredential(username=username, + password=password, + ntlm_hash=ntlm_hash, + lm_hash=lm_hash) + + +def _hash_to_string(hash): + if type(hash) == str: + return hash + if type(hash) == bytes: + return binascii.hexlify(bytearray(hash)).decode() + raise Exception(f"Can't convert hash to string, unsupported hash type {type(hash)}") diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py b/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py new file mode 100644 index 000000000..025f5d1dc --- /dev/null +++ b/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py @@ -0,0 +1,84 @@ +from unittest import TestCase + +from infection_monkey.system_info.windows_cred_collector.pypykatz_handler import _get_creds_from_pypykatz_session + + +class TestPypykatzHandler(TestCase): + # Made up credentials, but structure of dict should be roughly the same + PYPYKATZ_SESSION = { + 'authentication_id': 555555, 'session_id': 3, 'username': 'Monkey', + 'domainname': 'ReAlDoMaIn', 'logon_server': 'ReAlDoMaIn', + 'logon_time': '2020-06-02T04:53:45.256562+00:00', + 'sid': 'S-1-6-25-260123139-3611579848-5589493929-3021', 'luid': 123086, + 'msv_creds': [ + {'username': 'monkey', 'domainname': 'ReAlDoMaIn', + 'NThash': b'1\xb7 Dict: + return {'username': self.username, + 'password': self.password, + 'ntlm_hash': self.ntlm_hash, + 'lm_hash': self.lm_hash} diff --git a/monkey/infection_monkey/system_info/windows_info_collector.py b/monkey/infection_monkey/system_info/windows_info_collector.py index 857b42303..01d6c768e 100644 --- a/monkey/infection_monkey/system_info/windows_info_collector.py +++ b/monkey/infection_monkey/system_info/windows_info_collector.py @@ -2,12 +2,12 @@ import os import logging import sys +from infection_monkey.system_info.windows_cred_collector.windows_cred_collector import WindowsCredentialCollector + sys.coinit_flags = 0 # needed for proper destruction of the wmi python module # noinspection PyPep8 import infection_monkey.config # noinspection PyPep8 -from infection_monkey.system_info.mimikatz_collector import MimikatzCollector -# noinspection PyPep8 from infection_monkey.system_info import InfoCollector # noinspection PyPep8 from infection_monkey.system_info.wmi_consts import WMI_CLASSES @@ -61,12 +61,15 @@ class WindowsInfoCollector(InfoCollector): LOG.debug('finished get_wmi_info') def get_mimikatz_info(self): - mimikatz_collector = MimikatzCollector() - mimikatz_info = mimikatz_collector.get_logon_info() - if mimikatz_info: - if "credentials" in self.info: - self.info["credentials"].update(mimikatz_info) - self.info["mimikatz"] = mimikatz_collector.get_mimikatz_text() - LOG.info('Mimikatz info gathered successfully') - else: - LOG.info('No mimikatz info was gathered') + LOG.info("Gathering mimikatz info") + try: + credentials = WindowsCredentialCollector.get_creds() + if credentials: + if "credentials" in self.info: + self.info["credentials"].update(credentials) + self.info["mimikatz"] = credentials + LOG.info('Mimikatz info gathered successfully') + else: + LOG.info('No mimikatz info was gathered') + except Exception as e: + LOG.info(f"Pypykatz failed: {e}") From 9ea6718d37b5ea00ea46ab132fe681b552a21818 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Wed, 3 Jun 2020 16:18:19 +0300 Subject: [PATCH 09/21] Moved a function into common, since Monkey doesn't have ring as a dependency Also renamed it and added UTs --- monkey/common/network/network_utils.py | 12 ++++++++++++ monkey/common/network/test_network_utils.py | 12 ++++++++++++ ...tion_utils_test.py => test_segmentation_utils.py} | 0 monkey/infection_monkey/monkey.py | 4 ++-- monkey/monkey_island/cc/network_utils.py | 6 ------ 5 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 monkey/common/network/network_utils.py create mode 100644 monkey/common/network/test_network_utils.py rename monkey/common/network/{segmentation_utils_test.py => test_segmentation_utils.py} (100%) diff --git a/monkey/common/network/network_utils.py b/monkey/common/network/network_utils.py new file mode 100644 index 000000000..230e494bf --- /dev/null +++ b/monkey/common/network/network_utils.py @@ -0,0 +1,12 @@ +from urllib.parse import urlparse + + +def get_host_from_network_location(network_location: str) -> str: + """ + URL structure is ":///;?#" (https://tools.ietf.org/html/rfc1808.html) + And the net_loc is ":@:" (https://tools.ietf.org/html/rfc1738#section-3.1) + :param network_location: server network location + :return: host part of the network location + """ + url = urlparse("http://" + network_location) + return str(url.hostname) diff --git a/monkey/common/network/test_network_utils.py b/monkey/common/network/test_network_utils.py new file mode 100644 index 000000000..b4194aa27 --- /dev/null +++ b/monkey/common/network/test_network_utils.py @@ -0,0 +1,12 @@ +from unittest import TestCase + +from common.network.network_utils import get_host_from_network_location + + +class TestNetworkUtils(TestCase): + def test_remove_port_from_ip_string(self): + assert get_host_from_network_location("127.0.0.1:12345") == "127.0.0.1" + assert get_host_from_network_location("127.0.0.1:12345") == "127.0.0.1" + assert get_host_from_network_location("127.0.0.1") == "127.0.0.1" + assert get_host_from_network_location("www.google.com:8080") == "www.google.com" + assert get_host_from_network_location("user:password@host:8080") == "host" diff --git a/monkey/common/network/segmentation_utils_test.py b/monkey/common/network/test_segmentation_utils.py similarity index 100% rename from monkey/common/network/segmentation_utils_test.py rename to monkey/common/network/test_segmentation_utils.py diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index a31ea4d47..3495fd7cc 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -33,7 +33,7 @@ from infection_monkey.telemetry.attack.t1106_telem import T1106Telem from common.utils.attack_utils import ScanStatus, UsageEnum from common.version import get_version from infection_monkey.exploit.HostExploiter import HostExploiter -from monkey_island.cc.network_utils import remove_port_from_ip_string +from common.network.network_utils import get_host_from_network_location MAX_DEPTH_REACHED_MESSAGE = "Reached max depth, shutting down" @@ -386,7 +386,7 @@ class InfectionMonkey(object): LOG.debug("default server set to: %s" % self._default_server) def is_started_on_island(self): - island_ip = remove_port_from_ip_string(self._default_server) + island_ip = get_host_from_network_location(self._default_server) return is_running_on_server(island_ip) and WormConfiguration.depth == WormConfiguration.max_depth def log_arguments(self): diff --git a/monkey/monkey_island/cc/network_utils.py b/monkey/monkey_island/cc/network_utils.py index 903485723..d399d4255 100644 --- a/monkey/monkey_island/cc/network_utils.py +++ b/monkey/monkey_island/cc/network_utils.py @@ -5,7 +5,6 @@ import socket import struct import sys from typing import List -from urllib.parse import urlparse from netifaces import interfaces, ifaddresses, AF_INET from ring import lru @@ -86,8 +85,3 @@ def get_subnets(): ] ) return subnets - - -def remove_port_from_ip_string(ip_string: str) -> str: - url = urlparse("http://" + ip_string) - return str(url.hostname) From 192ac67159025a939f111430ec10f1a95ba36dea Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 09:27:09 +0300 Subject: [PATCH 10/21] Fixed typo in ScannedServers.js --- .../src/components/report-components/security/ScannedServers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js index 644d77f54..315a9adae 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js @@ -46,7 +46,7 @@ class ScannedServersComponent extends React.Component {

The Monkey discovered {scannedServicesAmount} - open {Pluralize('service', scannedServicesAmount)} + open {Pluralize('service', scannedServicesAmount)} on {scannedMachinesCount} {Pluralize('machine', scannedMachinesCount)}: From 895db8b446817272819eb7669c7c58b68503bcd5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 09:36:35 +0300 Subject: [PATCH 11/21] Fixed bugs and finished up pypykatz integration --- monkey/infection_monkey/exploit/wmiexec.py | 4 +- .../infection_monkey/system_info/__init__.py | 1 + .../windows_cred_collector.py | 5 +- .../cc/services/mimikatz_utils.py | 51 ------------------- .../cc/services/reporting/report.py | 10 ++-- .../monkey_island/cc/services/wmi_handler.py | 2 +- 6 files changed, 14 insertions(+), 59 deletions(-) delete mode 100644 monkey/monkey_island/cc/services/mimikatz_utils.py diff --git a/monkey/infection_monkey/exploit/wmiexec.py b/monkey/infection_monkey/exploit/wmiexec.py index 2100c23b0..ea2541381 100644 --- a/monkey/infection_monkey/exploit/wmiexec.py +++ b/monkey/infection_monkey/exploit/wmiexec.py @@ -39,9 +39,9 @@ class WmiExploiter(HostExploiter): for user, password, lm_hash, ntlm_hash in creds: password_hashed = self._config.hash_sensitive_data(password) lm_hash_hashed = self._config.hash_sensitive_data(lm_hash) - mtlm_hash_hashed = self._config.hash_sensitive_data(ntlm_hash) + ntlm_hash_hashed = self._config.hash_sensitive_data(ntlm_hash) creds_for_logging = "user, password (SHA-512), lm hash (SHA-512), ntlm hash (SHA-512): " \ - "({},{},{},{})".format(user, password_hashed, lm_hash_hashed, mtlm_hash_hashed) + "({},{},{},{})".format(user, password_hashed, lm_hash_hashed, ntlm_hash_hashed) LOG.debug(("Attempting to connect %r using WMI with " % self.host) + creds_for_logging) wmi_connection = WmiTools.WmiConnection() diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index 76bc40eb6..c619094b5 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -105,6 +105,7 @@ class InfoCollector(object): # we might be losing passwords in case of multiple reset attempts on same username # or in case another collector already filled in a password for this user self.info["credentials"][username]['password'] = password + self.info["credentials"][username]['username'] = username if len(azure_creds) != 0: self.info["Azure"] = {} self.info["Azure"]['usernames'] = [cred[0] for cred in azure_creds] diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py b/monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py index eadc684b4..3e462bbe0 100644 --- a/monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py +++ b/monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py @@ -18,5 +18,8 @@ class WindowsCredentialCollector(object): def cred_list_to_cred_dict(creds: List[WindowsCredential]): cred_dict = {} for cred in creds: - cred_dict.update({cred.username: cred.to_dict()}) + # Lets not use "." and "$" in keys, because it will confuse mongo. + # Ideally we should refactor island not to use a dict and simply parse credential list. + key = cred.username.replace(".", ",").replace("$", "") + cred_dict.update({key: cred.to_dict()}) return cred_dict diff --git a/monkey/monkey_island/cc/services/mimikatz_utils.py b/monkey/monkey_island/cc/services/mimikatz_utils.py deleted file mode 100644 index e2ab8ec10..000000000 --- a/monkey/monkey_island/cc/services/mimikatz_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -__author__ = 'maor.rayzin' - - -class MimikatzSecrets(object): - - def __init__(self): - # Static class - pass - - @staticmethod - def extract_sam_secrets(mim_string, users_dict): - users_secrets = mim_string.split("\n42.")[1].split("\nSAMKey :")[1].split("\n\n")[1:] - - if mim_string.count("\n42.") != 2: - return {} - - for sam_user_txt in users_secrets: - sam_user = dict([list(map(str.strip, line.split(":"))) for line in - [l for l in sam_user_txt.splitlines() if l.count(":") == 1]]) - username = sam_user.get("User") - users_dict[username] = {} - - ntlm = sam_user.get("NTLM") - if not ntlm or "[hashed secret]" not in ntlm: - continue - - users_dict[username]['SAM'] = ntlm.replace("[hashed secret]", "").strip() - - @staticmethod - def extract_ntlm_secrets(mim_string, users_dict): - - if mim_string.count("\n42.") != 2: - return {} - - ntds_users = mim_string.split("\n42.")[2].split("\nRID :")[1:] - - for ntds_user_txt in ntds_users: - user = ntds_user_txt.split("User :")[1].splitlines()[0].replace("User :", "").strip() - ntlm = ntds_user_txt.split("* Primary\n NTLM :")[1].splitlines()[0].replace("NTLM :", "").strip() - ntlm = ntlm.replace("[hashed secret]", "").strip() - users_dict[user] = {} - if ntlm: - users_dict[user]['ntlm'] = ntlm - - @staticmethod - def extract_secrets_from_mimikatz(mim_string): - users_dict = {} - MimikatzSecrets.extract_sam_secrets(mim_string, users_dict) - MimikatzSecrets.extract_ntlm_secrets(mim_string, users_dict) - - return users_dict diff --git a/monkey/monkey_island/cc/services/reporting/report.py b/monkey/monkey_island/cc/services/reporting/report.py index 195eac3d6..1685046c5 100644 --- a/monkey/monkey_island/cc/services/reporting/report.py +++ b/monkey/monkey_island/cc/services/reporting/report.py @@ -184,10 +184,13 @@ class ReportService: continue origin = NodeService.get_monkey_by_guid(telem['monkey_guid'])['hostname'] for user in monkey_creds: - for pass_type in monkey_creds[user]: + for pass_type in PASS_TYPE_DICT: + if pass_type not in monkey_creds[user] or not monkey_creds[user][pass_type]: + continue + username = monkey_creds[user]['username'] if 'username' in monkey_creds[user] else user cred_row = \ { - 'username': user.replace(',', '.'), + 'username': username, 'type': PASS_TYPE_DICT[pass_type], 'origin': origin } @@ -729,8 +732,7 @@ class ReportService: 'stolen_creds': ReportService.get_stolen_creds(), 'azure_passwords': ReportService.get_azure_creds(), 'ssh_keys': ReportService.get_ssh_keys(), - 'strong_users': PTHReportService.get_strong_users_on_crit_details(), - 'pth_map': PTHReportService.get_pth_map() + 'strong_users': PTHReportService.get_strong_users_on_crit_details() }, 'recommendations': { diff --git a/monkey/monkey_island/cc/services/wmi_handler.py b/monkey/monkey_island/cc/services/wmi_handler.py index a802aabf1..413a5f307 100644 --- a/monkey/monkey_island/cc/services/wmi_handler.py +++ b/monkey/monkey_island/cc/services/wmi_handler.py @@ -69,7 +69,7 @@ class WMIHandler(object): base_entity = self.build_entity_document(user) else: base_entity = self.build_entity_document(user, self.monkey_id) - base_entity['NTLM_secret'] = self.users_secrets.get(base_entity['name'], {}).get('ntlm') + base_entity['NTLM_secret'] = self.users_secrets.get(base_entity['name'], {}).get('ntlm_hash') base_entity['SAM_secret'] = self.users_secrets.get(base_entity['name'], {}).get('sam') base_entity['secret_location'] = [] From f5b37044fd415ef90f139bad00cd9361bf0fa03e Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 09:36:53 +0300 Subject: [PATCH 12/21] Removed PTH map --- .../cc/services/telemetry/processing/system_info.py | 10 +++------- .../src/components/report-components/SecurityReport.js | 3 +-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 844724163..375bd6cf6 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -1,8 +1,6 @@ import logging -from ipaddress import ip_address from monkey_island.cc.encryptor import encryptor -from monkey_island.cc.services import mimikatz_utils from monkey_island.cc.services.config import ConfigService from monkey_island.cc.services.node import NodeService from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import \ @@ -17,7 +15,7 @@ def process_system_info_telemetry(telemetry_json): telemetry_processing_stages = [ process_ssh_info, process_credential_info, - process_mimikatz_and_wmi_info, + process_wmi_info, dispatcher.dispatch_collector_results_to_relevant_processors ] @@ -93,11 +91,9 @@ def add_system_info_creds_to_config(creds): ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) -def process_mimikatz_and_wmi_info(telemetry_json): +def process_wmi_info(telemetry_json): users_secrets = {} - if 'mimikatz' in telemetry_json['data']: - users_secrets = mimikatz_utils.MimikatzSecrets. \ - extract_secrets_from_mimikatz(telemetry_json['data'].get('mimikatz', '')) + if 'wmi' in telemetry_json['data']: monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index d6891b5bb..1480786df 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -8,7 +8,6 @@ import StolenPasswords from 'components/report-components/security/StolenPasswor import CollapsibleWellComponent from 'components/report-components/security/CollapsibleWell'; import {Line} from 'rc-progress'; import AuthComponent from '../AuthComponent'; -import PassTheHashMapPageComponent from '../pages/PassTheHashMapPage'; import StrongUsers from 'components/report-components/security/StrongUsers'; import ReportHeader, {ReportTypes} from './common/ReportHeader'; import ReportLoader from './common/ReportLoader'; @@ -421,7 +420,7 @@ class ReportPageComponent extends AuthComponent {

- {this.generateReportPthMap()} + {/*this.generateReportPthMap()*/}
From 0be709958c3ea7fc11a28339d5c14d201b40f472 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 12:09:28 +0300 Subject: [PATCH 13/21] Improved scanned servers overview by inputting space character code --- .../report-components/security/ScannedServers.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js index 315a9adae..e70674700 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/security/ScannedServers.js @@ -44,11 +44,10 @@ class ScannedServersComponent extends React.Component { return ( <>

- The Monkey discovered - {scannedServicesAmount} - open {Pluralize('service', scannedServicesAmount)} - on - {scannedMachinesCount} + The Monkey discovered  + {scannedServicesAmount} open  + {Pluralize('service', scannedServicesAmount)} on  + {scannedMachinesCount}  {Pluralize('machine', scannedMachinesCount)}:

From c03c70ba28c7c71052023a18b72f3e1b615b75ee Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 14:40:58 +0300 Subject: [PATCH 14/21] Removed infrastructure related to mimikatz binary: deployment scripts and docs updated --- deployment_scripts/config.ps1 | 4 ---- deployment_scripts/deploy_windows.ps1 | 14 -------------- monkey/infection_monkey/monkey.spec | 9 --------- monkey/infection_monkey/readme.md | 21 +-------------------- 4 files changed, 1 insertion(+), 47 deletions(-) diff --git a/deployment_scripts/config.ps1 b/deployment_scripts/config.ps1 index b18b7c63c..e835ad633 100644 --- a/deployment_scripts/config.ps1 +++ b/deployment_scripts/config.ps1 @@ -29,8 +29,6 @@ $TRACEROUTE_32_BINARY_URL = $MONKEY_DOWNLOAD_URL + "traceroute32" $MONKEY_ISLAND_DIR = Join-Path "\monkey" -ChildPath "monkey_island" $MONKEY_DIR = Join-Path "\monkey" -ChildPath "infection_monkey" $SAMBA_BINARIES_DIR = Join-Path -Path $MONKEY_DIR -ChildPath "\bin" -$MK32_DLL = "mk32.zip" -$MK64_DLL = "mk64.zip" $TEMP_PYTHON_INSTALLER = ".\python.exe" $TEMP_MONGODB_ZIP = ".\mongodb.zip" $TEMP_OPEN_SSL_ZIP = ".\openssl.zip" @@ -44,6 +42,4 @@ $MONGODB_URL = "https://downloads.mongodb.org/win32/mongodb-win32-x86_64-2012plu $OPEN_SSL_URL = "https://indy.fulgan.com/SSL/openssl-1.0.2u-x64_86-win64.zip" $CPP_URL = "https://go.microsoft.com/fwlink/?LinkId=746572" $NPM_URL = "https://nodejs.org/dist/v12.14.1/node-v12.14.1-x64.msi" -$MK32_DLL_URL = "https://github.com/guardicore/mimikatz/releases/download/1.1.0/mk32.zip" -$MK64_DLL_URL = "https://github.com/guardicore/mimikatz/releases/download/1.1.0/mk64.zip" $UPX_URL = "https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win64.zip" diff --git a/deployment_scripts/deploy_windows.ps1 b/deployment_scripts/deploy_windows.ps1 index 6872f5c3a..3a57e9dcb 100644 --- a/deployment_scripts/deploy_windows.ps1 +++ b/deployment_scripts/deploy_windows.ps1 @@ -226,20 +226,6 @@ function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName, Remove-Item $TEMP_UPX_ZIP } - # Download mimikatz binaries - $mk32_path = Join-Path -Path $binDir -ChildPath $MK32_DLL - if (!(Test-Path -Path $mk32_path)) - { - "Downloading mimikatz 32 binary" - $webClient.DownloadFile($MK32_DLL_URL, $mk32_path) - } - $mk64_path = Join-Path -Path $binDir -ChildPath $MK64_DLL - if (!(Test-Path -Path $mk64_path)) - { - "Downloading mimikatz 64 binary" - $webClient.DownloadFile($MK64_DLL_URL, $mk64_path) - } - # Download sambacry binaries $samba_path = Join-Path -Path $monkey_home -ChildPath $SAMBA_BINARIES_DIR $samba32_path = Join-Path -Path $samba_path -ChildPath $SAMBA_32_BINARY_NAME diff --git a/monkey/infection_monkey/monkey.spec b/monkey/infection_monkey/monkey.spec index e5873c9c5..51bd4bb83 100644 --- a/monkey/infection_monkey/monkey.spec +++ b/monkey/infection_monkey/monkey.spec @@ -8,9 +8,6 @@ __author__ = 'itay.mizeretz' block_cipher = None -# Name of zip file in monkey. That's the name of the file in the _MEI folder -MIMIKATZ_ZIP_NAME = 'tmpzipfile123456.zip' - def main(): a = Analysis(['main.py'], @@ -66,7 +63,6 @@ def process_datas(orig_datas): datas = orig_datas if is_windows(): datas = [i for i in datas if i[0].find('Include') < 0] - datas += [(MIMIKATZ_ZIP_NAME, get_mimikatz_zip_path(), 'BINARY')] return datas @@ -118,9 +114,4 @@ def get_exe_icon(): return 'monkey.ico' if is_windows() else None -def get_mimikatz_zip_path(): - mk_filename = 'mk32.zip' if is_32_bit() else 'mk64.zip' - return os.path.join(get_bin_folder(), mk_filename) - - main() # We don't check if __main__ because this isn't the main script. diff --git a/monkey/infection_monkey/readme.md b/monkey/infection_monkey/readme.md index da865c35f..fa192c33e 100644 --- a/monkey/infection_monkey/readme.md +++ b/monkey/infection_monkey/readme.md @@ -7,7 +7,6 @@ The monkey is composed of three separate parts. - The Infection Monkey itself - PyInstaller compressed python archives - Sambacry binaries - Two linux binaries, 32/64 bit. -- Mimikatz binaries - Two windows binaries, 32/64 bit. - Traceroute binaries - Two linux binaries, 32/64bit. ## Windows @@ -28,7 +27,7 @@ The monkey is composed of three separate parts. `pip install -r requirements.txt` 4. Download and extract UPX binary to monkey\infection_monkey\bin\upx.exe: -5. Build/Download Sambacry and Mimikatz binaries +5. Build/Download Sambacry - Build/Download according to sections at the end of this readme. - Place the binaries under monkey\infection_monkey\bin 6. To build the final exe: @@ -83,24 +82,6 @@ Sambacry requires two standalone binaries to execute remotely. - 32bit: - 64bit: -### Mimikatz - -Mimikatz is required for the Monkey to be able to steal credentials on Windows. It's possible to either compile binaries from source (requires Visual Studio 2013 and up) or download them from our repository. - -1. Build Mimikatz yourself - - Building mimikatz requires Visual Studio 2013 and up - - Clone our version of mimikatz from - - Build using Visual Studio. - - Put each version in a zip file - 1. The zip should contain only the Mimikatz DLL named tmpzipfile123456.dll - 2. It should be protected using the password 'VTQpsJPXgZuXhX6x3V84G'. - 3. The zip file should be named mk32.zip/mk64.zip accordingly. - 4. Zipping with 7zip has been tested. Other zipping software may not work. - -2. Download our pre-built mimikatz binaries - - Download both 32 and 64 bit zipped DLLs from - - Place them under [code location]\infection_monkey\bin - ### Traceroute Traceroute requires two standalone binaries to execute remotely. From 6703e32ff28d45b5685256a8b9d9d7c330c7a8d7 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 14:46:12 +0300 Subject: [PATCH 15/21] UI bugs, related to PTH map hiding, fixed. --- .../ui/src/components/report-components/SecurityReport.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index 1480786df..0ca0b74f6 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -8,6 +8,7 @@ import StolenPasswords from 'components/report-components/security/StolenPasswor import CollapsibleWellComponent from 'components/report-components/security/CollapsibleWell'; import {Line} from 'rc-progress'; import AuthComponent from '../AuthComponent'; +import PassTheHashMapPageComponent from '../pages/PassTheHashMapPage'; import StrongUsers from 'components/report-components/security/StrongUsers'; import ReportHeader, {ReportTypes} from './common/ReportHeader'; import ReportLoader from './common/ReportLoader'; @@ -419,8 +420,9 @@ class ReportPageComponent extends AuthComponent {
-
- {/*this.generateReportPthMap()*/} +
+ {/*Disable PTH map until we fix it + this.generateReportPthMap()*/}
From 0dc864baa5fd5653b6ab596871c63adafc4cf772 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Fri, 5 Jun 2020 15:59:31 +0300 Subject: [PATCH 16/21] Fixed a bug that added empty credentials to configuration --- .../cc/services/telemetry/processing/system_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 375bd6cf6..c6caf4865 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -83,11 +83,11 @@ def replace_user_dot_with_comma(creds): def add_system_info_creds_to_config(creds): for user in creds: ConfigService.creds_add_username(user) - if 'password' in creds[user]: + if 'password' in creds[user] and creds[user]['password']: ConfigService.creds_add_password(creds[user]['password']) - if 'lm_hash' in creds[user]: + if 'lm_hash' in creds[user] and creds[user]['lm_hash']: ConfigService.creds_add_lm_hash(creds[user]['lm_hash']) - if 'ntlm_hash' in creds[user]: + if 'ntlm_hash' in creds[user] and creds[user]['ntlm_hash']: ConfigService.creds_add_ntlm_hash(creds[user]['ntlm_hash']) From fb595319699d16fc70fa089302ee8f9afacc4592 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 8 Jun 2020 10:29:04 +0300 Subject: [PATCH 17/21] Refactored EdgeService into a boundary object. --- monkey/monkey_island/cc/models/edge.py | 3 + monkey/monkey_island/cc/resources/monkey.py | 7 +- .../cc/services/edge/displayed_edge.py | 28 +++-- monkey/monkey_island/cc/services/edge/edge.py | 103 +++++++++++------- .../cc/services/edge/test_displayed_edge.py | 20 ++-- .../cc/services/edge/test_edge.py | 8 +- .../cc/services/netmap/net_edge.py | 6 +- monkey/monkey_island/cc/services/node.py | 9 +- .../services/telemetry/processing/exploit.py | 6 +- .../cc/services/telemetry/processing/scan.py | 4 +- 10 files changed, 111 insertions(+), 83 deletions(-) diff --git a/monkey/monkey_island/cc/models/edge.py b/monkey/monkey_island/cc/models/edge.py index 31a51598e..09af04680 100644 --- a/monkey/monkey_island/cc/models/edge.py +++ b/monkey/monkey_island/cc/models/edge.py @@ -2,6 +2,9 @@ from mongoengine import Document, ObjectIdField, ListField, DynamicField, Boolea class Edge(Document): + + meta = {'allow_inheritance': True} + # SCHEMA src_node_id = ObjectIdField(required=True) dst_node_id = ObjectIdField(required=True) diff --git a/monkey/monkey_island/cc/resources/monkey.py b/monkey/monkey_island/cc/resources/monkey.py index ae34c624d..a39aaf199 100644 --- a/monkey/monkey_island/cc/resources/monkey.py +++ b/monkey/monkey_island/cc/resources/monkey.py @@ -4,7 +4,6 @@ from datetime import datetime import dateutil.parser import flask_restful -from monkey_island.cc.models.edge import Edge from monkey_island.cc.resources.test.utils.telem_store import TestTelemStore from flask import request @@ -12,6 +11,7 @@ from monkey_island.cc.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECOND from monkey_island.cc.database import mongo from monkey_island.cc.models.monkey_ttl import create_monkey_ttl_document from monkey_island.cc.services.config import ConfigService +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.node import NodeService __author__ = 'Barak' @@ -133,9 +133,8 @@ class Monkey(flask_restful.Resource): if existing_node: node_id = existing_node["_id"] - for edge in Edge.objects(dst_node_id=node_id): - edge.dst_node_id = new_monkey_id - edge.save() + EdgeService.update_all_dst_nodes(old_dst_node_id=node_id, + new_dst_node_id=new_monkey_id) for creds in existing_node['creds']: NodeService.add_credentials_to_monkey(new_monkey_id, creds) mongo.db.node.remove({"_id": node_id}) diff --git a/monkey/monkey_island/cc/services/edge/displayed_edge.py b/monkey/monkey_island/cc/services/edge/displayed_edge.py index 8f94a6ffa..f7a0664bf 100644 --- a/monkey/monkey_island/cc/services/edge/displayed_edge.py +++ b/monkey/monkey_island/cc/services/edge/displayed_edge.py @@ -1,8 +1,8 @@ from copy import deepcopy +from typing import Dict from bson import ObjectId -from monkey_island.cc.models.edge import Edge from monkey_island.cc.services.edge.edge import EdgeService __author__ = "itay.mizeretz" @@ -11,18 +11,18 @@ __author__ = "itay.mizeretz" class DisplayedEdgeService: @staticmethod - def get_displayed_edges_by_dst(dst_id, for_report=False): - edges = Edge.objects(dst_node_id=ObjectId(dst_id)) + def get_displayed_edges_by_dst(dst_id: str, for_report=False): + edges = EdgeService.get_by_dst_node(dst_node_id=ObjectId(dst_id)) return [DisplayedEdgeService.edge_to_displayed_edge(edge, for_report) for edge in edges] @staticmethod - def get_displayed_edge_by_id(edge_id, for_report=False): - edge = Edge.objects.get(id=edge_id) + def get_displayed_edge_by_id(edge_id: str, for_report=False): + edge = EdgeService.get_edge_by_id(ObjectId(edge_id)) displayed_edge = DisplayedEdgeService.edge_to_displayed_edge(edge, for_report) return displayed_edge @staticmethod - def edge_to_displayed_edge(edge: Edge, for_report=False): + def edge_to_displayed_edge(edge: EdgeService, for_report=False): services = [] os = {} @@ -33,13 +33,13 @@ class DisplayedEdgeService: displayed_edge = DisplayedEdgeService.edge_to_net_edge(edge) - displayed_edge["ip_address"] = edge['ip_address'] + displayed_edge["ip_address"] = edge.ip_address displayed_edge["services"] = services displayed_edge["os"] = os # we need to deepcopy all mutable edge properties, because weak-reference link is made otherwise, # which is destroyed after method is exited and causes an error later. - displayed_edge["exploits"] = deepcopy(edge['exploits']) - displayed_edge["_label"] = EdgeService.get_edge_label(displayed_edge) + displayed_edge["exploits"] = deepcopy(edge.exploits) + displayed_edge["_label"] = edge.get_label() return displayed_edge @staticmethod @@ -53,9 +53,13 @@ class DisplayedEdgeService: "src_label": src_label, "dst_label": dst_label } - edge["_label"] = EdgeService.get_edge_label(edge) + edge["_label"] = DisplayedEdgeService.get_pseudo_label(edge) return edge + @staticmethod + def get_pseudo_label(edge: Dict): + return f"{edge['src_label']} {RIGHT_ARROW} {edge['dst_label']}" + @staticmethod def services_to_displayed_services(services, for_report=False): if for_report: @@ -64,13 +68,13 @@ class DisplayedEdgeService: return [x + ": " + (services[x]['name'] if 'name' in services[x] else 'unknown') for x in services] @staticmethod - def edge_to_net_edge(edge: Edge): + def edge_to_net_edge(edge: EdgeService): return \ { "id": edge.id, "from": edge.src_node_id, "to": edge.dst_node_id, - "group": EdgeService.get_edge_group(edge), + "group": edge.get_group(), "src_label": edge.src_label, "dst_label": edge.dst_label } diff --git a/monkey/monkey_island/cc/services/edge/edge.py b/monkey/monkey_island/cc/services/edge/edge.py index ab4c3c114..4c9ef57d7 100644 --- a/monkey/monkey_island/cc/services/edge/edge.py +++ b/monkey/monkey_island/cc/services/edge/edge.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import copy -from typing import Dict +from typing import Dict, List from bson import ObjectId from mongoengine import DoesNotExist @@ -9,35 +11,60 @@ from monkey_island.cc.models.edge import Edge RIGHT_ARROW = "\u2192" -class EdgeService: +class EdgeService(Edge): @staticmethod - def get_or_create_edge(src_node_id, dst_node_id, src_label, dst_label): - edge = False + def get_all_edges() -> List[EdgeService]: + return EdgeService.objects() + + @staticmethod + def get_or_create_edge(src_node_id, dst_node_id, src_label, dst_label) -> EdgeService: + edge = None try: - edge = Edge.objects.get(src_node_id=src_node_id, dst_node_id=dst_node_id) + edge = EdgeService.objects.get(src_node_id=src_node_id, dst_node_id=dst_node_id) except DoesNotExist: - edge = Edge(src_node_id=src_node_id, dst_node_id=dst_node_id) + edge = EdgeService(src_node_id=src_node_id, dst_node_id=dst_node_id) finally: if edge: - edge.src_label = src_label - edge.dst_label = dst_label - edge.save() + edge.update_label(node_id=src_node_id, label=src_label) + edge.update_label(node_id=dst_node_id, label=dst_label) return edge @staticmethod - def update_label(edge: Edge, node_id: ObjectId, label: str): - if edge.src_node_id == node_id: - edge.src_label = label - elif edge.dst_node_id == node_id: - edge.dst_label = label - else: - raise DoesNotExist("Node id provided does not match with any endpoint of an edge provided.") - edge.save() - pass + def get_by_dst_node(dst_node_id: ObjectId) -> List[EdgeService]: + return EdgeService.objects(dst_node_id=dst_node_id) @staticmethod - def update_based_on_scan_telemetry(edge: Edge, telemetry: Dict): + def get_edge_by_id(edge_id: ObjectId) -> EdgeService: + return EdgeService.objects.get(id=edge_id) + + def update_label(self, node_id: ObjectId, label: str): + if self.src_node_id == node_id: + self.src_label = label + elif self.dst_node_id == node_id: + self.dst_label = label + else: + raise DoesNotExist("Node id provided does not match with any endpoint of an self provided.") + self.save() + + @staticmethod + def update_all_dst_nodes(old_dst_node_id: ObjectId, new_dst_node_id: ObjectId): + for edge in EdgeService.objects(dst_node_id=old_dst_node_id): + edge.dst_node_id = new_dst_node_id + edge.save() + + @staticmethod + def get_tunnel_edges_by_src(src_node_id) -> List[EdgeService]: + try: + return EdgeService.objects(src_node_id=src_node_id, tunnel=True) + except DoesNotExist: + return [] + + def disable_tunnel(self): + self.tunnel = False + self.save() + + def update_based_on_scan_telemetry(self, telemetry: Dict): machine_info = copy.deepcopy(telemetry['data']['machine']) new_scan = \ { @@ -46,33 +73,29 @@ class EdgeService: } ip_address = machine_info.pop("ip_addr") domain_name = machine_info.pop("domain_name") - edge.scans.append(new_scan) - edge.ip_address = ip_address - edge.domain_name = domain_name - edge.save() + self.scans.append(new_scan) + self.ip_address = ip_address + self.domain_name = domain_name + self.save() - @staticmethod - def update_based_on_exploit(edge: Edge, exploit: Dict): - edge.exploits.append(exploit) - edge.save() + def update_based_on_exploit(self, exploit: Dict): + self.exploits.append(exploit) + self.save() if exploit['result']: - EdgeService.set_edge_exploited(edge) + self.set_exploited() - @staticmethod - def set_edge_exploited(edge: Edge): - edge.exploited = True - edge.save() + def set_exploited(self): + self.exploited = True + self.save() - @staticmethod - def get_edge_group(edge: Edge): - if edge.exploited: + def get_group(self) -> str: + if self.exploited: return "exploited" - if edge.tunnel: + if self.tunnel: return "tunnel" - if edge.scans or edge.exploits: + if self.scans or self.exploits: return "scan" return "empty" - @staticmethod - def get_edge_label(edge): - return "%s %s %s" % (edge['src_label'], RIGHT_ARROW, edge['dst_label']) + def get_label(self) -> str: + return f"{self.src_label} {RIGHT_ARROW} {self.dst_label}" diff --git a/monkey/monkey_island/cc/services/edge/test_displayed_edge.py b/monkey/monkey_island/cc/services/edge/test_displayed_edge.py index 340efe771..c134bce33 100644 --- a/monkey/monkey_island/cc/services/edge/test_displayed_edge.py +++ b/monkey/monkey_island/cc/services/edge/test_displayed_edge.py @@ -58,21 +58,21 @@ class TestDisplayedEdgeService(IslandTestCase): src_id2 = ObjectId() EdgeService.get_or_create_edge(src_id2, dst_id, "Ubuntu-4ubuntu3.2", "Ubuntu-4ubuntu2.8") - displayed_edges = DisplayedEdgeService.get_displayed_edges_by_dst(dst_id) + displayed_edges = DisplayedEdgeService.get_displayed_edges_by_dst(str(dst_id)) self.assertEqual(len(displayed_edges), 2) def test_edge_to_displayed_edge(self): src_node_id = ObjectId() dst_node_id = ObjectId() - edge = Edge(src_node_id=src_node_id, - dst_node_id=dst_node_id, - scans=SCAN_DATA_MOCK, - exploits=EXPLOIT_DATA_MOCK, - exploited=True, - domain_name=None, - ip_address="10.2.2.2", - dst_label="Ubuntu-4ubuntu2.8", - src_label="Ubuntu-4ubuntu3.2") + edge = EdgeService(src_node_id=src_node_id, + dst_node_id=dst_node_id, + scans=SCAN_DATA_MOCK, + exploits=EXPLOIT_DATA_MOCK, + exploited=True, + domain_name=None, + ip_address="10.2.2.2", + dst_label="Ubuntu-4ubuntu2.8", + src_label="Ubuntu-4ubuntu3.2") displayed_edge = DisplayedEdgeService.edge_to_displayed_edge(edge) diff --git a/monkey/monkey_island/cc/services/edge/test_edge.py b/monkey/monkey_island/cc/services/edge/test_edge.py index 8053f45e3..8ebf45343 100644 --- a/monkey/monkey_island/cc/services/edge/test_edge.py +++ b/monkey/monkey_island/cc/services/edge/test_edge.py @@ -45,16 +45,16 @@ class TestEdgeService(IslandTestCase): edge = Edge(src_node_id=ObjectId(), dst_node_id=ObjectId(), exploited=True) - self.assertEqual("exploited", EdgeService.get_edge_group(edge)) + self.assertEqual("exploited", EdgeService.get_group(edge)) edge.exploited = False edge.tunnel = True - self.assertEqual("tunnel", EdgeService.get_edge_group(edge)) + self.assertEqual("tunnel", EdgeService.get_group(edge)) edge.tunnel = False edge.exploits.append(["mock_exploit_data"]) - self.assertEqual("scan", EdgeService.get_edge_group(edge)) + self.assertEqual("scan", EdgeService.get_group(edge)) edge.exploits = [] edge.scans = [] - self.assertEqual("empty", EdgeService.get_edge_group(edge)) + self.assertEqual("empty", EdgeService.get_group(edge)) diff --git a/monkey/monkey_island/cc/services/netmap/net_edge.py b/monkey/monkey_island/cc/services/netmap/net_edge.py index f9d5e1932..44e097630 100644 --- a/monkey/monkey_island/cc/services/netmap/net_edge.py +++ b/monkey/monkey_island/cc/services/netmap/net_edge.py @@ -3,6 +3,7 @@ from bson import ObjectId from monkey_island.cc.models import Monkey from monkey_island.cc.models.edge import Edge from monkey_island.cc.services.edge.displayed_edge import DisplayedEdgeService +from monkey_island.cc.services.edge.edge import EdgeService from monkey_island.cc.services.node import NodeService @@ -20,7 +21,7 @@ class NetEdgeService: @staticmethod def _get_standard_net_edges(): - return [DisplayedEdgeService.edge_to_net_edge(x) for x in Edge.objects()] + return [DisplayedEdgeService.edge_to_net_edge(x) for x in EdgeService.get_all_edges()] @staticmethod def _get_uninfected_island_net_edges(): @@ -44,7 +45,8 @@ class NetEdgeService: @staticmethod def _get_infected_island_net_edges(monkey_island_monkey): - existing_ids = [x.src_node_id for x in Edge.objects(dst_node_id=monkey_island_monkey["_id"])] + existing_ids = [x.src_node_id for x + in EdgeService.get_by_dst_node(dst_node_id=monkey_island_monkey["_id"])] monkey_ids = [x.id for x in Monkey.objects() if ("tunnel" not in x) and (x.id not in existing_ids) and diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index ecce06c85..f6f5362c3 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -189,12 +189,9 @@ class NodeService: {'$unset': {'tunnel': ''}}, upsert=False) - try: - edge = Edge.objects.get(src_node_id=monkey_id, tunnel=True) - edge.tunnel = False - edge.save() - except DoesNotExist: - pass + edges = EdgeService.get_tunnel_edges_by_src(monkey_id) + for edge in edges: + edge.disable_tunnel() @staticmethod def set_monkey_tunnel(monkey_id, tunnel_host_ip): diff --git a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py index 8cb5db0d6..f8ea52b6e 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/exploit.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/exploit.py @@ -25,7 +25,7 @@ def process_exploit_telemetry(telemetry_json): timestamp=telemetry_json['timestamp']) -def update_node_credentials_from_successful_attempts(edge: Edge, telemetry_json): +def update_node_credentials_from_successful_attempts(edge: EdgeService, telemetry_json): for attempt in telemetry_json['data']['attempts']: if attempt['result']: found_creds = {'user': attempt['user']} @@ -35,13 +35,13 @@ def update_node_credentials_from_successful_attempts(edge: Edge, telemetry_json) NodeService.add_credentials_to_node(edge.dst_node_id, found_creds) -def update_network_with_exploit(edge: Edge, telemetry_json): +def update_network_with_exploit(edge: EdgeService, telemetry_json): telemetry_json['data']['info']['started'] = dateutil.parser.parse(telemetry_json['data']['info']['started']) telemetry_json['data']['info']['finished'] = dateutil.parser.parse(telemetry_json['data']['info']['finished']) new_exploit = copy.deepcopy(telemetry_json['data']) new_exploit.pop('machine') new_exploit['timestamp'] = telemetry_json['timestamp'] - EdgeService.update_based_on_exploit(edge, new_exploit) + edge.update_based_on_exploit(new_exploit) if new_exploit['result']: NodeService.set_node_exploited(edge.dst_node_id) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/scan.py b/monkey/monkey_island/cc/services/telemetry/processing/scan.py index 48c1f11c3..7a7f0b19c 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/scan.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/scan.py @@ -18,7 +18,7 @@ def process_scan_telemetry(telemetry_json): def update_edges_and_nodes_based_on_scan_telemetry(telemetry_json): edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) - EdgeService.update_based_on_scan_telemetry(edge, telemetry_json) + edge.update_based_on_scan_telemetry(telemetry_json) node = mongo.db.node.find_one({"_id": edge.dst_node_id}) if node is not None: @@ -32,4 +32,4 @@ def update_edges_and_nodes_based_on_scan_telemetry(telemetry_json): {"$set": {"os.version": scan_os["version"]}}, upsert=False) label = NodeService.get_label_for_endpoint(node["_id"]) - EdgeService.update_label(edge, node["_id"], label) + edge.update_label(node["_id"], label) From 4c8319669f29cecb3adceea8f66be642c524c68f Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 8 Jun 2020 10:31:31 +0300 Subject: [PATCH 18/21] Map jiggle small-fix: wait less for map to stop moving --- monkey/monkey_island/cc/ui/src/components/map/MapOptions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js b/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js index e09e93124..e09040b01 100644 --- a/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js +++ b/monkey/monkey_island/cc/ui/src/components/map/MapOptions.js @@ -45,7 +45,7 @@ export const basic_options = { springLength: 100, springConstant: 0.025 }, - minVelocity: 0.3, + minVelocity: 0.7, maxVelocity: 25 } }; From 3228bcf2c7da23a66ca8f56dc5e9113b230f3201 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 8 Jun 2020 14:23:39 +0300 Subject: [PATCH 19/21] CR comments fixed: renames and readability improvements --- ...ollector.py => mimikatz_cred_collector.py} | 12 +++--- .../pypykatz_handler.py | 40 +++++++++---------- .../test_pypykatz_handler.py | 3 +- ...s_credential.py => windows_credentials.py} | 2 +- .../system_info/windows_info_collector.py | 6 +-- .../report-components/SecurityReport.js | 5 --- 6 files changed, 31 insertions(+), 37 deletions(-) rename monkey/infection_monkey/system_info/windows_cred_collector/{windows_cred_collector.py => mimikatz_cred_collector.py} (61%) rename monkey/infection_monkey/system_info/windows_cred_collector/{windows_credential.py => windows_credentials.py} (94%) diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py b/monkey/infection_monkey/system_info/windows_cred_collector/mimikatz_cred_collector.py similarity index 61% rename from monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py rename to monkey/infection_monkey/system_info/windows_cred_collector/mimikatz_cred_collector.py index 3e462bbe0..96d3912e3 100644 --- a/monkey/infection_monkey/system_info/windows_cred_collector/windows_cred_collector.py +++ b/monkey/infection_monkey/system_info/windows_cred_collector/mimikatz_cred_collector.py @@ -1,21 +1,21 @@ import logging from typing import List -from infection_monkey.system_info.windows_cred_collector.pypykatz_handler import get_windows_creds -from infection_monkey.system_info.windows_cred_collector.windows_credential import WindowsCredential +from infection_monkey.system_info.windows_cred_collector import pypykatz_handler +from infection_monkey.system_info.windows_cred_collector.windows_credentials import WindowsCredentials LOG = logging.getLogger(__name__) -class WindowsCredentialCollector(object): +class MimikatzCredentialCollector(object): @staticmethod def get_creds(): - creds = get_windows_creds() - return WindowsCredentialCollector.cred_list_to_cred_dict(creds) + creds = pypykatz_handler.get_windows_creds() + return MimikatzCredentialCollector.cred_list_to_cred_dict(creds) @staticmethod - def cred_list_to_cred_dict(creds: List[WindowsCredential]): + def cred_list_to_cred_dict(creds: List[WindowsCredentials]): cred_dict = {} for cred in creds: # Lets not use "." and "$" in keys, because it will confuse mongo. diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py b/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py index 3e726a989..7688c8643 100644 --- a/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py +++ b/monkey/infection_monkey/system_info/windows_cred_collector/pypykatz_handler.py @@ -1,29 +1,29 @@ import binascii -from typing import Dict, List +from typing import Dict, List, NewType, Any from pypykatz.pypykatz import pypykatz -from infection_monkey.system_info.windows_cred_collector.windows_credential import WindowsCredential +from infection_monkey.system_info.windows_cred_collector.windows_credentials import WindowsCredentials CREDENTIAL_TYPES = ['msv_creds', 'wdigest_creds', 'ssp_creds', 'livessp_creds', 'dpapi_creds', 'kerberos_creds', 'credman_creds', 'tspkg_creds'] +PypykatzCredential = NewType('PypykatzCredential', Dict) - -def get_windows_creds(): +def get_windows_creds() -> List[WindowsCredentials]: pypy_handle = pypykatz.go_live() logon_data = pypy_handle.to_dict() windows_creds = _parse_pypykatz_results(logon_data) return windows_creds -def _parse_pypykatz_results(pypykatz_data: Dict) -> List: +def _parse_pypykatz_results(pypykatz_data: Dict) -> List[WindowsCredentials]: windows_creds = [] for session in pypykatz_data['logon_sessions'].values(): windows_creds.extend(_get_creds_from_pypykatz_session(session)) return windows_creds -def _get_creds_from_pypykatz_session(pypykatz_session: Dict): +def _get_creds_from_pypykatz_session(pypykatz_session: Dict) -> List[WindowsCredentials]: windows_creds = [] for cred_type_key in CREDENTIAL_TYPES: pypykatz_creds = pypykatz_session[cred_type_key] @@ -31,23 +31,23 @@ def _get_creds_from_pypykatz_session(pypykatz_session: Dict): return windows_creds -def _get_creds_from_pypykatz_creds(pypykatz_creds): +def _get_creds_from_pypykatz_creds(pypykatz_creds: List[PypykatzCredential]) -> List[WindowsCredentials]: creds = _filter_empty_creds(pypykatz_creds) return [_get_windows_cred(cred) for cred in creds] -def _filter_empty_creds(pypykatz_creds: List[Dict]): +def _filter_empty_creds(pypykatz_creds: List[PypykatzCredential]) -> List[PypykatzCredential]: return [cred for cred in pypykatz_creds if not _is_cred_empty(cred)] -def _is_cred_empty(pypykatz_cred: Dict): +def _is_cred_empty(pypykatz_cred: PypykatzCredential): password_empty = 'password' not in pypykatz_cred or not pypykatz_cred['password'] ntlm_hash_empty = 'NThash' not in pypykatz_cred or not pypykatz_cred['NThash'] lm_hash_empty = 'LMhash' not in pypykatz_cred or not pypykatz_cred['LMhash'] return password_empty and ntlm_hash_empty and lm_hash_empty -def _get_windows_cred(pypykatz_cred: Dict): +def _get_windows_cred(pypykatz_cred: PypykatzCredential): password = '' ntlm_hash = '' lm_hash = '' @@ -58,15 +58,15 @@ def _get_windows_cred(pypykatz_cred: Dict): ntlm_hash = _hash_to_string(pypykatz_cred['NThash']) if 'LMhash' in pypykatz_cred: lm_hash = _hash_to_string(pypykatz_cred['LMhash']) - return WindowsCredential(username=username, - password=password, - ntlm_hash=ntlm_hash, - lm_hash=lm_hash) + return WindowsCredentials(username=username, + password=password, + ntlm_hash=ntlm_hash, + lm_hash=lm_hash) -def _hash_to_string(hash): - if type(hash) == str: - return hash - if type(hash) == bytes: - return binascii.hexlify(bytearray(hash)).decode() - raise Exception(f"Can't convert hash to string, unsupported hash type {type(hash)}") +def _hash_to_string(hash_: Any): + if type(hash_) == str: + return hash_ + if type(hash_) == bytes: + return binascii.hexlify(bytearray(hash_)).decode() + raise Exception(f"Can't convert hash_ to string, unsupported hash_ type {type(hash_)}") diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py b/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py index 025f5d1dc..b0ae2d751 100644 --- a/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py +++ b/monkey/infection_monkey/system_info/windows_cred_collector/test_pypykatz_handler.py @@ -80,5 +80,4 @@ class TestPypykatzHandler(TestCase): 'lm_hash': ''}, ] results = [result.to_dict() for result in results] - for test_dict in test_dicts: - self.assertTrue(test_dict in results) + [self.assertTrue(test_dict in results) for test_dict in test_dicts] diff --git a/monkey/infection_monkey/system_info/windows_cred_collector/windows_credential.py b/monkey/infection_monkey/system_info/windows_cred_collector/windows_credentials.py similarity index 94% rename from monkey/infection_monkey/system_info/windows_cred_collector/windows_credential.py rename to monkey/infection_monkey/system_info/windows_cred_collector/windows_credentials.py index f8d1ecac9..8f57ce5c3 100644 --- a/monkey/infection_monkey/system_info/windows_cred_collector/windows_credential.py +++ b/monkey/infection_monkey/system_info/windows_cred_collector/windows_credentials.py @@ -1,7 +1,7 @@ from typing import Dict -class WindowsCredential: +class WindowsCredentials: def __init__(self, username: str, password="", ntlm_hash="", lm_hash=""): self.username = username self.password = password diff --git a/monkey/infection_monkey/system_info/windows_info_collector.py b/monkey/infection_monkey/system_info/windows_info_collector.py index 01d6c768e..13f0a5593 100644 --- a/monkey/infection_monkey/system_info/windows_info_collector.py +++ b/monkey/infection_monkey/system_info/windows_info_collector.py @@ -2,7 +2,7 @@ import os import logging import sys -from infection_monkey.system_info.windows_cred_collector.windows_cred_collector import WindowsCredentialCollector +from infection_monkey.system_info.windows_cred_collector.mimikatz_cred_collector import MimikatzCredentialCollector sys.coinit_flags = 0 # needed for proper destruction of the wmi python module # noinspection PyPep8 @@ -63,7 +63,7 @@ class WindowsInfoCollector(InfoCollector): def get_mimikatz_info(self): LOG.info("Gathering mimikatz info") try: - credentials = WindowsCredentialCollector.get_creds() + credentials = MimikatzCredentialCollector.get_creds() if credentials: if "credentials" in self.info: self.info["credentials"].update(credentials) @@ -72,4 +72,4 @@ class WindowsInfoCollector(InfoCollector): else: LOG.info('No mimikatz info was gathered') except Exception as e: - LOG.info(f"Pypykatz failed: {e}") + LOG.info(f"Mimikatz credential collector failed: {e}") diff --git a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js index 0ca0b74f6..87299edff 100644 --- a/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js +++ b/monkey/monkey_island/cc/ui/src/components/report-components/SecurityReport.js @@ -420,11 +420,6 @@ class ReportPageComponent extends AuthComponent {
-
- {/*Disable PTH map until we fix it - this.generateReportPthMap()*/} -
-
From 5669ae652cc08b715fd980f52075ad9af6f802a3 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 8 Jun 2020 14:24:16 +0300 Subject: [PATCH 20/21] Bugfix - username with "." character fix --- .../cc/services/telemetry/processing/system_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index c6caf4865..88eb9285c 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -82,7 +82,7 @@ def replace_user_dot_with_comma(creds): def add_system_info_creds_to_config(creds): for user in creds: - ConfigService.creds_add_username(user) + ConfigService.creds_add_username(creds[user]['username']) if 'password' in creds[user] and creds[user]['password']: ConfigService.creds_add_password(creds[user]['password']) if 'lm_hash' in creds[user] and creds[user]['lm_hash']: From 966599a0382515d6a62776768ddb84a407abc1ea Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 8 Jun 2020 15:12:40 +0300 Subject: [PATCH 21/21] Removed pass the hash map UI component --- .../components/pages/PassTheHashMapPage.js | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js diff --git a/monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js b/monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js deleted file mode 100644 index af102c57e..000000000 --- a/monkey/monkey_island/cc/ui/src/components/pages/PassTheHashMapPage.js +++ /dev/null @@ -1,52 +0,0 @@ -import React from 'react'; -import {ReactiveGraph} from 'components/reactive-graph/ReactiveGraph'; -import AuthComponent from '../AuthComponent'; -import {optionsPth} from '../map/MapOptions'; -import {Col} from 'react-bootstrap'; - -class PassTheHashMapPageComponent extends AuthComponent { - constructor(props) { - super(props); - this.state = { - graph: props.graph, - selected: null, - selectedType: null - }; - } - - events = { - select: event => this.selectionChanged(event) - }; - - selectionChanged(event) { - if (event.nodes.length === 1) { - let displayedNode = this.state.graph.nodes.find( - function (node) { - return node['id'] === event.nodes[0]; - }); - this.setState({selected: displayedNode, selectedType: 'node'}) - } else if (event.edges.length === 1) { - let displayedEdge = this.state.graph.edges.find( - function (edge) { - return edge['id'] === event.edges[0]; - }); - this.setState({selected: displayedEdge, selectedType: 'edge'}); - } else { - this.setState({selected: null, selectedType: null}); - } - } - - render() { - return ( -
- -
- -
- -
- ); - } -} - -export default PassTheHashMapPageComponent;