Merge pull request #892 from shreyamalviya/postgresql-fp

PostgreSQL Fingerprinter

monkey/common/common_consts/zero_trust_consts.py

@@ -22,6 +22,7 @@ STATUS_FAILED = "Failed"
 # Don't change order! The statuses are ordered by importance/severity.
 ORDERED_TEST_STATUSES = [STATUS_FAILED, STATUS_VERIFY, STATUS_PASSED, STATUS_UNEXECUTED]
 
+TEST_DATA_ENDPOINT_POSTGRESQL = "unencrypted_data_endpoint_postgresql"
 TEST_DATA_ENDPOINT_ELASTIC = "unencrypted_data_endpoint_elastic"
 TEST_DATA_ENDPOINT_HTTP = "unencrypted_data_endpoint_http"
 TEST_MACHINE_EXPLOITED = "machine_exploited"
@@ -47,6 +48,7 @@ TESTS = (
     TEST_MACHINE_EXPLOITED,
     TEST_DATA_ENDPOINT_HTTP,
     TEST_DATA_ENDPOINT_ELASTIC,
+    TEST_DATA_ENDPOINT_POSTGRESQL,
     TEST_TUNNELING,
     TEST_COMMUNICATE_AS_NEW_USER,
     TEST_SCOUTSUITE_PERMISSIVE_FIREWALL_RULES,
@@ -165,6 +167,17 @@ TESTS_MAP = {
         PILLARS_KEY: [DATA],
         POSSIBLE_STATUSES_KEY: [STATUS_UNEXECUTED, STATUS_FAILED, STATUS_PASSED]
     },
+    TEST_DATA_ENDPOINT_POSTGRESQL: {
+        TEST_EXPLANATION_KEY: "The Monkey scanned for unencrypted access to PostgreSQL servers.",
+        FINDING_EXPLANATION_BY_STATUS_KEY: {
+            STATUS_FAILED: "Monkey accessed PostgreSQL servers. Limit access to data by encrypting it in in-transit.",
+            STATUS_PASSED: "Monkey didn't find open PostgreSQL servers. If you have such servers, look for alerts that "
+                           "indicate attempts to access them. "
+        },
+        PRINCIPLE_KEY: PRINCIPLE_DATA_CONFIDENTIALITY,
+        PILLARS_KEY: [DATA],
+        POSSIBLE_STATUSES_KEY: [STATUS_UNEXECUTED, STATUS_FAILED, STATUS_PASSED]
+    },
     TEST_TUNNELING: {
         TEST_EXPLANATION_KEY: "The Monkey tried to tunnel traffic using other monkeys.",
         FINDING_EXPLANATION_BY_STATUS_KEY: {

monkey/infection_monkey/config.py

@@ -179,7 +179,8 @@ class Configuration(object):
                         443,
                         8008,
                         3306,
-                        9200]
+                        9200,
+                        5432]
     tcp_target_ports.extend(HTTP_PORTS)
     tcp_scan_timeout = 3000  # 3000 Milliseconds
     tcp_scan_interval = 0  # in milliseconds

monkey/infection_monkey/network/postgresql_finger.py

@@ -0,0 +1,155 @@
+import logging
+
+import psycopg2
+
+from common.common_consts.timeouts import MEDIUM_REQUEST_TIMEOUT
+from infection_monkey.model import ID_STRING
+from infection_monkey.network.HostFinger import HostFinger
+
+LOG = logging.getLogger(__name__)
+
+
+class PostgreSQLFinger(HostFinger):
+    """
+    Fingerprints PostgreSQL databases, only on port 5432
+    """
+
+    # Class related consts
+    _SCANNED_SERVICE = "PostgreSQL"
+    POSTGRESQL_DEFAULT_PORT = 5432
+    CREDS = {"username": ID_STRING, "password": ID_STRING}
+    CONNECTION_DETAILS = {
+        "ssl_conf": "SSL is configured on the PostgreSQL server.\n",
+        "ssl_not_conf": "SSL is NOT configured on the PostgreSQL server.\n",
+        "all_ssl": "SSL connections can be made by all.\n",
+        "all_non_ssl": "Non-SSL connections can be made by all.\n",
+        "selected_ssl": "SSL connections can be made by selected hosts only OR "
+        "non-SSL usage is forced.\n",
+        "selected_non_ssl": "Non-SSL connections can be made by selected hosts only OR "
+        "SSL usage is forced.\n",
+        "only_selected": "Only selected hosts can make connections (SSL or non-SSL).\n",
+    }
+    RELEVANT_EX_SUBSTRINGS = {
+        "no_auth": "password authentication failed",
+        "no_entry": "entry for host",  # "no pg_hba.conf entry for host" but filename may be diff
+    }
+
+    def get_host_fingerprint(self, host):
+        try:
+            psycopg2.connect(
+                host=host.ip_addr,
+                port=self.POSTGRESQL_DEFAULT_PORT,
+                user=self.CREDS["username"],
+                password=self.CREDS["password"],
+                sslmode="prefer",
+                connect_timeout=MEDIUM_REQUEST_TIMEOUT,
+            )  # don't need to worry about DB name; creds are wrong, won't check
+
+            # if it comes here, the creds worked
+            # this shouldn't happen since capital letters are not supported in postgres usernames
+            # perhaps the service is a honeypot
+            self.init_service(
+                host.services, self._SCANNED_SERVICE, self.POSTGRESQL_DEFAULT_PORT
+            )
+            host.services[self._SCANNED_SERVICE]["communication_encryption_details"] = (
+                "The PostgreSQL server was unexpectedly accessible with the credentials - "
+                + f"user: '{self.CREDS['username']}' and password: '{self.CREDS['password']}'. Is this a honeypot?"
+            )
+            return True
+
+        except psycopg2.OperationalError as ex:
+            # try block will throw an OperationalError since the credentials are wrong, which we then analyze
+            try:
+                exception_string = str(ex)
+
+                if not self._is_relevant_exception(exception_string):
+                    return False
+
+                # all's well; start analyzing errors
+                self.analyze_operational_error(host, exception_string)
+                return True
+
+            except Exception as err:
+                LOG.debug("Error getting PostgreSQL fingerprint: %s", err)
+
+            return False
+
+    def _is_relevant_exception(self, exception_string):
+        if not any(
+            substr in exception_string
+            for substr in self.RELEVANT_EX_SUBSTRINGS.values()
+        ):
+            # OperationalError due to some other reason - irrelevant exception
+            return False
+        return True
+
+    def analyze_operational_error(self, host, exception_string):
+        self.init_service(
+            host.services, self._SCANNED_SERVICE, self.POSTGRESQL_DEFAULT_PORT
+        )
+
+        exceptions = exception_string.split("\n")
+
+        self.ssl_connection_details = []
+        ssl_conf_on_server = self.is_ssl_configured(exceptions)
+
+        if ssl_conf_on_server:  # SSL configured
+            self.get_connection_details_ssl_configured(exceptions)
+        else:  # SSL not configured
+            self.get_connection_details_ssl_not_configured(exceptions)
+
+        host.services[self._SCANNED_SERVICE][
+            "communication_encryption_details"
+        ] = "".join(self.ssl_connection_details)
+
+    @staticmethod
+    def is_ssl_configured(exceptions):
+        # when trying to authenticate, it checks pg_hba.conf file:
+        # first, for a record where it can connect with SSL and second, without SSL
+        if (
+            len(exceptions) == 1
+        ):  # SSL not configured on server so only checks for non-SSL record
+            return False
+        elif len(exceptions) == 2:  # SSL configured so checks for both
+            return True
+
+    def get_connection_details_ssl_configured(self, exceptions):
+        self.ssl_connection_details.append(self.CONNECTION_DETAILS["ssl_conf"])
+        ssl_selected_comms_only = False
+
+        # check exception message for SSL connection
+        if self.found_entry_for_host_but_pwd_auth_failed(exceptions[0]):
+            self.ssl_connection_details.append(self.CONNECTION_DETAILS["all_ssl"])
+        else:
+            self.ssl_connection_details.append(self.CONNECTION_DETAILS["selected_ssl"])
+            ssl_selected_comms_only = True
+
+        # check exception message for non-SSL connection
+        if self.found_entry_for_host_but_pwd_auth_failed(exceptions[1]):
+            self.ssl_connection_details.append(self.CONNECTION_DETAILS["all_non_ssl"])
+        else:
+            if (
+                ssl_selected_comms_only
+            ):  # if only selected SSL allowed and only selected non-SSL allowed
+                self.ssl_connection_details[-1] = self.CONNECTION_DETAILS[
+                    "only_selected"
+                ]
+            else:
+                self.ssl_connection_details.append(
+                    self.CONNECTION_DETAILS["selected_non_ssl"]
+                )
+
+    def get_connection_details_ssl_not_configured(self, exceptions):
+        self.ssl_connection_details.append(self.CONNECTION_DETAILS["ssl_not_conf"])
+        if self.found_entry_for_host_but_pwd_auth_failed(exceptions[0]):
+            self.ssl_connection_details.append(self.CONNECTION_DETAILS["all_non_ssl"])
+        else:
+            self.ssl_connection_details.append(
+                self.CONNECTION_DETAILS["selected_non_ssl"]
+            )
+
+    @staticmethod
+    def found_entry_for_host_but_pwd_auth_failed(exception):
+        if PostgreSQLFinger.RELEVANT_EX_SUBSTRINGS["no_auth"] in exception:
+            return True  # entry found in pg_hba.conf file but password authentication failed
+        return False  # entry not found in pg_hba.conf file

monkey/infection_monkey/network/test_postgresql_finger.py

@@ -0,0 +1,185 @@
+import pytest
+
+import infection_monkey.network.postgresql_finger
+from infection_monkey.network.postgresql_finger import PostgreSQLFinger
+
+IRRELEVANT_EXCEPTION_STRING = "This is an irrelevant exception string."
+
+_RELEVANT_EXCEPTION_STRING_PARTS = {
+    "pwd_auth_failed": 'FATAL:  password authentication failed for user "root"',
+    "ssl_on_entry_not_found": 'FATAL:  no pg_hba.conf entry for host "127.0.0.1",'
+    'user "random", database "postgres", SSL on',
+    "ssl_off_entry_not_found": 'FATAL:  no pg_hba.conf entry for host "127.0.0.1",'
+    'user "random", database "postgres", SSL off',
+}
+
+_RELEVANT_EXCEPTION_STRINGS = {
+    "pwd_auth_failed": _RELEVANT_EXCEPTION_STRING_PARTS["pwd_auth_failed"],
+    "ssl_off_entry_not_found": _RELEVANT_EXCEPTION_STRING_PARTS[
+        "ssl_off_entry_not_found"
+    ],
+    "pwd_auth_failed_pwd_auth_failed": "\n".join(
+        [
+            _RELEVANT_EXCEPTION_STRING_PARTS["pwd_auth_failed"],
+            _RELEVANT_EXCEPTION_STRING_PARTS["pwd_auth_failed"],
+        ]
+    ),
+    "pwd_auth_failed_ssl_off_entry_not_found": "\n".join(
+        [
+            _RELEVANT_EXCEPTION_STRING_PARTS["pwd_auth_failed"],
+            _RELEVANT_EXCEPTION_STRING_PARTS["ssl_off_entry_not_found"],
+        ]
+    ),
+    "ssl_on_entry_not_found_pwd_auth_failed": "\n".join(
+        [
+            _RELEVANT_EXCEPTION_STRING_PARTS["ssl_on_entry_not_found"],
+            _RELEVANT_EXCEPTION_STRING_PARTS["pwd_auth_failed"],
+        ]
+    ),
+    "ssl_on_entry_not_found_ssl_off_entry_not_found": "\n".join(
+        [
+            _RELEVANT_EXCEPTION_STRING_PARTS["ssl_on_entry_not_found"],
+            _RELEVANT_EXCEPTION_STRING_PARTS["ssl_off_entry_not_found"],
+        ]
+    ),
+}
+
+_RESULT_STRINGS = {
+    "ssl_conf": "SSL is configured on the PostgreSQL server.\n",
+    "ssl_not_conf": "SSL is NOT configured on the PostgreSQL server.\n",
+    "all_ssl": "SSL connections can be made by all.\n",
+    "all_non_ssl": "Non-SSL connections can be made by all.\n",
+    "selected_ssl": "SSL connections can be made by selected hosts only OR "
+    "non-SSL usage is forced.\n",
+    "selected_non_ssl": "Non-SSL connections can be made by selected hosts only OR "
+    "SSL usage is forced.\n",
+    "only_selected": "Only selected hosts can make connections (SSL or non-SSL).\n",
+}
+
+RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS = {
+    # SSL not configured, all non-SSL allowed
+    _RELEVANT_EXCEPTION_STRINGS["pwd_auth_failed"]: [
+        _RESULT_STRINGS["ssl_not_conf"],
+        _RESULT_STRINGS["all_non_ssl"],
+    ],
+    # SSL not configured, selected non-SSL allowed
+    _RELEVANT_EXCEPTION_STRINGS["ssl_off_entry_not_found"]: [
+        _RESULT_STRINGS["ssl_not_conf"],
+        _RESULT_STRINGS["selected_non_ssl"],
+    ],
+    # all SSL allowed, all non-SSL allowed
+    _RELEVANT_EXCEPTION_STRINGS["pwd_auth_failed_pwd_auth_failed"]: [
+        _RESULT_STRINGS["ssl_conf"],
+        _RESULT_STRINGS["all_ssl"],
+        _RESULT_STRINGS["all_non_ssl"],
+    ],
+    # all SSL allowed, selected non-SSL allowed
+    _RELEVANT_EXCEPTION_STRINGS["pwd_auth_failed_ssl_off_entry_not_found"]: [
+        _RESULT_STRINGS["ssl_conf"],
+        _RESULT_STRINGS["all_ssl"],
+        _RESULT_STRINGS["selected_non_ssl"],
+    ],
+    # selected SSL allowed, all non-SSL allowed
+    _RELEVANT_EXCEPTION_STRINGS["ssl_on_entry_not_found_pwd_auth_failed"]: [
+        _RESULT_STRINGS["ssl_conf"],
+        _RESULT_STRINGS["selected_ssl"],
+        _RESULT_STRINGS["all_non_ssl"],
+    ],
+    # selected SSL allowed, selected non-SSL allowed
+    _RELEVANT_EXCEPTION_STRINGS["ssl_on_entry_not_found_ssl_off_entry_not_found"]: [
+        _RESULT_STRINGS["ssl_conf"],
+        _RESULT_STRINGS["only_selected"],
+    ],
+}
+
+
+@pytest.fixture
+def mock_PostgreSQLFinger():
+    return PostgreSQLFinger()
+
+
+class DummyHost:
+    def __init__(self):
+        self.services = {}
+
+
+@pytest.fixture
+def host():
+    return DummyHost()
+
+
+def test_irrelevant_exception(mock_PostgreSQLFinger):
+    assert (
+        mock_PostgreSQLFinger._is_relevant_exception(IRRELEVANT_EXCEPTION_STRING)
+        is False
+    )
+
+
+def test_exception_ssl_not_configured_all_non_ssl_allowed(mock_PostgreSQLFinger, host):
+    exception = _RELEVANT_EXCEPTION_STRINGS["pwd_auth_failed"]
+    assert mock_PostgreSQLFinger._is_relevant_exception(exception) is True
+
+    mock_PostgreSQLFinger.analyze_operational_error(host, exception)
+    assert host.services[mock_PostgreSQLFinger._SCANNED_SERVICE][
+        "communication_encryption_details"
+    ] == "".join(RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS[exception])
+
+
+def test_exception_ssl_not_configured_selected_non_ssl_allowed(
+    mock_PostgreSQLFinger, host
+):
+    exception = _RELEVANT_EXCEPTION_STRINGS["ssl_off_entry_not_found"]
+    assert mock_PostgreSQLFinger._is_relevant_exception(exception) is True
+
+    mock_PostgreSQLFinger.analyze_operational_error(host, exception)
+    assert host.services[mock_PostgreSQLFinger._SCANNED_SERVICE][
+        "communication_encryption_details"
+    ] == "".join(RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS[exception])
+
+
+def test_exception_all_ssl_allowed_all_non_ssl_allowed(mock_PostgreSQLFinger, host):
+    exception = _RELEVANT_EXCEPTION_STRINGS["pwd_auth_failed_pwd_auth_failed"]
+    assert mock_PostgreSQLFinger._is_relevant_exception(exception) is True
+
+    mock_PostgreSQLFinger.analyze_operational_error(host, exception)
+    assert host.services[mock_PostgreSQLFinger._SCANNED_SERVICE][
+        "communication_encryption_details"
+    ] == "".join(RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS[exception])
+
+
+def test_exception_all_ssl_allowed_selected_non_ssl_allowed(
+    mock_PostgreSQLFinger, host
+):
+    exception = _RELEVANT_EXCEPTION_STRINGS["pwd_auth_failed_ssl_off_entry_not_found"]
+    assert mock_PostgreSQLFinger._is_relevant_exception(exception) is True
+
+    mock_PostgreSQLFinger.analyze_operational_error(host, exception)
+    assert host.services[mock_PostgreSQLFinger._SCANNED_SERVICE][
+        "communication_encryption_details"
+    ] == "".join(RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS[exception])
+
+
+def test_exception_selected_ssl_allowed_all_non_ssl_allowed(
+    mock_PostgreSQLFinger, host
+):
+    exception = _RELEVANT_EXCEPTION_STRINGS["ssl_on_entry_not_found_pwd_auth_failed"]
+    assert mock_PostgreSQLFinger._is_relevant_exception(exception) is True
+
+    mock_PostgreSQLFinger.analyze_operational_error(host, exception)
+    assert host.services[mock_PostgreSQLFinger._SCANNED_SERVICE][
+        "communication_encryption_details"
+    ] == "".join(RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS[exception])
+
+
+def test_exception_selected_ssl_allowed_selected_non_ssl_allowed(
+    mock_PostgreSQLFinger, host
+):
+    exception = _RELEVANT_EXCEPTION_STRINGS[
+        "ssl_on_entry_not_found_ssl_off_entry_not_found"
+    ]
+    assert mock_PostgreSQLFinger._is_relevant_exception(exception) is True
+
+    mock_PostgreSQLFinger.analyze_operational_error(host, exception)
+    assert host.services[mock_PostgreSQLFinger._SCANNED_SERVICE][
+        "communication_encryption_details"
+    ] == "".join(RELEVANT_EXCEPTIONS_WITH_EXPECTED_RESULTS[exception])

monkey/infection_monkey/requirements.txt

@@ -9,6 +9,7 @@ netifaces>=0.10.9
 odict==1.7.0
 paramiko>=2.7.1
 psutil>=5.7.0
+psycopg2-binary==2.8.6
 pycryptodome==3.9.8
 pyftpdlib==1.5.6
 pymssql<3.0

monkey/monkey_island/cc/services/config_schema/definitions/finger_classes.py

@@ -71,6 +71,15 @@ FINGER_CLASSES = {
             "safe": True,
             "info": "Checks if ElasticSearch is running and attempts to find it's version.",
             "attack_techniques": ["T1210"]
+        },
+        {
+            "type": "string",
+            "enum": [
+                "PostgreSQLFinger"
+            ],
+            "title": "PostgreSQLFinger",
+            "info": "Checks if PostgreSQL service is running and if its communication is encrypted.",
+            "attack_techniques": ["T1210"]
         }
     ]
 }

monkey/monkey_island/cc/services/config_schema/internal.py

@@ -239,7 +239,8 @@ INTERNAL = {
                         "HTTPFinger",
                         "MySQLFinger",
                         "MSSQLFinger",
-                        "ElasticFinger"
+                        "ElasticFinger",
+                        "PostgreSQLFinger"
                     ]
                 }
             }

monkey/monkey_island/cc/services/telemetry/zero_trust_checks/data_endpoints.py

@@ -7,6 +7,7 @@ from monkey_island.cc.models.zero_trust.event import Event
 from monkey_island.cc.services.zero_trust.monkey_findings.monkey_zt_finding_service import MonkeyZTFindingService
 
 HTTP_SERVERS_SERVICES_NAMES = ['tcp-80']
+POSTGRESQL_SERVER_SERVICE_NAME = 'PostgreSQL'
 
 
 def check_open_data_endpoints(telemetry_json):
@@ -14,6 +15,7 @@ def check_open_data_endpoints(telemetry_json):
     current_monkey = Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid'])
     found_http_server_status = zero_trust_consts.STATUS_PASSED
     found_elastic_search_server = zero_trust_consts.STATUS_PASSED
+    found_postgresql_server = zero_trust_consts.STATUS_PASSED
 
     events = [
         Event.create_event(
@@ -54,6 +56,17 @@ def check_open_data_endpoints(telemetry_json):
                 ),
                 event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK
             ))
+        if service_name == POSTGRESQL_SERVER_SERVICE_NAME:
+            found_postgresql_server = zero_trust_consts.STATUS_FAILED
+            events.append(Event.create_event(
+                title="Scan telemetry analysis",
+                message="Service {} on {} recognized as an open data endpoint! Service details: {}".format(
+                    service_data["display_name"],
+                    telemetry_json["data"]["machine"]["ip_addr"],
+                    json.dumps(service_data)
+                ),
+                event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK
+            ))
 
     MonkeyZTFindingService.create_or_add_to_existing(test=zero_trust_consts.TEST_DATA_ENDPOINT_HTTP,
                                                      status=found_http_server_status, events=events)
@@ -61,4 +74,7 @@ def check_open_data_endpoints(telemetry_json):
     MonkeyZTFindingService.create_or_add_to_existing(test=zero_trust_consts.TEST_DATA_ENDPOINT_ELASTIC,
                                                      status=found_elastic_search_server, events=events)
 
+    MonkeyZTFindingService.create_or_add_to_existing(test=zero_trust_consts.TEST_DATA_ENDPOINT_POSTGRESQL,
+                                                     status=found_postgresql_server, events=events)
+
     MonkeyZTFindingService.add_malicious_activity_to_timeline(events)