From 017519954012617bf4f69199be8bd508f731ba0e Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 11:35:08 +0200 Subject: [PATCH 1/9] Island, Agent: remove environment collector --- .swm/OwcKMnALpn7tuBaJY1US.swm | 12 +------- .../adding-system-info-collectors.md | 6 ++-- .../collectors/environment_collector.py | 24 ---------------- .../system_info_collectors/environment.py | 12 -------- .../system_info_telemetry_dispatcher.py | 5 ---- .../test_environment_telemetry_processing.py | 28 ------------------- vulture_allowlist.py | 1 - 7 files changed, 4 insertions(+), 84 deletions(-) delete mode 100644 monkey/infection_monkey/system_info/collectors/environment_collector.py delete mode 100644 monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py delete mode 100644 monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_environment_telemetry_processing.py diff --git a/.swm/OwcKMnALpn7tuBaJY1US.swm b/.swm/OwcKMnALpn7tuBaJY1US.swm index a091073c5..5555018fa 100644 --- a/.swm/OwcKMnALpn7tuBaJY1US.swm +++ b/.swm/OwcKMnALpn7tuBaJY1US.swm @@ -5,7 +5,7 @@ "dod": "Add a system info collector that collects the machine hostname.", "tests": [], "hints": [ - "First thing you should do is take a look at a different collector (like EnvironmentCollector) and 100% understand how it runs, how results are relayed back to the server, and how the server processes the data.", + "First thing you should do is take a look at a different collector (like HostnameCollector) and 100% understand how it runs, how results are relayed back to the server, and how the server processes the data.", "Try to run \"socket.getfqdn()\".", "Take a look at SystemInfoCollector - that's the base class you'll need to implement.", "Make sure you add the new collector to the configuration in all relevant places, including making it ON by default!" @@ -25,7 +25,6 @@ " AWS_COLLECTOR = \"AwsCollector\"", "*HOSTNAME_COLLECTOR = \"HostnameCollector\"", "+# SWIMMER: Collector name goes here.", - " ENVIRONMENT_COLLECTOR = \"EnvironmentCollector\"", " PROCESS_LIST_COLLECTOR = \"ProcessListCollector\"", " MIMIKATZ_COLLECTOR = \"MimikatzCollector\"" ] @@ -60,7 +59,6 @@ "comments": [], "firstLineNumber": 4, "lines": [ - " ENVIRONMENT_COLLECTOR,", "* HOSTNAME_COLLECTOR,", " MIMIKATZ_COLLECTOR,", " PROCESS_LIST_COLLECTOR,", @@ -97,7 +95,6 @@ "lines": [ " from common.common_consts.system_info_collectors_names import (", " AWS_COLLECTOR,", - " ENVIRONMENT_COLLECTOR,", "* HOSTNAME_COLLECTOR,", " MIMIKATZ_COLLECTOR,", " PROCESS_LIST_COLLECTOR,", @@ -111,7 +108,6 @@ "firstLineNumber": 91, "lines": [ " \"default\": [", - " ENVIRONMENT_COLLECTOR,", " AWS_COLLECTOR,", "* HOSTNAME_COLLECTOR,", " PROCESS_LIST_COLLECTOR,", @@ -149,7 +145,6 @@ " ", " from common.common_consts.system_info_collectors_names import (", " AWS_COLLECTOR,", - " ENVIRONMENT_COLLECTOR,", "* HOSTNAME_COLLECTOR," ] }, @@ -161,7 +156,6 @@ "lines": [ " SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS = {", " AWS_COLLECTOR: [process_aws_telemetry],", - " ENVIRONMENT_COLLECTOR: [process_environment_telemetry],", "* HOSTNAME_COLLECTOR: [process_hostname_telemetry],", " PROCESS_LIST_COLLECTOR: [check_antivirus_existence],", " }", @@ -171,10 +165,6 @@ { "type": "snippet", "lines": [ - " )", - " from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import (", - " process_environment_telemetry,", - " )", "*from monkey_island.cc.services.telemetry.processing.system_info_collectors.hostname import (", "* process_hostname_telemetry,", "*)", diff --git a/docs/content/development/adding-system-info-collectors.md b/docs/content/development/adding-system-info-collectors.md index 71cea6000..3e924bd4e 100644 --- a/docs/content/development/adding-system-info-collectors.md +++ b/docs/content/development/adding-system-info-collectors.md @@ -39,7 +39,7 @@ class MyNewCollector(SystemInfoCollector): #### Implementation -Override the `collect` method with your own implementation. See the `EnvironmentCollector.py` System Info Collector for reference. You can log during collection as well. +Override the `collect` method with your own implementation. See the `hostname_collector.py` System Info Collector for reference. You can log during collection as well. ### Modify the Monkey Island @@ -57,7 +57,7 @@ You'll need to add your Sytem Info Collector to the `monkey_island/cc/services/c { "type": "string", "enum": [ - "EnvironmentCollector" + "HostnameCollector" ], "title": "Which Environment this machine is on (on prem/cloud)", "attack_techniques": [] @@ -87,7 +87,7 @@ Also, you can add the System Info Collector to be used by default by adding it t "$ref": "#/definitions/system_info_collectors_classes" }, "default": [ - "EnvironmentCollector", + "HostnameCollector", "MyNewCollector" <================================= ], "description": "Determines which system information collectors will collect information." diff --git a/monkey/infection_monkey/system_info/collectors/environment_collector.py b/monkey/infection_monkey/system_info/collectors/environment_collector.py deleted file mode 100644 index 039ede6f5..000000000 --- a/monkey/infection_monkey/system_info/collectors/environment_collector.py +++ /dev/null @@ -1,24 +0,0 @@ -from common.cloud.all_instances import get_all_cloud_instances -from common.cloud.environment_names import Environment -from common.common_consts.system_info_collectors_names import ENVIRONMENT_COLLECTOR -from infection_monkey.system_info.system_info_collector import SystemInfoCollector - - -def get_monkey_environment() -> str: - """ - Get the Monkey's running environment. - :return: One of the cloud providers if on cloud; otherwise, assumes "on premise". - """ - for instance in get_all_cloud_instances(): - if instance.is_instance(): - return instance.get_cloud_provider_name().value - - return Environment.ON_PREMISE.value - - -class EnvironmentCollector(SystemInfoCollector): - def __init__(self): - super().__init__(name=ENVIRONMENT_COLLECTOR) - - def collect(self) -> dict: - return {"environment": get_monkey_environment()} diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py deleted file mode 100644 index 4c685a01b..000000000 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging - -from monkey_island.cc.models.monkey import Monkey - -logger = logging.getLogger(__name__) - - -def process_environment_telemetry(collector_results, monkey_guid): - relevant_monkey = Monkey.get_single_monkey_by_guid(monkey_guid) - relevant_monkey.environment = collector_results["environment"] - relevant_monkey.save() - logger.debug("Updated Monkey {} with env {}".format(str(relevant_monkey), collector_results)) diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py index 7ce4b6fcf..7683cac6f 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/system_info_telemetry_dispatcher.py @@ -3,16 +3,12 @@ import typing from common.common_consts.system_info_collectors_names import ( AWS_COLLECTOR, - ENVIRONMENT_COLLECTOR, HOSTNAME_COLLECTOR, PROCESS_LIST_COLLECTOR, ) from monkey_island.cc.services.telemetry.processing.system_info_collectors.aws import ( process_aws_telemetry, ) -from monkey_island.cc.services.telemetry.processing.system_info_collectors.environment import ( - process_environment_telemetry, -) from monkey_island.cc.services.telemetry.processing.system_info_collectors.hostname import ( process_hostname_telemetry, ) @@ -24,7 +20,6 @@ logger = logging.getLogger(__name__) SYSTEM_INFO_COLLECTOR_TO_TELEMETRY_PROCESSORS = { AWS_COLLECTOR: [process_aws_telemetry], - ENVIRONMENT_COLLECTOR: [process_environment_telemetry], HOSTNAME_COLLECTOR: [process_hostname_telemetry], PROCESS_LIST_COLLECTOR: [check_antivirus_existence], } diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_environment_telemetry_processing.py b/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_environment_telemetry_processing.py deleted file mode 100644 index 042f5b874..000000000 --- a/monkey/tests/unit_tests/monkey_island/cc/services/telemetry/processing/system_info_collectors/test_environment_telemetry_processing.py +++ /dev/null @@ -1,28 +0,0 @@ -import uuid - -from monkey_island.cc.models import Monkey -from monkey_island.cc.services.telemetry.processing.system_info_collectors.system_info_telemetry_dispatcher import ( # noqa: E501 - SystemInfoTelemetryDispatcher, -) - - -class TestEnvironmentTelemetryProcessing: - def test_process_environment_telemetry(self): - # Arrange - monkey_guid = str(uuid.uuid4()) - a_monkey = Monkey(guid=monkey_guid) - a_monkey.save() - dispatcher = SystemInfoTelemetryDispatcher() - - on_premise = "On Premise" - telem_json = { - "data": { - "collectors": { - "EnvironmentCollector": {"environment": on_premise}, - } - }, - "monkey_guid": monkey_guid, - } - dispatcher.dispatch_collector_results_to_relevant_processors(telem_json) - - assert Monkey.get_single_monkey_by_guid(monkey_guid).environment == on_premise diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 114178ea7..ae4f95b55 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -199,7 +199,6 @@ LOG_DIR_NAME # unused variable (envs/monkey_zoo/blackbox/log_handlers/test_logs delete_logs # unused function (envs/monkey_zoo/blackbox/test_blackbox.py:85) MongoQueryJSONEncoder # unused class (envs/monkey_zoo/blackbox/utils/json_encoder.py:6) environment # unused variable (monkey/monkey_island/cc/models/monkey.py:59) -_.environment # unused attribute (monkey/monkey_island/cc/services/telemetry/processing/system_info_collectors/environment.py:10) _.instance_name # unused attribute (monkey/common/cloud/azure/azure_instance.py:35) _.instance_name # unused attribute (monkey/common/cloud/azure/azure_instance.py:64) GCPHandler # unused function (envs/monkey_zoo/blackbox/test_blackbox.py:57) From f3c2a2d01254927a05774c0670c968bd166bcef5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 11:44:31 +0200 Subject: [PATCH 2/9] Common, UT: remove unused GCP cloud instance infrastructure. This infrastructure/classes store information about cloud instances monkeys are ran upon, but this data isn't used anywhere --- monkey/common/cloud/azure/__init__.py | 0 monkey/common/cloud/gcp/__init__.py | 0 monkey/common/cloud/gcp/gcp_instance.py | 54 ------------------ .../common/cloud/gcp/test_gcp_instance.py | 55 ------------------- 4 files changed, 109 deletions(-) delete mode 100644 monkey/common/cloud/azure/__init__.py delete mode 100644 monkey/common/cloud/gcp/__init__.py delete mode 100644 monkey/common/cloud/gcp/gcp_instance.py delete mode 100644 monkey/tests/unit_tests/common/cloud/gcp/test_gcp_instance.py diff --git a/monkey/common/cloud/azure/__init__.py b/monkey/common/cloud/azure/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/monkey/common/cloud/gcp/__init__.py b/monkey/common/cloud/gcp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/monkey/common/cloud/gcp/gcp_instance.py b/monkey/common/cloud/gcp/gcp_instance.py deleted file mode 100644 index 1fc208165..000000000 --- a/monkey/common/cloud/gcp/gcp_instance.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging - -import requests - -from common.cloud.environment_names import Environment -from common.cloud.instance import CloudInstance -from common.common_consts.timeouts import SHORT_REQUEST_TIMEOUT - -logger = logging.getLogger(__name__) - -GCP_METADATA_SERVICE_URL = "http://metadata.google.internal/" - - -class GcpInstance(CloudInstance): - """ - Used to determine if on GCP. See https://cloud.google.com/compute/docs/storing-retrieving - -metadata#runninggce - """ - - def is_instance(self): - return self._on_gcp - - def get_cloud_provider_name(self) -> Environment: - return Environment.GCP - - def __init__(self): - self._on_gcp = False - - try: - # If not on GCP, this domain shouldn't resolve. - response = requests.get(GCP_METADATA_SERVICE_URL, timeout=SHORT_REQUEST_TIMEOUT) - - if response: - logger.debug("Got ok metadata response: on GCP") - self._on_gcp = True - - if "Metadata-Flavor" not in response.headers: - logger.warning("Got unexpected GCP Metadata format") - else: - if not response.headers["Metadata-Flavor"] == "Google": - logger.warning( - "Got unexpected Metadata flavor: {}".format( - response.headers["Metadata-Flavor"] - ) - ) - else: - logger.warning( - "On GCP, but metadata response not ok: {}".format(response.status_code) - ) - except requests.RequestException: - logger.debug( - "Failed to get response from GCP metadata service: This instance is not on GCP" - ) - self._on_gcp = False diff --git a/monkey/tests/unit_tests/common/cloud/gcp/test_gcp_instance.py b/monkey/tests/unit_tests/common/cloud/gcp/test_gcp_instance.py deleted file mode 100644 index 9170b81c5..000000000 --- a/monkey/tests/unit_tests/common/cloud/gcp/test_gcp_instance.py +++ /dev/null @@ -1,55 +0,0 @@ -import pytest -import requests -import requests_mock - -from common.cloud.environment_names import Environment -from common.cloud.gcp.gcp_instance import GCP_METADATA_SERVICE_URL, GcpInstance - - -def get_test_gcp_instance(url, **kwargs): - with requests_mock.Mocker() as m: - m.get(url, **kwargs) - test_gcp_instance_object = GcpInstance() - return test_gcp_instance_object - - -# good request -@pytest.fixture -def good_request_mock_instance(): - return get_test_gcp_instance(GCP_METADATA_SERVICE_URL) - - -def test_is_instance_good_request(good_request_mock_instance): - assert good_request_mock_instance.is_instance() - - -def test_get_cloud_provider_name_good_request(good_request_mock_instance): - assert good_request_mock_instance.get_cloud_provider_name() == Environment.GCP - - -# bad request -@pytest.fixture -def bad_request_mock_instance(): - return get_test_gcp_instance(GCP_METADATA_SERVICE_URL, exc=requests.RequestException) - - -def test_is_instance_bad_request(bad_request_mock_instance): - assert bad_request_mock_instance.is_instance() is False - - -def test_get_cloud_provider_name_bad_request(bad_request_mock_instance): - assert bad_request_mock_instance.get_cloud_provider_name() == Environment.GCP - - -# not found request -@pytest.fixture -def not_found_request_mock_instance(): - return get_test_gcp_instance(GCP_METADATA_SERVICE_URL, status_code=404) - - -def test_is_instance_not_found_request(not_found_request_mock_instance): - assert not_found_request_mock_instance.is_instance() is False - - -def test_get_cloud_provider_name_not_found_request(not_found_request_mock_instance): - assert not_found_request_mock_instance.get_cloud_provider_name() == Environment.GCP From 2b4a51dc974d095496abcc1c9f3b8474abc75679 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 11:45:11 +0200 Subject: [PATCH 3/9] Common, UT: remove unused azure cloud instance infrastructure. This infrastructure/classes store information about cloud instances monkeys are ran upon, but this data isn't used anywhere --- monkey/common/cloud/azure/azure_instance.py | 69 ------ .../common/cloud/azure/test_azure_instance.py | 223 ------------------ 2 files changed, 292 deletions(-) delete mode 100644 monkey/common/cloud/azure/azure_instance.py delete mode 100644 monkey/tests/unit_tests/common/cloud/azure/test_azure_instance.py diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py deleted file mode 100644 index 859ab279f..000000000 --- a/monkey/common/cloud/azure/azure_instance.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging - -import requests -import simplejson - -from common.cloud.environment_names import Environment -from common.cloud.instance import CloudInstance -from common.common_consts.timeouts import SHORT_REQUEST_TIMEOUT - -LATEST_AZURE_METADATA_API_VERSION = "2019-04-30" -AZURE_METADATA_SERVICE_URL = ( - "http://169.254.169.254/metadata/instance?api-version=%s" % LATEST_AZURE_METADATA_API_VERSION -) - -logger = logging.getLogger(__name__) - - -class AzureInstance(CloudInstance): - """ - Access to useful information about the current machine if it's an Azure VM. - Based on Azure metadata service: - https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service - """ - - def is_instance(self): - return self._on_azure - - def get_cloud_provider_name(self) -> Environment: - return Environment.AZURE - - def __init__(self): - """ - Determines if on Azure and if so, gets some basic metadata on this instance. - """ - self.instance_name = None - self.instance_id = None - self.location = None - self._on_azure = False - - try: - response = requests.get( - AZURE_METADATA_SERVICE_URL, - headers={"Metadata": "true"}, - timeout=SHORT_REQUEST_TIMEOUT, - ) - - # If not on cloud, the metadata URL is non-routable and the connection will fail. - # If on AWS, should get 404 since the metadata service URL is different, - # so bool(response) will be false. - if response: - logger.debug("Trying to parse Azure metadata.") - self.try_parse_response(response) - else: - logger.warning(f"Metadata response not ok: {response.status_code}") - except requests.RequestException: - logger.debug( - "Failed to get response from Azure metadata service: This instance is not on " - "Azure." - ) - - def try_parse_response(self, response): - try: - response_data = response.json() - self.instance_name = response_data["compute"]["name"] - self.instance_id = response_data["compute"]["vmId"] - self.location = response_data["compute"]["location"] - self._on_azure = True - except (KeyError, simplejson.errors.JSONDecodeError) as e: - logger.exception(f"Error while parsing response from Azure metadata service: {e}") diff --git a/monkey/tests/unit_tests/common/cloud/azure/test_azure_instance.py b/monkey/tests/unit_tests/common/cloud/azure/test_azure_instance.py deleted file mode 100644 index a7bed81dd..000000000 --- a/monkey/tests/unit_tests/common/cloud/azure/test_azure_instance.py +++ /dev/null @@ -1,223 +0,0 @@ -import pytest -import requests -import requests_mock -import simplejson - -from common.cloud.azure.azure_instance import AZURE_METADATA_SERVICE_URL, AzureInstance -from common.cloud.environment_names import Environment - -GOOD_DATA = { - "compute": { - "azEnvironment": "AZUREPUBLICCLOUD", - "isHostCompatibilityLayerVm": "true", - "licenseType": "Windows_Client", - "location": "westus", - "name": "examplevmname", - "offer": "Windows", - "osProfile": { - "adminUsername": "admin", - "computerName": "examplevmname", - "disablePasswordAuthentication": "true", - }, - "osType": "linux", - "placementGroupId": "f67c14ab-e92c-408c-ae2d-da15866ec79a", - "plan": {"name": "planName", "product": "planProduct", "publisher": "planPublisher"}, - "platformFaultDomain": "36", - "platformUpdateDomain": "42", - "publicKeys": [ - {"keyData": "ssh-rsa 0", "path": "/home/user/.ssh/authorized_keys0"}, - {"keyData": "ssh-rsa 1", "path": "/home/user/.ssh/authorized_keys1"}, - ], - "publisher": "RDFE-Test-Microsoft-Windows-Server-Group", - "resourceGroupName": "macikgo-test-may-23", - "resourceId": "/subscriptions/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx/resourceGroups/macikgo-test" - "-may-23/" - "providers/Microsoft.Compute/virtualMachines/examplevmname", - "securityProfile": {"secureBootEnabled": "true", "virtualTpmEnabled": "false"}, - "sku": "Windows-Server-2012-R2-Datacenter", - "storageProfile": { - "dataDisks": [ - { - "caching": "None", - "createOption": "Empty", - "diskSizeGB": "1024", - "image": {"uri": ""}, - "lun": "0", - "managedDisk": { - "id": "/subscriptions/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx/" - "resourceGroups/macikgo-test-may-23/providers/" - "Microsoft.Compute/disks/exampledatadiskname", - "storageAccountType": "Standard_LRS", - }, - "name": "exampledatadiskname", - "vhd": {"uri": ""}, - "writeAcceleratorEnabled": "false", - } - ], - "imageReference": { - "id": "", - "offer": "UbuntuServer", - "publisher": "Canonical", - "sku": "16.04.0-LTS", - "version": "latest", - }, - "osDisk": { - "caching": "ReadWrite", - "createOption": "FromImage", - "diskSizeGB": "30", - "diffDiskSettings": {"option": "Local"}, - "encryptionSettings": {"enabled": "false"}, - "image": {"uri": ""}, - "managedDisk": { - "id": "/subscriptions/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx/" - "resourceGroups/macikgo-test-may-23/providers/" - "Microsoft.Compute/disks/exampleosdiskname", - "storageAccountType": "Standard_LRS", - }, - "name": "exampleosdiskname", - "osType": "Linux", - "vhd": {"uri": ""}, - "writeAcceleratorEnabled": "false", - }, - }, - "subscriptionId": "xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx", - "tags": "baz:bash;foo:bar", - "version": "15.05.22", - "vmId": "02aab8a4-74ef-476e-8182-f6d2ba4166a6", - "vmScaleSetName": "crpteste9vflji9", - "vmSize": "Standard_A3", - "zone": "", - }, - "network": { - "interface": [ - { - "ipv4": { - "ipAddress": [{"privateIpAddress": "10.144.133.132", "publicIpAddress": ""}], - "subnet": [{"address": "10.144.133.128", "prefix": "26"}], - }, - "ipv6": {"ipAddress": []}, - "macAddress": "0011AAFFBB22", - } - ] - }, -} - -BAD_DATA_NOT_JSON = ( - '\n\n\n\n\nWaiting...\n\n\n " - "\n\n" -) - -BAD_DATA_JSON = {"": ""} - - -def get_test_azure_instance(url, **kwargs): - with requests_mock.Mocker() as m: - m.get(url, **kwargs) - test_azure_instance_object = AzureInstance() - return test_azure_instance_object - - -# good request, good data -@pytest.fixture -def good_data_mock_instance(): - return get_test_azure_instance(AZURE_METADATA_SERVICE_URL, text=simplejson.dumps(GOOD_DATA)) - - -def test_is_instance_good_data(good_data_mock_instance): - assert good_data_mock_instance.is_instance() - - -def test_get_cloud_provider_name_good_data(good_data_mock_instance): - assert good_data_mock_instance.get_cloud_provider_name() == Environment.AZURE - - -def test_try_parse_response_good_data(good_data_mock_instance): - assert good_data_mock_instance.instance_name == GOOD_DATA["compute"]["name"] - assert good_data_mock_instance.instance_id == GOOD_DATA["compute"]["vmId"] - assert good_data_mock_instance.location == GOOD_DATA["compute"]["location"] - - -# good request, bad data (json) -@pytest.fixture -def bad_data_json_mock_instance(): - return get_test_azure_instance(AZURE_METADATA_SERVICE_URL, text=simplejson.dumps(BAD_DATA_JSON)) - - -def test_is_instance_bad_data_json(bad_data_json_mock_instance): - assert bad_data_json_mock_instance.is_instance() is False - - -def test_get_cloud_provider_name_bad_data_json(bad_data_json_mock_instance): - assert bad_data_json_mock_instance.get_cloud_provider_name() == Environment.AZURE - - -def test_instance_attributes_bad_data_json(bad_data_json_mock_instance): - assert bad_data_json_mock_instance.instance_name is None - assert bad_data_json_mock_instance.instance_id is None - assert bad_data_json_mock_instance.location is None - - -# good request, bad data (not json) -@pytest.fixture -def bad_data_not_json_mock_instance(): - return get_test_azure_instance(AZURE_METADATA_SERVICE_URL, text=BAD_DATA_NOT_JSON) - - -def test_is_instance_bad_data_not_json(bad_data_not_json_mock_instance): - assert bad_data_not_json_mock_instance.is_instance() is False - - -def test_get_cloud_provider_name_bad_data_not_json(bad_data_not_json_mock_instance): - assert bad_data_not_json_mock_instance.get_cloud_provider_name() == Environment.AZURE - - -def test_instance_attributes_bad_data_not_json(bad_data_not_json_mock_instance): - assert bad_data_not_json_mock_instance.instance_name is None - assert bad_data_not_json_mock_instance.instance_id is None - assert bad_data_not_json_mock_instance.location is None - - -# bad request -@pytest.fixture -def bad_request_mock_instance(): - return get_test_azure_instance(AZURE_METADATA_SERVICE_URL, exc=requests.RequestException) - - -def test_is_instance_bad_request(bad_request_mock_instance): - assert bad_request_mock_instance.is_instance() is False - - -def test_get_cloud_provider_name_bad_request(bad_request_mock_instance): - assert bad_request_mock_instance.get_cloud_provider_name() == Environment.AZURE - - -def test_instance_attributes_bad_request(bad_request_mock_instance): - assert bad_request_mock_instance.instance_name is None - assert bad_request_mock_instance.instance_id is None - assert bad_request_mock_instance.location is None - - -# not found request -@pytest.fixture -def not_found_request_mock_instance(): - return get_test_azure_instance(AZURE_METADATA_SERVICE_URL, status_code=404) - - -def test_is_instance_not_found_request(not_found_request_mock_instance): - assert not_found_request_mock_instance.is_instance() is False - - -def test_get_cloud_provider_name_not_found_request(not_found_request_mock_instance): - assert not_found_request_mock_instance.get_cloud_provider_name() == Environment.AZURE - - -def test_instance_attributes_not_found_request(not_found_request_mock_instance): - assert not_found_request_mock_instance.instance_name is None - assert not_found_request_mock_instance.instance_id is None - assert not_found_request_mock_instance.location is None From 6a472b524f258b7831805dd14a9681929b0f0b52 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 11:50:30 +0200 Subject: [PATCH 4/9] Common, UT: remove unused AWS cloud instance infrastructure. This infrastructure/classes store information about cloud instances monkeys are ran upon, but this data isn't used anywhere --- monkey/common/cloud/aws/aws_instance.py | 4 --- .../common/cloud/aws/test_aws_instance.py | 32 ------------------- 2 files changed, 36 deletions(-) diff --git a/monkey/common/cloud/aws/aws_instance.py b/monkey/common/cloud/aws/aws_instance.py index 09d112480..4bdc89bf3 100644 --- a/monkey/common/cloud/aws/aws_instance.py +++ b/monkey/common/cloud/aws/aws_instance.py @@ -4,7 +4,6 @@ import re import requests -from common.cloud.environment_names import Environment from common.cloud.instance import CloudInstance AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS = "169.254.169.254" @@ -22,9 +21,6 @@ class AwsInstance(CloudInstance): def is_instance(self): return self.instance_id is not None - def get_cloud_provider_name(self) -> Environment: - return Environment.AWS - def __init__(self): self.instance_id = None self.region = None diff --git a/monkey/tests/unit_tests/common/cloud/aws/test_aws_instance.py b/monkey/tests/unit_tests/common/cloud/aws/test_aws_instance.py index 74ef5dd15..04421920f 100644 --- a/monkey/tests/unit_tests/common/cloud/aws/test_aws_instance.py +++ b/monkey/tests/unit_tests/common/cloud/aws/test_aws_instance.py @@ -3,7 +3,6 @@ import requests import requests_mock from common.cloud.aws.aws_instance import AWS_LATEST_METADATA_URI_PREFIX, AwsInstance -from common.cloud.environment_names import Environment INSTANCE_ID_RESPONSE = "i-1234567890abcdef0" @@ -78,10 +77,6 @@ def test_is_instance_good_data(good_data_mock_instance): assert good_data_mock_instance.is_instance() -def test_get_cloud_provider_name_good_data(good_data_mock_instance): - assert good_data_mock_instance.get_cloud_provider_name() == Environment.AWS - - def test_get_instance_id_good_data(good_data_mock_instance): assert good_data_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID @@ -110,10 +105,6 @@ def test_is_instance_bad_region_data(bad_region_data_mock_instance): assert bad_region_data_mock_instance.is_instance() -def test_get_cloud_provider_name_bad_region_data(bad_region_data_mock_instance): - assert bad_region_data_mock_instance.get_cloud_provider_name() == Environment.AWS - - def test_get_instance_id_bad_region_data(bad_region_data_mock_instance): assert bad_region_data_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID @@ -142,10 +133,6 @@ def test_is_instance_bad_account_id_data(bad_account_id_data_mock_instance): assert bad_account_id_data_mock_instance.is_instance() -def test_get_cloud_provider_name_bad_account_id_data(bad_account_id_data_mock_instance): - assert bad_account_id_data_mock_instance.get_cloud_provider_name() == Environment.AWS - - def test_get_instance_id_bad_account_id_data(bad_account_id_data_mock_instance): assert bad_account_id_data_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID @@ -176,11 +163,6 @@ def test_is_instance_bad_instance_id_request(bad_instance_id_request_mock_instan assert bad_instance_id_request_mock_instance.is_instance() is False -@pytest.mark.parametrize("instance_id_exception", [requests.RequestException, IOError]) -def test_get_cloud_provider_name_bad_instance_id_request(bad_instance_id_request_mock_instance): - assert bad_instance_id_request_mock_instance.get_cloud_provider_name() == Environment.AWS - - @pytest.mark.parametrize("instance_id_exception", [requests.RequestException, IOError]) def test_get_instance_id_bad_instance_id_request(bad_instance_id_request_mock_instance): assert bad_instance_id_request_mock_instance.get_instance_id() is None @@ -214,11 +196,6 @@ def test_is_instance_bad_region_request(bad_region_request_mock_instance): assert bad_region_request_mock_instance.is_instance() -@pytest.mark.parametrize("region_exception", [requests.RequestException, IOError]) -def test_get_cloud_provider_name_bad_region_request(bad_region_request_mock_instance): - assert bad_region_request_mock_instance.get_cloud_provider_name() == Environment.AWS - - @pytest.mark.parametrize("region_exception", [requests.RequestException, IOError]) def test_get_instance_id_bad_region_request(bad_region_request_mock_instance): assert bad_region_request_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID @@ -252,11 +229,6 @@ def test_is_instance_bad_account_id_request(bad_account_id_request_mock_instance assert bad_account_id_request_mock_instance.is_instance() -@pytest.mark.parametrize("account_id_exception", [requests.RequestException, IOError]) -def test_get_cloud_provider_name_bad_account_id_request(bad_account_id_request_mock_instance): - assert bad_account_id_request_mock_instance.get_cloud_provider_name() == Environment.AWS - - @pytest.mark.parametrize("account_id_exception", [requests.RequestException, IOError]) def test_get_instance_id_bad_account_id_request(bad_account_id_request_mock_instance): assert bad_account_id_request_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID @@ -296,10 +268,6 @@ def test_is_instance_not_found_request(not_found_request_mock_instance): assert not_found_request_mock_instance.is_instance() is False -def test_get_cloud_provider_name_not_found_request(not_found_request_mock_instance): - assert not_found_request_mock_instance.get_cloud_provider_name() == Environment.AWS - - def test_get_instance_id_not_found_request(not_found_request_mock_instance): assert not_found_request_mock_instance.get_instance_id() is None From 90a063c47cf2cdcbe75a0e6322d1874e0bca55c4 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 11:57:46 +0200 Subject: [PATCH 5/9] Island: remove environment property from monkey model because it's unused --- monkey/monkey_island/cc/models/monkey.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index 4bfaa1759..a6d5b2138 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -55,14 +55,8 @@ class Monkey(Document): tunnel = ReferenceField("self") command_control_channel = EmbeddedDocumentField(CommandControlChannel) - # Environment related fields - environment = StringField( - default=environment_names.Environment.UNKNOWN.value, - choices=environment_names.ALL_ENVIRONMENTS_NAMES, - ) - aws_instance_id = StringField( - required=False - ) # This field only exists when the monkey is running on an AWS + # This field only exists when the monkey is running on an AWS + aws_instance_id = StringField(required=False) # instance. See https://github.com/guardicore/monkey/issues/426. From 5b96f172b4e542162b9bd85e9086c20a77456917 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 12:15:01 +0200 Subject: [PATCH 6/9] Common: remove unused cloud environment names, CloudInstance class property and getter for all cloud instances --- monkey/common/cloud/all_instances.py | 12 ------------ monkey/common/cloud/environment_names.py | 10 ---------- monkey/common/cloud/instance.py | 6 ------ monkey/common/cloud/scoutsuite_consts.py | 4 ---- monkey/monkey_island/cc/models/monkey.py | 1 - 5 files changed, 33 deletions(-) delete mode 100644 monkey/common/cloud/all_instances.py diff --git a/monkey/common/cloud/all_instances.py b/monkey/common/cloud/all_instances.py deleted file mode 100644 index 6387730f6..000000000 --- a/monkey/common/cloud/all_instances.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import List - -from common.cloud.aws.aws_instance import AwsInstance -from common.cloud.azure.azure_instance import AzureInstance -from common.cloud.gcp.gcp_instance import GcpInstance -from common.cloud.instance import CloudInstance - -all_cloud_instances = [AwsInstance(), AzureInstance(), GcpInstance()] - - -def get_all_cloud_instances() -> List[CloudInstance]: - return all_cloud_instances diff --git a/monkey/common/cloud/environment_names.py b/monkey/common/cloud/environment_names.py index 945d438ce..438c64379 100644 --- a/monkey/common/cloud/environment_names.py +++ b/monkey/common/cloud/environment_names.py @@ -2,14 +2,4 @@ from enum import Enum class Environment(Enum): - UNKNOWN = "Unknown" - ON_PREMISE = "On Premise" - AZURE = "Azure" AWS = "AWS" - GCP = "GCP" - ALIBABA = "Alibaba Cloud" - IBM = "IBM Cloud" - DigitalOcean = "Digital Ocean" - - -ALL_ENVIRONMENTS_NAMES = [x.value for x in Environment] diff --git a/monkey/common/cloud/instance.py b/monkey/common/cloud/instance.py index f0da19359..77376ee8e 100644 --- a/monkey/common/cloud/instance.py +++ b/monkey/common/cloud/instance.py @@ -1,6 +1,3 @@ -from common.cloud.environment_names import Environment - - class CloudInstance(object): """ This is an abstract class which represents a cloud instance. @@ -10,6 +7,3 @@ class CloudInstance(object): def is_instance(self) -> bool: raise NotImplementedError() - - def get_cloud_provider_name(self) -> Environment: - raise NotImplementedError() diff --git a/monkey/common/cloud/scoutsuite_consts.py b/monkey/common/cloud/scoutsuite_consts.py index 091b51114..e2d0c1664 100644 --- a/monkey/common/cloud/scoutsuite_consts.py +++ b/monkey/common/cloud/scoutsuite_consts.py @@ -3,7 +3,3 @@ from enum import Enum class CloudProviders(Enum): AWS = "aws" - AZURE = "azure" - GCP = "gcp" - ALIBABA = "aliyun" - ORACLE = "oci" diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index a6d5b2138..888d1c569 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -14,7 +14,6 @@ from mongoengine import ( StringField, ) -from common.cloud import environment_names from monkey_island.cc.models.command_control_channel import CommandControlChannel from monkey_island.cc.models.monkey_ttl import MonkeyTtl, create_monkey_ttl_document from monkey_island.cc.server_utils.consts import DEFAULT_MONKEY_TTL_EXPIRY_DURATION_IN_SECONDS From 0b8af5e78f8b78cd4144ffa49f65b086b4143e93 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 12:28:40 +0200 Subject: [PATCH 7/9] Changelog: add entry about removed environment system info collector --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02a9d386d..f9b9ee010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Changelog](https://keepachangelog.com/en/1.0.0/). - Remove serialization of config. #1537 - Checkbox that gave the option to not try to first move the dropper file. #1537 - Custom singleton mutex name config option. #1589 +- Removed environment system info collector #1535 - Azure credential collector, because it was broken (not gathering credentials). #1535 - Custom monkey directory name config option. #1537 From ab35774d8af6210d489633dd61e5c4ea3225616b Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 15:30:22 +0200 Subject: [PATCH 8/9] Common, Island: remove environment collector from config schema --- .../common/common_consts/system_info_collectors_names.py | 1 - .../definitions/system_info_collector_classes.py | 9 --------- monkey/monkey_island/cc/services/config_schema/monkey.py | 2 -- 3 files changed, 12 deletions(-) diff --git a/monkey/common/common_consts/system_info_collectors_names.py b/monkey/common/common_consts/system_info_collectors_names.py index afd9e3321..f87fff4bd 100644 --- a/monkey/common/common_consts/system_info_collectors_names.py +++ b/monkey/common/common_consts/system_info_collectors_names.py @@ -1,5 +1,4 @@ AWS_COLLECTOR = "AwsCollector" HOSTNAME_COLLECTOR = "HostnameCollector" -ENVIRONMENT_COLLECTOR = "EnvironmentCollector" PROCESS_LIST_COLLECTOR = "ProcessListCollector" MIMIKATZ_COLLECTOR = "MimikatzCollector" diff --git a/monkey/monkey_island/cc/services/config_schema/definitions/system_info_collector_classes.py b/monkey/monkey_island/cc/services/config_schema/definitions/system_info_collector_classes.py index 128503078..514ee3183 100644 --- a/monkey/monkey_island/cc/services/config_schema/definitions/system_info_collector_classes.py +++ b/monkey/monkey_island/cc/services/config_schema/definitions/system_info_collector_classes.py @@ -1,6 +1,5 @@ from common.common_consts.system_info_collectors_names import ( AWS_COLLECTOR, - ENVIRONMENT_COLLECTOR, HOSTNAME_COLLECTOR, MIMIKATZ_COLLECTOR, PROCESS_LIST_COLLECTOR, @@ -11,14 +10,6 @@ SYSTEM_INFO_COLLECTOR_CLASSES = { "description": "Click on a system info collector to find out what it collects.", "type": "string", "anyOf": [ - { - "type": "string", - "enum": [ENVIRONMENT_COLLECTOR], - "title": "Environment Collector", - "safe": True, - "info": "Collects information about machine's environment (on " "premise/GCP/AWS).", - "attack_techniques": ["T1082"], - }, { "type": "string", "enum": [MIMIKATZ_COLLECTOR], diff --git a/monkey/monkey_island/cc/services/config_schema/monkey.py b/monkey/monkey_island/cc/services/config_schema/monkey.py index ddd14a3d0..68155970f 100644 --- a/monkey/monkey_island/cc/services/config_schema/monkey.py +++ b/monkey/monkey_island/cc/services/config_schema/monkey.py @@ -1,6 +1,5 @@ from common.common_consts.system_info_collectors_names import ( AWS_COLLECTOR, - ENVIRONMENT_COLLECTOR, HOSTNAME_COLLECTOR, MIMIKATZ_COLLECTOR, PROCESS_LIST_COLLECTOR, @@ -88,7 +87,6 @@ MONKEY = { "uniqueItems": True, "items": {"$ref": "#/definitions/system_info_collector_classes"}, "default": [ - ENVIRONMENT_COLLECTOR, AWS_COLLECTOR, HOSTNAME_COLLECTOR, PROCESS_LIST_COLLECTOR, From 77a38780c1110b037bc444ae4614ef0ee06e3eb4 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 16 Nov 2021 15:55:20 +0200 Subject: [PATCH 9/9] UI: fix the link in AWS run options page to point to the correct documentation --- .../components/pages/RunMonkeyPage/RunOnAWS/AWSRunOptions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage/RunOnAWS/AWSRunOptions.js b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage/RunOnAWS/AWSRunOptions.js index a1c3cb491..e6c2290b2 100644 --- a/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage/RunOnAWS/AWSRunOptions.js +++ b/monkey/monkey_island/cc/ui/src/components/pages/RunMonkeyPage/RunOnAWS/AWSRunOptions.js @@ -81,7 +81,7 @@ const getContents = (props) => {

Not sure what this is? Not seeing your AWS EC2 instances? Read the documentation!