diff --git a/monkey/common/cloud/aws/aws_instance.py b/monkey/common/cloud/aws/aws_instance.py index d09169407..75dee4ce9 100644 --- a/monkey/common/cloud/aws/aws_instance.py +++ b/monkey/common/cloud/aws/aws_instance.py @@ -1,8 +1,7 @@ import json import logging import re -import urllib.error -import urllib.request +import requests from common.cloud.environment_names import Environment from common.cloud.instance import CloudInstance @@ -33,19 +32,17 @@ class AwsInstance(CloudInstance): self.account_id = None try: - self.instance_id = urllib.request.urlopen( - AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/instance-id', timeout=2).read().decode() + response = requests.get(AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/instance-id', timeout=2) + self.instance_id = response.text if response else None self.region = self._parse_region( - urllib.request.urlopen( - AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/placement/availability-zone').read().decode()) - except (urllib.error.URLError, IOError) as e: + requests.get(AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/placement/availability-zone').text) + except (requests.RequestException, IOError) as e: logger.debug("Failed init of AwsInstance while getting metadata: {}".format(e)) try: self.account_id = self._extract_account_id( - urllib.request.urlopen( - AWS_LATEST_METADATA_URI_PREFIX + 'dynamic/instance-identity/document', timeout=2).read().decode()) - except (urllib.error.URLError, IOError) as e: + requests.get(AWS_LATEST_METADATA_URI_PREFIX + 'dynamic/instance-identity/document', timeout=2).text) + except (requests.RequestException, json.decoder.JSONDecodeError, IOError) as e: logger.debug("Failed init of AwsInstance while getting dynamic instance data: {}".format(e)) @staticmethod diff --git a/monkey/common/cloud/aws/test_aws_instance.py b/monkey/common/cloud/aws/test_aws_instance.py new file mode 100644 index 000000000..0353a0b9f --- /dev/null +++ b/monkey/common/cloud/aws/test_aws_instance.py @@ -0,0 +1,300 @@ +import json + +import pytest +import requests +import requests_mock + +from common.cloud.aws.aws_instance import (AWS_LATEST_METADATA_URI_PREFIX, + AwsInstance) +from common.cloud.environment_names import Environment + + +INSTANCE_ID_RESPONSE = 'i-1234567890abcdef0' + +AVAILABILITY_ZONE_RESPONSE = 'us-west-2b' + +# from https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html +INSTANCE_IDENTITY_DOCUMENT_RESPONSE = """ +{ + "devpayProductCodes": null, + "marketplaceProductCodes": ["1abc2defghijklm3nopqrs4tu"], + "availabilityZone": "us-west-2b", + "privateIp": "10.158.112.84", + "version": "2017-09-30", + "instanceId": "i-1234567890abcdef0", + "billingProducts": null, + "instanceType": "t2.micro", + "accountId": "123456789012", + "imageId": "ami-5fb8c835", + "pendingTime": "2016-11-19T16:32:11Z", + "architecture": "x86_64", + "kernelId": null, + "ramdiskId": null, + "region": "us-west-2" +} +""" + + +EXPECTED_INSTANCE_ID = 'i-1234567890abcdef0' + +EXPECTED_REGION = 'us-west-2' + +EXPECTED_ACCOUNT_ID = '123456789012' + + +def get_test_aws_instance(text={'instance_id': None, + 'region': None, + 'account_id': None}, + exception={'instance_id': None, + 'region': None, + 'account_id': None}): + with requests_mock.Mocker() as m: + # request made to get instance_id + url = f'{AWS_LATEST_METADATA_URI_PREFIX}meta-data/instance-id' + m.get(url, text=text['instance_id']) if text['instance_id'] else m.get( + url, exc=exception['instance_id']) + + # request made to get region + url = f'{AWS_LATEST_METADATA_URI_PREFIX}meta-data/placement/availability-zone' + m.get(url, text=text['region']) if text['region'] else m.get( + url, exc=exception['region']) + + # request made to get account_id + url = f'{AWS_LATEST_METADATA_URI_PREFIX}dynamic/instance-identity/document' + m.get(url, text=text['account_id']) if text['account_id'] else m.get( + url, exc=exception['account_id']) + + test_aws_instance_object = AwsInstance() + return test_aws_instance_object + + +# all good data +@pytest.fixture +def good_data_mock_instance(): + return get_test_aws_instance(text={'instance_id': INSTANCE_ID_RESPONSE, + 'region': AVAILABILITY_ZONE_RESPONSE, + 'account_id': INSTANCE_IDENTITY_DOCUMENT_RESPONSE}) + + +def test_is_instance_good_data(good_data_mock_instance): + assert good_data_mock_instance.is_instance() + + +def test_get_cloud_provider_name_good_data(good_data_mock_instance): + assert good_data_mock_instance.get_cloud_provider_name() == Environment.AWS + + +def test_get_instance_id_good_data(good_data_mock_instance): + assert good_data_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID + + +def test_get_region_good_data(good_data_mock_instance): + assert good_data_mock_instance.get_region() == EXPECTED_REGION + + +def test_get_account_id_good_data(good_data_mock_instance): + assert good_data_mock_instance.get_account_id() == EXPECTED_ACCOUNT_ID + + +# 'region' bad data +@pytest.fixture +def bad_region_data_mock_instance(): + return get_test_aws_instance(text={'instance_id': INSTANCE_ID_RESPONSE, + 'region': 'in-a-different-world', + 'account_id': INSTANCE_IDENTITY_DOCUMENT_RESPONSE}) + + +def test_is_instance_bad_region_data(bad_region_data_mock_instance): + assert bad_region_data_mock_instance.is_instance() + + +def test_get_cloud_provider_name_bad_region_data(bad_region_data_mock_instance): + assert bad_region_data_mock_instance.get_cloud_provider_name() == Environment.AWS + + +def test_get_instance_id_bad_region_data(bad_region_data_mock_instance): + assert bad_region_data_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID + + +def test_get_region_bad_region_data(bad_region_data_mock_instance): + assert bad_region_data_mock_instance.get_region() is None + + +def test_get_account_id_bad_region_data(bad_region_data_mock_instance): + assert bad_region_data_mock_instance.get_account_id() == EXPECTED_ACCOUNT_ID + + +# 'account_id' bad data +@pytest.fixture +def bad_account_id_data_mock_instance(): + return get_test_aws_instance(text={'instance_id': INSTANCE_ID_RESPONSE, + 'region': AVAILABILITY_ZONE_RESPONSE, + 'account_id': 'who-am-i'}) + + +def test_is_instance_bad_account_id_data(bad_account_id_data_mock_instance): + assert bad_account_id_data_mock_instance.is_instance() + + +def test_get_cloud_provider_name_bad_account_id_data(bad_account_id_data_mock_instance): + assert bad_account_id_data_mock_instance.get_cloud_provider_name() == Environment.AWS + + +def test_get_instance_id_bad_account_id_data(bad_account_id_data_mock_instance): + assert bad_account_id_data_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID + + +def test_get_region_bad_account_id_data(bad_account_id_data_mock_instance): + assert bad_account_id_data_mock_instance.get_region() == EXPECTED_REGION + + +def test_get_account_id_data_bad_account_id_data(bad_account_id_data_mock_instance): + assert bad_account_id_data_mock_instance.get_account_id() is None + + +# 'instance_id' bad requests +@pytest.fixture +def bad_instance_id_request_mock_instance(instance_id_exception): + return get_test_aws_instance(text={'instance_id': None, + 'region': AVAILABILITY_ZONE_RESPONSE, + 'account_id': INSTANCE_IDENTITY_DOCUMENT_RESPONSE}, + exception={'instance_id': instance_id_exception, + 'region': None, + 'account_id': None}) + + +@pytest.mark.parametrize('instance_id_exception', [requests.RequestException, IOError]) +def test_is_instance_bad_instance_id_request(bad_instance_id_request_mock_instance): + assert bad_instance_id_request_mock_instance.is_instance() is False + + +@pytest.mark.parametrize('instance_id_exception', [requests.RequestException, IOError]) +def test_get_cloud_provider_name_bad_instance_id_request(bad_instance_id_request_mock_instance): + assert bad_instance_id_request_mock_instance.get_cloud_provider_name() == Environment.AWS + + +@pytest.mark.parametrize('instance_id_exception', [requests.RequestException, IOError]) +def test_get_instance_id_bad_instance_id_request(bad_instance_id_request_mock_instance): + assert bad_instance_id_request_mock_instance.get_instance_id() is None + + +@pytest.mark.parametrize('instance_id_exception', [requests.RequestException, IOError]) +def test_get_region_bad_instance_id_request(bad_instance_id_request_mock_instance): + assert bad_instance_id_request_mock_instance.get_region() is None + + +@pytest.mark.parametrize('instance_id_exception', [requests.RequestException, IOError]) +def test_get_account_id_bad_instance_id_request(bad_instance_id_request_mock_instance): + assert bad_instance_id_request_mock_instance.get_account_id() == EXPECTED_ACCOUNT_ID + + +# 'region' bad requests +@pytest.fixture +def bad_region_request_mock_instance(region_exception): + return get_test_aws_instance(text={'instance_id': INSTANCE_ID_RESPONSE, + 'region': None, + 'account_id': INSTANCE_IDENTITY_DOCUMENT_RESPONSE}, + exception={'instance_id': None, + 'region': region_exception, + 'account_id': None}) + + +@pytest.mark.parametrize('region_exception', [requests.RequestException, IOError]) +def test_is_instance_bad_region_request(bad_region_request_mock_instance): + assert bad_region_request_mock_instance.is_instance() + + +@pytest.mark.parametrize('region_exception', [requests.RequestException, IOError]) +def test_get_cloud_provider_name_bad_region_request(bad_region_request_mock_instance): + assert bad_region_request_mock_instance.get_cloud_provider_name() == Environment.AWS + + +@pytest.mark.parametrize('region_exception', [requests.RequestException, IOError]) +def test_get_instance_id_bad_region_request(bad_region_request_mock_instance): + assert bad_region_request_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID + + +@pytest.mark.parametrize('region_exception', [requests.RequestException, IOError]) +def test_get_region_bad_region_request(bad_region_request_mock_instance): + assert bad_region_request_mock_instance.get_region() is None + + +@pytest.mark.parametrize('region_exception', [requests.RequestException, IOError]) +def test_get_account_id_bad_region_request(bad_region_request_mock_instance): + assert bad_region_request_mock_instance.get_account_id() == EXPECTED_ACCOUNT_ID + + +# 'account_id' bad requests +@pytest.fixture +def bad_account_id_request_mock_instance(account_id_exception): + return get_test_aws_instance(text={'instance_id': INSTANCE_ID_RESPONSE, + 'region': AVAILABILITY_ZONE_RESPONSE, + 'account_id': None}, + exception={'instance_id': None, + 'region': None, + 'account_id': account_id_exception}) + + +@pytest.mark.parametrize('account_id_exception', [requests.RequestException, IOError]) +def test_is_instance_bad_account_id_request(bad_account_id_request_mock_instance): + assert bad_account_id_request_mock_instance.is_instance() + + +@pytest.mark.parametrize('account_id_exception', [requests.RequestException, IOError]) +def test_get_cloud_provider_name_bad_account_id_request(bad_account_id_request_mock_instance): + assert bad_account_id_request_mock_instance.get_cloud_provider_name() == Environment.AWS + + +@pytest.mark.parametrize('account_id_exception', [requests.RequestException, IOError]) +def test_get_instance_id_bad_account_id_request(bad_account_id_request_mock_instance): + assert bad_account_id_request_mock_instance.get_instance_id() == EXPECTED_INSTANCE_ID + + +@pytest.mark.parametrize('account_id_exception', [requests.RequestException, IOError]) +def test_get_region_bad_account_id_request(bad_account_id_request_mock_instance): + assert bad_account_id_request_mock_instance.get_region() == EXPECTED_REGION + + +@pytest.mark.parametrize('account_id_exception', [requests.RequestException, IOError]) +def test_get_account_id_bad_account_id_request(bad_account_id_request_mock_instance): + assert bad_account_id_request_mock_instance.get_account_id() is None + + +# not found request +@pytest.fixture +def not_found_request_mock_instance(): + with requests_mock.Mocker() as m: + # request made to get instance_id + url = f'{AWS_LATEST_METADATA_URI_PREFIX}meta-data/instance-id' + m.get(url, status_code=404) + + # request made to get region + url = f'{AWS_LATEST_METADATA_URI_PREFIX}meta-data/placement/availability-zone' + m.get(url) + + # request made to get account_id + url = f'{AWS_LATEST_METADATA_URI_PREFIX}dynamic/instance-identity/document' + m.get(url) + + not_found_aws_instance_object = AwsInstance() + return not_found_aws_instance_object + + +def test_is_instance_not_found_request(not_found_request_mock_instance): + assert not_found_request_mock_instance.is_instance() is False + + +def test_get_cloud_provider_name_not_found_request(not_found_request_mock_instance): + assert not_found_request_mock_instance.get_cloud_provider_name() == Environment.AWS + + +def test_get_instance_id_not_found_request(not_found_request_mock_instance): + assert not_found_request_mock_instance.get_instance_id() is None + + +def test_get_region_not_found_request(not_found_request_mock_instance): + assert not_found_request_mock_instance.get_region() is None + + +def test_get_account_id_not_found_request(not_found_request_mock_instance): + assert not_found_request_mock_instance.get_account_id() is None diff --git a/monkey/common/cloud/aws/aws_service_test.py b/monkey/common/cloud/aws/test_aws_service.py similarity index 100% rename from monkey/common/cloud/aws/aws_service_test.py rename to monkey/common/cloud/aws/test_aws_service.py diff --git a/monkey/common/cloud/azure/azure_instance.py b/monkey/common/cloud/azure/azure_instance.py index af6c85460..969e4a8ca 100644 --- a/monkey/common/cloud/azure/azure_instance.py +++ b/monkey/common/cloud/azure/azure_instance.py @@ -1,6 +1,7 @@ import logging import requests +import simplejson from common.cloud.environment_names import Environment from common.cloud.instance import CloudInstance @@ -18,7 +19,7 @@ class AzureInstance(CloudInstance): Based on Azure metadata service: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service """ def is_instance(self): - return self.on_azure + return self._on_azure def get_cloud_provider_name(self) -> Environment: return Environment.AZURE @@ -30,24 +31,22 @@ class AzureInstance(CloudInstance): self.instance_name = None self.instance_id = None self.location = None - self.on_azure = False + self._on_azure = False try: response = requests.get(AZURE_METADATA_SERVICE_URL, headers={"Metadata": "true"}, timeout=SHORT_REQUEST_TIMEOUT) - self.on_azure = True # If not on cloud, the metadata URL is non-routable and the connection will fail. # If on AWS, should get 404 since the metadata service URL is different, so bool(response) will be false. if response: - logger.debug("On Azure. Trying to parse metadata.") + logger.debug("Trying to parse Azure metadata.") self.try_parse_response(response) else: - logger.warning("On Azure, but metadata response not ok: {}".format(response.status_code)) + logger.warning(f"Metadata response not ok: {response.status_code}") except requests.RequestException: logger.debug("Failed to get response from Azure metadata service: This instance is not on Azure.") - self.on_azure = False def try_parse_response(self, response): try: @@ -55,5 +54,6 @@ class AzureInstance(CloudInstance): self.instance_name = response_data["compute"]["name"] self.instance_id = response_data["compute"]["vmId"] self.location = response_data["compute"]["location"] - except KeyError: - logger.exception("Error while parsing response from Azure metadata service.") + self._on_azure = True + except (KeyError, simplejson.errors.JSONDecodeError) as e: + logger.exception(f"Error while parsing response from Azure metadata service: {e}") diff --git a/monkey/common/cloud/azure/test_azure_instance.py b/monkey/common/cloud/azure/test_azure_instance.py new file mode 100644 index 000000000..680af90ed --- /dev/null +++ b/monkey/common/cloud/azure/test_azure_instance.py @@ -0,0 +1,199 @@ +import pytest +import requests +import requests_mock +import simplejson + +from common.cloud.azure.azure_instance import (AZURE_METADATA_SERVICE_URL, + AzureInstance) +from common.cloud.environment_names import Environment + + +GOOD_DATA = { + 'compute': {'azEnvironment': 'AZUREPUBLICCLOUD', + 'isHostCompatibilityLayerVm': 'true', + 'licenseType': 'Windows_Client', + 'location': 'westus', + 'name': 'examplevmname', + 'offer': 'Windows', + 'osProfile': {'adminUsername': 'admin', + 'computerName': 'examplevmname', + 'disablePasswordAuthentication': 'true'}, + 'osType': 'linux', + 'placementGroupId': 'f67c14ab-e92c-408c-ae2d-da15866ec79a', + 'plan': {'name': 'planName', + 'product': 'planProduct', + 'publisher': 'planPublisher'}, + 'platformFaultDomain': '36', + 'platformUpdateDomain': '42', + 'publicKeys': [{'keyData': 'ssh-rsa 0', + 'path': '/home/user/.ssh/authorized_keys0'}, + {'keyData': 'ssh-rsa 1', + 'path': '/home/user/.ssh/authorized_keys1'}], + 'publisher': 'RDFE-Test-Microsoft-Windows-Server-Group', + 'resourceGroupName': 'macikgo-test-may-23', + 'resourceId': '/subscriptions/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx/resourceGroups/macikgo-test-may-23/' + 'providers/Microsoft.Compute/virtualMachines/examplevmname', + 'securityProfile': {'secureBootEnabled': 'true', + 'virtualTpmEnabled': 'false'}, + 'sku': 'Windows-Server-2012-R2-Datacenter', + 'storageProfile': {'dataDisks': [{'caching': 'None', + 'createOption': 'Empty', + 'diskSizeGB': '1024', + 'image': {'uri': ''}, + 'lun': '0', + 'managedDisk': {'id': '/subscriptions/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx/' + 'resourceGroups/macikgo-test-may-23/providers/' + 'Microsoft.Compute/disks/exampledatadiskname', + 'storageAccountType': 'Standard_LRS'}, + 'name': 'exampledatadiskname', + 'vhd': {'uri': ''}, + 'writeAcceleratorEnabled': 'false'}], + 'imageReference': {'id': '', + 'offer': 'UbuntuServer', + 'publisher': 'Canonical', + 'sku': '16.04.0-LTS', + 'version': 'latest'}, + 'osDisk': {'caching': 'ReadWrite', + 'createOption': 'FromImage', + 'diskSizeGB': '30', + 'diffDiskSettings': {'option': 'Local'}, + 'encryptionSettings': {'enabled': 'false'}, + 'image': {'uri': ''}, + 'managedDisk': {'id': '/subscriptions/xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx/' + 'resourceGroups/macikgo-test-may-23/providers/' + 'Microsoft.Compute/disks/exampleosdiskname', + 'storageAccountType': 'Standard_LRS'}, + 'name': 'exampleosdiskname', + 'osType': 'Linux', + 'vhd': {'uri': ''}, + 'writeAcceleratorEnabled': 'false'}}, + 'subscriptionId': 'xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx', + 'tags': 'baz:bash;foo:bar', + 'version': '15.05.22', + 'vmId': '02aab8a4-74ef-476e-8182-f6d2ba4166a6', + 'vmScaleSetName': 'crpteste9vflji9', + 'vmSize': 'Standard_A3', + 'zone': ''}, + 'network': {'interface': [{'ipv4': {'ipAddress': [{'privateIpAddress': '10.144.133.132', + 'publicIpAddress': ''}], + 'subnet': [{'address': '10.144.133.128', + 'prefix': '26'}]}, + 'ipv6': {'ipAddress': []}, + 'macAddress': '0011AAFFBB22'}]} + } + + +BAD_DATA_NOT_JSON = '\n\n
\n\n\n