Merge pull request #1078 from guardicore/flake8

Flake8 improvements

.flake8

@@ -1,11 +1,14 @@
 [flake8]
 ## Warn about linter issues.
 
-exclude = ../monkey/monkey_island/cc/ui,
+exclude = ../monkey/monkey_island/cc/ui
-          ../monkey/common/cloud
 show-source = True
 max-complexity = 10
-max-line-length = 127
+max-line-length = 100
+
+### ignore "whitespace before ':'", "line break before binary operator" for
+### compatibility with black, and cyclomatic complexity (for now).
+extend-ignore = E203, W503, C901
 
 ### --statistics Count the number of occurrences of each error/warning code and print a report.
 statistics = True

.travis.yml

@@ -24,7 +24,7 @@ install:
 # Python
 - pip freeze
 - pip install -r monkey/monkey_island/requirements.txt  # for unit tests
-- pip install flake8 pytest pytest-cov dlint isort # for next stages
+- pip install flake8 pytest pytest-cov isort # for next stages
 - pip install coverage  # for code coverage
 - pip install -r monkey/infection_monkey/requirements.txt  # for unit tests
 - pip install pipdeptree
@@ -55,17 +55,7 @@ install:
 script:
 # Check Python code
 ## Check syntax errors and fail the build if any are found.
-- flake8 ./monkey --config=./ci_scripts/flake8_syntax_check.ini
+- flake8 ./monkey
-
-## Warn about linter issues.
-### --exit-zero forces Flake8 to use the exit status code 0 even if there are errors, which means this will NOT fail the build.
-### The output is redirected to a file.
-- flake8 ./monkey --exit-zero --config=./ci_scripts/flake8_linter_check.ini > ./ci_scripts/flake8_warnings.txt
-## Display the linter issues
-- cat ./ci_scripts/flake8_warnings.txt
-## Make sure that we haven't increased the amount of warnings.
-- PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=80
-- if [ $(tail -n 1 ./ci_scripts/flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi
 
 ## Check import order
 - python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg

ci_scripts/flake8_syntax_check.ini

@@ -1,15 +0,0 @@
-[flake8]
-
-## Check syntax errors and fail the build if any are found.
-exclude =
-    ../monkey/monkey_island/cc/ui,
-    ../monkey/common/cloud
-select =
-    E901,
-    E999,
-    F821,
-    F822,
-    F823
-count = True
-show-source = True
-statistics = True

monkey/common/cloud/aws/test_aws_instance.py

@@ -1,5 +1,3 @@
-import json
-
 import pytest
 import requests
 import requests_mock

monkey/common/network/network_utils.py

@@ -16,5 +16,5 @@ def get_host_from_network_location(network_location: str) -> str:
 def remove_port(url):
     parsed = urlparse(url)
     with_port = f'{parsed.scheme}://{parsed.netloc}'
-    without_port = re.sub(':[0-9]+(?=$|\/)', '', with_port)
+    without_port = re.sub(':[0-9]+(?=$|/)', '', with_port)
     return without_port

monkey/common/utils/mongo_utils.py

@@ -37,12 +37,12 @@ class MongoUtils:
                 # ISWbemObjectEx interface. Class Uint8Array ?
                 if str(o._oleobj_.GetTypeInfo().GetTypeAttr().iid) == "{269AD56A-8A67-4129-BC8C-0506DCFE9880}":
                     return o.Value
-            except:
+            except Exception:
                 pass
 
             try:
                 return o.GetObjectText_()
-            except:
+            except Exception:
                 pass
 
             return repr(o)

monkey/infection_monkey/exploit/smbexec.py

@@ -7,7 +7,8 @@ from common.utils.exploit_enum import ExploitType
 from infection_monkey.exploit.HostExploiter import HostExploiter
 from infection_monkey.exploit.tools.helpers import build_monkey_commandline, get_monkey_depth, get_target_monkey
 from infection_monkey.exploit.tools.smb_tools import SmbTools
-from infection_monkey.model import DROPPER_CMDLINE_DETACHED_WINDOWS, MONKEY_CMDLINE_DETACHED_WINDOWS, VictimHost
+from infection_monkey.model import (DROPPER_CMDLINE_DETACHED_WINDOWS,
+                                    MONKEY_CMDLINE_DETACHED_WINDOWS)
 from infection_monkey.network.smbfinger import SMBFinger
 from infection_monkey.network.tools import check_tcp_port
 from infection_monkey.telemetry.attack.t1035_telem import T1035Telem
@@ -148,7 +149,7 @@ class SmbExploiter(HostExploiter):
         try:
             scmr.hRStartServiceW(scmr_rpc, service)
             status = ScanStatus.USED
-        except:
+        except Exception:
             status = ScanStatus.SCANNED
             pass
         T1035Telem(status, UsageEnum.SMB).send()

monkey/infection_monkey/exploit/tools/smb_tools.py

@@ -42,7 +42,7 @@ class SmbTools(object):
 
             try:
                 smb.logoff()
-            except:
+            except Exception:
                 pass
 
             return None
@@ -113,7 +113,7 @@ class SmbTools(object):
                     return None
 
             try:
-                tid = smb.connectTree(share_name)
+                smb.connectTree(share_name)
             except Exception as exc:
                 LOG.debug("Error connecting tree to share '%s' on victim %r: %s",
                           share_name, host, exc)
@@ -134,7 +134,7 @@ class SmbTools(object):
                             return remote_full_path
 
                         LOG.debug("Remote monkey file is found but different, moving along with attack")
-                except:
+                except Exception:
                     pass  # file isn't found on remote victim, moving on
 
             try:
@@ -163,7 +163,7 @@ class SmbTools(object):
             finally:
                 try:
                     smb.logoff()
-                except:
+                except Exception:
                     pass
 
                 smb = None

monkey/infection_monkey/exploit/tools/wmi_tools.py

@@ -58,7 +58,7 @@ class WmiTools(object):
             try:
                 self._iWbemServices = iWbemLevel1Login.NTLMLogin('//./root/cimv2', NULL, NULL)
                 self._dcom = dcom
-            except:
+            except Exception:
                 dcom.disconnect()
 
                 raise

monkey/infection_monkey/exploit/web_rce.py

@@ -235,7 +235,7 @@ class WebRCE(HostExploiter):
             resp = self.exploit(url, GET_ARCH_LINUX)
             if resp:
                 # Pulls architecture string
-                arch = re.search('(?<=Architecture:)\s+(\w+)', resp)
+                arch = re.search(r'(?<=Architecture:)\s+(\w+)', resp)
                 try:
                     arch = arch.group(1)
                 except AttributeError:

monkey/infection_monkey/exploit/win_ms08_067.py

@@ -227,7 +227,7 @@ class Ms08_067_Exploiter(HostExploiter):
                             self._config.remote_user_pass,
                             self._config.user_to_add).encode())
                 time.sleep(2)
-                reply = sock.recv(1000)
+                sock.recv(1000)
 
                 LOG.debug("Exploited into %r using MS08-067", self.host)
                 exploited = True

monkey/infection_monkey/network/firewall.py

@@ -41,12 +41,12 @@ class WinAdvFirewall(FirewallApp):
             cmd = subprocess.Popen('netsh advfirewall show currentprofile', stdout=subprocess.PIPE)
             out = cmd.stdout.readlines()
 
-            for l in out:
+            for line in out:
-                if l.startswith('State'):
+                if line.startswith('State'):
-                    state = l.split()[-1].strip()
+                    state = line.split()[-1].strip()
 
             return state == "ON"
-        except:
+        except Exception:
             return None
 
     def add_firewall_rule(self, name="Firewall", direction="in", action="allow", program=sys.executable, **kwargs):
@@ -61,7 +61,7 @@ class WinAdvFirewall(FirewallApp):
                 return True
             else:
                 return False
-        except:
+        except Exception:
             return None
 
     def remove_firewall_rule(self, name="Firewall", **kwargs):
@@ -75,7 +75,7 @@ class WinAdvFirewall(FirewallApp):
                 return True
             else:
                 return False
-        except:
+        except Exception:
             return None
 
     def listen_allowed(self, **kwargs):
@@ -94,7 +94,7 @@ class WinAdvFirewall(FirewallApp):
         try:
             for rule in list(self._rules.keys()):
                 self.remove_firewall_rule(name=rule)
-        except:
+        except Exception:
             pass
 
 
@@ -107,14 +107,14 @@ class WinFirewall(FirewallApp):
             cmd = subprocess.Popen('netsh firewall show state', stdout=subprocess.PIPE)
             out = cmd.stdout.readlines()
 
-            for l in out:
+            for line in out:
-                if l.startswith('Operational mode'):
+                if line.startswith('Operational mode'):
-                    state = l.split('=')[-1].strip()
+                    state = line.split('=')[-1].strip()
-                elif l.startswith('The service has not been started.'):
+                elif line.startswith('The service has not been started.'):
                     return False
 
             return state == "Enable"
-        except:
+        except Exception:
             return None
 
     def add_firewall_rule(self, rule='allowedprogram', name="Firewall", mode="ENABLE", program=sys.executable,
@@ -131,7 +131,7 @@ class WinFirewall(FirewallApp):
                 return True
             else:
                 return False
-        except:
+        except Exception:
             return None
 
     def remove_firewall_rule(self, rule='allowedprogram', name="Firewall", mode="ENABLE", program=sys.executable,
@@ -145,7 +145,7 @@ class WinFirewall(FirewallApp):
                 return True
             else:
                 return False
-        except:
+        except Exception:
             return None
 
     def listen_allowed(self, **kwargs):
@@ -161,14 +161,14 @@ class WinFirewall(FirewallApp):
         try:
             for rule in list(self._rules.values()):
                 self.remove_firewall_rule(**rule)
-        except:
+        except Exception:
             pass
 
 
 if sys.platform == "win32":
     try:
         win_ver = int(platform.version().split('.')[0])
-    except:
+    except Exception:
         win_ver = 0
     if win_ver > 5:
         app = WinAdvFirewall()

monkey/infection_monkey/network/info.py

@@ -1,4 +1,3 @@
-import ipaddress
 import itertools
 import socket
 import struct
@@ -76,8 +75,8 @@ else:
             ifaddr = socket.inet_ntoa(ifreq[20:24])
             routes.append((dst, msk, "0.0.0.0", LOOPBACK_NAME, ifaddr))
 
-        for l in f.readlines()[1:]:
+        for line in f.readlines()[1:]:
-            iff, dst, gw, flags, x, x, x, msk, x, x, x = [var.encode() for var in l.split()]
+            iff, dst, gw, flags, x, x, x, msk, x, x, x = [var.encode() for var in line.split()]
             flags = int(flags, 16)
             if flags & RTF_UP == 0:
                 continue
@@ -145,7 +144,6 @@ def get_interfaces_ranges():
     for net_interface in ifs:
         address_str = net_interface['addr']
         netmask_str = net_interface['netmask']
-        ip_interface = ipaddress.ip_interface("%s/%s" % (address_str, netmask_str))
         # limit subnet scans to class C only
         res.append(CidrRange(cidr_range="%s/%s" % (address_str, netmask_str)))
     return res

monkey/infection_monkey/network/test_postgresql_finger.py

@@ -1,6 +1,5 @@
 import pytest
 
-import infection_monkey.network.postgresql_finger
 from infection_monkey.network.postgresql_finger import PostgreSQLFinger
 
 IRRELEVANT_EXCEPTION_STRING = "This is an irrelevant exception string."

monkey/infection_monkey/post_breach/tests/actions/test_users_custom_pba.py

@@ -1,7 +1,6 @@
 import pytest
 
-from infection_monkey.post_breach.actions.users_custom_pba import (
+from infection_monkey.post_breach.actions.users_custom_pba import UsersPBA
-    DIR_CHANGE_LINUX, DIR_CHANGE_WINDOWS, UsersPBA)
 
 MONKEY_DIR_PATH = "/dir/to/monkey/"
 CUSTOM_LINUX_CMD = "command-for-linux"

monkey/infection_monkey/telemetry/tests/attack/test_victim_host_telem.py

@@ -2,7 +2,7 @@ import json
 
 import pytest
 
-from common.utils.attack_utils import ScanStatus, UsageEnum
+from common.utils.attack_utils import ScanStatus
 from infection_monkey.model import VictimHost
 from infection_monkey.telemetry.attack.victim_host_telem import VictimHostTelem
 

monkey/infection_monkey/transport/http.py

@@ -47,7 +47,7 @@ class FileServHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
                     chunk = end_range - start_range
                 try:
                     self.wfile.write(f.read(chunk))
-                except:
+                except Exception:
                     break
                 total += chunk
                 start_range += chunk

monkey/infection_monkey/transport/tcp.py

@@ -32,13 +32,13 @@ class SocketsPipe(Thread):
                 other = self.dest if r is self.source else self.source
                 try:
                     data = r.recv(READ_BUFFER_SIZE)
-                except:
+                except Exception:
                     break
                 if data:
                     try:
                         other.sendall(data)
                         update_last_serve_time()
-                    except:
+                    except Exception:
                         break
                     self._keep_connection = True
 

monkey/monkey_island/cc/environment/environment_singleton.py

@@ -2,7 +2,7 @@ import logging
 
 import monkey_island.cc.resources.auth.user_store as user_store
 from monkey_island.cc.environment import (EnvironmentConfig, aws, password,
-                                          standard, testing)
+                                          standard)
 from monkey_island.cc.server_utils.consts import DEFAULT_SERVER_CONFIG_PATH
 
 __author__ = 'itay.mizeretz'

monkey/monkey_island/cc/services/tests/test_config.py

@@ -1,6 +1,5 @@
 import pytest
 
-import monkey_island.cc.services.config
 from monkey_island.cc.environment import Environment
 from monkey_island.cc.services.config import ConfigService