Merge pull request #1865 from guardicore/1613-agent-timeouts

1613 agent timeouts

monkey/infection_monkey/config.py

@@ -87,7 +87,7 @@ class Configuration(object):
     # Configuration servers to try to connect to, in this order.
     command_servers = []
 
-    keep_tunnel_open_time = 60
+    keep_tunnel_open_time = 30
 
     ###########################
     # testing configuration

monkey/infection_monkey/exploit/sshexec.py

@@ -86,13 +86,15 @@ class SSHExploiter(HostExploiter):
                 self.exploit_result.exploitation_success = True
                 self.report_login_attempt(True, user, ssh_key=ssh_string)
                 return ssh
-            except Exception:
+            except paramiko.AuthenticationException as err:
                 ssh.close()
-                logger.debug(
-                    "Error logging into victim %r with %s" " private key", self.host, ssh_string
+                logger.info(
+                    f"Failed logging into victim {self.host} with {ssh_string} private key: {err}",
                 )
                 self.report_login_attempt(False, user, ssh_key=ssh_string)
                 continue
+            except Exception as err:
+                logger.error(f"Unknown error while attempting to login with ssh key: {err}")
         raise FailedExploitationError
 
     def exploit_with_login_creds(self, port) -> paramiko.SSHClient:
@@ -130,16 +132,18 @@ class SSHExploiter(HostExploiter):
                 self.report_login_attempt(True, user, current_password)
                 return ssh
 
-            except Exception as exc:
+            except paramiko.AuthenticationException as err:
                 logger.debug(
-                    "Error logging into victim %r with user" " %s: (%s)",
+                    "Failed logging into victim %r with user" " %s: (%s)",
                     self.host,
                     user,
-                    exc,
+                    err,
                 )
                 self.report_login_attempt(False, user, current_password)
                 ssh.close()
                 continue
+            except Exception as err:
+                logger.error(f"Unknown error occurred while trying to login to ssh: {err}")
         raise FailedExploitationError
 
     def _exploit_host(self) -> ExploiterResultData:

monkey/infection_monkey/monkey.py

@@ -343,6 +343,7 @@ class InfectionMonkey:
 
     def cleanup(self):
         logger.info("Monkey cleanup started")
+        deleted = None
         try:
             if self._master:
                 self._master.cleanup()
@@ -357,7 +358,7 @@ class InfectionMonkey:
                 firewall.remove_firewall_rule()
                 firewall.close()
 
-            InfectionMonkey._self_delete()
+            deleted = InfectionMonkey._self_delete()
 
             InfectionMonkey._send_log()
 
@@ -365,13 +366,12 @@ class InfectionMonkey:
                 is_done=True, version=get_version()
             ).send()  # Signal the server (before closing the tunnel)
 
-            # TODO: Determine how long between when we
-            #  send telemetry and the monkey actually exits
             InfectionMonkey._close_tunnel()
             self._singleton.unlock()
         except Exception as e:
             logger.error(f"An error occurred while cleaning up the monkey agent: {e}")
-            InfectionMonkey._self_delete()
+            if deleted is None:
+                InfectionMonkey._self_delete()
 
         logger.info("Monkey is shutting down")
 
@@ -400,9 +400,10 @@ class InfectionMonkey:
         ControlClient.send_log(log)
 
     @staticmethod
-    def _self_delete():
+    def _self_delete() -> bool:
         status = ScanStatus.USED if remove_monkey_dir() else ScanStatus.SCANNED
         T1107Telem(status, get_monkey_dir_path()).send()
+        deleted = False
 
         if -1 == sys.executable.find("python"):
             try:
@@ -421,11 +422,14 @@ class InfectionMonkey:
                         close_fds=True,
                         startupinfo=startupinfo,
                     )
+                    deleted = True
                 else:
                     os.remove(sys.executable)
                     status = ScanStatus.USED
+                    deleted = True
             except Exception as exc:
                 logger.error("Exception in self delete: %s", exc)
                 status = ScanStatus.SCANNED
             if status:
                 T1107Telem(status, sys.executable).send()
+            return deleted

monkey/infection_monkey/network/firewall.py

@@ -1,18 +1,23 @@
+import logging
 import platform
 import subprocess
 import sys
 
+from common.common_consts.timeouts import SHORT_REQUEST_TIMEOUT
+
+logger = logging.getLogger(__name__)
+
 
 def _run_netsh_cmd(command, args):
-    cmd = subprocess.Popen(
+    output = subprocess.check_output(
         "netsh %s %s"
         % (
             command,
             " ".join(['%s="%s"' % (key, value) for key, value in list(args.items()) if value]),
         ),
-        stdout=subprocess.PIPE,
+        timeout=SHORT_REQUEST_TIMEOUT,
     )
-    return cmd.stdout.read().strip().lower().endswith("ok.")
+    return output.strip().lower().endswith(b"ok.")
 
 
 class FirewallApp(object):
@@ -44,19 +49,23 @@ class WinAdvFirewall(FirewallApp):
 
     def is_enabled(self):
         try:
-            cmd = subprocess.Popen("netsh advfirewall show currentprofile", stdout=subprocess.PIPE)
-            out = cmd.stdout.readlines()
-
-            for line in out:
-                if line.startswith("State"):
-                    state = line.split()[-1].strip()
-
-            return state == "ON"
+            out = subprocess.check_output(
+                "netsh advfirewall show currentprofile", timeout=SHORT_REQUEST_TIMEOUT
+            )
+        except subprocess.TimeoutExpired:
+            return None
         except Exception:
             return None
 
+        for line in out.decode().splitlines():
+            if line.startswith("State"):
+                state = line.split()[-1].strip()
+                return state == "ON"
+
+        return None
+
     def add_firewall_rule(
-        self, name="Firewall", direction="in", action="allow", program=sys.executable, **kwargs
+        self, name="MonkeyRule", direction="in", action="allow", program=sys.executable, **kwargs
     ):
         netsh_args = {"name": name, "dir": direction, "action": action, "program": program}
         netsh_args.update(kwargs)
@@ -66,8 +75,11 @@ class WinAdvFirewall(FirewallApp):
                 return True
             else:
                 return False
-        except Exception:
-            return None
+        except subprocess.CalledProcessError as err:
+            logger.info(f"Failed adding a firewall rule: {err.stdout}")
+        except subprocess.TimeoutExpired:
+            logger.info("Timeout expired trying to add a firewall rule.")
+        return None
 
     def remove_firewall_rule(self, name="Firewall", **kwargs):
         netsh_args = {"name": name}

monkey/infection_monkey/transport/base.py

@@ -2,6 +2,7 @@ import time
 from threading import Thread
 
 g_last_served = None
+PROXY_TIMEOUT = 2.5
 
 
 class TransportProxyBase(Thread):

monkey/infection_monkey/transport/http.py

@@ -7,7 +7,11 @@ from logging import getLogger
 from urllib.parse import urlsplit
 
 from infection_monkey.network.tools import get_interface_to_target
-from infection_monkey.transport.base import TransportProxyBase, update_last_serve_time
+from infection_monkey.transport.base import (
+    PROXY_TIMEOUT,
+    TransportProxyBase,
+    update_last_serve_time,
+)
 
 logger = getLogger(__name__)
 
@@ -227,6 +231,6 @@ class LockedHTTPServer(threading.Thread):
 class HTTPConnectProxy(TransportProxyBase):
     def run(self):
         httpd = http.server.HTTPServer((self.local_host, self.local_port), HTTPConnectProxyHandler)
-        httpd.timeout = 10
+        httpd.timeout = PROXY_TIMEOUT
         while not self._stopped:
             httpd.handle_request()

monkey/infection_monkey/transport/tcp.py

@@ -3,16 +3,20 @@ import socket
 from logging import getLogger
 from threading import Thread
 
-from infection_monkey.transport.base import TransportProxyBase, update_last_serve_time
+from infection_monkey.transport.base import (
+    PROXY_TIMEOUT,
+    TransportProxyBase,
+    update_last_serve_time,
+)
 
 READ_BUFFER_SIZE = 8192
-DEFAULT_TIMEOUT = 10
+SOCKET_READ_TIMEOUT = 10
 
 logger = getLogger(__name__)
 
 
 class SocketsPipe(Thread):
-    def __init__(self, source, dest, timeout=DEFAULT_TIMEOUT):
+    def __init__(self, source, dest, timeout=SOCKET_READ_TIMEOUT):
         Thread.__init__(self)
         self.source = source
         self.dest = dest
@@ -51,7 +55,7 @@ class TcpProxy(TransportProxyBase):
         pipes = []
         l_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         l_socket.bind((self.local_host, self.local_port))
-        l_socket.settimeout(DEFAULT_TIMEOUT)
+        l_socket.settimeout(PROXY_TIMEOUT)
         l_socket.listen(5)
 
         while not self._stopped:

monkey/monkey_island/cc/services/config_schema/internal.py

@@ -9,7 +9,7 @@ INTERNAL = {
                 "keep_tunnel_open_time": {
                     "title": "Keep tunnel open time",
                     "type": "integer",
-                    "default": 60,
+                    "default": 30,
                     "description": "Time to keep tunnel open before going down after last exploit "
                     "(in seconds)",
                 },