From 17dc764f3762a34941e278e5f4de8e4eee33d7dd Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 9 Sep 2019 11:36:28 +0300
Subject: [PATCH 01/19] Add support for MP in Windows PyInstaller

---
 monkey/infection_monkey/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/monkey/infection_monkey/main.py b/monkey/infection_monkey/main.py
index 2ddf9127e..ca5bb9832 100644
--- a/monkey/infection_monkey/main.py
+++ b/monkey/infection_monkey/main.py
@@ -7,6 +7,7 @@ import logging.config
 import os
 import sys
 import traceback
+from multiprocessing import freeze_support
 
 import infection_monkey.utils as utils
 from infection_monkey.config import WormConfiguration, EXTERNAL_CONFIG_FILE
@@ -43,7 +44,7 @@ def main():
 
     if 2 > len(sys.argv):
         return True
-
+    freeze_support() # required for multiprocessing + pyinstaller on windows
     monkey_mode = sys.argv[1]
 
     if not (monkey_mode in [MONKEY_ARG, DROPPER_ARG]):

From 7357c8c168dd66dde73ee672691f0e3044c536a6 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 9 Sep 2019 10:21:02 +0300
Subject: [PATCH 02/19] Iterate over ranges as chunks rather than discrete
 addresses

---
 .../network/network_scanner.py                | 72 ++++++++++++-------
 1 file changed, 45 insertions(+), 27 deletions(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index 837add48a..49d6f88c4 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -1,3 +1,4 @@
+import itertools
 import time
 
 from common.network.network_range import *
@@ -11,6 +12,22 @@ __author__ = 'itamar'
 LOG = logging.getLogger(__name__)
 
 SCAN_DELAY = 0
+ITERATION_BLOCK_SIZE = 5
+
+
+def _grouper(iterables, chunk_size):
+    """
+    Goes over an iterable using chunks
+    :param iterables: a sequence of iterable objects
+    :param chunk_size:  Chunk size, last chunk may be smaller
+    :return:
+    """
+    iterable = itertools.chain(*iterables)
+    while True:
+        group = tuple(itertools.islice(iterable, chunk_size))
+        if not group:
+            break
+        yield group
 
 
 class NetworkScanner(object):
@@ -69,35 +86,37 @@ class NetworkScanner(object):
         :return: yields a sequence of VictimHost instances
         """
 
-        TCPscan = TcpScanner()
-        Pinger = PingScanner()
+        tcp_scan = TcpScanner()
+        ping_scan = PingScanner()
         victims_count = 0
+        for network_chunk in _grouper(self._ranges, ITERATION_BLOCK_SIZE):
+            LOG.debug("Scanning for potential victims in chunk %r", network_chunk)
+            victim_chunk = []
+            for address in network_chunk:
+                #if hasattr(net_range, 'domain_name'):
+                #    victim = VictimHost(address, net_range.domain_name)
+                #else:
+                victim = VictimHost(address)
 
-        for net_range in self._ranges:
-            LOG.debug("Scanning for potential victims in the network %r", net_range)
-            for ip_addr in net_range:
-                if hasattr(net_range, 'domain_name'):
-                    victim = VictimHost(ip_addr, net_range.domain_name)
-                else:
-                    victim = VictimHost(ip_addr)
-                if stop_callback and stop_callback():
-                    LOG.debug("Got stop signal")
-                    break
+                victim_chunk.append(victim)
+            # skip self IP addresses
+            victim_chunk = [x for x in victim_chunk if x.ip_addr not in self._ip_addresses]
+            # skip IPs marked as blocked
 
-                # skip self IP address
-                if victim.ip_addr in self._ip_addresses:
-                    continue
+            bad_victims = [x for x in victim_chunk if x.ip_addr in WormConfiguration.blocked_ips]
+            for victim in bad_victims:
+                LOG.info("Skipping %s due to blacklist" % victim)
+            victim_chunk = [x for x in victim_chunk if x.ip_addr not in WormConfiguration.blocked_ips]
 
-                # skip IPs marked as blocked
-                if victim.ip_addr in WormConfiguration.blocked_ips:
-                    LOG.info("Skipping %s due to blacklist" % victim)
-                    continue
+            # check before running scans
+            if stop_callback and stop_callback():
+                LOG.debug("Got stop signal")
+                break
 
+            for victim in victim_chunk:
                 LOG.debug("Scanning %r...", victim)
-                pingAlive = Pinger.is_host_alive(victim)
-                tcpAlive = TCPscan.is_host_alive(victim)
-
-                # if scanner detect machine is up, add it to victims list
+                pingAlive = ping_scan.is_host_alive(victim)
+                tcpAlive = tcp_scan.is_host_alive(victim)
                 if pingAlive or tcpAlive:
                     LOG.debug("Found potential victim: %r", victim)
                     victims_count += 1
@@ -107,10 +126,9 @@ class NetworkScanner(object):
                         LOG.debug("Found max needed victims (%d), stopping scan", max_find)
 
                         break
-
-                if WormConfiguration.tcp_scan_interval:
-                    # time.sleep uses seconds, while config is in milliseconds
-                    time.sleep(WormConfiguration.tcp_scan_interval/float(1000))
+            if WormConfiguration.tcp_scan_interval:
+                # time.sleep uses seconds, while config is in milliseconds
+                time.sleep(WormConfiguration.tcp_scan_interval / float(1000))
 
     @staticmethod
     def _is_any_ip_in_subnet(ip_addresses, subnet_str):

From 6c5d6a5ecc76e572d3636816eef15bea00f2c641 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 9 Sep 2019 11:37:14 +0300
Subject: [PATCH 03/19] Move scanners to be instance variable. Add MP support
 (threading/process) for scanning victims in chunks

---
 .../network/network_scanner.py                | 48 ++++++++++++-------
 1 file changed, 31 insertions(+), 17 deletions(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index 49d6f88c4..90418218a 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -1,6 +1,10 @@
+import sys
 import itertools
 import time
-
+if sys.platform.startswith("win"):
+    from multiprocessing.dummy import Pool
+else:
+    from multiprocessing import Pool
 from common.network.network_range import *
 from infection_monkey.config import WormConfiguration
 from infection_monkey.network.info import local_ips, get_interfaces_ranges
@@ -34,6 +38,7 @@ class NetworkScanner(object):
     def __init__(self):
         self._ip_addresses = None
         self._ranges = None
+        self.scanners = [TcpScanner(), PingScanner()]
 
     def initialize(self):
         """
@@ -85,17 +90,15 @@ class NetworkScanner(object):
         :param stop_callback: A callback to check at any point if we should stop scanning
         :return: yields a sequence of VictimHost instances
         """
-
-        tcp_scan = TcpScanner()
-        ping_scan = PingScanner()
+        pool = Pool()
         victims_count = 0
         for network_chunk in _grouper(self._ranges, ITERATION_BLOCK_SIZE):
             LOG.debug("Scanning for potential victims in chunk %r", network_chunk)
             victim_chunk = []
             for address in network_chunk:
-                #if hasattr(net_range, 'domain_name'):
+                # if hasattr(net_range, 'domain_name'):
                 #    victim = VictimHost(address, net_range.domain_name)
-                #else:
+                # else:
                 victim = VictimHost(address)
 
                 victim_chunk.append(victim)
@@ -113,19 +116,17 @@ class NetworkScanner(object):
                 LOG.debug("Got stop signal")
                 break
 
-            for victim in victim_chunk:
-                LOG.debug("Scanning %r...", victim)
-                pingAlive = ping_scan.is_host_alive(victim)
-                tcpAlive = tcp_scan.is_host_alive(victim)
-                if pingAlive or tcpAlive:
-                    LOG.debug("Found potential victim: %r", victim)
-                    victims_count += 1
-                    yield victim
+            results = pool.map(self.scan_machine, victim_chunk)
+            resulting_victims = [x for x in results if x]  # filter out dead addresses
+            for victim in resulting_victims:
+                LOG.debug("Found potential victim: %r", victim)
+                victims_count += 1
+                yield victim
 
-                    if victims_count >= max_find:
-                        LOG.debug("Found max needed victims (%d), stopping scan", max_find)
+                if victims_count >= max_find:
+                    LOG.debug("Found max needed victims (%d), stopping scan", max_find)
 
-                        break
+                    break
             if WormConfiguration.tcp_scan_interval:
                 # time.sleep uses seconds, while config is in milliseconds
                 time.sleep(WormConfiguration.tcp_scan_interval / float(1000))
@@ -137,5 +138,18 @@ class NetworkScanner(object):
                 return True
         return False
 
+    def scan_machine(self, victim):
+        """
+        Scans specific machine using given scanner
+        :param victim: VictimHost machine
+        :return: Victim or None if victim isn't alive
+        """
+        LOG.debug("Scanning target address: %r", victim)
+        if any([scanner.is_host_alive(victim.ip_addr) for scanner in self.scanners]):
+            LOG.debug("Found potential target_ip: %r", victim)
+            return victim
+        else:
+            return None
+
     def on_island(self, server):
         return bool([x for x in self._ip_addresses if x in server])

From 2b14878a9c5549a54d2c7f28c72856294cba82d0 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 9 Sep 2019 12:06:30 +0300
Subject: [PATCH 04/19] Bugfix, used wrong logger!

---
 monkey/infection_monkey/network/network_scanner.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index 90418218a..68abcb786 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -1,11 +1,13 @@
 import sys
 import itertools
 import time
+import logging
+
 if sys.platform.startswith("win"):
     from multiprocessing.dummy import Pool
 else:
     from multiprocessing import Pool
-from common.network.network_range import *
+from common.network.network_range import NetworkRange
 from infection_monkey.config import WormConfiguration
 from infection_monkey.network.info import local_ips, get_interfaces_ranges
 from infection_monkey.model import VictimHost

From e11be48b80bda7f518be95ab87a6fea7f1895636 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 9 Sep 2019 14:17:07 +0300
Subject: [PATCH 05/19] Add documentation for TCP scan interval in Monkey
 configuration

---
 monkey/infection_monkey/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py
index cb5bf881b..09ce4b8de 100644
--- a/monkey/infection_monkey/config.py
+++ b/monkey/infection_monkey/config.py
@@ -199,7 +199,7 @@ class Configuration(object):
                         9200]
     tcp_target_ports.extend(HTTP_PORTS)
     tcp_scan_timeout = 3000  # 3000 Milliseconds
-    tcp_scan_interval = 0
+    tcp_scan_interval = 0  # in milliseconds
     tcp_scan_get_banner = True
 
     # Ping Scanner

From d8bac57eb5c9adfff40cb9418f1cbeb5162ac473 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 16 Sep 2019 15:14:53 +0300
Subject: [PATCH 06/19] Change grouper to be a VictimHost generator

---
 .../network/network_scanner.py                | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index 68abcb786..b751e99d7 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -21,19 +21,24 @@ SCAN_DELAY = 0
 ITERATION_BLOCK_SIZE = 5
 
 
-def _grouper(iterables, chunk_size):
+def generate_victims(net_ranges, chunk_size):
     """
-    Goes over an iterable using chunks
-    :param iterables: a sequence of iterable objects
-    :param chunk_size:  Chunk size, last chunk may be smaller
+    Generates VictimHosts in chunks from all the netranges
+    :param net_ranges: Iterable of network ranges
+    :param chunk_size: Maximum size of each chunk
     :return:
     """
-    iterable = itertools.chain(*iterables)
-    while True:
-        group = tuple(itertools.islice(iterable, chunk_size))
-        if not group:
-            break
-        yield group
+    chunk = []
+    for net_range in net_ranges:
+        for address in net_range:
+            if hasattr(net_range, 'domain_name'):
+                victim = VictimHost(address, net_range.domain_name)
+            else:
+                victim = VictimHost(address)
+            chunk.append(victim)
+            if len(chunk) == chunk_size:
+                yield chunk
+    yield chunk
 
 
 class NetworkScanner(object):
@@ -94,16 +99,9 @@ class NetworkScanner(object):
         """
         pool = Pool()
         victims_count = 0
-        for network_chunk in _grouper(self._ranges, ITERATION_BLOCK_SIZE):
-            LOG.debug("Scanning for potential victims in chunk %r", network_chunk)
-            victim_chunk = []
-            for address in network_chunk:
-                # if hasattr(net_range, 'domain_name'):
-                #    victim = VictimHost(address, net_range.domain_name)
-                # else:
-                victim = VictimHost(address)
+        for victim_chunk in generate_victims(self._ranges, ITERATION_BLOCK_SIZE):
+            LOG.debug("Scanning for potential victims in chunk %r", victim_chunk)
 
-                victim_chunk.append(victim)
             # skip self IP addresses
             victim_chunk = [x for x in victim_chunk if x.ip_addr not in self._ip_addresses]
             # skip IPs marked as blocked
@@ -133,6 +131,7 @@ class NetworkScanner(object):
                 # time.sleep uses seconds, while config is in milliseconds
                 time.sleep(WormConfiguration.tcp_scan_interval / float(1000))
 
+
     @staticmethod
     def _is_any_ip_in_subnet(ip_addresses, subnet_str):
         for ip_address in ip_addresses:
@@ -140,6 +139,7 @@ class NetworkScanner(object):
                 return True
         return False
 
+
     def scan_machine(self, victim):
         """
         Scans specific machine using given scanner
@@ -153,5 +153,6 @@ class NetworkScanner(object):
         else:
             return None
 
+
     def on_island(self, server):
         return bool([x for x in self._ip_addresses if x in server])

From d8d55cb546d77239289db032365a7b400c70ec32 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 16 Sep 2019 16:32:13 +0300
Subject: [PATCH 07/19] PEP8 changes Documentation improvements

---
 monkey/infection_monkey/main.py               |  2 +-
 .../network/network_scanner.py                | 25 ++++++++++---------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/monkey/infection_monkey/main.py b/monkey/infection_monkey/main.py
index ca5bb9832..9c9d24d73 100644
--- a/monkey/infection_monkey/main.py
+++ b/monkey/infection_monkey/main.py
@@ -44,7 +44,7 @@ def main():
 
     if 2 > len(sys.argv):
         return True
-    freeze_support() # required for multiprocessing + pyinstaller on windows
+    freeze_support()  # required for multiprocessing + pyinstaller on windows
     monkey_mode = sys.argv[1]
 
     if not (monkey_mode in [MONKEY_ARG, DROPPER_ARG]):
diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index b751e99d7..a89920c72 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -1,17 +1,18 @@
 import sys
-import itertools
 import time
 import logging
 
-if sys.platform.startswith("win"):
-    from multiprocessing.dummy import Pool
-else:
-    from multiprocessing import Pool
 from common.network.network_range import NetworkRange
 from infection_monkey.config import WormConfiguration
 from infection_monkey.network.info import local_ips, get_interfaces_ranges
 from infection_monkey.model import VictimHost
 from infection_monkey.network import TcpScanner, PingScanner
+from infection_monkey.utils import is_windows_os
+
+if is_windows_os():
+    from multiprocessing.dummy import Pool
+else:
+    from multiprocessing import Pool
 
 __author__ = 'itamar'
 
@@ -26,7 +27,6 @@ def generate_victims(net_ranges, chunk_size):
     Generates VictimHosts in chunks from all the netranges
     :param net_ranges: Iterable of network ranges
     :param chunk_size: Maximum size of each chunk
-    :return:
     """
     chunk = []
     for net_range in net_ranges:
@@ -51,7 +51,6 @@ class NetworkScanner(object):
         """
         Set up scanning.
         based on configuration: scans local network and/or scans fixed list of IPs/subnets.
-        :return:
         """
         # get local ip addresses
         self._ip_addresses = local_ips()
@@ -97,7 +96,12 @@ class NetworkScanner(object):
         :param stop_callback: A callback to check at any point if we should stop scanning
         :return: yields a sequence of VictimHost instances
         """
-        pool = Pool()
+        # We currently use the ITERATION_BLOCK_SIZE as the pool size, however, this may not be the best decision
+        # However, the decision what ITERATION_BLOCK_SIZE also requires balancing network usage (pps and bw)
+        # Because we are using this to spread out IO heavy tasks, we can probably go a lot higher than CPU core size
+        # But again, balance
+        pool = Pool(ITERATION_BLOCK_SIZE)
+
         victims_count = 0
         for victim_chunk in generate_victims(self._ranges, ITERATION_BLOCK_SIZE):
             LOG.debug("Scanning for potential victims in chunk %r", victim_chunk)
@@ -131,7 +135,6 @@ class NetworkScanner(object):
                 # time.sleep uses seconds, while config is in milliseconds
                 time.sleep(WormConfiguration.tcp_scan_interval / float(1000))
 
-
     @staticmethod
     def _is_any_ip_in_subnet(ip_addresses, subnet_str):
         for ip_address in ip_addresses:
@@ -139,10 +142,9 @@ class NetworkScanner(object):
                 return True
         return False
 
-
     def scan_machine(self, victim):
         """
-        Scans specific machine using given scanner
+        Scans specific machine using instance scanners
         :param victim: VictimHost machine
         :return: Victim or None if victim isn't alive
         """
@@ -153,6 +155,5 @@ class NetworkScanner(object):
         else:
             return None
 
-
     def on_island(self, server):
         return bool([x for x in self._ip_addresses if x in server])

From 032ee2ee0edc2b721ec8af3d93f348781b5c6445 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Wed, 18 Sep 2019 11:08:24 +0300
Subject: [PATCH 08/19] Bugfix in generator

---
 monkey/infection_monkey/network/network_scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index a89920c72..dea7707ad 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -38,7 +38,7 @@ def generate_victims(net_ranges, chunk_size):
             chunk.append(victim)
             if len(chunk) == chunk_size:
                 yield chunk
-    yield chunk
+                chunk = []
 
 
 class NetworkScanner(object):

From 4dcc919b49586c600f48d47d8b680e76155e23ac Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 20 Sep 2019 22:55:10 +0300
Subject: [PATCH 09/19] Remove unused import, remove author

---
 monkey/infection_monkey/network/network_scanner.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index dea7707ad..a0b57c07b 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -1,4 +1,3 @@
-import sys
 import time
 import logging
 
@@ -14,8 +13,6 @@ if is_windows_os():
 else:
     from multiprocessing import Pool
 
-__author__ = 'itamar'
-
 LOG = logging.getLogger(__name__)
 
 SCAN_DELAY = 0

From a32a78325743b6d4d63d7d17b38d801efc711298 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 20 Sep 2019 22:58:20 +0300
Subject: [PATCH 10/19] Factor out generating VictimHosts from NetworkRange
 object

---
 monkey/infection_monkey/model/host.py              | 14 ++++++++++++++
 monkey/infection_monkey/network/network_scanner.py | 10 ++++------
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/monkey/infection_monkey/model/host.py b/monkey/infection_monkey/model/host.py
index dcc6e7455..ebafb09f5 100644
--- a/monkey/infection_monkey/model/host.py
+++ b/monkey/infection_monkey/model/host.py
@@ -46,3 +46,17 @@ class VictimHost(object):
 
     def set_default_server(self, default_server):
         self.default_server = default_server
+
+
+def generate_victims_from_range(net_range):
+    """
+    Generates VictimHosts from a given netrange
+    :param net_range: Network range object
+    :return: Generator of VictimHost objects
+    """
+    for address in net_range:
+        if hasattr(net_range, 'domain_name'):
+            victim = VictimHost(address, net_range.domain_name)
+        else:
+            victim = VictimHost(address)
+        yield victim
diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index a0b57c07b..fb2b5269e 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -3,8 +3,8 @@ import logging
 
 from common.network.network_range import NetworkRange
 from infection_monkey.config import WormConfiguration
+from infection_monkey.model.host import generate_victims_from_range
 from infection_monkey.network.info import local_ips, get_interfaces_ranges
-from infection_monkey.model import VictimHost
 from infection_monkey.network import TcpScanner, PingScanner
 from infection_monkey.utils import is_windows_os
 
@@ -27,15 +27,13 @@ def generate_victims(net_ranges, chunk_size):
     """
     chunk = []
     for net_range in net_ranges:
-        for address in net_range:
-            if hasattr(net_range, 'domain_name'):
-                victim = VictimHost(address, net_range.domain_name)
-            else:
-                victim = VictimHost(address)
+        for victim in generate_victims_from_range(net_range):
             chunk.append(victim)
             if len(chunk) == chunk_size:
                 yield chunk
                 chunk = []
+    if chunk:  # finished with number of victims < chunk_size
+        yield chunk
 
 
 class NetworkScanner(object):

From c4ec6683a1de6195e77c31e617d42d3cb56c26d3 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Sun, 22 Sep 2019 16:44:20 +0300
Subject: [PATCH 11/19] Silly bugfix in scanning

---
 monkey/infection_monkey/network/network_scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index fb2b5269e..c807a2fe3 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -144,7 +144,7 @@ class NetworkScanner(object):
         :return: Victim or None if victim isn't alive
         """
         LOG.debug("Scanning target address: %r", victim)
-        if any([scanner.is_host_alive(victim.ip_addr) for scanner in self.scanners]):
+        if any([scanner.is_host_alive(victim) for scanner in self.scanners]):
             LOG.debug("Found potential target_ip: %r", victim)
             return victim
         else:

From ccc6c50a7f3f92f93aa0d395b7df3e9738bb4dda Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Sun, 22 Sep 2019 16:46:55 +0300
Subject: [PATCH 12/19] Remove unused constant

---
 monkey/infection_monkey/network/network_scanner.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index c807a2fe3..be424692c 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -15,7 +15,6 @@ else:
 
 LOG = logging.getLogger(__name__)
 
-SCAN_DELAY = 0
 ITERATION_BLOCK_SIZE = 5
 
 

From c76cc72821ab180edd1f1ba5b6e52db4a224d388 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 23 Sep 2019 17:45:17 +0300
Subject: [PATCH 13/19] Fixed horrible bug where we would return more victims
 than we needed to

---
 monkey/infection_monkey/network/network_scanner.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index be424692c..eb7c29572 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -112,7 +112,7 @@ class NetworkScanner(object):
             # check before running scans
             if stop_callback and stop_callback():
                 LOG.debug("Got stop signal")
-                break
+                return
 
             results = pool.map(self.scan_machine, victim_chunk)
             resulting_victims = [x for x in results if x]  # filter out dead addresses
@@ -123,8 +123,7 @@ class NetworkScanner(object):
 
                 if victims_count >= max_find:
                     LOG.debug("Found max needed victims (%d), stopping scan", max_find)
-
-                    break
+                    return
             if WormConfiguration.tcp_scan_interval:
                 # time.sleep uses seconds, while config is in milliseconds
                 time.sleep(WormConfiguration.tcp_scan_interval / float(1000))

From 8c55d2acd48dbe349c15221b83b3aceede1271b7 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Mon, 23 Sep 2019 18:01:39 +0300
Subject: [PATCH 14/19] Refactor victim generation. Now we have a VictimHost
 generator that handles all the filtering.

---
 monkey/infection_monkey/model/host.py         | 14 ------
 .../model/victim_host_generator.py            | 46 +++++++++++++++++++
 .../network/network_scanner.py                | 31 ++-----------
 3 files changed, 49 insertions(+), 42 deletions(-)
 create mode 100644 monkey/infection_monkey/model/victim_host_generator.py

diff --git a/monkey/infection_monkey/model/host.py b/monkey/infection_monkey/model/host.py
index ebafb09f5..dcc6e7455 100644
--- a/monkey/infection_monkey/model/host.py
+++ b/monkey/infection_monkey/model/host.py
@@ -46,17 +46,3 @@ class VictimHost(object):
 
     def set_default_server(self, default_server):
         self.default_server = default_server
-
-
-def generate_victims_from_range(net_range):
-    """
-    Generates VictimHosts from a given netrange
-    :param net_range: Network range object
-    :return: Generator of VictimHost objects
-    """
-    for address in net_range:
-        if hasattr(net_range, 'domain_name'):
-            victim = VictimHost(address, net_range.domain_name)
-        else:
-            victim = VictimHost(address)
-        yield victim
diff --git a/monkey/infection_monkey/model/victim_host_generator.py b/monkey/infection_monkey/model/victim_host_generator.py
new file mode 100644
index 000000000..1309278c8
--- /dev/null
+++ b/monkey/infection_monkey/model/victim_host_generator.py
@@ -0,0 +1,46 @@
+from infection_monkey.model.host import VictimHost
+from infection_monkey.network.info import local_ips
+
+
+class VictimHostGenerator(object):
+    def __init__(self, network_ranges, blocked_ips):
+        self._ip_addresses = local_ips()
+        self.blocked_ips = blocked_ips
+        self.ranges = network_ranges
+
+    def generate_victims(self, chunk_size):
+        """
+        Generates VictimHosts in chunks from all the instances network ranges
+        :param chunk_size: Maximum size of each chunk
+        """
+        chunk = []
+        for net_range in self.ranges:
+            for victim in self.generate_victims_from_range(net_range):
+                chunk.append(victim)
+                if len(chunk) == chunk_size:
+                    yield chunk
+                    chunk = []
+        if chunk:  # finished with number of victims < chunk_size
+            yield chunk
+
+    def generate_victims_from_range(self, net_range):
+        """
+        Generates VictimHosts from a given netrange
+        :param net_range: Network range object
+        :return: Generator of VictimHost objects
+        """
+        for address in net_range:
+            if not self.is_ip_scannable(address):  # check if the IP should be skipped
+                continue
+            if hasattr(net_range, 'domain_name'):
+                victim = VictimHost(address, net_range.domain_name)
+            else:
+                victim = VictimHost(address)
+            yield victim
+
+    def is_ip_scannable(self, ip_address):
+        if ip_address in self._ip_addresses:
+            return False
+        if ip_address in self.blocked_ips:
+            return False
+        return True
diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index eb7c29572..da3d88e5f 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -3,7 +3,7 @@ import logging
 
 from common.network.network_range import NetworkRange
 from infection_monkey.config import WormConfiguration
-from infection_monkey.model.host import generate_victims_from_range
+from infection_monkey.model.victim_host_generator import VictimHostGenerator
 from infection_monkey.network.info import local_ips, get_interfaces_ranges
 from infection_monkey.network import TcpScanner, PingScanner
 from infection_monkey.utils import is_windows_os
@@ -18,23 +18,6 @@ LOG = logging.getLogger(__name__)
 ITERATION_BLOCK_SIZE = 5
 
 
-def generate_victims(net_ranges, chunk_size):
-    """
-    Generates VictimHosts in chunks from all the netranges
-    :param net_ranges: Iterable of network ranges
-    :param chunk_size: Maximum size of each chunk
-    """
-    chunk = []
-    for net_range in net_ranges:
-        for victim in generate_victims_from_range(net_range):
-            chunk.append(victim)
-            if len(chunk) == chunk_size:
-                yield chunk
-                chunk = []
-    if chunk:  # finished with number of victims < chunk_size
-        yield chunk
-
-
 class NetworkScanner(object):
     def __init__(self):
         self._ip_addresses = None
@@ -95,20 +78,12 @@ class NetworkScanner(object):
         # Because we are using this to spread out IO heavy tasks, we can probably go a lot higher than CPU core size
         # But again, balance
         pool = Pool(ITERATION_BLOCK_SIZE)
+        victim_generator = VictimHostGenerator(self._ranges, WormConfiguration.blocked_ips)
 
         victims_count = 0
-        for victim_chunk in generate_victims(self._ranges, ITERATION_BLOCK_SIZE):
+        for victim_chunk in victim_generator.generate_victims(ITERATION_BLOCK_SIZE):
             LOG.debug("Scanning for potential victims in chunk %r", victim_chunk)
 
-            # skip self IP addresses
-            victim_chunk = [x for x in victim_chunk if x.ip_addr not in self._ip_addresses]
-            # skip IPs marked as blocked
-
-            bad_victims = [x for x in victim_chunk if x.ip_addr in WormConfiguration.blocked_ips]
-            for victim in bad_victims:
-                LOG.info("Skipping %s due to blacklist" % victim)
-            victim_chunk = [x for x in victim_chunk if x.ip_addr not in WormConfiguration.blocked_ips]
-
             # check before running scans
             if stop_callback and stop_callback():
                 LOG.debug("Got stop signal")

From a1d631b39eca6bf75b0ea6d8d3d1366164d1b73f Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 27 Sep 2019 16:33:29 +0300
Subject: [PATCH 15/19] Remove list comprehension

---
 monkey/infection_monkey/network/network_scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index da3d88e5f..3e447b85e 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -90,7 +90,7 @@ class NetworkScanner(object):
                 return
 
             results = pool.map(self.scan_machine, victim_chunk)
-            resulting_victims = [x for x in results if x]  # filter out dead addresses
+            resulting_victims = filter(lambda x: x is not None, results)
             for victim in resulting_victims:
                 LOG.debug("Found potential victim: %r", victim)
                 victims_count += 1

From 2f25e5b127dd8bbe23ca36485c4ff0c247c2f80a Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 27 Sep 2019 16:51:55 +0300
Subject: [PATCH 16/19] Added basic tests for VictimHostGenerator

---
 .../model/victim_host_generator_test.py       | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 monkey/infection_monkey/model/victim_host_generator_test.py

diff --git a/monkey/infection_monkey/model/victim_host_generator_test.py b/monkey/infection_monkey/model/victim_host_generator_test.py
new file mode 100644
index 000000000..849196ff6
--- /dev/null
+++ b/monkey/infection_monkey/model/victim_host_generator_test.py
@@ -0,0 +1,35 @@
+from unittest import TestCase
+from infection_monkey.model.victim_host_generator import VictimHostGenerator
+from common.network.network_range import CidrRange, SingleIpRange
+
+
+class TestPayload(TestCase):
+
+    def setUp(self):
+        self.test_ranges = [CidrRange("10.0.0.0/28", False),
+                            SingleIpRange('41.50.13.37'),
+                            SingleIpRange('localhost')
+                            ]
+        self.generator = VictimHostGenerator(self.test_ranges, '10.0.0.1')
+        self.generator._ip_addresses = []  # test later on
+
+    def test_remove_blocked_ip(self):
+        victims = list(self.generator.generate_victims_from_range(self.test_ranges[0]))
+        self.assertEqual(len(victims), 14)  # 15 minus the 1 we blocked
+
+    def test_remove_local_ips(self):
+        self.generator._ip_addresses = ['127.0.0.1']
+        victims = list(self.generator.generate_victims_from_range(self.test_ranges[-1]))
+        self.assertEqual(len(victims), 0)  # block the local IP
+
+    def test_generate_domain_victim(self):
+        # domain name victim
+        self.generator._ip_addresses = []
+        victims = list(self.generator.generate_victims_from_range(self.test_ranges[-1]))
+        self.assertEqual(len(victims), 1)
+        self.assertEqual(victims[0].domain_name, 'localhost')
+
+        # don't generate for other victims
+        victims = list(self.generator.generate_victims_from_range(self.test_ranges[1]))
+        self.assertEqual(len(victims), 1)
+        self.assertEqual(victims[0].domain_name, '')

From 0a61e83a154efb9d0955d1db3558111dcdb21771 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 27 Sep 2019 16:57:36 +0300
Subject: [PATCH 17/19] Add chunking test and some basic docs

---
 .../model/victim_host_generator_test.py             | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/monkey/infection_monkey/model/victim_host_generator_test.py b/monkey/infection_monkey/model/victim_host_generator_test.py
index 849196ff6..2414bd794 100644
--- a/monkey/infection_monkey/model/victim_host_generator_test.py
+++ b/monkey/infection_monkey/model/victim_host_generator_test.py
@@ -3,16 +3,25 @@ from infection_monkey.model.victim_host_generator import VictimHostGenerator
 from common.network.network_range import CidrRange, SingleIpRange
 
 
-class TestPayload(TestCase):
+class VictimHostGeneratorTester(TestCase):
 
     def setUp(self):
-        self.test_ranges = [CidrRange("10.0.0.0/28", False),
+        self.test_ranges = [CidrRange("10.0.0.0/28", False),  # this gives us 15 hosts
                             SingleIpRange('41.50.13.37'),
                             SingleIpRange('localhost')
                             ]
         self.generator = VictimHostGenerator(self.test_ranges, '10.0.0.1')
         self.generator._ip_addresses = []  # test later on
 
+    def test_chunking(self):
+        chunk_size = 3
+        # current test setup is 15+1+1-1 hosts
+        victims = self.generator.generate_victims(chunk_size)
+        for i in range(5):  # quickly check the equally sided chunks
+            self.assertEqual(len(victims.next()), chunk_size)
+        victim_chunk_last = victims.next()
+        self.assertEqual(len(victim_chunk_last), 1)
+
     def test_remove_blocked_ip(self):
         victims = list(self.generator.generate_victims_from_range(self.test_ranges[0]))
         self.assertEqual(len(victims), 14)  # 15 minus the 1 we blocked

From 297686dc53852c64897e30d0aa740c01492a319c Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 27 Sep 2019 17:00:18 +0300
Subject: [PATCH 18/19] Changed default scanning size to be bigger.

---
 monkey/infection_monkey/config.py                 | 4 ++--
 monkey/infection_monkey/example.conf              | 4 ++--
 monkey/monkey_island/cc/services/config_schema.py | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py
index 09ce4b8de..63358492d 100644
--- a/monkey/infection_monkey/config.py
+++ b/monkey/infection_monkey/config.py
@@ -141,10 +141,10 @@ class Configuration(object):
     exploiter_classes = []
 
     # how many victims to look for in a single scan iteration
-    victims_max_find = 30
+    victims_max_find = 100
 
     # how many victims to exploit before stopping
-    victims_max_exploit = 7
+    victims_max_exploit = 15
 
     # depth of propagation
     depth = 2
diff --git a/monkey/infection_monkey/example.conf b/monkey/infection_monkey/example.conf
index 8dba50352..4c0b82f48 100644
--- a/monkey/infection_monkey/example.conf
+++ b/monkey/infection_monkey/example.conf
@@ -98,8 +98,8 @@
     ],
     "timeout_between_iterations": 10,
     "use_file_logging": true,
-    "victims_max_exploit": 7,
-    "victims_max_find": 30,
+    "victims_max_exploit": 15,
+    "victims_max_find": 100,
     "post_breach_actions" : []
     custom_PBA_linux_cmd = ""
     custom_PBA_windows_cmd = ""
diff --git a/monkey/monkey_island/cc/services/config_schema.py b/monkey/monkey_island/cc/services/config_schema.py
index 2fa1a3aff..f4925e2cc 100644
--- a/monkey/monkey_island/cc/services/config_schema.py
+++ b/monkey/monkey_island/cc/services/config_schema.py
@@ -447,13 +447,13 @@ SCHEMA = {
                         "victims_max_find": {
                             "title": "Max victims to find",
                             "type": "integer",
-                            "default": 30,
+                            "default": 100,
                             "description": "Determines the maximum number of machines the monkey is allowed to scan"
                         },
                         "victims_max_exploit": {
                             "title": "Max victims to exploit",
                             "type": "integer",
-                            "default": 7,
+                            "default": 15,
                             "description":
                                 "Determines the maximum number of machines the monkey"
                                 " is allowed to successfully exploit. " + WARNING_SIGN

From f55a3e483b8f761d00218ad62f8f2a02d96c4380 Mon Sep 17 00:00:00 2001
From: Daniel Goldberg <danielg@guardicore.com>
Date: Fri, 27 Sep 2019 18:10:59 +0300
Subject: [PATCH 19/19] Changed VictimHostGenerator to accept the local
 addresses rather than generating them itself. Changed UTs to be independent.

---
 .../model/victim_host_generator.py            |  7 ++---
 .../model/victim_host_generator_test.py       | 28 ++++++++++---------
 .../network/network_scanner.py                |  2 +-
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/monkey/infection_monkey/model/victim_host_generator.py b/monkey/infection_monkey/model/victim_host_generator.py
index 1309278c8..1e9eba9c2 100644
--- a/monkey/infection_monkey/model/victim_host_generator.py
+++ b/monkey/infection_monkey/model/victim_host_generator.py
@@ -1,12 +1,11 @@
 from infection_monkey.model.host import VictimHost
-from infection_monkey.network.info import local_ips
 
 
 class VictimHostGenerator(object):
-    def __init__(self, network_ranges, blocked_ips):
-        self._ip_addresses = local_ips()
+    def __init__(self, network_ranges, blocked_ips, same_machine_ips):
         self.blocked_ips = blocked_ips
         self.ranges = network_ranges
+        self.local_addresses = same_machine_ips
 
     def generate_victims(self, chunk_size):
         """
@@ -39,7 +38,7 @@ class VictimHostGenerator(object):
             yield victim
 
     def is_ip_scannable(self, ip_address):
-        if ip_address in self._ip_addresses:
+        if ip_address in self.local_addresses:
             return False
         if ip_address in self.blocked_ips:
             return False
diff --git a/monkey/infection_monkey/model/victim_host_generator_test.py b/monkey/infection_monkey/model/victim_host_generator_test.py
index 2414bd794..102014d45 100644
--- a/monkey/infection_monkey/model/victim_host_generator_test.py
+++ b/monkey/infection_monkey/model/victim_host_generator_test.py
@@ -6,39 +6,41 @@ from common.network.network_range import CidrRange, SingleIpRange
 class VictimHostGeneratorTester(TestCase):
 
     def setUp(self):
-        self.test_ranges = [CidrRange("10.0.0.0/28", False),  # this gives us 15 hosts
-                            SingleIpRange('41.50.13.37'),
-                            SingleIpRange('localhost')
-                            ]
-        self.generator = VictimHostGenerator(self.test_ranges, '10.0.0.1')
-        self.generator._ip_addresses = []  # test later on
+        self.cidr_range = CidrRange("10.0.0.0/28", False)  # this gives us 15 hosts
+        self.local_host_range = SingleIpRange('localhost')
+        self.random_single_ip_range = SingleIpRange('41.50.13.37')
 
     def test_chunking(self):
         chunk_size = 3
         # current test setup is 15+1+1-1 hosts
-        victims = self.generator.generate_victims(chunk_size)
+        test_ranges = [self.cidr_range, self.local_host_range, self.random_single_ip_range]
+        generator = VictimHostGenerator(test_ranges, '10.0.0.1', [])
+        victims = generator.generate_victims(chunk_size)
         for i in range(5):  # quickly check the equally sided chunks
             self.assertEqual(len(victims.next()), chunk_size)
         victim_chunk_last = victims.next()
         self.assertEqual(len(victim_chunk_last), 1)
 
     def test_remove_blocked_ip(self):
-        victims = list(self.generator.generate_victims_from_range(self.test_ranges[0]))
+        generator = VictimHostGenerator(self.cidr_range, ['10.0.0.1'], [])
+
+        victims = list(generator.generate_victims_from_range(self.cidr_range))
         self.assertEqual(len(victims), 14)  # 15 minus the 1 we blocked
 
     def test_remove_local_ips(self):
-        self.generator._ip_addresses = ['127.0.0.1']
-        victims = list(self.generator.generate_victims_from_range(self.test_ranges[-1]))
+        generator = VictimHostGenerator([], [], [])
+        generator.local_addresses = ['127.0.0.1']
+        victims = list(generator.generate_victims_from_range(self.local_host_range))
         self.assertEqual(len(victims), 0)  # block the local IP
 
     def test_generate_domain_victim(self):
         # domain name victim
-        self.generator._ip_addresses = []
-        victims = list(self.generator.generate_victims_from_range(self.test_ranges[-1]))
+        generator = VictimHostGenerator([], [], [])  # dummy object
+        victims = list(generator.generate_victims_from_range(self.local_host_range))
         self.assertEqual(len(victims), 1)
         self.assertEqual(victims[0].domain_name, 'localhost')
 
         # don't generate for other victims
-        victims = list(self.generator.generate_victims_from_range(self.test_ranges[1]))
+        victims = list(generator.generate_victims_from_range(self.random_single_ip_range))
         self.assertEqual(len(victims), 1)
         self.assertEqual(victims[0].domain_name, '')
diff --git a/monkey/infection_monkey/network/network_scanner.py b/monkey/infection_monkey/network/network_scanner.py
index 3e447b85e..9452a3fb8 100644
--- a/monkey/infection_monkey/network/network_scanner.py
+++ b/monkey/infection_monkey/network/network_scanner.py
@@ -78,7 +78,7 @@ class NetworkScanner(object):
         # Because we are using this to spread out IO heavy tasks, we can probably go a lot higher than CPU core size
         # But again, balance
         pool = Pool(ITERATION_BLOCK_SIZE)
-        victim_generator = VictimHostGenerator(self._ranges, WormConfiguration.blocked_ips)
+        victim_generator = VictimHostGenerator(self._ranges, WormConfiguration.blocked_ips, local_ips())
 
         victims_count = 0
         for victim_chunk in victim_generator.generate_victims(ITERATION_BLOCK_SIZE):