Merge pull request #469 from guardicore/393/python-3-ci

393/python 3 ci

.travis.yml

@@ -1,18 +1,29 @@
+# Infection Monkey travis.yml. See Travis documentation for information about this file structure.
+
 group: travis_latest
 language: python
 cache: pip
 python:
-    - 2.7
+- 3.7
 install:
-    #- pip install -r requirements.txt
-    - pip install flake8  # pytest  # add another testing frameworks later
+- pip install -r monkey/monkey_island/requirements.txt  # for unit tests
+- pip install flake8 pytest dlint  # for next stages
+- pip install -r monkey/infection_monkey/requirements_linux.txt  # for unit tests
 before_script:
-    # stop the build if there are Python syntax errors or undefined names
-    - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
-    # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-    - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics  # Check syntax errors
+- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics  # warn about linter issues. --exit-zero
+  # means this stage will not fail the build. This is (hopefully) a temporary measure until all warnings are suppressed.
+- python monkey/monkey_island/cc/set_server_config.py testing  # Set the server config to `testing`, for the UTs to use
+  # mongomaock and pass.
 script:
-    - true  # pytest --capture=sys  # add other tests here
+- cd monkey  # This is our source dir
+- python -m pytest  # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path.
 notifications:
-    on_success: change
-    on_failure: change # `always` will be the setting once code changes slow down
+  slack:  # Notify to slack
+      rooms:
+      - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#ci  # room: #ci
+      on_success: change
+      on_failure: always
+  email:
+      on_success: change
+      on_failure: always

README.md

@@ -1,13 +1,18 @@
 Infection Monkey
 ====================
+[![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey)
+[![GitHub release (latest by date)](https://img.shields.io/github/v/release/guardicore/monkey)](https://github.com/guardicore/monkey/releases)
+![GitHub stars](https://img.shields.io/github/stars/guardicore/monkey)
+![GitHub commit activity](https://img.shields.io/github/commit-activity/m/guardicore/monkey)
 
-### Data center Security Testing Tool
+## Data center Security Testing Tool
 ------------------------
 
 Welcome to the Infection Monkey! 
 
 The Infection Monkey is an open source security tool for testing a data center's resiliency to perimeter breaches and internal server infection. The Monkey uses various methods to self propagate across a data center and reports success to a centralized Monkey Island server.
 
+
 <img src=".github/map-full.png" >
 
 <img src=".github/Security-overview.png" width="800" height="500">
@@ -50,6 +55,12 @@ If you only want to build the monkey from source, see [Setup](https://github.com
 and follow the instructions at the readme files under [infection_monkey](infection_monkey) and [monkey_island](monkey_island). 
 
 
+### Build status
+| Branch | Status |
+| ------ | :----: |
+| Develop | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) |
+| Master | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) | 
+
 License
 =======
 Copyright (c) Guardicore Ltd

monkey/common/network/segmentation_utils_test.py

@@ -11,20 +11,20 @@ class TestSegmentationUtils(IslandTestCase):
 
         # IP not in both
         self.assertIsNone(get_ip_in_src_and_not_in_dst(
-            [text_type("3.3.3.3"), text_type("4.4.4.4")], source, target
+            ["3.3.3.3", "4.4.4.4"], source, target
         ))
 
         # IP not in source, in target
         self.assertIsNone(get_ip_in_src_and_not_in_dst(
-            [text_type("2.2.2.2")], source, target
+            ["2.2.2.2"], source, target
         ))
 
         # IP in source, not in target
         self.assertIsNotNone(get_ip_in_src_and_not_in_dst(
-            [text_type("8.8.8.8"), text_type("1.1.1.1")], source, target
+            ["8.8.8.8", "1.1.1.1"], source, target
         ))
 
         # IP in both subnets
         self.assertIsNone(get_ip_in_src_and_not_in_dst(
-            [text_type("8.8.8.8"), text_type("1.1.1.1")], source, source
+            ["8.8.8.8", "1.1.1.1"], source, source
         ))

monkey/monkey_island/cc/models/monkey.py

@@ -123,7 +123,8 @@ class Monkey(Document):
         self.save()
 
 
-# Can't make following methods static under Monkey class due to ring bug
+# TODO Can't make following methods static under Monkey class due to ring bug. When ring will support static methods, we
+# should move to static methods in the Monkey class.
 @ring.lru(
     expire=1  # data has TTL of 1 second. This is useful for rapid calls for report generation.
 )

monkey/monkey_island/cc/models/test_monkey.py

@@ -1,11 +1,15 @@
 import uuid
+import logging
 from time import sleep
 
-from .monkey import Monkey
-from monkey_island.cc.models.monkey import MonkeyNotFoundError, is_monkey, get_monkey_label_by_id
+import pytest
+
+from monkey_island.cc.models.monkey import Monkey, MonkeyNotFoundError, is_monkey, get_monkey_label_by_id
 from monkey_island.cc.testing.IslandTestCase import IslandTestCase
 from .monkey_ttl import MonkeyTtl
 
+logger = logging.getLogger(__name__)
+
 
 class TestMonkey(IslandTestCase):
     """
@@ -32,7 +36,7 @@ class TestMonkey(IslandTestCase):
         # MIA stands for Missing In Action
         mia_monkey_ttl = MonkeyTtl.create_ttl_expire_in(30)
         mia_monkey_ttl.save()
-        mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl)
+        mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl.id)
         mia_monkey.save()
         # Emulate timeout - ttl is manually deleted here, since we're using mongomock and not a real mongo instance.
         sleep(1)
@@ -70,8 +74,10 @@ class TestMonkey(IslandTestCase):
         # Act + assert
         # Find the existing one
         self.assertIsNotNone(Monkey.get_single_monkey_by_id(a_monkey.id))
+
         # Raise on non-existent monkey
-        self.assertRaises(MonkeyNotFoundError, Monkey.get_single_monkey_by_id, "abcdefabcdefabcdefabcdef")
+        with pytest.raises(MonkeyNotFoundError) as e_info:
+            _ = Monkey.get_single_monkey_by_id("abcdefabcdefabcdefabcdef")
 
     def test_get_os(self):
         self.fail_if_not_testing_env()
@@ -125,29 +131,41 @@ class TestMonkey(IslandTestCase):
                               ip_addresses=[ip_example])
         linux_monkey.save()
 
+        logger.debug(id(get_monkey_label_by_id))
+
         cache_info_before_query = get_monkey_label_by_id.storage.backend.cache_info()
         self.assertEqual(cache_info_before_query.hits, 0)
+        self.assertEqual(cache_info_before_query.misses, 0)
 
         # not cached
         label = get_monkey_label_by_id(linux_monkey.id)
+        cache_info_after_query_1 = get_monkey_label_by_id.storage.backend.cache_info()
+        self.assertEqual(cache_info_after_query_1.hits, 0)
+        self.assertEqual(cache_info_after_query_1.misses, 1)
+        logger.info("1) ID: {} label: {}".format(linux_monkey.id, label))
 
         self.assertIsNotNone(label)
         self.assertIn(hostname_example, label)
         self.assertIn(ip_example, label)
 
         # should be cached
-        _ = get_monkey_label_by_id(linux_monkey.id)
-        cache_info_after_query = get_monkey_label_by_id.storage.backend.cache_info()
-        self.assertEqual(cache_info_after_query.hits, 1)
+        label = get_monkey_label_by_id(linux_monkey.id)
+        logger.info("2) ID: {} label: {}".format(linux_monkey.id, label))
+        cache_info_after_query_2 = get_monkey_label_by_id.storage.backend.cache_info()
+        self.assertEqual(cache_info_after_query_2.hits, 1)
+        self.assertEqual(cache_info_after_query_2.misses, 1)
 
+        # set hostname deletes the id from the cache.
         linux_monkey.set_hostname("Another hostname")
 
         # should be a miss
         label = get_monkey_label_by_id(linux_monkey.id)
-        cache_info_after_second_query = get_monkey_label_by_id.storage.backend.cache_info()
+        logger.info("3) ID: {} label: {}".format(linux_monkey.id, label))
+        cache_info_after_query_3 = get_monkey_label_by_id.storage.backend.cache_info()
+        logger.debug("Cache info: {}".format(str(cache_info_after_query_3)))
         # still 1 hit only
-        self.assertEqual(cache_info_after_second_query.hits, 1)
-        self.assertEqual(cache_info_after_second_query.misses, 2)
+        self.assertEqual(cache_info_after_query_3.hits, 1)
+        self.assertEqual(cache_info_after_query_3.misses, 2)
 
     def test_is_monkey(self):
         self.fail_if_not_testing_env()

monkey/monkey_island/cc/server_config.json

@@ -1,4 +1,4 @@
 {
-  "server_config": "standard",
-  "deployment": "develop"
+    "server_config": "standard",
+    "deployment": "develop"
 }

monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py

@@ -1,9 +1,151 @@
-from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
-
 from common.data.zero_trust_consts import *
 from monkey_island.cc.models.zero_trust.finding import Finding
+from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService
 from monkey_island.cc.testing.IslandTestCase import IslandTestCase
 
+EXPECTED_DICT = {
+    AUTOMATION_ORCHESTRATION: [],
+    DATA: [
+        {
+            "principle": PRINCIPLES[PRINCIPLE_DATA_TRANSIT],
+            "status": STATUS_FAILED,
+            "tests": [
+                {
+                    "status": STATUS_FAILED,
+                    "test": TESTS_MAP[TEST_DATA_ENDPOINT_HTTP][TEST_EXPLANATION_KEY]
+                },
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_DATA_ENDPOINT_ELASTIC][TEST_EXPLANATION_KEY]
+                },
+            ]
+        }
+    ],
+    DEVICES: [
+        {
+            "principle": PRINCIPLES[PRINCIPLE_ENDPOINT_SECURITY],
+            "status": STATUS_FAILED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_MACHINE_EXPLOITED][TEST_EXPLANATION_KEY]
+                },
+                {
+                    "status": STATUS_FAILED,
+                    "test": TESTS_MAP[TEST_ENDPOINT_SECURITY_EXISTS][TEST_EXPLANATION_KEY]
+                },
+            ]
+        }
+    ],
+    NETWORKS: [
+        {
+            "principle": PRINCIPLES[PRINCIPLE_SEGMENTATION],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_SEGMENTATION][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR],
+            "status": STATUS_VERIFY,
+            "tests": [
+                {
+                    "status": STATUS_VERIFY,
+                    "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+    ],
+    PEOPLE: [
+        {
+            "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR],
+            "status": STATUS_VERIFY,
+            "tests": [
+                {
+                    "status": STATUS_VERIFY,
+                    "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY]
+                }
+            ]
+        }
+    ],
+    VISIBILITY_ANALYTICS: [
+        {
+            "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+        {
+            "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES],
+            "status": STATUS_UNEXECUTED,
+            "tests": [
+                {
+                    "status": STATUS_UNEXECUTED,
+                    "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY]
+                }
+            ]
+        },
+    ],
+    WORKLOADS: []
+}
+
 
 def save_example_findings():
     # arrange
@@ -106,151 +248,24 @@ class TestZeroTrustService(IslandTestCase):
 
         save_example_findings()
 
-        expected = {
-            AUTOMATION_ORCHESTRATION: [],
-            DATA: [
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_DATA_TRANSIT],
-                    "status": STATUS_FAILED,
-                    "tests": [
-                        {
-                            "status": STATUS_FAILED,
-                            "test": TESTS_MAP[TEST_DATA_ENDPOINT_HTTP][TEST_EXPLANATION_KEY]
-                        },
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_DATA_ENDPOINT_ELASTIC][TEST_EXPLANATION_KEY]
-                        },
-                    ]
-                }
-            ],
-            DEVICES: [
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_ENDPOINT_SECURITY],
-                    "status": STATUS_FAILED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_MACHINE_EXPLOITED][TEST_EXPLANATION_KEY]
-                        },
-                        {
-                            "status": STATUS_FAILED,
-                            "test": TESTS_MAP[TEST_ENDPOINT_SECURITY_EXISTS][TEST_EXPLANATION_KEY]
-                        },
-                    ]
-                }
-            ],
-            NETWORKS: [
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_SEGMENTATION],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_SEGMENTATION][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR],
-                    "status": STATUS_VERIFY,
-                    "tests": [
-                        {
-                            "status": STATUS_VERIFY,
-                            "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-            ],
-            PEOPLE: [
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR],
-                    "status": STATUS_VERIFY,
-                    "tests": [
-                        {
-                            "status": STATUS_VERIFY,
-                            "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                }
-            ],
-            VISIBILITY_ANALYTICS: [
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-                {
-                    "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES],
-                    "status": STATUS_UNEXECUTED,
-                    "tests": [
-                        {
-                            "status": STATUS_UNEXECUTED,
-                            "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY]
-                        }
-                    ]
-                },
-            ],
-            WORKLOADS: []
-        }
+        expected = dict(EXPECTED_DICT)  # new mutable
 
         result = ZeroTrustService.get_principles_status()
-        self.assertEqual(result, expected)
+        # Compare expected and result, no order:
+        for pillar_name, pillar_principles_status_result in result.items():
+            for index, pillar_principle_status_expected in enumerate(expected.get(pillar_name)):
+                correct_one = None
+                for pillar_principle_status_result in pillar_principles_status_result:
+                    if pillar_principle_status_result["principle"] == pillar_principle_status_expected["principle"]:
+                        correct_one = pillar_principle_status_result
+                        break
+
+                # Compare tests no order
+                self.assertTrue(compare_lists_no_order(correct_one["tests"], pillar_principle_status_expected["tests"]))
+                # Compare the rest
+                del pillar_principle_status_expected["tests"]
+                del correct_one["tests"]
+                self.assertEqual(sorted(correct_one), sorted(pillar_principle_status_expected))
 
     def test_get_pillars_to_statuses(self):
         self.fail_if_not_testing_env()
@@ -283,3 +298,13 @@ class TestZeroTrustService(IslandTestCase):
         }
 
         self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected)
+
+
+def compare_lists_no_order(s, t):
+    t = list(t)   # make a mutable copy
+    try:
+        for elem in s:
+            t.remove(elem)
+    except ValueError:
+        return False
+    return not t

monkey/monkey_island/cc/set_server_config.py

@@ -0,0 +1,46 @@
+import argparse
+import json
+import logging
+from pathlib import Path
+
+SERVER_CONFIG = "server_config"
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.DEBUG)
+
+
+def main():
+    args = parse_args()
+    file_path = get_config_file_path(args)
+
+    # Read config
+    with open(file_path) as config_file:
+        config_data = json.load(config_file)
+
+    # Edit the config
+    config_data[SERVER_CONFIG] = args.server_config
+
+    # Write new config
+    logger.info("Writing the following config: {}".format(json.dumps(config_data, indent=4)))
+    with open(file_path, "w") as config_file:
+        json.dump(config_data, config_file, indent=4)
+        config_file.write("\n")  # Have to add newline at end of file, since json.dump does not.
+
+
+def get_config_file_path(args):
+    file_path = Path(__file__).parent.joinpath(args.file_name)
+    logger.info("Config file path: {}".format(file_path))
+    return file_path
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("server_config", choices=["standard", "testing", "password"])
+    parser.add_argument("-f", "--file_name", required=False, default="server_config.json")
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    main()

monkey/monkey_island/requirements.txt

@@ -1,3 +1,4 @@
+pytest
 bson
 python-dateutil
 tornado

monkey/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s
+log_cli_date_format=%H:%M:%S
+addopts = -v --capture=sys