From 05fbdafd9dbc68bc6e2f2778c0919761db294e43 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 12:13:26 +0200 Subject: [PATCH 01/17] Updated travis file, let's test it --- .travis.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b14482939..9e942af58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,17 +2,20 @@ group: travis_latest language: python cache: pip python: - - 2.7 +- 3.7 install: - #- pip install -r requirements.txt - - pip install flake8 # pytest # add another testing frameworks later +- pip install -r monkey/monkey_island/requirements.txt +- pip install flake8 pytest dlint before_script: - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics +- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics script: - - true # pytest --capture=sys # add other tests here +- pytest --capture=sys notifications: + on_success: change + on_failure: always + slack: + rooms: + secure: wFddByqGGfwpjuisZhXQGAI7Y/7yZ+ON58R19D2ff+p1lT26BIdE+gSWQFWTbSNcyjVhAMCkbKgRblL2o0WcuiCyYjlcIoT/fEMuypxNTnlC1FtHlmEO/1JAyDVskzRWvu7q98szcDP+yKjUZSjUrLMnzUYtdB7hhAp8iRvEUTgtKQi27kmgBX9lLqAf93WO1Ocp2+fmDkNmegx2bBa9PQS4FYOtqN7DYZ0cHM7wLffyHZmSXZkwCcq+u7mSONrfQZprCvzQu9kntx/zEnjTuNHUZ8L0SNu035Yg2MtxnKDY1GPSox5ax5rZJID4aN+f7vDkIgIskMwVcjFoMS4bvp8xH1liWleR/VdOYhTP1bx5GcLvkUFC/xAwW730AZnU9Ihn3iSLK8iiowf7UMOYLB371KDlZWj2gwRvccRb7VwismLzcYt/So/5/xDa4IgDNNUu/fb9J+TPc+3jKPt3E6KSHseBcyk2ov7iC14pUz/AWEcWga2s5cRiWqBibUuiJUXZf0zExyEnj9dic5jF/UW8EVpDygy5TaAVq9VD30zslzPcIDbl1slEh2Nd6LXQWCiz9UMDN85OHkuXSgNyuSpna6W0+/qNv3SQLrcwTKTNj/ZSLTj5lZIyQ7l11xW36H+uyx4D8rYOpHw8HxrGXnh5hyWhOa4GCGzgSgFZlU8= on_success: change - on_failure: change # `always` will be the setting once code changes slow down + on_failure: always From c11d78c6a078ce179e5042c6dd81eeffde1e00f0 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 12:24:46 +0200 Subject: [PATCH 02/17] cd-ing to source root --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9e942af58..a5c2da172 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ install: - pip install -r monkey/monkey_island/requirements.txt - pip install flake8 pytest dlint before_script: +- cd monkey - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics script: From cc674cac5731a8009d116846bb3d29747807777d Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 12:29:52 +0200 Subject: [PATCH 03/17] Trying to fix slack notifications from travis --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5c2da172..dccf11dff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ notifications: on_success: change on_failure: always slack: - rooms: - secure: wFddByqGGfwpjuisZhXQGAI7Y/7yZ+ON58R19D2ff+p1lT26BIdE+gSWQFWTbSNcyjVhAMCkbKgRblL2o0WcuiCyYjlcIoT/fEMuypxNTnlC1FtHlmEO/1JAyDVskzRWvu7q98szcDP+yKjUZSjUrLMnzUYtdB7hhAp8iRvEUTgtKQi27kmgBX9lLqAf93WO1Ocp2+fmDkNmegx2bBa9PQS4FYOtqN7DYZ0cHM7wLffyHZmSXZkwCcq+u7mSONrfQZprCvzQu9kntx/zEnjTuNHUZ8L0SNu035Yg2MtxnKDY1GPSox5ax5rZJID4aN+f7vDkIgIskMwVcjFoMS4bvp8xH1liWleR/VdOYhTP1bx5GcLvkUFC/xAwW730AZnU9Ihn3iSLK8iiowf7UMOYLB371KDlZWj2gwRvccRb7VwismLzcYt/So/5/xDa4IgDNNUu/fb9J+TPc+3jKPt3E6KSHseBcyk2ov7iC14pUz/AWEcWga2s5cRiWqBibUuiJUXZf0zExyEnj9dic5jF/UW8EVpDygy5TaAVq9VD30zslzPcIDbl1slEh2Nd6LXQWCiz9UMDN85OHkuXSgNyuSpna6W0+/qNv3SQLrcwTKTNj/ZSLTj5lZIyQ7l11xW36H+uyx4D8rYOpHw8HxrGXnh5hyWhOa4GCGzgSgFZlU8= - on_success: change - on_failure: always + rooms: + - secure: wFddByqGGfwpjuisZhXQGAI7Y/7yZ+ON58R19D2ff+p1lT26BIdE+gSWQFWTbSNcyjVhAMCkbKgRblL2o0WcuiCyYjlcIoT/fEMuypxNTnlC1FtHlmEO/1JAyDVskzRWvu7q98szcDP+yKjUZSjUrLMnzUYtdB7hhAp8iRvEUTgtKQi27kmgBX9lLqAf93WO1Ocp2+fmDkNmegx2bBa9PQS4FYOtqN7DYZ0cHM7wLffyHZmSXZkwCcq+u7mSONrfQZprCvzQu9kntx/zEnjTuNHUZ8L0SNu035Yg2MtxnKDY1GPSox5ax5rZJID4aN+f7vDkIgIskMwVcjFoMS4bvp8xH1liWleR/VdOYhTP1bx5GcLvkUFC/xAwW730AZnU9Ihn3iSLK8iiowf7UMOYLB371KDlZWj2gwRvccRb7VwismLzcYt/So/5/xDa4IgDNNUu/fb9J+TPc+3jKPt3E6KSHseBcyk2ov7iC14pUz/AWEcWga2s5cRiWqBibUuiJUXZf0zExyEnj9dic5jF/UW8EVpDygy5TaAVq9VD30zslzPcIDbl1slEh2Nd6LXQWCiz9UMDN85OHkuXSgNyuSpna6W0+/qNv3SQLrcwTKTNj/ZSLTj5lZIyQ7l11xW36H+uyx4D8rYOpHw8HxrGXnh5hyWhOa4GCGzgSgFZlU8= + on_success: change + on_failure: always From ee1e913291657dc2c032269c28081bb233e75a36 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 13:48:09 +0200 Subject: [PATCH 04/17] using python -m for tests - and trying still to make slack integ work --- .travis.yml | 9 +++++---- monkey/monkey_island/cc/server_config.json | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index dccf11dff..3ab2b95a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,17 +6,18 @@ python: install: - pip install -r monkey/monkey_island/requirements.txt - pip install flake8 pytest dlint +- pip install -r monkey/infection_monkey/requirements-linux.txt before_script: -- cd monkey - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics script: -- pytest --capture=sys +- cd monkey # This is our source dir +- python -m pytest --capture=sys # Have to do this to add monkey to sys.path. notifications: on_success: change on_failure: always slack: rooms: - - secure: wFddByqGGfwpjuisZhXQGAI7Y/7yZ+ON58R19D2ff+p1lT26BIdE+gSWQFWTbSNcyjVhAMCkbKgRblL2o0WcuiCyYjlcIoT/fEMuypxNTnlC1FtHlmEO/1JAyDVskzRWvu7q98szcDP+yKjUZSjUrLMnzUYtdB7hhAp8iRvEUTgtKQi27kmgBX9lLqAf93WO1Ocp2+fmDkNmegx2bBa9PQS4FYOtqN7DYZ0cHM7wLffyHZmSXZkwCcq+u7mSONrfQZprCvzQu9kntx/zEnjTuNHUZ8L0SNu035Yg2MtxnKDY1GPSox5ax5rZJID4aN+f7vDkIgIskMwVcjFoMS4bvp8xH1liWleR/VdOYhTP1bx5GcLvkUFC/xAwW730AZnU9Ihn3iSLK8iiowf7UMOYLB371KDlZWj2gwRvccRb7VwismLzcYt/So/5/xDa4IgDNNUu/fb9J+TPc+3jKPt3E6KSHseBcyk2ov7iC14pUz/AWEcWga2s5cRiWqBibUuiJUXZf0zExyEnj9dic5jF/UW8EVpDygy5TaAVq9VD30zslzPcIDbl1slEh2Nd6LXQWCiz9UMDN85OHkuXSgNyuSpna6W0+/qNv3SQLrcwTKTNj/ZSLTj5lZIyQ7l11xW36H+uyx4D8rYOpHw8HxrGXnh5hyWhOa4GCGzgSgFZlU8= - on_success: change + - secure: wFddByqGGfwpjuisZhXQGAI7Y/7yZ+ON58R19D2ff+p1lT26BIdE+gSWQFWTbSNcyjVhAMCkbKgRblL2o0WcuiCyYjlcIoT/fEMuypxNTnlC1FtHlmEO/1JAyDVskzRWvu7q98szcDP+yKjUZSjUrLMnzUYtdB7hhAp8iRvEUTgtKQi27kmgBX9lLqAf93WO1Ocp2+fmDkNmegx2bBa9PQS4FYOtqN7DYZ0cHM7wLffyHZmSXZkwCcq+u7mSONrfQZprCvzQu9kntx/zEnjTuNHUZ8L0SNu035Yg2MtxnKDY1GPSox5ax5rZJID4aN+f7vDkIgIskMwVcjFoMS4bvp8xH1liWleR/VdOYhTP1bx5GcLvkUFC/xAwW730AZnU9Ihn3iSLK8iiowf7UMOYLB371KDlZWj2gwRvccRb7VwismLzcYt/So/5/xDa4IgDNNUu/fb9J+TPc+3jKPt3E6KSHseBcyk2ov7iC14pUz/AWEcWga2s5cRiWqBibUuiJUXZf0zExyEnj9dic5jF/UW8EVpDygy5TaAVq9VD30zslzPcIDbl1slEh2Nd6LXQWCiz9UMDN85OHkuXSgNyuSpna6W0+/qNv3SQLrcwTKTNj/ZSLTj5lZIyQ7l11xW36H+uyx4D8rYOpHw8HxrGXnh5hyWhOa4GCGzgSgFZlU8= + on_success: always on_failure: always diff --git a/monkey/monkey_island/cc/server_config.json b/monkey/monkey_island/cc/server_config.json index 420f1b303..7bf106194 100644 --- a/monkey/monkey_island/cc/server_config.json +++ b/monkey/monkey_island/cc/server_config.json @@ -1,4 +1,4 @@ { - "server_config": "standard", + "server_config": "testing", "deployment": "develop" } From 43048329d030c73c6ecfb41f507f0f853861e157 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 13:50:13 +0200 Subject: [PATCH 05/17] Fix requirements path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3ab2b95a4..501814e6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: install: - pip install -r monkey/monkey_island/requirements.txt - pip install flake8 pytest dlint -- pip install -r monkey/infection_monkey/requirements-linux.txt +- pip install -r monkey/infection_monkey/requirements_linux.txt before_script: - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics From ceca7ce1277407f90ee1fe39e3c15bd2d44bb5a5 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 17:55:24 +0200 Subject: [PATCH 06/17] Adding pytest to requirements and pytest.ini file for logging purposes --- monkey/monkey_island/requirements.txt | 1 + monkey/pytest.ini | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 monkey/pytest.ini diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index 49c1e37a5..77ff9a620 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -1,3 +1,4 @@ +pytest bson python-dateutil tornado diff --git a/monkey/pytest.ini b/monkey/pytest.ini new file mode 100644 index 000000000..3d355a4ac --- /dev/null +++ b/monkey/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +log_cli = 1 +log_cli_level = DEBUG +log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s +log_cli_date_format=%H:%M:%S +addopts = -v --capture=sys From 7b153d29b25d4437c097bf9657f3ed19686c6f89 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 17:55:34 +0200 Subject: [PATCH 07/17] Fix segmentation utils test --- monkey/common/network/segmentation_utils_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/common/network/segmentation_utils_test.py b/monkey/common/network/segmentation_utils_test.py index 56a560922..221f1d9bf 100644 --- a/monkey/common/network/segmentation_utils_test.py +++ b/monkey/common/network/segmentation_utils_test.py @@ -11,20 +11,20 @@ class TestSegmentationUtils(IslandTestCase): # IP not in both self.assertIsNone(get_ip_in_src_and_not_in_dst( - [text_type("3.3.3.3"), text_type("4.4.4.4")], source, target + ["3.3.3.3", "4.4.4.4"], source, target )) # IP not in source, in target self.assertIsNone(get_ip_in_src_and_not_in_dst( - [text_type("2.2.2.2")], source, target + ["2.2.2.2"], source, target )) # IP in source, not in target self.assertIsNotNone(get_ip_in_src_and_not_in_dst( - [text_type("8.8.8.8"), text_type("1.1.1.1")], source, target + ["8.8.8.8", "1.1.1.1"], source, target )) # IP in both subnets self.assertIsNone(get_ip_in_src_and_not_in_dst( - [text_type("8.8.8.8"), text_type("1.1.1.1")], source, source + ["8.8.8.8", "1.1.1.1"], source, source )) From f2297de6610b50029bd539d913e5aabad812649d Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 17:55:51 +0200 Subject: [PATCH 08/17] Fix TestMonkey to pytest --- monkey/monkey_island/cc/models/monkey.py | 3 +- monkey/monkey_island/cc/models/test_monkey.py | 38 ++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index daeb9ea5b..1a0e872f6 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -123,7 +123,8 @@ class Monkey(Document): self.save() -# Can't make following methods static under Monkey class due to ring bug +# TODO Can't make following methods static under Monkey class due to ring bug. When ring will support static methods, we +# should move to static methods in the Monkey class. @ring.lru( expire=1 # data has TTL of 1 second. This is useful for rapid calls for report generation. ) diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index fb9b329b1..472c5770b 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -1,11 +1,15 @@ import uuid +import logging from time import sleep -from .monkey import Monkey -from monkey_island.cc.models.monkey import MonkeyNotFoundError, is_monkey, get_monkey_label_by_id +import pytest + +from monkey_island.cc.models.monkey import Monkey, MonkeyNotFoundError, is_monkey, get_monkey_label_by_id from monkey_island.cc.testing.IslandTestCase import IslandTestCase from .monkey_ttl import MonkeyTtl +logger = logging.getLogger(__name__) + class TestMonkey(IslandTestCase): """ @@ -32,7 +36,7 @@ class TestMonkey(IslandTestCase): # MIA stands for Missing In Action mia_monkey_ttl = MonkeyTtl.create_ttl_expire_in(30) mia_monkey_ttl.save() - mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl) + mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl.id) mia_monkey.save() # Emulate timeout - ttl is manually deleted here, since we're using mongomock and not a real mongo instance. sleep(1) @@ -70,8 +74,10 @@ class TestMonkey(IslandTestCase): # Act + assert # Find the existing one self.assertIsNotNone(Monkey.get_single_monkey_by_id(a_monkey.id)) + # Raise on non-existent monkey - self.assertRaises(MonkeyNotFoundError, Monkey.get_single_monkey_by_id, "abcdefabcdefabcdefabcdef") + with pytest.raises(MonkeyNotFoundError) as e_info: + _ = Monkey.get_single_monkey_by_id("abcdefabcdefabcdefabcdef") def test_get_os(self): self.fail_if_not_testing_env() @@ -125,29 +131,41 @@ class TestMonkey(IslandTestCase): ip_addresses=[ip_example]) linux_monkey.save() + logger.debug(id(get_monkey_label_by_id)) + cache_info_before_query = get_monkey_label_by_id.storage.backend.cache_info() self.assertEqual(cache_info_before_query.hits, 0) + self.assertEqual(cache_info_before_query.misses, 0) # not cached label = get_monkey_label_by_id(linux_monkey.id) + cache_info_after_query_1 = get_monkey_label_by_id.storage.backend.cache_info() + self.assertEqual(cache_info_after_query_1.hits, 0) + self.assertEqual(cache_info_after_query_1.misses, 1) + logger.info("1) ID: {} label: {}".format(linux_monkey.id, label)) self.assertIsNotNone(label) self.assertIn(hostname_example, label) self.assertIn(ip_example, label) # should be cached - _ = get_monkey_label_by_id(linux_monkey.id) - cache_info_after_query = get_monkey_label_by_id.storage.backend.cache_info() - self.assertEqual(cache_info_after_query.hits, 1) + label = get_monkey_label_by_id(linux_monkey.id) + logger.info("2) ID: {} label: {}".format(linux_monkey.id, label)) + cache_info_after_query_2 = get_monkey_label_by_id.storage.backend.cache_info() + self.assertEqual(cache_info_after_query_2.hits, 1) + self.assertEqual(cache_info_after_query_2.misses, 1) + # set hostname deletes the id from the cache. linux_monkey.set_hostname("Another hostname") # should be a miss label = get_monkey_label_by_id(linux_monkey.id) - cache_info_after_second_query = get_monkey_label_by_id.storage.backend.cache_info() + logger.info("3) ID: {} label: {}".format(linux_monkey.id, label)) + cache_info_after_query_3 = get_monkey_label_by_id.storage.backend.cache_info() + logger.debug("Cache info: {}".format(str(cache_info_after_query_3))) # still 1 hit only - self.assertEqual(cache_info_after_second_query.hits, 1) - self.assertEqual(cache_info_after_second_query.misses, 2) + self.assertEqual(cache_info_after_query_3.hits, 1) + self.assertEqual(cache_info_after_query_3.misses, 2) def test_is_monkey(self): self.fail_if_not_testing_env() From ab348bb12a5330ffc1d967e5717be38a62b1ce3b Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 17:56:12 +0200 Subject: [PATCH 09/17] Fix zero_trust_service tests (comparison order problems) --- .../reporting/test_zero_trust_service.py | 315 ++++++++++-------- 1 file changed, 170 insertions(+), 145 deletions(-) diff --git a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py index 06a730e05..d77e67aad 100644 --- a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py @@ -1,9 +1,151 @@ -from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService - from common.data.zero_trust_consts import * from monkey_island.cc.models.zero_trust.finding import Finding +from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService from monkey_island.cc.testing.IslandTestCase import IslandTestCase +EXPECTED_DICT = { + AUTOMATION_ORCHESTRATION: [], + DATA: [ + { + "principle": PRINCIPLES[PRINCIPLE_DATA_TRANSIT], + "status": STATUS_FAILED, + "tests": [ + { + "status": STATUS_FAILED, + "test": TESTS_MAP[TEST_DATA_ENDPOINT_HTTP][TEST_EXPLANATION_KEY] + }, + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_DATA_ENDPOINT_ELASTIC][TEST_EXPLANATION_KEY] + }, + ] + } + ], + DEVICES: [ + { + "principle": PRINCIPLES[PRINCIPLE_ENDPOINT_SECURITY], + "status": STATUS_FAILED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_MACHINE_EXPLOITED][TEST_EXPLANATION_KEY] + }, + { + "status": STATUS_FAILED, + "test": TESTS_MAP[TEST_ENDPOINT_SECURITY_EXISTS][TEST_EXPLANATION_KEY] + }, + ] + } + ], + NETWORKS: [ + { + "principle": PRINCIPLES[PRINCIPLE_SEGMENTATION], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_SEGMENTATION][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], + "status": STATUS_VERIFY, + "tests": [ + { + "status": STATUS_VERIFY, + "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] + } + ] + }, + ], + PEOPLE: [ + { + "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], + "status": STATUS_VERIFY, + "tests": [ + { + "status": STATUS_VERIFY, + "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] + } + ] + } + ], + VISIBILITY_ANALYTICS: [ + { + "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] + } + ] + }, + ], + WORKLOADS: [] +} + def save_example_findings(): # arrange @@ -106,151 +248,24 @@ class TestZeroTrustService(IslandTestCase): save_example_findings() - expected = { - AUTOMATION_ORCHESTRATION: [], - DATA: [ - { - "principle": PRINCIPLES[PRINCIPLE_DATA_TRANSIT], - "status": STATUS_FAILED, - "tests": [ - { - "status": STATUS_FAILED, - "test": TESTS_MAP[TEST_DATA_ENDPOINT_HTTP][TEST_EXPLANATION_KEY] - }, - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_DATA_ENDPOINT_ELASTIC][TEST_EXPLANATION_KEY] - }, - ] - } - ], - DEVICES: [ - { - "principle": PRINCIPLES[PRINCIPLE_ENDPOINT_SECURITY], - "status": STATUS_FAILED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_MACHINE_EXPLOITED][TEST_EXPLANATION_KEY] - }, - { - "status": STATUS_FAILED, - "test": TESTS_MAP[TEST_ENDPOINT_SECURITY_EXISTS][TEST_EXPLANATION_KEY] - }, - ] - } - ], - NETWORKS: [ - { - "principle": PRINCIPLES[PRINCIPLE_SEGMENTATION], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_SEGMENTATION][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], - "status": STATUS_VERIFY, - "tests": [ - { - "status": STATUS_VERIFY, - "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] - } - ] - }, - ], - PEOPLE: [ - { - "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], - "status": STATUS_VERIFY, - "tests": [ - { - "status": STATUS_VERIFY, - "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] - } - ] - } - ], - VISIBILITY_ANALYTICS: [ - { - "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] - } - ] - }, - ], - WORKLOADS: [] - } + expected = dict(EXPECTED_DICT) # new mutable result = ZeroTrustService.get_principles_status() - self.assertEqual(result, expected) + # Compare expected and result, no order: + for pillar_name, pillar_principles_status_result in result.items(): + for index, pillar_principle_status_expected in enumerate(expected.get(pillar_name)): + correct_one = None + for pillar_principle_status_result in pillar_principles_status_result: + if pillar_principle_status_result["principle"] == pillar_principle_status_expected["principle"]: + correct_one = pillar_principle_status_result + break + + # Compare tests no order + self.assertTrue(compare_lists_no_order(correct_one["tests"], pillar_principle_status_expected["tests"])) + # Compare the rest + del pillar_principle_status_expected["tests"] + del correct_one["tests"] + self.assertEqual(sorted(correct_one), sorted(pillar_principle_status_expected)) def test_get_pillars_to_statuses(self): self.fail_if_not_testing_env() @@ -283,3 +298,13 @@ class TestZeroTrustService(IslandTestCase): } self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected) + + +def compare_lists_no_order(s, t): + t = list(t) # make a mutable copy + try: + for elem in s: + t.remove(elem) + except ValueError: + return False + return not t From 7c23065efafd10219e2dd9374f216b307d2e23e7 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 18:31:44 +0200 Subject: [PATCH 10/17] Trying to get slack notifications to work. --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 501814e6f..b05dfbe94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,14 @@ before_script: - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics script: - cd monkey # This is our source dir -- python -m pytest --capture=sys # Have to do this to add monkey to sys.path. +- python -m pytest # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path. notifications: - on_success: change - on_failure: always slack: rooms: - - secure: wFddByqGGfwpjuisZhXQGAI7Y/7yZ+ON58R19D2ff+p1lT26BIdE+gSWQFWTbSNcyjVhAMCkbKgRblL2o0WcuiCyYjlcIoT/fEMuypxNTnlC1FtHlmEO/1JAyDVskzRWvu7q98szcDP+yKjUZSjUrLMnzUYtdB7hhAp8iRvEUTgtKQi27kmgBX9lLqAf93WO1Ocp2+fmDkNmegx2bBa9PQS4FYOtqN7DYZ0cHM7wLffyHZmSXZkwCcq+u7mSONrfQZprCvzQu9kntx/zEnjTuNHUZ8L0SNu035Yg2MtxnKDY1GPSox5ax5rZJID4aN+f7vDkIgIskMwVcjFoMS4bvp8xH1liWleR/VdOYhTP1bx5GcLvkUFC/xAwW730AZnU9Ihn3iSLK8iiowf7UMOYLB371KDlZWj2gwRvccRb7VwismLzcYt/So/5/xDa4IgDNNUu/fb9J+TPc+3jKPt3E6KSHseBcyk2ov7iC14pUz/AWEcWga2s5cRiWqBibUuiJUXZf0zExyEnj9dic5jF/UW8EVpDygy5TaAVq9VD30zslzPcIDbl1slEh2Nd6LXQWCiz9UMDN85OHkuXSgNyuSpna6W0+/qNv3SQLrcwTKTNj/ZSLTj5lZIyQ7l11xW36H+uyx4D8rYOpHw8HxrGXnh5hyWhOa4GCGzgSgFZlU8= + - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#ci + - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#github on_success: always on_failure: always + email: + on_success: change + on_failure: always From 681c0396c4a24fa2a16798b5de4abf2c797a5f26 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 18:35:38 +0200 Subject: [PATCH 11/17] Added travis badges to readme --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 67b5b2e8b..b10ebbf8b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,11 @@ Welcome to the Infection Monkey! The Infection Monkey is an open source security tool for testing a data center's resiliency to perimeter breaches and internal server infection. The Monkey uses various methods to self propagate across a data center and reports success to a centralized Monkey Island server. +#### Build status +* Development branch [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) +* Master [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) + + From 984a280b66dc3a57956dc701c27ea42add6e832e Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 18:39:54 +0200 Subject: [PATCH 12/17] Changed bullets to table --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b10ebbf8b..070d6243d 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,10 @@ Welcome to the Infection Monkey! The Infection Monkey is an open source security tool for testing a data center's resiliency to perimeter breaches and internal server infection. The Monkey uses various methods to self propagate across a data center and reports success to a centralized Monkey Island server. #### Build status -* Development branch [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) -* Master [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) +| Branch | Status | +| ------ | :----: | +| Develop | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) | +| Master | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) | From bb536755bf0034ee53f7faf2ebd239cdffae1e37 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Sun, 27 Oct 2019 18:44:38 +0200 Subject: [PATCH 13/17] reordered readme with badges --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 070d6243d..462383969 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,14 @@ Infection Monkey ==================== +[![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) -### Data center Security Testing Tool +## Data center Security Testing Tool ------------------------ Welcome to the Infection Monkey! The Infection Monkey is an open source security tool for testing a data center's resiliency to perimeter breaches and internal server infection. The Monkey uses various methods to self propagate across a data center and reports success to a centralized Monkey Island server. -#### Build status -| Branch | Status | -| ------ | :----: | -| Develop | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) | -| Master | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) | - @@ -57,6 +52,12 @@ If you only want to build the monkey from source, see [Setup](https://github.com and follow the instructions at the readme files under [infection_monkey](infection_monkey) and [monkey_island](monkey_island). +### Build status +| Branch | Status | +| ------ | :----: | +| Develop | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) | +| Master | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) | + License ======= Copyright (c) Guardicore Ltd From 59a779822bde75c58eaf46ebb35717b29dfff8d9 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 28 Oct 2019 09:31:00 +0200 Subject: [PATCH 14/17] Added some badges (since we added the build badge and I don't want it to look alone) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 462383969..2d7490bfe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ Infection Monkey ==================== [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) +[![GitHub release (latest by date)](https://img.shields.io/github/v/release/guardicore/monkey)](https://github.com/guardicore/monkey/releases) +![GitHub stars](https://img.shields.io/github/stars/guardicore/monkey) +![GitHub commit activity](https://img.shields.io/github/commit-activity/m/guardicore/monkey) ## Data center Security Testing Tool ------------------------ From 827c4942d910bca8ea5f8406440457f77ff44ce7 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 28 Oct 2019 13:37:18 +0200 Subject: [PATCH 15/17] Added script which changes the server_config to testing in travis so the default will be standard (for running) --- .travis.yml | 3 +- monkey/monkey_island/cc/server_config.json | 4 +- monkey/monkey_island/cc/set_server_config.py | 46 ++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 monkey/monkey_island/cc/set_server_config.py diff --git a/.travis.yml b/.travis.yml index b05dfbe94..06511e74b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,12 @@ python: - 3.7 install: - pip install -r monkey/monkey_island/requirements.txt -- pip install flake8 pytest dlint +- pip install flake8 pytest dlint pylint - pip install -r monkey/infection_monkey/requirements_linux.txt before_script: - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +- monkey/monkey_island/cc/set_server_config.py testing script: - cd monkey # This is our source dir - python -m pytest # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path. diff --git a/monkey/monkey_island/cc/server_config.json b/monkey/monkey_island/cc/server_config.json index 7bf106194..0b28d0b74 100644 --- a/monkey/monkey_island/cc/server_config.json +++ b/monkey/monkey_island/cc/server_config.json @@ -1,4 +1,4 @@ { - "server_config": "testing", - "deployment": "develop" + "server_config": "standard", + "deployment": "develop" } diff --git a/monkey/monkey_island/cc/set_server_config.py b/monkey/monkey_island/cc/set_server_config.py new file mode 100644 index 000000000..fc20747c9 --- /dev/null +++ b/monkey/monkey_island/cc/set_server_config.py @@ -0,0 +1,46 @@ +import argparse +import json +import logging +from pathlib import Path + +SERVER_CONFIG = "server_config" + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.DEBUG) + + +def main(): + args = parse_args() + file_path = get_config_file_path(args) + + # Read config + with open(file_path) as config_file: + config_data = json.load(config_file) + + # Edit the config + config_data[SERVER_CONFIG] = args.server_config + + # Write new config + logger.info("Writing the following config: {}".format(json.dumps(config_data, indent=4))) + with open(file_path, "w") as config_file: + json.dump(config_data, config_file, indent=4) + config_file.write("\n") # Have to add newline at end of file, since json.dump does not. + + +def get_config_file_path(args): + file_path = Path(__file__).parent.joinpath(args.file_name) + logger.info("Config file path: {}".format(file_path)) + return file_path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("server_config", choices=["standard", "testing", "password"]) + parser.add_argument("-f", "--file_name", required=False, default="server_config.json") + args = parser.parse_args() + return args + + +if __name__ == '__main__': + main() From f8cf78a2928d1d7d078418053564f3c2eb932205 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 28 Oct 2019 13:40:22 +0200 Subject: [PATCH 16/17] Forgot python as command for running the script --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 06511e74b..608c8924f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: before_script: - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics -- monkey/monkey_island/cc/set_server_config.py testing +- python monkey/monkey_island/cc/set_server_config.py testing script: - cd monkey # This is our source dir - python -m pytest # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path. From 97baaabdd7086d324aaaca22eeaa0426d70ef551 Mon Sep 17 00:00:00 2001 From: Shay Nehmad Date: Mon, 28 Oct 2019 13:54:07 +0200 Subject: [PATCH 17/17] Remove pylint from travis installation + added some doc + fixed notifications for travis --- .travis.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 608c8924f..d5103b989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,28 @@ +# Infection Monkey travis.yml. See Travis documentation for information about this file structure. + group: travis_latest language: python cache: pip python: - 3.7 install: -- pip install -r monkey/monkey_island/requirements.txt -- pip install flake8 pytest dlint pylint -- pip install -r monkey/infection_monkey/requirements_linux.txt +- pip install -r monkey/monkey_island/requirements.txt # for unit tests +- pip install flake8 pytest dlint # for next stages +- pip install -r monkey/infection_monkey/requirements_linux.txt # for unit tests before_script: -- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics -- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics -- python monkey/monkey_island/cc/set_server_config.py testing +- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics # Check syntax errors +- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics # warn about linter issues. --exit-zero + # means this stage will not fail the build. This is (hopefully) a temporary measure until all warnings are suppressed. +- python monkey/monkey_island/cc/set_server_config.py testing # Set the server config to `testing`, for the UTs to use + # mongomaock and pass. script: - cd monkey # This is our source dir - python -m pytest # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path. notifications: - slack: + slack: # Notify to slack rooms: - - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#ci - - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#github - on_success: always + - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#ci # room: #ci + on_success: change on_failure: always email: on_success: change