diff --git a/.travis.yml b/.travis.yml index b14482939..d5103b989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,29 @@ +# Infection Monkey travis.yml. See Travis documentation for information about this file structure. + group: travis_latest language: python cache: pip python: - - 2.7 +- 3.7 install: - #- pip install -r requirements.txt - - pip install flake8 # pytest # add another testing frameworks later +- pip install -r monkey/monkey_island/requirements.txt # for unit tests +- pip install flake8 pytest dlint # for next stages +- pip install -r monkey/infection_monkey/requirements_linux.txt # for unit tests before_script: - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics # Check syntax errors +- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics # warn about linter issues. --exit-zero + # means this stage will not fail the build. This is (hopefully) a temporary measure until all warnings are suppressed. +- python monkey/monkey_island/cc/set_server_config.py testing # Set the server config to `testing`, for the UTs to use + # mongomaock and pass. script: - - true # pytest --capture=sys # add other tests here +- cd monkey # This is our source dir +- python -m pytest # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path. notifications: - on_success: change - on_failure: change # `always` will be the setting once code changes slow down + slack: # Notify to slack + rooms: + - infectionmonkey:QaXbsx4g7tHFJW0lhtiBmoAg#ci # room: #ci + on_success: change + on_failure: always + email: + on_success: change + on_failure: always diff --git a/README.md b/README.md index 67b5b2e8b..2d7490bfe 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,18 @@ Infection Monkey ==================== +[![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) +[![GitHub release (latest by date)](https://img.shields.io/github/v/release/guardicore/monkey)](https://github.com/guardicore/monkey/releases) +![GitHub stars](https://img.shields.io/github/stars/guardicore/monkey) +![GitHub commit activity](https://img.shields.io/github/commit-activity/m/guardicore/monkey) -### Data center Security Testing Tool +## Data center Security Testing Tool ------------------------ Welcome to the Infection Monkey! The Infection Monkey is an open source security tool for testing a data center's resiliency to perimeter breaches and internal server infection. The Monkey uses various methods to self propagate across a data center and reports success to a centralized Monkey Island server. + @@ -50,6 +55,12 @@ If you only want to build the monkey from source, see [Setup](https://github.com and follow the instructions at the readme files under [infection_monkey](infection_monkey) and [monkey_island](monkey_island). +### Build status +| Branch | Status | +| ------ | :----: | +| Develop | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=develop)](https://travis-ci.com/guardicore/monkey) | +| Master | [![Build Status](https://travis-ci.com/guardicore/monkey.svg?branch=master)](https://travis-ci.com/guardicore/monkey) | + License ======= Copyright (c) Guardicore Ltd diff --git a/monkey/common/network/segmentation_utils_test.py b/monkey/common/network/segmentation_utils_test.py index 56a560922..221f1d9bf 100644 --- a/monkey/common/network/segmentation_utils_test.py +++ b/monkey/common/network/segmentation_utils_test.py @@ -11,20 +11,20 @@ class TestSegmentationUtils(IslandTestCase): # IP not in both self.assertIsNone(get_ip_in_src_and_not_in_dst( - [text_type("3.3.3.3"), text_type("4.4.4.4")], source, target + ["3.3.3.3", "4.4.4.4"], source, target )) # IP not in source, in target self.assertIsNone(get_ip_in_src_and_not_in_dst( - [text_type("2.2.2.2")], source, target + ["2.2.2.2"], source, target )) # IP in source, not in target self.assertIsNotNone(get_ip_in_src_and_not_in_dst( - [text_type("8.8.8.8"), text_type("1.1.1.1")], source, target + ["8.8.8.8", "1.1.1.1"], source, target )) # IP in both subnets self.assertIsNone(get_ip_in_src_and_not_in_dst( - [text_type("8.8.8.8"), text_type("1.1.1.1")], source, source + ["8.8.8.8", "1.1.1.1"], source, source )) diff --git a/monkey/monkey_island/cc/models/monkey.py b/monkey/monkey_island/cc/models/monkey.py index daeb9ea5b..1a0e872f6 100644 --- a/monkey/monkey_island/cc/models/monkey.py +++ b/monkey/monkey_island/cc/models/monkey.py @@ -123,7 +123,8 @@ class Monkey(Document): self.save() -# Can't make following methods static under Monkey class due to ring bug +# TODO Can't make following methods static under Monkey class due to ring bug. When ring will support static methods, we +# should move to static methods in the Monkey class. @ring.lru( expire=1 # data has TTL of 1 second. This is useful for rapid calls for report generation. ) diff --git a/monkey/monkey_island/cc/models/test_monkey.py b/monkey/monkey_island/cc/models/test_monkey.py index fb9b329b1..472c5770b 100644 --- a/monkey/monkey_island/cc/models/test_monkey.py +++ b/monkey/monkey_island/cc/models/test_monkey.py @@ -1,11 +1,15 @@ import uuid +import logging from time import sleep -from .monkey import Monkey -from monkey_island.cc.models.monkey import MonkeyNotFoundError, is_monkey, get_monkey_label_by_id +import pytest + +from monkey_island.cc.models.monkey import Monkey, MonkeyNotFoundError, is_monkey, get_monkey_label_by_id from monkey_island.cc.testing.IslandTestCase import IslandTestCase from .monkey_ttl import MonkeyTtl +logger = logging.getLogger(__name__) + class TestMonkey(IslandTestCase): """ @@ -32,7 +36,7 @@ class TestMonkey(IslandTestCase): # MIA stands for Missing In Action mia_monkey_ttl = MonkeyTtl.create_ttl_expire_in(30) mia_monkey_ttl.save() - mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl) + mia_monkey = Monkey(guid=str(uuid.uuid4()), dead=False, ttl_ref=mia_monkey_ttl.id) mia_monkey.save() # Emulate timeout - ttl is manually deleted here, since we're using mongomock and not a real mongo instance. sleep(1) @@ -70,8 +74,10 @@ class TestMonkey(IslandTestCase): # Act + assert # Find the existing one self.assertIsNotNone(Monkey.get_single_monkey_by_id(a_monkey.id)) + # Raise on non-existent monkey - self.assertRaises(MonkeyNotFoundError, Monkey.get_single_monkey_by_id, "abcdefabcdefabcdefabcdef") + with pytest.raises(MonkeyNotFoundError) as e_info: + _ = Monkey.get_single_monkey_by_id("abcdefabcdefabcdefabcdef") def test_get_os(self): self.fail_if_not_testing_env() @@ -125,29 +131,41 @@ class TestMonkey(IslandTestCase): ip_addresses=[ip_example]) linux_monkey.save() + logger.debug(id(get_monkey_label_by_id)) + cache_info_before_query = get_monkey_label_by_id.storage.backend.cache_info() self.assertEqual(cache_info_before_query.hits, 0) + self.assertEqual(cache_info_before_query.misses, 0) # not cached label = get_monkey_label_by_id(linux_monkey.id) + cache_info_after_query_1 = get_monkey_label_by_id.storage.backend.cache_info() + self.assertEqual(cache_info_after_query_1.hits, 0) + self.assertEqual(cache_info_after_query_1.misses, 1) + logger.info("1) ID: {} label: {}".format(linux_monkey.id, label)) self.assertIsNotNone(label) self.assertIn(hostname_example, label) self.assertIn(ip_example, label) # should be cached - _ = get_monkey_label_by_id(linux_monkey.id) - cache_info_after_query = get_monkey_label_by_id.storage.backend.cache_info() - self.assertEqual(cache_info_after_query.hits, 1) + label = get_monkey_label_by_id(linux_monkey.id) + logger.info("2) ID: {} label: {}".format(linux_monkey.id, label)) + cache_info_after_query_2 = get_monkey_label_by_id.storage.backend.cache_info() + self.assertEqual(cache_info_after_query_2.hits, 1) + self.assertEqual(cache_info_after_query_2.misses, 1) + # set hostname deletes the id from the cache. linux_monkey.set_hostname("Another hostname") # should be a miss label = get_monkey_label_by_id(linux_monkey.id) - cache_info_after_second_query = get_monkey_label_by_id.storage.backend.cache_info() + logger.info("3) ID: {} label: {}".format(linux_monkey.id, label)) + cache_info_after_query_3 = get_monkey_label_by_id.storage.backend.cache_info() + logger.debug("Cache info: {}".format(str(cache_info_after_query_3))) # still 1 hit only - self.assertEqual(cache_info_after_second_query.hits, 1) - self.assertEqual(cache_info_after_second_query.misses, 2) + self.assertEqual(cache_info_after_query_3.hits, 1) + self.assertEqual(cache_info_after_query_3.misses, 2) def test_is_monkey(self): self.fail_if_not_testing_env() diff --git a/monkey/monkey_island/cc/server_config.json b/monkey/monkey_island/cc/server_config.json index 420f1b303..0b28d0b74 100644 --- a/monkey/monkey_island/cc/server_config.json +++ b/monkey/monkey_island/cc/server_config.json @@ -1,4 +1,4 @@ { - "server_config": "standard", - "deployment": "develop" + "server_config": "standard", + "deployment": "develop" } diff --git a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py index 06a730e05..d77e67aad 100644 --- a/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py +++ b/monkey/monkey_island/cc/services/reporting/test_zero_trust_service.py @@ -1,9 +1,151 @@ -from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService - from common.data.zero_trust_consts import * from monkey_island.cc.models.zero_trust.finding import Finding +from monkey_island.cc.services.reporting.zero_trust_service import ZeroTrustService from monkey_island.cc.testing.IslandTestCase import IslandTestCase +EXPECTED_DICT = { + AUTOMATION_ORCHESTRATION: [], + DATA: [ + { + "principle": PRINCIPLES[PRINCIPLE_DATA_TRANSIT], + "status": STATUS_FAILED, + "tests": [ + { + "status": STATUS_FAILED, + "test": TESTS_MAP[TEST_DATA_ENDPOINT_HTTP][TEST_EXPLANATION_KEY] + }, + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_DATA_ENDPOINT_ELASTIC][TEST_EXPLANATION_KEY] + }, + ] + } + ], + DEVICES: [ + { + "principle": PRINCIPLES[PRINCIPLE_ENDPOINT_SECURITY], + "status": STATUS_FAILED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_MACHINE_EXPLOITED][TEST_EXPLANATION_KEY] + }, + { + "status": STATUS_FAILED, + "test": TESTS_MAP[TEST_ENDPOINT_SECURITY_EXISTS][TEST_EXPLANATION_KEY] + }, + ] + } + ], + NETWORKS: [ + { + "principle": PRINCIPLES[PRINCIPLE_SEGMENTATION], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_SEGMENTATION][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], + "status": STATUS_VERIFY, + "tests": [ + { + "status": STATUS_VERIFY, + "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] + } + ] + }, + ], + PEOPLE: [ + { + "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], + "status": STATUS_VERIFY, + "tests": [ + { + "status": STATUS_VERIFY, + "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] + } + ] + } + ], + VISIBILITY_ANALYTICS: [ + { + "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] + } + ] + }, + { + "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], + "status": STATUS_UNEXECUTED, + "tests": [ + { + "status": STATUS_UNEXECUTED, + "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] + } + ] + }, + ], + WORKLOADS: [] +} + def save_example_findings(): # arrange @@ -106,151 +248,24 @@ class TestZeroTrustService(IslandTestCase): save_example_findings() - expected = { - AUTOMATION_ORCHESTRATION: [], - DATA: [ - { - "principle": PRINCIPLES[PRINCIPLE_DATA_TRANSIT], - "status": STATUS_FAILED, - "tests": [ - { - "status": STATUS_FAILED, - "test": TESTS_MAP[TEST_DATA_ENDPOINT_HTTP][TEST_EXPLANATION_KEY] - }, - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_DATA_ENDPOINT_ELASTIC][TEST_EXPLANATION_KEY] - }, - ] - } - ], - DEVICES: [ - { - "principle": PRINCIPLES[PRINCIPLE_ENDPOINT_SECURITY], - "status": STATUS_FAILED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_MACHINE_EXPLOITED][TEST_EXPLANATION_KEY] - }, - { - "status": STATUS_FAILED, - "test": TESTS_MAP[TEST_ENDPOINT_SECURITY_EXISTS][TEST_EXPLANATION_KEY] - }, - ] - } - ], - NETWORKS: [ - { - "principle": PRINCIPLES[PRINCIPLE_SEGMENTATION], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_SEGMENTATION][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], - "status": STATUS_VERIFY, - "tests": [ - { - "status": STATUS_VERIFY, - "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] - } - ] - }, - ], - PEOPLE: [ - { - "principle": PRINCIPLES[PRINCIPLE_USER_BEHAVIOUR], - "status": STATUS_VERIFY, - "tests": [ - { - "status": STATUS_VERIFY, - "test": TESTS_MAP[TEST_SCHEDULED_EXECUTION][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] - } - ] - } - ], - VISIBILITY_ANALYTICS: [ - { - "principle": PRINCIPLES[PRINCIPLE_USERS_MAC_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_COMMUNICATE_AS_NEW_USER][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_ANALYZE_NETWORK_TRAFFIC], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_MALICIOUS_ACTIVITY_TIMELINE][TEST_EXPLANATION_KEY] - } - ] - }, - { - "principle": PRINCIPLES[PRINCIPLE_RESTRICTIVE_NETWORK_POLICIES], - "status": STATUS_UNEXECUTED, - "tests": [ - { - "status": STATUS_UNEXECUTED, - "test": TESTS_MAP[TEST_TUNNELING][TEST_EXPLANATION_KEY] - } - ] - }, - ], - WORKLOADS: [] - } + expected = dict(EXPECTED_DICT) # new mutable result = ZeroTrustService.get_principles_status() - self.assertEqual(result, expected) + # Compare expected and result, no order: + for pillar_name, pillar_principles_status_result in result.items(): + for index, pillar_principle_status_expected in enumerate(expected.get(pillar_name)): + correct_one = None + for pillar_principle_status_result in pillar_principles_status_result: + if pillar_principle_status_result["principle"] == pillar_principle_status_expected["principle"]: + correct_one = pillar_principle_status_result + break + + # Compare tests no order + self.assertTrue(compare_lists_no_order(correct_one["tests"], pillar_principle_status_expected["tests"])) + # Compare the rest + del pillar_principle_status_expected["tests"] + del correct_one["tests"] + self.assertEqual(sorted(correct_one), sorted(pillar_principle_status_expected)) def test_get_pillars_to_statuses(self): self.fail_if_not_testing_env() @@ -283,3 +298,13 @@ class TestZeroTrustService(IslandTestCase): } self.assertEqual(ZeroTrustService.get_pillars_to_statuses(), expected) + + +def compare_lists_no_order(s, t): + t = list(t) # make a mutable copy + try: + for elem in s: + t.remove(elem) + except ValueError: + return False + return not t diff --git a/monkey/monkey_island/cc/set_server_config.py b/monkey/monkey_island/cc/set_server_config.py new file mode 100644 index 000000000..fc20747c9 --- /dev/null +++ b/monkey/monkey_island/cc/set_server_config.py @@ -0,0 +1,46 @@ +import argparse +import json +import logging +from pathlib import Path + +SERVER_CONFIG = "server_config" + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.DEBUG) + + +def main(): + args = parse_args() + file_path = get_config_file_path(args) + + # Read config + with open(file_path) as config_file: + config_data = json.load(config_file) + + # Edit the config + config_data[SERVER_CONFIG] = args.server_config + + # Write new config + logger.info("Writing the following config: {}".format(json.dumps(config_data, indent=4))) + with open(file_path, "w") as config_file: + json.dump(config_data, config_file, indent=4) + config_file.write("\n") # Have to add newline at end of file, since json.dump does not. + + +def get_config_file_path(args): + file_path = Path(__file__).parent.joinpath(args.file_name) + logger.info("Config file path: {}".format(file_path)) + return file_path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("server_config", choices=["standard", "testing", "password"]) + parser.add_argument("-f", "--file_name", required=False, default="server_config.json") + args = parser.parse_args() + return args + + +if __name__ == '__main__': + main() diff --git a/monkey/monkey_island/requirements.txt b/monkey/monkey_island/requirements.txt index 49c1e37a5..77ff9a620 100644 --- a/monkey/monkey_island/requirements.txt +++ b/monkey/monkey_island/requirements.txt @@ -1,3 +1,4 @@ +pytest bson python-dateutil tornado diff --git a/monkey/pytest.ini b/monkey/pytest.ini new file mode 100644 index 000000000..3d355a4ac --- /dev/null +++ b/monkey/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +log_cli = 1 +log_cli_level = DEBUG +log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s +log_cli_date_format=%H:%M:%S +addopts = -v --capture=sys