Merge branch 'unit-tests' into develop

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+default_stages: [commit]
 repos:
   - repo: https://github.com/pycqa/isort
     rev: 5.8.0
@@ -33,3 +34,12 @@ repos:
     hooks:
       - id: eslint
         args: ["monkey/monkey_island/cc/ui/src/", "--fix", "--max-warnings=0"]
+  - repo: local
+    hooks:
+    -   id: pytest
+        name: pytest
+        entry: bash -c "cd monkey && pytest"
+        language: system
+        files: "monkey/"
+        exclude: "monkey/monkey_island/cc/ui"
+        stages: [push]

deployment_scripts/README.md

@@ -81,4 +81,4 @@ been run or all issues have not been resolved.
 
 To install and configure pre-commit manually, run `pip install --user
 pre-commit`. Next, go to the top level directory of this repository and run
-`pre-commit install` Now, pre-commit will automatically run whenever you `git commit`.
+`pre-commit install -t pre-commit -t pre-push` Now, pre-commit will automatically run whenever you `git commit`.

deployment_scripts/deploy_linux.sh

@@ -26,7 +26,7 @@ log_message() {
 configure_precommit() {
     $1 -m pip install --user pre-commit
     pushd "$2"
-    $HOME/.local/bin/pre-commit install
+    $HOME/.local/bin/pre-commit install -t pre-commit -t pre-push
     popd
 }
 

deployment_scripts/deploy_windows.ps1

@@ -18,7 +18,7 @@ function Configure-precommit([String] $git_repo_dir)
 	if ($LastExitCode) {
 		exit
 	}
-    pre-commit install
+    pre-commit install -t pre-commit -t pre-push
 	if ($LastExitCode) {
 		exit
 	}

docs/content/development/setup-development-environment.md

@@ -30,4 +30,4 @@ Pre-commit is a multi-language package manager for pre-commit hooks. It will run
 
 Our CI system runs the same checks when pull requests are submitted. This system may report that the build has failed if the pre-commit hooks have not been run or all issues have not been resolved.
 
-To install and configure pre-commit, run `pip install --user pre-commit`. Next, go to the top level directory of this repository and run `pre-commit install`. Pre-commit will now run automatically whenever you `git commit`.
+To install and configure pre-commit, run `pip install --user pre-commit`. Next, go to the top level directory of this repository and run `pre-commit install -t pre-commit -t pre-push`. Pre-commit will now run automatically whenever you `git commit`.

monkey/monkey_island/cc/services/reporting/test_report.py

@@ -1,62 +0,0 @@
-import datetime
-from copy import deepcopy
-
-from monkey_island.cc.services.reporting.report import ReportService
-
-NODE_DICT = {
-    "id": "602f62118e30cf35830ff8e4",
-    "label": "WinDev2010Eval.mshome.net",
-    "group": "monkey_windows",
-    "os": "windows",
-    "dead": True,
-    "exploits": [
-        {
-            "result": True,
-            "exploiter": "DrupalExploiter",
-            "info": {
-                "display_name": "Drupal Server",
-                "started": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000),
-                "finished": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000),
-                "vulnerable_urls": [],
-                "vulnerable_ports": [],
-                "executed_cmds": [],
-            },
-            "attempts": [],
-            "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000),
-            "origin": "MonkeyIsland : 192.168.56.1",
-        },
-        {
-            "result": True,
-            "exploiter": "ElasticGroovyExploiter",
-            "info": {
-                "display_name": "Elastic search",
-                "started": datetime.datetime(2021, 2, 19, 9, 0, 15, 16000),
-                "finished": datetime.datetime(2021, 2, 19, 9, 0, 15, 17000),
-                "vulnerable_urls": [],
-                "vulnerable_ports": [],
-                "executed_cmds": [],
-            },
-            "attempts": [],
-            "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 15, 60000),
-            "origin": "MonkeyIsland : 192.168.56.1",
-        },
-    ],
-}
-
-NODE_DICT_DUPLICATE_EXPLOITS = deepcopy(NODE_DICT)
-NODE_DICT_DUPLICATE_EXPLOITS["exploits"][1] = NODE_DICT_DUPLICATE_EXPLOITS["exploits"][0]
-
-NODE_DICT_FAILED_EXPLOITS = deepcopy(NODE_DICT)
-NODE_DICT_FAILED_EXPLOITS["exploits"][0]["result"] = False
-NODE_DICT_FAILED_EXPLOITS["exploits"][1]["result"] = False
-
-
-def test_get_exploits_used_on_node():
-    exploits = ReportService.get_exploits_used_on_node(NODE_DICT)
-    assert sorted(exploits) == sorted(["Elastic Groovy Exploiter", "Drupal Server Exploiter"])
-
-    exploits = ReportService.get_exploits_used_on_node(NODE_DICT_DUPLICATE_EXPLOITS)
-    assert exploits == ["Drupal Server Exploiter"]
-
-    exploits = ReportService.get_exploits_used_on_node(NODE_DICT_FAILED_EXPLOITS)
-    assert exploits == []

monkey/pytest.ini

@@ -1,7 +0,0 @@
-[pytest]
-log_cli = 1
-log_cli_level = DEBUG
-log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s
-log_cli_date_format=%H:%M:%S
-addopts = -v --capture=sys --ignore=common/cloud/scoutsuite
-norecursedirs = node_modules dist

monkey/tests/common/cloud/aws/test_aws_service.py

@@ -1,7 +1,7 @@
 import json
 from unittest import TestCase
 
-from .aws_service import filter_instance_data_from_aws_response
+from common.cloud.aws.aws_service import filter_instance_data_from_aws_response
 
 __author__ = "shay.nehmad"
 

monkey/tests/conftest.py

@@ -0,0 +1,48 @@
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+MONKEY_BASE_PATH = str(Path(__file__).parent.parent)
+sys.path.insert(0, MONKEY_BASE_PATH)
+
+
+@pytest.fixture(scope="session")
+def resources_dir(pytestconfig):
+    return os.path.join(pytestconfig.rootdir, "monkey", "tests", "resources")
+
+
+@pytest.fixture(scope="session")
+def environment_resources_dir(resources_dir):
+    return os.path.join(resources_dir, "environment")
+
+
+@pytest.fixture(scope="session")
+def with_credentials(environment_resources_dir):
+    return os.path.join(environment_resources_dir, "server_config_with_credentials.json")
+
+
+@pytest.fixture(scope="session")
+def no_credentials(environment_resources_dir):
+    return os.path.join(environment_resources_dir, "server_config_no_credentials.json")
+
+
+@pytest.fixture(scope="session")
+def partial_credentials(environment_resources_dir):
+    return os.path.join(environment_resources_dir, "server_config_partial_credentials.json")
+
+
+@pytest.fixture(scope="session")
+def standard_with_credentials(environment_resources_dir):
+    return os.path.join(environment_resources_dir, "server_config_standard_with_credentials.json")
+
+
+@pytest.fixture(scope="session")
+def with_data_dir(environment_resources_dir):
+    return os.path.join(environment_resources_dir, "server_config_with_data_dir.json")
+
+
+@pytest.fixture(scope="session")
+def with_data_dir_home(environment_resources_dir):
+    return os.path.join(environment_resources_dir, "server_config_with_data_dir_home.json")

monkey/tests/infection_monkey/exploit/tools/test_payload.py

@@ -1,6 +1,6 @@
 from unittest import TestCase
 
-from .payload_parsing import LimitedSizePayload, Payload
+from infection_monkey.exploit.tools.payload_parsing import LimitedSizePayload, Payload
 
 
 class TestPayload(TestCase):

monkey/tests/monkey_island/cc/environment/test_environment.py

@@ -4,6 +4,8 @@ from typing import Dict
 from unittest import TestCase
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from common.utils.exceptions import (
     AlreadyRegisteredError,
     CredentialsNotRequiredError,
@@ -11,17 +13,35 @@ from common.utils.exceptions import (
     RegistrationNotNeededError,
 )
 from monkey_island.cc.environment import Environment, EnvironmentConfig, UserCreds
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
 
-TEST_RESOURCES_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment")
+WITH_CREDENTIALS = None
+NO_CREDENTIALS = None
+PARTIAL_CREDENTIALS = None
+STANDARD_WITH_CREDENTIALS = None
+STANDARD_ENV = None
 
-WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_with_credentials.json")
+
-NO_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_no_credentials.json")
+# This fixture is a dirty hack that can be removed once these tests are converted from
-PARTIAL_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_partial_credentials.json")
+# unittest to pytest. Instead, the appropriate fixtures from conftest.py can be used.
-STANDARD_WITH_CREDENTIALS = os.path.join(
+@pytest.fixture(scope="module", autouse=True)
-    TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json"
+def configure_resources(environment_resources_dir):
-)
+    global WITH_CREDENTIALS
-STANDARD_ENV = os.path.join(TEST_RESOURCES_DIR, "server_config_standard_env.json")
+    global NO_CREDENTIALS
+    global PARTIAL_CREDENTIALS
+    global STANDARD_WITH_CREDENTIALS
+    global STANDARD_ENV
+
+    WITH_CREDENTIALS = os.path.join(
+        environment_resources_dir, "server_config_with_credentials.json"
+    )
+    NO_CREDENTIALS = os.path.join(environment_resources_dir, "server_config_no_credentials.json")
+    PARTIAL_CREDENTIALS = os.path.join(
+        environment_resources_dir, "server_config_partial_credentials.json"
+    )
+    STANDARD_WITH_CREDENTIALS = os.path.join(
+        environment_resources_dir, "server_config_standard_with_credentials.json"
+    )
+    STANDARD_ENV = os.path.join(environment_resources_dir, "server_config_standard_env.json")
 
 
 def get_tmp_file():

monkey/tests/monkey_island/cc/environment/test_environment_config.py

@@ -6,18 +6,7 @@ import pytest
 
 from monkey_island.cc.environment.environment_config import EnvironmentConfig
 from monkey_island.cc.environment.user_creds import UserCreds
-from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, MONKEY_ISLAND_ABS_PATH
+from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR
-
-TEST_RESOURCES_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing", "environment")
-
-WITH_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_with_credentials.json")
-NO_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_no_credentials.json")
-PARTIAL_CREDENTIALS = os.path.join(TEST_RESOURCES_DIR, "server_config_partial_credentials.json")
-STANDARD_WITH_CREDENTIALS = os.path.join(
-    TEST_RESOURCES_DIR, "server_config_standard_with_credentials.json"
-)
-WITH_DATA_DIR = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir.json")
-WITH_DATA_DIR_HOME = os.path.join(TEST_RESOURCES_DIR, "server_config_with_data_dir_home.json")
 
 
 @pytest.fixture
@@ -25,8 +14,8 @@ def config_file(tmpdir):
     return os.path.join(tmpdir, "test_config.json")
 
 
-def test_get_with_credentials():
+def test_get_with_credentials(with_credentials):
-    config_dict = EnvironmentConfig(WITH_CREDENTIALS).to_dict()
+    config_dict = EnvironmentConfig(with_credentials).to_dict()
 
     assert len(config_dict.keys()) == 5
     assert config_dict["server_config"] == "password"
@@ -36,8 +25,8 @@ def test_get_with_credentials():
     assert config_dict["data_dir"] == DEFAULT_DATA_DIR
 
 
-def test_get_with_no_credentials():
+def test_get_with_no_credentials(no_credentials):
-    config_dict = EnvironmentConfig(NO_CREDENTIALS).to_dict()
+    config_dict = EnvironmentConfig(no_credentials).to_dict()
 
     assert len(config_dict.keys()) == 3
     assert config_dict["server_config"] == "password"
@@ -45,8 +34,8 @@ def test_get_with_no_credentials():
     assert config_dict["data_dir"] == DEFAULT_DATA_DIR
 
 
-def test_get_with_partial_credentials():
+def test_get_with_partial_credentials(partial_credentials):
-    config_dict = EnvironmentConfig(PARTIAL_CREDENTIALS).to_dict()
+    config_dict = EnvironmentConfig(partial_credentials).to_dict()
 
     assert len(config_dict.keys()) == 4
     assert config_dict["server_config"] == "password"
@@ -55,8 +44,8 @@ def test_get_with_partial_credentials():
     assert config_dict["data_dir"] == DEFAULT_DATA_DIR
 
 
-def test_save_to_file(config_file):
+def test_save_to_file(config_file, standard_with_credentials):
-    shutil.copyfile(STANDARD_WITH_CREDENTIALS, config_file)
+    shutil.copyfile(standard_with_credentials, config_file)
 
     environment_config = EnvironmentConfig(config_file)
     environment_config.aws = "test_aws"
@@ -74,12 +63,12 @@ def test_save_to_file(config_file):
     assert from_file["data_dir"] == DEFAULT_DATA_DIR
 
 
-def test_add_user(config_file):
+def test_add_user(config_file, standard_with_credentials):
     new_user = "new_user"
     new_password_hash = "fedcba"
     new_user_creds = UserCreds(new_user, new_password_hash)
 
-    shutil.copyfile(STANDARD_WITH_CREDENTIALS, config_file)
+    shutil.copyfile(standard_with_credentials, config_file)
 
     environment_config = EnvironmentConfig(config_file)
     environment_config.add_user(new_user_creds)
@@ -92,8 +81,8 @@ def test_add_user(config_file):
     assert from_file["password_hash"] == new_password_hash
 
 
-def test_get_users():
+def test_get_users(standard_with_credentials):
-    environment_config = EnvironmentConfig(STANDARD_WITH_CREDENTIALS)
+    environment_config = EnvironmentConfig(standard_with_credentials)
     users = environment_config.get_users()
 
     assert len(users) == 1
@@ -115,8 +104,8 @@ def test_generate_default_file(config_file):
     assert environment_config.data_dir == DEFAULT_DATA_DIR
 
 
-def test_data_dir():
+def test_data_dir(with_data_dir):
-    environment_config = EnvironmentConfig(WITH_DATA_DIR)
+    environment_config = EnvironmentConfig(with_data_dir)
     assert environment_config.data_dir == "/test/data/dir"
 
 
@@ -124,8 +113,8 @@ def set_home_env(monkeypatch, tmpdir):
     monkeypatch.setenv("HOME", str(tmpdir))
 
 
-def test_data_dir_abs_path_from_file(monkeypatch, tmpdir):
+def test_data_dir_abs_path_from_file(monkeypatch, tmpdir, with_data_dir_home):
     set_home_env(monkeypatch, tmpdir)
 
-    config = EnvironmentConfig(WITH_DATA_DIR_HOME)
+    config = EnvironmentConfig(with_data_dir_home)
     assert config.data_dir_abs_path == os.path.join(tmpdir, "data_dir")

monkey/tests/monkey_island/cc/models/test_monkey.py

@@ -5,9 +5,8 @@ from time import sleep
 import pytest
 
 from monkey_island.cc.models.monkey import Monkey, MonkeyNotFoundError
-
+from monkey_island.cc.models.monkey_ttl import MonkeyTtl
-from ..test_common.fixtures import FixtureEnum
+from monkey_island.cc.test_common.fixtures import FixtureEnum
-from .monkey_ttl import MonkeyTtl
 
 logger = logging.getLogger(__name__)
 

monkey/tests/monkey_island/cc/server_utils/test_island_logger.py

@@ -3,12 +3,12 @@ import os
 
 import pytest
 
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
 from monkey_island.cc.server_utils.island_logger import json_setup_logging
 
-TEST_LOGGER_CONFIG_PATH = os.path.join(
+
-    MONKEY_ISLAND_ABS_PATH, "cc", "testing", "logger_config.json"
+@pytest.fixture()
-)
+def test_logger_config_path(resources_dir):
+    return os.path.join(resources_dir, "logger_config.json")
 
 
 # TODO move into monkey/monkey_island/cc/test_common/fixtures after rebase/backmerge
@@ -17,11 +17,11 @@ def mock_home_env(monkeypatch, tmpdir):
     monkeypatch.setenv("HOME", str(tmpdir))
 
 
-def test_expanduser_filename(mock_home_env, tmpdir):
+def test_expanduser_filename(mock_home_env, tmpdir, test_logger_config_path):
     INFO_LOG = os.path.join(tmpdir, "info.log")
     TEST_STRING = "Hello, Monkey!"
 
-    json_setup_logging(TEST_LOGGER_CONFIG_PATH)
+    json_setup_logging(test_logger_config_path)
 
     logger = logging.getLogger("TestLogger")
     logger.info(TEST_STRING)

monkey/tests/monkey_island/cc/services/reporting/test_report.py

@@ -1,3 +1,6 @@
+import datetime
+from copy import deepcopy
+
 import mongomock
 import pytest
 from bson import ObjectId
@@ -73,6 +76,53 @@ NO_CREDS_TELEMETRY_TELEM = {
 
 MONKEY_TELEM = {"_id": TELEM_ID["monkey"], "guid": MONKEY_GUID, "hostname": HOSTNAME}
 
+NODE_DICT = {
+    "id": "602f62118e30cf35830ff8e4",
+    "label": "WinDev2010Eval.mshome.net",
+    "group": "monkey_windows",
+    "os": "windows",
+    "dead": True,
+    "exploits": [
+        {
+            "result": True,
+            "exploiter": "DrupalExploiter",
+            "info": {
+                "display_name": "Drupal Server",
+                "started": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000),
+                "finished": datetime.datetime(2021, 2, 19, 9, 0, 14, 950000),
+                "vulnerable_urls": [],
+                "vulnerable_ports": [],
+                "executed_cmds": [],
+            },
+            "attempts": [],
+            "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 14, 984000),
+            "origin": "MonkeyIsland : 192.168.56.1",
+        },
+        {
+            "result": True,
+            "exploiter": "ElasticGroovyExploiter",
+            "info": {
+                "display_name": "Elastic search",
+                "started": datetime.datetime(2021, 2, 19, 9, 0, 15, 16000),
+                "finished": datetime.datetime(2021, 2, 19, 9, 0, 15, 17000),
+                "vulnerable_urls": [],
+                "vulnerable_ports": [],
+                "executed_cmds": [],
+            },
+            "attempts": [],
+            "timestamp": datetime.datetime(2021, 2, 19, 9, 0, 15, 60000),
+            "origin": "MonkeyIsland : 192.168.56.1",
+        },
+    ],
+}
+
+NODE_DICT_DUPLICATE_EXPLOITS = deepcopy(NODE_DICT)
+NODE_DICT_DUPLICATE_EXPLOITS["exploits"][1] = NODE_DICT_DUPLICATE_EXPLOITS["exploits"][0]
+
+NODE_DICT_FAILED_EXPLOITS = deepcopy(NODE_DICT)
+NODE_DICT_FAILED_EXPLOITS["exploits"][0]["result"] = False
+NODE_DICT_FAILED_EXPLOITS["exploits"][1]["result"] = False
+
 
 @pytest.fixture
 def fake_mongo(monkeypatch):
@@ -115,3 +165,14 @@ def test_get_stolen_creds_no_creds(fake_mongo):
     expected_stolen_creds_no_creds = []
 
     assert expected_stolen_creds_no_creds == stolen_creds_no_creds
+
+
+def test_get_exploits_used_on_node():
+    exploits = ReportService.get_exploits_used_on_node(NODE_DICT)
+    assert sorted(exploits) == sorted(["Elastic Groovy Exploiter", "Drupal Server Exploiter"])
+
+    exploits = ReportService.get_exploits_used_on_node(NODE_DICT_DUPLICATE_EXPLOITS)
+    assert exploits == ["Drupal Server Exploiter"]
+
+    exploits = ReportService.get_exploits_used_on_node(NODE_DICT_FAILED_EXPLOITS)
+    assert exploits == []

monkey/tests/monkey_island/cc/services/telemetry/processing/test_post_breach.py

@@ -1,8 +1,7 @@
 from unittest.mock import Mock
 
 import monkey_island.cc.services.telemetry.processing.post_breach as post_breach
-
+from monkey_island.cc.services.telemetry.processing.post_breach import EXECUTION_WITHOUT_OUTPUT
-from .post_breach import EXECUTION_WITHOUT_OUTPUT
 
 original_telem_multiple_results = {
     "data": {

monkey/tests/monkey_island/cc/test_encryptor.py

@@ -1,29 +1,27 @@
 import os
 
-from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
 from monkey_island.cc.server_utils.encryptor import get_encryptor, initialize_encryptor
 
-TEST_DATA_DIR = os.path.join(MONKEY_ISLAND_ABS_PATH, "cc", "testing")
 PASSWORD_FILENAME = "mongo_key.bin"
 
 PLAINTEXT = "Hello, Monkey!"
 CYPHERTEXT = "vKgvD6SjRyIh1dh2AM/rnTa0NI/vjfwnbZLbMocWtE4e42WJmSUz2ordtbQrH1Fq"
 
 
-def test_aes_cbc_encryption():
+def test_aes_cbc_encryption(resources_dir):
-    initialize_encryptor(TEST_DATA_DIR)
+    initialize_encryptor(resources_dir)
 
     assert get_encryptor().enc(PLAINTEXT) != PLAINTEXT
 
 
-def test_aes_cbc_decryption():
+def test_aes_cbc_decryption(resources_dir):
-    initialize_encryptor(TEST_DATA_DIR)
+    initialize_encryptor(resources_dir)
 
     assert get_encryptor().dec(CYPHERTEXT) == PLAINTEXT
 
 
-def test_aes_cbc_enc_dec():
+def test_aes_cbc_enc_dec(resources_dir):
-    initialize_encryptor(TEST_DATA_DIR)
+    initialize_encryptor(resources_dir)
 
     assert get_encryptor().dec(get_encryptor().enc(PLAINTEXT)) == PLAINTEXT
 

pyproject.toml

@@ -12,3 +12,12 @@ include_trailing_comma = true
 force_grid_wrap = 0
 use_parentheses = true
 ensure_newline_before_comments = true
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+log_cli = 1
+log_cli_level = "DEBUG"
+log_cli_format = "%(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s"
+log_cli_date_format = "%H:%M:%S"
+addopts = "-v --capture=sys tests"
+norecursedirs = "node_modules dist"