From 70b9a9f6b73a0314ad34ef333b804410246a18dd Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 11:05:15 +0300 Subject: [PATCH 1/5] Refactored config_loader.py and server_config_handler.py into a single file, with server config write/read operations --- .../cc/environment/server_config_generator.py | 21 ---------- .../cc/environment/server_config_handler.py | 39 +++++++++++++++++++ .../monkey_island/cc/server_utils/consts.py | 2 + monkey/monkey_island/config_loader.py | 25 ------------ .../monkey_island/cc/environment/__init__.py | 0 .../test_server_config_handler.py} | 8 ++-- 6 files changed, 45 insertions(+), 50 deletions(-) delete mode 100644 monkey/monkey_island/cc/environment/server_config_generator.py create mode 100644 monkey/monkey_island/cc/environment/server_config_handler.py delete mode 100644 monkey/monkey_island/config_loader.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py rename monkey/tests/unit_tests/monkey_island/{test_config_loader.py => cc/environment/test_server_config_handler.py} (64%) diff --git a/monkey/monkey_island/cc/environment/server_config_generator.py b/monkey/monkey_island/cc/environment/server_config_generator.py deleted file mode 100644 index 17ee4a50c..000000000 --- a/monkey/monkey_island/cc/environment/server_config_generator.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from pathlib import Path - -from monkey_island.cc.environment.data_dir_generator import create_data_dir -from monkey_island.cc.server_utils.consts import ( - DEFAULT_DATA_DIR, - DEFAULT_DEVELOP_SERVER_CONFIG_PATH, - DEFAULT_SERVER_CONFIG_PATH, -) - - -def create_default_server_config_file() -> str: - if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): - create_data_dir(DEFAULT_DATA_DIR, False) - write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) - return DEFAULT_SERVER_CONFIG_PATH - - -def write_default_server_config_to_file(path: str) -> None: - default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() - Path(path).write_text(default_config) diff --git a/monkey/monkey_island/cc/environment/server_config_handler.py b/monkey/monkey_island/cc/environment/server_config_handler.py new file mode 100644 index 000000000..35cc91f3e --- /dev/null +++ b/monkey/monkey_island/cc/environment/server_config_handler.py @@ -0,0 +1,39 @@ +import json +import os +from pathlib import Path + +from monkey_island.cc.server_utils.consts import ( + DEFAULT_DATA_DIR, + DEFAULT_DEVELOP_SERVER_CONFIG_PATH, + DEFAULT_LOG_LEVEL, + DEFAULT_SERVER_CONFIG_PATH, +) + + +def create_default_server_config_file() -> None: + if not os.path.isfile(DEFAULT_SERVER_CONFIG_PATH): + write_default_server_config_to_file(DEFAULT_SERVER_CONFIG_PATH) + + +def write_default_server_config_to_file(path: str) -> None: + default_config = Path(DEFAULT_DEVELOP_SERVER_CONFIG_PATH).read_text() + Path(path).write_text(default_config) + + +def load_server_config_from_file(server_config_path): + with open(server_config_path, "r") as f: + config_content = f.read() + config = json.loads(config_content) + add_default_values_to_config(config) + + return config + + +def add_default_values_to_config(config): + config["data_dir"] = os.path.abspath( + os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR))) + ) + + config.setdefault("log_level", DEFAULT_LOG_LEVEL) + + return config diff --git a/monkey/monkey_island/cc/server_utils/consts.py b/monkey/monkey_island/cc/server_utils/consts.py index 8b88b000b..a62bb98ab 100644 --- a/monkey/monkey_island/cc/server_utils/consts.py +++ b/monkey/monkey_island/cc/server_utils/consts.py @@ -14,6 +14,8 @@ def get_default_data_dir() -> str: SERVER_CONFIG_FILENAME = "server_config.json" +DEFAULT_LOG_LEVEL = "INFO" + MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island") DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir()) diff --git a/monkey/monkey_island/config_loader.py b/monkey/monkey_island/config_loader.py deleted file mode 100644 index aaa9185d7..000000000 --- a/monkey/monkey_island/config_loader.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import os - -from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR - -DEFAULT_LOG_LEVEL = "INFO" - - -def load_server_config_from_file(server_config_path): - with open(server_config_path, "r") as f: - config_content = f.read() - config = json.loads(config_content) - add_default_values_to_config(config) - - return config - - -def add_default_values_to_config(config): - config["data_dir"] = os.path.abspath( - os.path.expanduser(os.path.expandvars(config.get("data_dir", DEFAULT_DATA_DIR))) - ) - - config.setdefault("log_level", DEFAULT_LOG_LEVEL) - - return config diff --git a/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/environment/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/tests/unit_tests/monkey_island/test_config_loader.py b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py similarity index 64% rename from monkey/tests/unit_tests/monkey_island/test_config_loader.py rename to monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py index 20c330f6a..acd89d84f 100644 --- a/monkey/tests/unit_tests/monkey_island/test_config_loader.py +++ b/monkey/tests/unit_tests/monkey_island/cc/environment/test_server_config_handler.py @@ -1,11 +1,11 @@ import os -from monkey_island import config_loader +from monkey_island.cc.environment import server_config_handler from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR def test_load_server_config_from_file(test_server_config, mock_home_env): - config = config_loader.load_server_config_from_file(test_server_config) + config = server_config_handler.load_server_config_from_file(test_server_config) assert config["data_dir"] == os.path.join(mock_home_env, ".monkey_island") assert config["log_level"] == "NOTICE" @@ -13,7 +13,7 @@ def test_load_server_config_from_file(test_server_config, mock_home_env): def test_default_log_level(): test_config = {} - config = config_loader.add_default_values_to_config(test_config) + config = server_config_handler.add_default_values_to_config(test_config) assert "log_level" in config assert config["log_level"] == "INFO" @@ -21,7 +21,7 @@ def test_default_log_level(): def test_default_data_dir(mock_home_env): test_config = {} - config = config_loader.add_default_values_to_config(test_config) + config = server_config_handler.add_default_values_to_config(test_config) assert "data_dir" in config assert config["data_dir"] == DEFAULT_DATA_DIR From 8afe93747f47d73141330d4fe00c6cc6bdf573ca Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 10:27:16 +0300 Subject: [PATCH 2/5] Improved monkey_island.py setup readability by extracting workflows of server_config setup by cmd_arguments and setup of server_config using defaults --- monkey/monkey_island.py | 17 +++++------------ monkey/monkey_island/setup/config_setup.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 monkey/monkey_island/setup/config_setup.py diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 76399b6ff..662ebe892 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -1,32 +1,25 @@ from gevent import monkey as gevent_monkey from monkey_island.cc.arg_parser import parse_cli_args +from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config gevent_monkey.patch_all() import json # noqa: E402 -import os # noqa: E402 -import monkey_island.cc.environment.server_config_generator as server_config_generator # noqa: E402 -from monkey_island import config_loader # noqa: E402 -from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E402 if "__main__" == __name__: island_args = parse_cli_args() - # This is here in order to catch EVERYTHING, some functions are being called on - # imports, so the log init needs to be first. try: if island_args.server_config: - server_config_path = os.path.expanduser(island_args.server_config) + config, server_config_path = setup_config_by_cmd_arg(island_args.server_config) else: - server_config_path = server_config_generator.create_default_server_config_file() - - config = config_loader.load_server_config_from_file(server_config_path) - - create_data_dir(config["data_dir"], True) + config, server_config_path = setup_default_config() + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. setup_logging(config["data_dir"], config["log_level"]) except OSError as ex: diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py new file mode 100644 index 000000000..9980c66cd --- /dev/null +++ b/monkey/monkey_island/setup/config_setup.py @@ -0,0 +1,21 @@ +import os +from typing import Tuple + +from monkey_island.cc.environment import server_config_handler +from monkey_island.cc.environment.data_dir_generator import create_data_dir # noqa: E402 +from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH + + +def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]: + server_config_path = os.path.expanduser(server_config_path) + config = server_config_handler.load_server_config_from_file(server_config_path) + create_data_dir(config["data_dir"], create_parent_dirs=True) + return config, server_config_path + + +def setup_default_config() -> Tuple[dict, str]: + server_config_path = DEFAULT_SERVER_CONFIG_PATH + create_data_dir(DEFAULT_DATA_DIR, create_parent_dirs=False) + server_config_handler.create_default_server_config_file() + config = server_config_handler.load_server_config_from_file(server_config_path) + return config, server_config_path From 5f88f6f04be429be9f356dbafdee578c35a875e2 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 12:13:33 +0300 Subject: [PATCH 3/5] Moved a comment to a proper place --- monkey/monkey_island.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island.py b/monkey/monkey_island.py index 662ebe892..e3d8348d1 100644 --- a/monkey/monkey_island.py +++ b/monkey/monkey_island.py @@ -12,14 +12,14 @@ from monkey_island.cc.server_utils.island_logger import setup_logging # noqa: E if "__main__" == __name__: island_args = parse_cli_args() + # This is here in order to catch EVERYTHING, some functions are being called on + # imports, so the log init needs to be first. try: if island_args.server_config: config, server_config_path = setup_config_by_cmd_arg(island_args.server_config) else: config, server_config_path = setup_default_config() - # This is here in order to catch EVERYTHING, some functions are being called on - # imports, so the log init needs to be first. setup_logging(config["data_dir"], config["log_level"]) except OSError as ex: From 3098ac1459f841aaead2d587beda14c44bdef8b5 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Mon, 24 May 2021 12:25:10 +0300 Subject: [PATCH 4/5] Fixed a failing environment setup in unit tests --- monkey/monkey_island/cc/models/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/models/__init__.py b/monkey/monkey_island/cc/models/__init__.py index a46e5bd76..9b244a974 100644 --- a/monkey/monkey_island/cc/models/__init__.py +++ b/monkey/monkey_island/cc/models/__init__.py @@ -2,11 +2,12 @@ from mongoengine import connect # Needed so that a server_config.json file exists at the default path, # otherwise, unit tests will error while importing `env_singleton` below. -from monkey_island.cc.environment.server_config_generator import ( # noqa: E402 - create_default_server_config_file, -) +from monkey_island.cc.environment import data_dir_generator, server_config_handler # noqa: E402 -create_default_server_config_file() +from ..server_utils.consts import DEFAULT_DATA_DIR + +data_dir_generator.create_data_dir(DEFAULT_DATA_DIR, False) +server_config_handler.create_default_server_config_file() import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402 From d33c5d68d861640f63fcef01defeab3bd24ab3d3 Mon Sep 17 00:00:00 2001 From: Shreya Date: Mon, 24 May 2021 15:17:22 +0530 Subject: [PATCH 5/5] Use `os.path.expandvars()` on server config argument --- monkey/monkey_island/setup/config_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/setup/config_setup.py b/monkey/monkey_island/setup/config_setup.py index 9980c66cd..05f410fa8 100644 --- a/monkey/monkey_island/setup/config_setup.py +++ b/monkey/monkey_island/setup/config_setup.py @@ -7,7 +7,7 @@ from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVE def setup_config_by_cmd_arg(server_config_path) -> Tuple[dict, str]: - server_config_path = os.path.expanduser(server_config_path) + server_config_path = os.path.expandvars(os.path.expanduser(server_config_path)) config = server_config_handler.load_server_config_from_file(server_config_path) create_data_dir(config["data_dir"], create_parent_dirs=True) return config, server_config_path