forked from p15670423/monkey
Merge branch 'refactor-island-start' into develop
This commit is contained in:
commit
4e888b00b1
|
@ -25,5 +25,5 @@ do
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
(PYTHONHOME="${APPDIR}/opt/python3.7" exec "/bin/bash" "${APPDIR}/usr/src/monkey_island/linux/run_appimage.sh")
|
(PYTHONHOME="${APPDIR}/opt/python3.7" exec "${APPDIR}/opt/python3.7/bin/python3.7" "${APPDIR}/usr/src/monkey_island.py" $@)
|
||||||
exit "$?"
|
exit "$?"
|
||||||
|
|
|
@ -165,7 +165,6 @@ copy_monkey_island_to_appdir() {
|
||||||
cp "$1"/monkey_island.py "$INSTALL_DIR"
|
cp "$1"/monkey_island.py "$INSTALL_DIR"
|
||||||
cp -r "$1"/common "$INSTALL_DIR/"
|
cp -r "$1"/common "$INSTALL_DIR/"
|
||||||
cp -r "$1"/monkey_island "$INSTALL_DIR/"
|
cp -r "$1"/monkey_island "$INSTALL_DIR/"
|
||||||
cp ./run_appimage.sh "$INSTALL_DIR"/monkey_island/linux/
|
|
||||||
cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/
|
cp ./server_config.json.standard "$INSTALL_DIR"/monkey_island/cc/
|
||||||
|
|
||||||
# TODO: This is a workaround that may be able to be removed after PR #848 is
|
# TODO: This is a workaround that may be able to be removed after PR #848 is
|
||||||
|
|
|
@ -1,14 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
PYTHON_CMD="$APPDIR"/opt/python3.7/bin/python3.7
|
|
||||||
DOT_MONKEY="$HOME"/.monkey_island/
|
|
||||||
|
|
||||||
# shellcheck disable=SC2174
|
|
||||||
mkdir --mode=0700 --parents "$DOT_MONKEY"
|
|
||||||
|
|
||||||
DB_DIR="$DOT_MONKEY"/db
|
|
||||||
mkdir --parents "$DB_DIR"
|
|
||||||
|
|
||||||
cd "$APPDIR"/usr/src || exit 1
|
|
||||||
./monkey_island/bin/mongodb/bin/mongod --dbpath "$DB_DIR" &
|
|
||||||
${PYTHON_CMD} ./monkey_island.py --server-config "$DOT_MONKEY"/server_config.json
|
|
|
@ -1,4 +1,4 @@
|
||||||
from monkey_island.startup import start_island
|
from monkey_island.main import main
|
||||||
|
|
||||||
if "__main__" == __name__:
|
if "__main__" == __name__:
|
||||||
start_island()
|
main()
|
||||||
|
|
|
@ -6,7 +6,7 @@ from monkey_island.cc.server_utils.consts import (
|
||||||
DEFAULT_DEVELOP_SERVER_CONFIG_PATH,
|
DEFAULT_DEVELOP_SERVER_CONFIG_PATH,
|
||||||
DEFAULT_SERVER_CONFIG_PATH,
|
DEFAULT_SERVER_CONFIG_PATH,
|
||||||
)
|
)
|
||||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
from monkey_island.cc.setup.island_config_options import IslandConfigOptions
|
||||||
|
|
||||||
|
|
||||||
def create_default_server_config_file() -> None:
|
def create_default_server_config_file() -> None:
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
|
from typing import Tuple
|
||||||
|
|
||||||
from gevent.pywsgi import WSGIServer
|
from gevent.pywsgi import WSGIServer
|
||||||
|
|
||||||
|
@ -13,41 +15,71 @@ if str(MONKEY_ISLAND_DIR_BASE_PATH) not in sys.path:
|
||||||
sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH)
|
sys.path.insert(0, MONKEY_ISLAND_DIR_BASE_PATH)
|
||||||
|
|
||||||
import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402
|
import monkey_island.cc.environment.environment_singleton as env_singleton # noqa: E402
|
||||||
|
import monkey_island.cc.setup.config_setup as config_setup # noqa: E402
|
||||||
from common.version import get_version # noqa: E402
|
from common.version import get_version # noqa: E402
|
||||||
from monkey_island.cc.app import init_app # noqa: E402
|
from monkey_island.cc.app import init_app # noqa: E402
|
||||||
|
from monkey_island.cc.arg_parser import IslandCmdArgs # noqa: E402
|
||||||
|
from monkey_island.cc.arg_parser import parse_cli_args # noqa: E402
|
||||||
from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402
|
from monkey_island.cc.resources.monkey_download import MonkeyDownload # noqa: E402
|
||||||
from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402
|
from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer # noqa: E402
|
||||||
from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH # noqa: E402
|
from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH # noqa: E402
|
||||||
from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402
|
from monkey_island.cc.server_utils.encryptor import initialize_encryptor # noqa: E402
|
||||||
|
from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging # noqa: E402
|
||||||
from monkey_island.cc.services.initialize import initialize_services # noqa: E402
|
from monkey_island.cc.services.initialize import initialize_services # noqa: E402
|
||||||
from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402
|
from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list # noqa: E402
|
||||||
from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402
|
from monkey_island.cc.services.utils.network_utils import local_ip_addresses # noqa: E402
|
||||||
|
from monkey_island.cc.setup.island_config_options import IslandConfigOptions # noqa: E402
|
||||||
from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402
|
from monkey_island.cc.setup.mongo.database_initializer import init_collections # noqa: E402
|
||||||
from monkey_island.cc.setup.mongo.mongo_setup import MONGO_URL, setup_mongodb # noqa: E402
|
from monkey_island.cc.setup.mongo.mongo_setup import ( # noqa: E402
|
||||||
from monkey_island.setup.island_config_options import IslandConfigOptions # noqa: E402
|
MONGO_URL,
|
||||||
|
connect_to_mongodb,
|
||||||
|
register_mongo_shutdown_callback,
|
||||||
|
start_mongodb,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def setup_island(setup_only: bool, config_options: IslandConfigOptions, server_config_path: str):
|
def run_monkey_island():
|
||||||
|
island_args = parse_cli_args()
|
||||||
|
config_options, server_config_path = _setup_data_dir(island_args)
|
||||||
|
|
||||||
|
_configure_logging(config_options)
|
||||||
|
_initialize_globals(config_options, server_config_path)
|
||||||
|
|
||||||
|
if config_options.start_mongodb:
|
||||||
|
mongo_db_process = start_mongodb(config_options.data_dir)
|
||||||
|
register_mongo_shutdown_callback(mongo_db_process)
|
||||||
|
|
||||||
|
connect_to_mongodb()
|
||||||
|
|
||||||
|
_start_island_server(island_args.setup_only, config_options)
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_data_dir(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]:
|
||||||
|
try:
|
||||||
|
return config_setup.setup_data_dir(island_args)
|
||||||
|
except OSError as ex:
|
||||||
|
print(f"Error opening server config file: {ex}")
|
||||||
|
exit(1)
|
||||||
|
except json.JSONDecodeError as ex:
|
||||||
|
print(f"Error loading server config: {ex}")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def _configure_logging(config_options):
|
||||||
|
reset_logger()
|
||||||
|
setup_logging(config_options.data_dir, config_options.log_level)
|
||||||
|
|
||||||
|
|
||||||
|
def _initialize_globals(config_options: IslandConfigOptions, server_config_path: str):
|
||||||
env_singleton.initialize_from_file(server_config_path)
|
env_singleton.initialize_from_file(server_config_path)
|
||||||
|
|
||||||
initialize_encryptor(config_options.data_dir)
|
initialize_encryptor(config_options.data_dir)
|
||||||
initialize_services(config_options.data_dir)
|
initialize_services(config_options.data_dir)
|
||||||
|
|
||||||
bootloader_server_thread = Thread(
|
|
||||||
target=BootloaderHttpServer(MONGO_URL).serve_forever, daemon=True
|
|
||||||
)
|
|
||||||
|
|
||||||
bootloader_server_thread.start()
|
|
||||||
_start_island_server(setup_only, config_options)
|
|
||||||
bootloader_server_thread.join()
|
|
||||||
|
|
||||||
|
|
||||||
def _start_island_server(should_setup_only, config_options: IslandConfigOptions):
|
def _start_island_server(should_setup_only, config_options: IslandConfigOptions):
|
||||||
|
|
||||||
setup_mongodb(config_options)
|
|
||||||
|
|
||||||
populate_exporter_list()
|
populate_exporter_list()
|
||||||
app = init_app(MONGO_URL)
|
app = init_app(MONGO_URL)
|
||||||
|
|
||||||
|
@ -60,6 +92,8 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions)
|
||||||
logger.warning("Setup only flag passed. Exiting.")
|
logger.warning("Setup only flag passed. Exiting.")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
bootloader_server_thread = _start_bootloader_server()
|
||||||
|
|
||||||
if env_singleton.env.is_debug():
|
if env_singleton.env.is_debug():
|
||||||
app.run(host="0.0.0.0", debug=True, ssl_context=(crt_path, key_path))
|
app.run(host="0.0.0.0", debug=True, ssl_context=(crt_path, key_path))
|
||||||
else:
|
else:
|
||||||
|
@ -72,6 +106,16 @@ def _start_island_server(should_setup_only, config_options: IslandConfigOptions)
|
||||||
_log_init_info()
|
_log_init_info()
|
||||||
http_server.serve_forever()
|
http_server.serve_forever()
|
||||||
|
|
||||||
|
bootloader_server_thread.join()
|
||||||
|
|
||||||
|
|
||||||
|
def _start_bootloader_server() -> Thread:
|
||||||
|
bootloader_server_thread = Thread(target=BootloaderHttpServer().serve_forever, daemon=True)
|
||||||
|
|
||||||
|
bootloader_server_thread.start()
|
||||||
|
|
||||||
|
return bootloader_server_thread
|
||||||
|
|
||||||
|
|
||||||
def _log_init_info():
|
def _log_init_info():
|
||||||
logger.info("Monkey Island Server is running!")
|
logger.info("Monkey Island Server is running!")
|
||||||
|
|
|
@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class BootloaderHttpServer(ThreadingMixIn, HTTPServer):
|
class BootloaderHttpServer(ThreadingMixIn, HTTPServer):
|
||||||
def __init__(self, mongo_url):
|
def __init__(self):
|
||||||
server_address = ("", 5001)
|
server_address = ("", 5001)
|
||||||
super().__init__(server_address, BootloaderHTTPRequestHandler)
|
super().__init__(server_address, BootloaderHTTPRequestHandler)
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from monkey_island.cc.environment.utils import is_windows_os
|
from monkey_island.cc.environment.utils import is_windows_os
|
||||||
|
|
||||||
|
@ -12,9 +13,17 @@ def get_default_data_dir() -> str:
|
||||||
return r"$HOME/.monkey_island"
|
return r"$HOME/.monkey_island"
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: Figure out why windows requires the use of `os.getcwd()`. See issue #1207.
|
||||||
|
def _get_monkey_island_abs_path() -> str:
|
||||||
|
if is_windows_os():
|
||||||
|
return os.path.join(os.getcwd(), "monkey_island")
|
||||||
|
else:
|
||||||
|
return str(Path(__file__).resolve().parent.parent.parent)
|
||||||
|
|
||||||
|
|
||||||
SERVER_CONFIG_FILENAME = "server_config.json"
|
SERVER_CONFIG_FILENAME = "server_config.json"
|
||||||
|
|
||||||
MONKEY_ISLAND_ABS_PATH = os.path.join(os.getcwd(), "monkey_island")
|
MONKEY_ISLAND_ABS_PATH = _get_monkey_island_abs_path()
|
||||||
|
|
||||||
DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())
|
DEFAULT_DATA_DIR = os.path.expandvars(get_default_data_dir())
|
||||||
|
|
||||||
|
|
|
@ -1,38 +1,15 @@
|
||||||
import logging.config
|
import logging
|
||||||
|
import logging.handlers
|
||||||
import os
|
import os
|
||||||
from copy import deepcopy
|
import sys
|
||||||
|
|
||||||
ISLAND_LOG_FILENAME = "monkey_island.log"
|
ISLAND_LOG_FILENAME = "monkey_island.log"
|
||||||
|
LOG_FORMAT = (
|
||||||
LOGGER_CONFIG_DICT = {
|
"%(asctime)s - %(filename)s:%(lineno)s - %(funcName)10s() - %(levelname)s - %(message)s"
|
||||||
"version": 1,
|
)
|
||||||
"disable_existing_loggers": False,
|
FILE_MAX_BYTES = 10485760
|
||||||
"formatters": {
|
FILE_BACKUP_COUNT = 20
|
||||||
"simple": {
|
FILE_ENCODING = "utf8"
|
||||||
"format": "%(asctime)s - %(filename)s:%(lineno)s - "
|
|
||||||
+ "%(funcName)10s() - %(levelname)s - %(message)s"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"handlers": {
|
|
||||||
"console_handler": {
|
|
||||||
"class": "logging.StreamHandler",
|
|
||||||
"formatter": "simple",
|
|
||||||
"stream": "ext://sys.stdout",
|
|
||||||
},
|
|
||||||
"file_handler": {
|
|
||||||
"class": "logging.handlers.RotatingFileHandler",
|
|
||||||
"formatter": "simple",
|
|
||||||
"filename": None, # set in setup_logging()
|
|
||||||
"maxBytes": 10485760,
|
|
||||||
"backupCount": 20,
|
|
||||||
"encoding": "utf8",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"root": {
|
|
||||||
"level": None, # set in setup_logging()
|
|
||||||
"handlers": ["console_handler", "file_handler"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def setup_logging(data_dir_path, log_level):
|
def setup_logging(data_dir_path, log_level):
|
||||||
|
@ -42,12 +19,48 @@ def setup_logging(data_dir_path, log_level):
|
||||||
:param log_level: level to log from
|
:param log_level: level to log from
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
|
logger = logging.getLogger()
|
||||||
|
logger.setLevel(log_level.upper())
|
||||||
|
|
||||||
logger_configuration = deepcopy(LOGGER_CONFIG_DICT)
|
formatter = _get_log_formatter()
|
||||||
|
|
||||||
logger_configuration["handlers"]["file_handler"]["filename"] = os.path.join(
|
log_file_path = os.path.join(data_dir_path, ISLAND_LOG_FILENAME)
|
||||||
data_dir_path, ISLAND_LOG_FILENAME
|
_add_file_handler(logger, formatter, log_file_path)
|
||||||
|
|
||||||
|
_add_console_handler(logger, formatter)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_default_failsafe_logging():
|
||||||
|
logger = logging.getLogger()
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
formatter = _get_log_formatter()
|
||||||
|
|
||||||
|
_add_console_handler(logger, formatter)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_log_formatter():
|
||||||
|
return logging.Formatter(LOG_FORMAT)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_file_handler(logger, formatter, file_path):
|
||||||
|
fh = logging.handlers.RotatingFileHandler(
|
||||||
|
file_path, maxBytes=FILE_MAX_BYTES, backupCount=FILE_BACKUP_COUNT, encoding=FILE_ENCODING
|
||||||
)
|
)
|
||||||
logger_configuration["root"]["level"] = log_level.upper()
|
fh.setFormatter(formatter)
|
||||||
|
|
||||||
logging.config.dictConfig(logger_configuration)
|
logger.addHandler(fh)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_console_handler(logger, formatter):
|
||||||
|
ch = logging.StreamHandler(stream=sys.stdout)
|
||||||
|
ch.setFormatter(formatter)
|
||||||
|
|
||||||
|
logger.addHandler(ch)
|
||||||
|
|
||||||
|
|
||||||
|
def reset_logger():
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
|
for handler in logger.handlers:
|
||||||
|
logger.removeHandler(handler)
|
||||||
|
|
|
@ -1,10 +1,15 @@
|
||||||
|
import os
|
||||||
from typing import Dict, List
|
from typing import Dict, List
|
||||||
|
|
||||||
from stix2 import AttackPattern, CourseOfAction, FileSystemSource, Filter
|
from stix2 import AttackPattern, CourseOfAction, FileSystemSource, Filter
|
||||||
|
|
||||||
|
from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH
|
||||||
|
|
||||||
|
|
||||||
class MitreApiInterface:
|
class MitreApiInterface:
|
||||||
ATTACK_DATA_PATH = "monkey_island/cc/services/attack/attack_data/enterprise-attack"
|
ATTACK_DATA_PATH = os.path.join(
|
||||||
|
MONKEY_ISLAND_ABS_PATH, "cc", "services", "attack", "attack_data", "enterprise-attack"
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_all_mitigations() -> Dict[str, CourseOfAction]:
|
def get_all_mitigations() -> Dict[str, CourseOfAction]:
|
||||||
|
|
|
@ -1,20 +1,28 @@
|
||||||
import os
|
import os
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
|
from monkey_island.cc.arg_parser import IslandCmdArgs
|
||||||
from monkey_island.cc.environment import server_config_handler
|
from monkey_island.cc.environment import server_config_handler
|
||||||
from monkey_island.cc.environment.utils import create_secure_directory
|
from monkey_island.cc.environment.utils import create_secure_directory
|
||||||
from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH
|
from monkey_island.cc.server_utils.consts import DEFAULT_DATA_DIR, DEFAULT_SERVER_CONFIG_PATH
|
||||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
from monkey_island.cc.setup.island_config_options import IslandConfigOptions
|
||||||
|
|
||||||
|
|
||||||
def setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]:
|
def setup_data_dir(island_args: IslandCmdArgs) -> Tuple[IslandConfigOptions, str]:
|
||||||
|
if island_args.server_config_path:
|
||||||
|
return _setup_config_by_cmd_arg(island_args.server_config_path)
|
||||||
|
|
||||||
|
return _setup_default_config()
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_config_by_cmd_arg(server_config_path) -> Tuple[IslandConfigOptions, str]:
|
||||||
server_config_path = os.path.expandvars(os.path.expanduser(server_config_path))
|
server_config_path = os.path.expandvars(os.path.expanduser(server_config_path))
|
||||||
config = server_config_handler.load_server_config_from_file(server_config_path)
|
config = server_config_handler.load_server_config_from_file(server_config_path)
|
||||||
create_secure_directory(config.data_dir, create_parent_dirs=True)
|
create_secure_directory(config.data_dir, create_parent_dirs=True)
|
||||||
return config, server_config_path
|
return config, server_config_path
|
||||||
|
|
||||||
|
|
||||||
def setup_default_config() -> Tuple[IslandConfigOptions, str]:
|
def _setup_default_config() -> Tuple[IslandConfigOptions, str]:
|
||||||
server_config_path = DEFAULT_SERVER_CONFIG_PATH
|
server_config_path = DEFAULT_SERVER_CONFIG_PATH
|
||||||
create_secure_directory(DEFAULT_DATA_DIR, create_parent_dirs=False)
|
create_secure_directory(DEFAULT_DATA_DIR, create_parent_dirs=False)
|
||||||
server_config_handler.create_default_server_config_file()
|
server_config_handler.create_default_server_config_file()
|
|
@ -0,0 +1,49 @@
|
||||||
|
import logging
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DB_DIR_PARAM = "--dbpath"
|
||||||
|
TERMINATE_TIMEOUT = 10
|
||||||
|
|
||||||
|
|
||||||
|
class MongoDbProcess:
|
||||||
|
def __init__(self, db_dir: str, log_file: str):
|
||||||
|
"""
|
||||||
|
@param db_dir: Path where a folder for database contents will be created
|
||||||
|
@param log_file: Path to the file that will contain mongodb logs
|
||||||
|
"""
|
||||||
|
self._mongo_run_cmd = [MONGO_EXECUTABLE_PATH, DB_DIR_PARAM, db_dir]
|
||||||
|
self._log_file = log_file
|
||||||
|
self._process = None
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
logger.info("Starting MongoDB process.")
|
||||||
|
logger.debug(f"MongoDB will be launched with command: {' '.join(self._mongo_run_cmd)}.")
|
||||||
|
logger.info(f"MongoDB log will be available at {self._log_file}.")
|
||||||
|
|
||||||
|
with open(self._log_file, "w") as log:
|
||||||
|
self._process = subprocess.Popen(
|
||||||
|
self._mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("MongoDB launched successfully!")
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
if not self._process:
|
||||||
|
logger.warning("Failed to stop MongoDB process: No process found")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("Terminating MongoDB process")
|
||||||
|
self._process.terminate()
|
||||||
|
|
||||||
|
try:
|
||||||
|
self._process.wait(timeout=TERMINATE_TIMEOUT)
|
||||||
|
logger.info("MongoDB process terminated successfully")
|
||||||
|
except subprocess.TimeoutExpired as te:
|
||||||
|
logger.warning(
|
||||||
|
f"MongoDB did not terminate gracefully and will be forcefully killed: {te}"
|
||||||
|
)
|
||||||
|
self._process.kill()
|
|
@ -1,50 +0,0 @@
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
from typing import List
|
|
||||||
|
|
||||||
from monkey_island.cc.environment.utils import create_secure_directory
|
|
||||||
from monkey_island.cc.server_utils.consts import MONGO_EXECUTABLE_PATH
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DB_DIR_NAME = "db"
|
|
||||||
DB_DIR_PARAM = "--dbpath"
|
|
||||||
MONGO_LOG_FILENAME = "mongo_log.txt"
|
|
||||||
|
|
||||||
|
|
||||||
class MongoDbRunner:
|
|
||||||
def __init__(self, db_dir_parent_path: str, logging_dir_path: str):
|
|
||||||
"""
|
|
||||||
@param db_dir_parent_path: Path where a folder for database contents will be created
|
|
||||||
@param logging_dir_path: Path to a folder where mongodb logs will be created
|
|
||||||
"""
|
|
||||||
self.db_dir_parent_path = db_dir_parent_path
|
|
||||||
self.logging_dir_path = logging_dir_path
|
|
||||||
|
|
||||||
def launch_mongodb(self):
|
|
||||||
db_path = self._create_db_dir()
|
|
||||||
self._start_mongodb_process(db_path)
|
|
||||||
|
|
||||||
def _create_db_dir(self) -> str:
|
|
||||||
db_path = os.path.join(self.db_dir_parent_path, DB_DIR_NAME)
|
|
||||||
logger.info(f"Database content directory: {db_path}.")
|
|
||||||
create_secure_directory(db_path, create_parent_dirs=False)
|
|
||||||
return db_path
|
|
||||||
|
|
||||||
def _start_mongodb_process(self, db_dir_path: str):
|
|
||||||
logger.info("Starting MongoDb process.")
|
|
||||||
|
|
||||||
mongo_run_cmd = MongoDbRunner._build_mongo_launch_cmd(MONGO_EXECUTABLE_PATH, db_dir_path)
|
|
||||||
logger.info(f"Mongodb will be launched with command: {' '.join(mongo_run_cmd)}.")
|
|
||||||
|
|
||||||
mongo_log_path = os.path.join(self.logging_dir_path, MONGO_LOG_FILENAME)
|
|
||||||
logger.info(f"Mongodb log will be available at {mongo_log_path}.")
|
|
||||||
|
|
||||||
with open(mongo_log_path, "w") as log:
|
|
||||||
subprocess.Popen(mongo_run_cmd, stderr=subprocess.STDOUT, stdout=log)
|
|
||||||
logger.info("MongoDb launched successfully!")
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _build_mongo_launch_cmd(exec_path: str, db_path: str) -> List[str]:
|
|
||||||
return [exec_path, DB_DIR_PARAM, db_path]
|
|
|
@ -1,14 +1,17 @@
|
||||||
|
import atexit
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from monkey_island.cc.database import get_db_version, is_db_server_up
|
from monkey_island.cc.database import get_db_version, is_db_server_up
|
||||||
|
from monkey_island.cc.environment.utils import create_secure_directory
|
||||||
from monkey_island.cc.setup.mongo import mongo_connector
|
from monkey_island.cc.setup.mongo import mongo_connector
|
||||||
from monkey_island.cc.setup.mongo.mongo_connector import MONGO_DB_HOST, MONGO_DB_NAME, MONGO_DB_PORT
|
from monkey_island.cc.setup.mongo.mongo_connector import MONGO_DB_HOST, MONGO_DB_NAME, MONGO_DB_PORT
|
||||||
from monkey_island.cc.setup.mongo.mongo_process_runner import MongoDbRunner
|
from monkey_island.cc.setup.mongo.mongo_db_process import MongoDbProcess
|
||||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
|
||||||
|
|
||||||
|
DB_DIR_NAME = "db"
|
||||||
|
MONGO_LOG_FILENAME = "mongodb.log"
|
||||||
MONGO_URL = os.environ.get(
|
MONGO_URL = os.environ.get(
|
||||||
"MONKEY_MONGO_URL",
|
"MONKEY_MONGO_URL",
|
||||||
"mongodb://{0}:{1}/{2}".format(MONGO_DB_HOST, MONGO_DB_PORT, MONGO_DB_NAME),
|
"mongodb://{0}:{1}/{2}".format(MONGO_DB_HOST, MONGO_DB_PORT, MONGO_DB_NAME),
|
||||||
|
@ -18,23 +21,41 @@ MINIMUM_MONGO_DB_VERSION_REQUIRED = "4.2.0"
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def setup_mongodb(config_options: IslandConfigOptions):
|
def start_mongodb(data_dir: str) -> MongoDbProcess:
|
||||||
if config_options.start_mongodb:
|
db_dir = _create_db_dir(data_dir)
|
||||||
MongoDbRunner(
|
log_file = os.path.join(data_dir, MONGO_LOG_FILENAME)
|
||||||
db_dir_parent_path=config_options.data_dir, logging_dir_path=config_options.data_dir
|
|
||||||
).launch_mongodb()
|
mongo_db_process = MongoDbProcess(db_dir=db_dir, log_file=log_file)
|
||||||
wait_for_mongo_db_server(MONGO_URL)
|
mongo_db_process.start()
|
||||||
assert_mongo_db_version(MONGO_URL)
|
|
||||||
|
return mongo_db_process
|
||||||
|
|
||||||
|
|
||||||
|
def _create_db_dir(db_dir_parent_path) -> str:
|
||||||
|
db_dir = os.path.join(db_dir_parent_path, DB_DIR_NAME)
|
||||||
|
logger.info(f"Database content directory: {db_dir}.")
|
||||||
|
|
||||||
|
create_secure_directory(db_dir, create_parent_dirs=False)
|
||||||
|
return db_dir
|
||||||
|
|
||||||
|
|
||||||
|
def register_mongo_shutdown_callback(mongo_db_process: MongoDbProcess):
|
||||||
|
atexit.register(mongo_db_process.stop)
|
||||||
|
|
||||||
|
|
||||||
|
def connect_to_mongodb():
|
||||||
|
_wait_for_mongo_db_server(MONGO_URL)
|
||||||
|
_assert_mongo_db_version(MONGO_URL)
|
||||||
mongo_connector.connect_dal_to_mongodb()
|
mongo_connector.connect_dal_to_mongodb()
|
||||||
|
|
||||||
|
|
||||||
def wait_for_mongo_db_server(mongo_url):
|
def _wait_for_mongo_db_server(mongo_url):
|
||||||
while not is_db_server_up(mongo_url):
|
while not is_db_server_up(mongo_url):
|
||||||
logger.info("Waiting for MongoDB server on {0}".format(mongo_url))
|
logger.info("Waiting for MongoDB server on {0}".format(mongo_url))
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
def assert_mongo_db_version(mongo_url):
|
def _assert_mongo_db_version(mongo_url):
|
||||||
"""
|
"""
|
||||||
Checks if the mongodb version is new enough for running the app.
|
Checks if the mongodb version is new enough for running the app.
|
||||||
If the DB is too old, quits.
|
If the DB is too old, quits.
|
||||||
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
# This import patches other imports and needs to be first
|
||||||
|
import monkey_island.setup.gevent_setup # noqa: F401 isort:skip
|
||||||
|
|
||||||
|
from monkey_island.cc.server_utils.island_logger import setup_default_failsafe_logging
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# This is here in order to catch EVERYTHING, some functions are being called on
|
||||||
|
# imports, so the log init needs to be first.
|
||||||
|
try:
|
||||||
|
setup_default_failsafe_logging()
|
||||||
|
except Exception as ex:
|
||||||
|
print(f"Error configuring logging: {ex}")
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
from monkey_island.cc.server_setup import run_monkey_island # noqa: E402
|
||||||
|
|
||||||
|
run_monkey_island()
|
||||||
|
|
||||||
|
|
||||||
|
if "__main__" == __name__:
|
||||||
|
main()
|
|
@ -16,7 +16,7 @@ def main():
|
||||||
("../monkey_island/cc/services/attack/attack_data", "/monkey_island/cc/services/attack/attack_data")
|
("../monkey_island/cc/services/attack/attack_data", "/monkey_island/cc/services/attack/attack_data")
|
||||||
]
|
]
|
||||||
|
|
||||||
a = Analysis(['startup.py'],
|
a = Analysis(['main.py'],
|
||||||
pathex=['..'],
|
pathex=['..'],
|
||||||
hiddenimports=get_hidden_imports(),
|
hiddenimports=get_hidden_imports(),
|
||||||
hookspath=[os.path.join(".", "pyinstaller_hooks")],
|
hookspath=[os.path.join(".", "pyinstaller_hooks")],
|
||||||
|
|
|
@ -1,37 +0,0 @@
|
||||||
# This import patches other imports and needs to be first
|
|
||||||
import monkey_island.setup.gevent_setup # noqa: F401 isort:skip
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
from monkey_island.cc.arg_parser import parse_cli_args
|
|
||||||
from monkey_island.cc.server_setup import setup_island
|
|
||||||
from monkey_island.cc.server_utils.island_logger import setup_logging
|
|
||||||
from monkey_island.setup.config_setup import setup_config_by_cmd_arg, setup_default_config
|
|
||||||
|
|
||||||
|
|
||||||
def start_island():
|
|
||||||
island_args = parse_cli_args()
|
|
||||||
|
|
||||||
# This is here in order to catch EVERYTHING, some functions are being called on
|
|
||||||
# imports, so the log init needs to be first.
|
|
||||||
try:
|
|
||||||
if island_args.server_config_path:
|
|
||||||
config, server_config_path = setup_config_by_cmd_arg(island_args.server_config_path)
|
|
||||||
else:
|
|
||||||
config, server_config_path = setup_default_config()
|
|
||||||
|
|
||||||
setup_logging(config.data_dir, config.log_level)
|
|
||||||
|
|
||||||
except OSError as ex:
|
|
||||||
print(f"Error opening server config file: {ex}")
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
except json.JSONDecodeError as ex:
|
|
||||||
print(f"Error loading server config: {ex}")
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
setup_island(island_args.setup_only, config, server_config_path)
|
|
||||||
|
|
||||||
|
|
||||||
if "__main__" == __name__:
|
|
||||||
start_island()
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
from monkey_island.cc.server_utils import consts
|
||||||
|
|
||||||
|
|
||||||
|
def test_monkey_island_abs_path():
|
||||||
|
assert consts.MONKEY_ISLAND_ABS_PATH.endswith("monkey_island")
|
|
@ -1,16 +1,25 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from monkey_island.cc.server_utils.island_logger import ISLAND_LOG_FILENAME, setup_logging
|
import pytest
|
||||||
|
|
||||||
|
import monkey_island.cc.server_utils.island_logger as island_logger
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def reset_logger():
|
||||||
|
yield
|
||||||
|
|
||||||
|
island_logger.reset_logger()
|
||||||
|
|
||||||
|
|
||||||
def test_setup_logging_file_log_level_debug(tmpdir):
|
def test_setup_logging_file_log_level_debug(tmpdir):
|
||||||
DATA_DIR = tmpdir
|
DATA_DIR = tmpdir
|
||||||
LOG_FILE = os.path.join(DATA_DIR, ISLAND_LOG_FILENAME)
|
LOG_FILE = os.path.join(DATA_DIR, island_logger.ISLAND_LOG_FILENAME)
|
||||||
LOG_LEVEL = "DEBUG"
|
LOG_LEVEL = "DEBUG"
|
||||||
TEST_STRING = "Hello, Monkey! (File; Log level: debug)"
|
TEST_STRING = "Hello, Monkey! (File; Log level: debug)"
|
||||||
|
|
||||||
setup_logging(DATA_DIR, LOG_LEVEL)
|
island_logger.setup_logging(DATA_DIR, LOG_LEVEL)
|
||||||
|
|
||||||
logger = logging.getLogger("TestLogger")
|
logger = logging.getLogger("TestLogger")
|
||||||
logger.debug(TEST_STRING)
|
logger.debug(TEST_STRING)
|
||||||
|
@ -23,11 +32,11 @@ def test_setup_logging_file_log_level_debug(tmpdir):
|
||||||
|
|
||||||
def test_setup_logging_file_log_level_info(tmpdir):
|
def test_setup_logging_file_log_level_info(tmpdir):
|
||||||
DATA_DIR = tmpdir
|
DATA_DIR = tmpdir
|
||||||
LOG_FILE = os.path.join(DATA_DIR, ISLAND_LOG_FILENAME)
|
LOG_FILE = os.path.join(DATA_DIR, island_logger.ISLAND_LOG_FILENAME)
|
||||||
LOG_LEVEL = "INFO"
|
LOG_LEVEL = "INFO"
|
||||||
TEST_STRING = "Hello, Monkey! (File; Log level: info)"
|
TEST_STRING = "Hello, Monkey! (File; Log level: info)"
|
||||||
|
|
||||||
setup_logging(DATA_DIR, LOG_LEVEL)
|
island_logger.setup_logging(DATA_DIR, LOG_LEVEL)
|
||||||
|
|
||||||
logger = logging.getLogger("TestLogger")
|
logger = logging.getLogger("TestLogger")
|
||||||
logger.debug(TEST_STRING)
|
logger.debug(TEST_STRING)
|
||||||
|
@ -43,7 +52,7 @@ def test_setup_logging_console_log_level_debug(capsys, tmpdir):
|
||||||
LOG_LEVEL = "DEBUG"
|
LOG_LEVEL = "DEBUG"
|
||||||
TEST_STRING = "Hello, Monkey! (Console; Log level: debug)"
|
TEST_STRING = "Hello, Monkey! (Console; Log level: debug)"
|
||||||
|
|
||||||
setup_logging(DATA_DIR, LOG_LEVEL)
|
island_logger.setup_logging(DATA_DIR, LOG_LEVEL)
|
||||||
|
|
||||||
logger = logging.getLogger("TestLogger")
|
logger = logging.getLogger("TestLogger")
|
||||||
logger.debug(TEST_STRING)
|
logger.debug(TEST_STRING)
|
||||||
|
@ -57,7 +66,7 @@ def test_setup_logging_console_log_level_info(capsys, tmpdir):
|
||||||
LOG_LEVEL = "INFO"
|
LOG_LEVEL = "INFO"
|
||||||
TEST_STRING = "Hello, Monkey! (Console; Log level: info)"
|
TEST_STRING = "Hello, Monkey! (Console; Log level: info)"
|
||||||
|
|
||||||
setup_logging(DATA_DIR, LOG_LEVEL)
|
island_logger.setup_logging(DATA_DIR, LOG_LEVEL)
|
||||||
|
|
||||||
logger = logging.getLogger("TestLogger")
|
logger = logging.getLogger("TestLogger")
|
||||||
logger.debug(TEST_STRING)
|
logger.debug(TEST_STRING)
|
||||||
|
@ -71,10 +80,22 @@ def test_setup_logging_console_log_level_lower_case(capsys, tmpdir):
|
||||||
LOG_LEVEL = "debug"
|
LOG_LEVEL = "debug"
|
||||||
TEST_STRING = "Hello, Monkey! (Console; Log level: debug)"
|
TEST_STRING = "Hello, Monkey! (Console; Log level: debug)"
|
||||||
|
|
||||||
setup_logging(DATA_DIR, LOG_LEVEL)
|
island_logger.setup_logging(DATA_DIR, LOG_LEVEL)
|
||||||
|
|
||||||
logger = logging.getLogger("TestLogger")
|
logger = logging.getLogger("TestLogger")
|
||||||
logger.debug(TEST_STRING)
|
logger.debug(TEST_STRING)
|
||||||
|
|
||||||
captured = capsys.readouterr()
|
captured = capsys.readouterr()
|
||||||
assert TEST_STRING in captured.out
|
assert TEST_STRING in captured.out
|
||||||
|
|
||||||
|
|
||||||
|
def test_setup_defailt_failsafe_logging(capsys):
|
||||||
|
TEST_STRING = "Hello, Monkey! (Console; Log level: debug)"
|
||||||
|
|
||||||
|
island_logger.setup_default_failsafe_logging()
|
||||||
|
logger = logging.getLogger("TestLogger")
|
||||||
|
logger.debug(TEST_STRING)
|
||||||
|
|
||||||
|
captured = capsys.readouterr()
|
||||||
|
assert TEST_STRING in captured.out
|
||||||
|
assert "DEBUG" in captured.out
|
||||||
|
|
|
@ -5,7 +5,7 @@ from monkey_island.cc.server_utils.consts import (
|
||||||
DEFAULT_LOG_LEVEL,
|
DEFAULT_LOG_LEVEL,
|
||||||
DEFAULT_START_MONGO_DB,
|
DEFAULT_START_MONGO_DB,
|
||||||
)
|
)
|
||||||
from monkey_island.setup.island_config_options import IslandConfigOptions
|
from monkey_island.cc.setup.island_config_options import IslandConfigOptions
|
||||||
|
|
||||||
TEST_CONFIG_FILE_CONTENTS_SPECIFIED = {
|
TEST_CONFIG_FILE_CONTENTS_SPECIFIED = {
|
||||||
"data_dir": "/tmp",
|
"data_dir": "/tmp",
|
Loading…
Reference in New Issue