diff --git a/.travis.yml b/.travis.yml index 504263456..ca03d20b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,23 +60,20 @@ before_script: script: # Check Python code -## Check syntax errors and fail the build if any are found. -- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics +- flake8 ./monkey --config=./ci_scripts/flake8_syntax_check.ini ## Warn about linter issues. ### --exit-zero forces Flake8 to use the exit status code 0 even if there are errors, which means this will NOT fail the build. -### --count will print the total number of errors. -### --statistics Count the number of occurrences of each error/warning code and print a report. ### The output is redirected to a file. -- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics > flake8_warnings.txt +- flake8 ./monkey --exit-zero --config=./ci_scripts/flake8_linter_check.ini > ./ci_scripts/flake8_warnings.txt ## Display the linter issues -- cat flake8_warnings.txt +- cat ./ci_scripts/flake8_warnings.txt ## Make sure that we haven't increased the amount of warnings. - PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=80 -- if [ $(tail -n 1 flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi +- if [ $(tail -n 1 ./ci_scripts/flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi ## Check import order -- python -m isort . -c -p common -p infection_monkey -p monkey_island +- python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg ## Run unit tests - cd monkey # This is our source dir diff --git a/ci_scripts/.gitignore b/ci_scripts/.gitignore new file mode 100644 index 000000000..67f93fcdc --- /dev/null +++ b/ci_scripts/.gitignore @@ -0,0 +1,2 @@ +./validation-env +./flake8_warnings.txt diff --git a/ci_scripts/README.md b/ci_scripts/README.md new file mode 100644 index 000000000..09330d298 --- /dev/null +++ b/ci_scripts/README.md @@ -0,0 +1,8 @@ +# About + +Run this script to validate your code locally and auto fix/format the problems before pushing. + +# Usage + +You've got to manually download swimm for swimm validation. +run from `infection_monkey` directory: `powershell .\ci_scripts\validate.ps1` diff --git a/ci_scripts/flake8_linter_check.ini b/ci_scripts/flake8_linter_check.ini new file mode 100644 index 000000000..b8daeaf70 --- /dev/null +++ b/ci_scripts/flake8_linter_check.ini @@ -0,0 +1,15 @@ +[flake8] +## Warn about linter issues. + +exclude = ../monkey/monkey_island/cc/ui, + ../monkey/common/cloud +show-source = True +max-complexity = 10 +max-line-length = 127 + +### --statistics Count the number of occurrences of each error/warning code and print a report. +statistics = True + +### --count will print the total number of errors. +count = True + diff --git a/ci_scripts/flake8_syntax_check.ini b/ci_scripts/flake8_syntax_check.ini new file mode 100644 index 000000000..969379326 --- /dev/null +++ b/ci_scripts/flake8_syntax_check.ini @@ -0,0 +1,15 @@ +[flake8] + +## Check syntax errors and fail the build if any are found. +exclude = + ../monkey/monkey_island/cc/ui, + ../monkey/common/cloud +select = + E901, + E999, + F821, + F822, + F823 +count = True +show-source = True +statistics = True diff --git a/ci_scripts/install_requirements.ps1 b/ci_scripts/install_requirements.ps1 new file mode 100644 index 000000000..de42d8599 --- /dev/null +++ b/ci_scripts/install_requirements.ps1 @@ -0,0 +1,5 @@ +python -m venv validation-env +.\validation-env\Scripts\activate.ps1 +python -m pip install -r .\requirements.txt +npm i -g eslint +deactivate diff --git a/ci_scripts/isort.cfg b/ci_scripts/isort.cfg new file mode 100644 index 000000000..d8651febd --- /dev/null +++ b/ci_scripts/isort.cfg @@ -0,0 +1,6 @@ +[isort] + +# Possible options: https://pycqa.github.io/isort/docs/configuration/options/ + +known_first_party=common,infection_monkey,monkey_island +skip=monkey/common/cloud/scoutsuite,monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/rule_path_creators_list.py,monkey/monkey_island/cc/ui,monkey/common/cloud/scoutsuite diff --git a/ci_scripts/requirements.txt b/ci_scripts/requirements.txt new file mode 100644 index 000000000..2b7db1909 --- /dev/null +++ b/ci_scripts/requirements.txt @@ -0,0 +1,6 @@ +flake8 +pytest +dlint +isort +coverage +black diff --git a/ci_scripts/validate.ps1 b/ci_scripts/validate.ps1 new file mode 100644 index 000000000..d85da6a2a --- /dev/null +++ b/ci_scripts/validate.ps1 @@ -0,0 +1,39 @@ +.\ci_scripts\validation-env\Scripts\activate.ps1 +$ErrorActionPreference = "Stop" +python -m pip install -r monkey/monkey_island/requirements.txt +python -m pip install -r monkey/infection_monkey/requirements.txt +flake8 ./monkey --config ./ci_scripts/flake8_syntax_check.cfg +flake8 ./monkey --exit-zero --config ./ci_scripts/flake8_linter_check.cfg | Out-File -FilePath .\ci_scripts\flake8_warnings.txt +Get-Content -Path .\ci_scripts\flake8_warnings.txt +$PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT = 80 +if ((Get-Item -Path .\ci_scripts\flake8_warnings.txt | Get-Content -Tail 1) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT){ + "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " + exit +} +python -m isort ./monkey -c --settings-file ./ci_scripts/isort.cfg +if (!$?) { + $confirmation = Read-Host "Isort found errors. Do you want to attmpt to fix them automatically? (y/n)" + if ($confirmation -eq 'y') { + python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg + } +} +Push-Location -Path ./monkey +python ./monkey_island/cc/environment/set_server_config.py testing +python -m pytest +$lastCommandSucceeded = $? +python ./monkey_island/cc/environment/set_server_config.py restore +Pop-Location + +if (!$lastCommandSucceeded) { + exit +} + +Push-Location -Path .\monkey\monkey_island\cc\ui +eslint ./src -c ./.eslintrc +Pop-Location + +swimm verify + +Write-Host "Script finished. Press any key to continue" +$null = $Host.UI.RawUI.ReadKey('NoEcho,IncludeKeyDown'); +deactivate diff --git a/monkey/monkey_island/cc/environment/set_server_config.py b/monkey/monkey_island/cc/environment/set_server_config.py index d31dd6916..8c5c12b63 100644 --- a/monkey/monkey_island/cc/environment/set_server_config.py +++ b/monkey/monkey_island/cc/environment/set_server_config.py @@ -3,6 +3,7 @@ import json import logging import sys from pathlib import Path +from shutil import move def add_monkey_dir_to_sys_path(): @@ -16,6 +17,7 @@ add_monkey_dir_to_sys_path() from monkey_island.cc.environment.environment_config import EnvironmentConfig # noqa: E402 isort:skip SERVER_CONFIG = "server_config" +BACKUP_CONFIG_FILENAME = "./server_config.backup" logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) @@ -26,10 +28,19 @@ def main(): args = parse_args() file_path = EnvironmentConfig.get_config_file_path() + if args.server_config == "restore": + restore_previous_config(file_path) + quit() + # Read config with open(file_path) as config_file: config_data = json.load(config_file) + # Backup the config + with open(BACKUP_CONFIG_FILENAME, "w") as backup_file: + json.dump(config_data, backup_file, indent=4) + backup_file.write("\n") + # Edit the config config_data[SERVER_CONFIG] = args.server_config @@ -42,10 +53,14 @@ def main(): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("server_config", choices=["standard", "testing", "password"]) + parser.add_argument("server_config", choices=["standard", "testing", "password", "restore"]) args = parser.parse_args() return args +def restore_previous_config(config_path): + move(BACKUP_CONFIG_FILENAME, config_path) + + if __name__ == '__main__': main() diff --git a/monkey/pytest.ini b/monkey/pytest.ini index 3596bf5f6..9b1766fc2 100644 --- a/monkey/pytest.ini +++ b/monkey/pytest.ini @@ -3,5 +3,5 @@ log_cli = 1 log_cli_level = DEBUG log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s log_cli_date_format=%H:%M:%S -addopts = -v --capture=sys +addopts = -v --capture=sys --ignore=common/cloud/scoutsuite norecursedirs = node_modules dist