From 2e35b9ac4334038128b45e4642412863ccf0b1a7 Mon Sep 17 00:00:00 2001
From: VakarisZ <vakarisz@yahoo.com>
Date: Thu, 5 Nov 2020 14:57:50 +0200
Subject: [PATCH 1/4] Initial implementation of validation script

---
 .travis.yml                         | 11 ++++-------
 ci_scripts/.gitignore               |  2 ++
 ci_scripts/README.md                |  7 +++++++
 ci_scripts/flake8_linter_check.cfg  | 15 +++++++++++++++
 ci_scripts/flake8_syntax_check.cfg  |  9 +++++++++
 ci_scripts/install_requirements.ps1 |  5 +++++
 ci_scripts/isort.cfg                |  5 +++++
 ci_scripts/requirements.txt         |  6 ++++++
 ci_scripts/validate.ps1             | 20 ++++++++++++++++++++
 9 files changed, 73 insertions(+), 7 deletions(-)
 create mode 100644 ci_scripts/.gitignore
 create mode 100644 ci_scripts/README.md
 create mode 100644 ci_scripts/flake8_linter_check.cfg
 create mode 100644 ci_scripts/flake8_syntax_check.cfg
 create mode 100644 ci_scripts/install_requirements.ps1
 create mode 100644 ci_scripts/isort.cfg
 create mode 100644 ci_scripts/requirements.txt
 create mode 100644 ci_scripts/validate.ps1

diff --git a/.travis.yml b/.travis.yml
index 887b7cc67..35f6210cb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -60,20 +60,17 @@ before_script:
 
 script:
 # Check Python code
-## Check syntax errors and fail the build if any are found.
-- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
+- flake8 ./monkey --config ./ci_scripts/flake8_syntax_check.config
 
 ## Warn about linter issues.
 ### --exit-zero forces Flake8 to use the exit status code 0 even if there are errors, which means this will NOT fail the build.
-### --count will print the total number of errors.
-### --statistics Count the number of occurrences of each error/warning code and print a report.
 ### The output is redirected to a file.
-- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics > flake8_warnings.txt
+- flake8 ./monkey --exit-zero --config ./ci_scripts/flake8_linter_check.config > ./ci_scripts/flake8_warnings.txt
 ## Display the linter issues
-- cat flake8_warnings.txt
+- cat ./ci_scripts/flake8_warnings.txt
 ## Make sure that we haven't increased the amount of warnings.
 - PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT=80
-- if [ $(tail -n 1 flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi
+- if [ $(tail -n 1 ./ci_scripts/flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi
 
 ## Check import order
 - python -m isort . -c -p common -p infection_monkey -p monkey_island
diff --git a/ci_scripts/.gitignore b/ci_scripts/.gitignore
new file mode 100644
index 000000000..67f93fcdc
--- /dev/null
+++ b/ci_scripts/.gitignore
@@ -0,0 +1,2 @@
+./validation-env
+./flake8_warnings.txt
diff --git a/ci_scripts/README.md b/ci_scripts/README.md
new file mode 100644
index 000000000..af555a7dd
--- /dev/null
+++ b/ci_scripts/README.md
@@ -0,0 +1,7 @@
+# About
+
+Run this script to validate your code locally and auto fix/format the problems before pushing.
+
+# Usage
+
+run from `infection_monkey` directory: `powershell .\ci_scripts\validate.ps1`
diff --git a/ci_scripts/flake8_linter_check.cfg b/ci_scripts/flake8_linter_check.cfg
new file mode 100644
index 000000000..b8daeaf70
--- /dev/null
+++ b/ci_scripts/flake8_linter_check.cfg
@@ -0,0 +1,15 @@
+[flake8]
+## Warn about linter issues.
+
+exclude = ../monkey/monkey_island/cc/ui,
+          ../monkey/common/cloud
+show-source = True
+max-complexity = 10
+max-line-length = 127
+
+### --statistics Count the number of occurrences of each error/warning code and print a report.
+statistics = True
+
+### --count will print the total number of errors.
+count = True
+
diff --git a/ci_scripts/flake8_syntax_check.cfg b/ci_scripts/flake8_syntax_check.cfg
new file mode 100644
index 000000000..7171e3db6
--- /dev/null
+++ b/ci_scripts/flake8_syntax_check.cfg
@@ -0,0 +1,9 @@
+[flake8]
+
+## Check syntax errors and fail the build if any are found.
+exclude = ../monkey/monkey_island/cc/ui,
+          ../monkey/common/cloud
+select = E901,E999,F821,F822,F823
+count = True
+show-source = True
+statistics = True
diff --git a/ci_scripts/install_requirements.ps1 b/ci_scripts/install_requirements.ps1
new file mode 100644
index 000000000..de42d8599
--- /dev/null
+++ b/ci_scripts/install_requirements.ps1
@@ -0,0 +1,5 @@
+python -m venv validation-env
+.\validation-env\Scripts\activate.ps1
+python -m pip install -r .\requirements.txt
+npm i -g eslint
+deactivate
diff --git a/ci_scripts/isort.cfg b/ci_scripts/isort.cfg
new file mode 100644
index 000000000..f9a62b0ed
--- /dev/null
+++ b/ci_scripts/isort.cfg
@@ -0,0 +1,5 @@
+[settings]
+project=common,infection_monkey,monkey_island
+skip=./monkey/common/cloud/scoutsuite,./monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/rule_path_creators_list.py,./monkey/monkey_island/cc/ui
+line-length=120
+wrap-length=120
diff --git a/ci_scripts/requirements.txt b/ci_scripts/requirements.txt
new file mode 100644
index 000000000..2b7db1909
--- /dev/null
+++ b/ci_scripts/requirements.txt
@@ -0,0 +1,6 @@
+flake8
+pytest
+dlint
+isort
+coverage
+black
diff --git a/ci_scripts/validate.ps1 b/ci_scripts/validate.ps1
new file mode 100644
index 000000000..98a0ab4dd
--- /dev/null
+++ b/ci_scripts/validate.ps1
@@ -0,0 +1,20 @@
+.\ci_scripts\validation-env\Scripts\activate.ps1
+flake8 ./monkey --config ./ci_scripts/flake8_syntax_check.cfg
+flake8 ./monkey --exit-zero --config ./ci_scripts/flake8_linter_check.cfg | Out-File -FilePath .\ci_scripts\flake8_warnings.txt
+Get-Content -Path .\ci_scripts\flake8_warnings.txt
+$PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT = 80
+if ((Get-Item -Path .\ci_scripts\flake8_warnings.txt | Get-Content -Tail 1) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT){
+    "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. "
+    exit
+}
+python -m isort ./monkey -c --settings-file ./ci_scripts/isort.cfg
+# python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg
+python monkey_island/cc/environment/set_server_config.py testing
+python -m pytest
+Push-Location -Path .\monkey_island\cc\ui
+eslint ./src -c ./.eslintrc
+
+Pop-Location
+Pop-Location
+$null = $Host.UI.RawUI.ReadKey('NoEcho,IncludeKeyDown');
+deactivate

From 636fb1be89a63f413fa58bcf5ba6e8c1e8323d72 Mon Sep 17 00:00:00 2001
From: VakarisZ <vakarisz@yahoo.com>
Date: Fri, 6 Nov 2020 11:47:38 +0200
Subject: [PATCH 2/4] Added option to rollback the changes done in
 "set_server_config"

---
 .../cc/environment/set_server_config.py         | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/monkey/monkey_island/cc/environment/set_server_config.py b/monkey/monkey_island/cc/environment/set_server_config.py
index d31dd6916..8c5c12b63 100644
--- a/monkey/monkey_island/cc/environment/set_server_config.py
+++ b/monkey/monkey_island/cc/environment/set_server_config.py
@@ -3,6 +3,7 @@ import json
 import logging
 import sys
 from pathlib import Path
+from shutil import move
 
 
 def add_monkey_dir_to_sys_path():
@@ -16,6 +17,7 @@ add_monkey_dir_to_sys_path()
 from monkey_island.cc.environment.environment_config import EnvironmentConfig  # noqa: E402 isort:skip
 
 SERVER_CONFIG = "server_config"
+BACKUP_CONFIG_FILENAME = "./server_config.backup"
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
@@ -26,10 +28,19 @@ def main():
     args = parse_args()
     file_path = EnvironmentConfig.get_config_file_path()
 
+    if args.server_config == "restore":
+        restore_previous_config(file_path)
+        quit()
+
     # Read config
     with open(file_path) as config_file:
         config_data = json.load(config_file)
 
+    # Backup the config
+    with open(BACKUP_CONFIG_FILENAME, "w") as backup_file:
+        json.dump(config_data, backup_file, indent=4)
+        backup_file.write("\n")
+
     # Edit the config
     config_data[SERVER_CONFIG] = args.server_config
 
@@ -42,10 +53,14 @@ def main():
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("server_config", choices=["standard", "testing", "password"])
+    parser.add_argument("server_config", choices=["standard", "testing", "password", "restore"])
     args = parser.parse_args()
     return args
 
 
+def restore_previous_config(config_path):
+    move(BACKUP_CONFIG_FILENAME, config_path)
+
+
 if __name__ == '__main__':
     main()

From 1f1682fad8be3934d2893c22a48ca74ed3af3918 Mon Sep 17 00:00:00 2001
From: VakarisZ <vakarisz@yahoo.com>
Date: Fri, 6 Nov 2020 11:48:52 +0200
Subject: [PATCH 3/4] Exported multiple configurations of CI tools to separate
 configurations, improved script

---
 .travis.yml             |  2 +-
 ci_scripts/README.md    |  1 +
 ci_scripts/isort.cfg    | 11 ++++++-----
 ci_scripts/validate.ps1 | 29 ++++++++++++++++++++++++-----
 monkey/pytest.ini       |  2 +-
 5 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 35f6210cb..3e4d3104a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -73,7 +73,7 @@ script:
 - if [ $(tail -n 1 ./ci_scripts/flake8_warnings.txt) -gt $PYTHON_WARNINGS_AMOUNT_UPPER_LIMIT ]; then echo "Too many python linter warnings! Failing this build. Lower the amount of linter errors in this and try again. " && exit 1; fi
 
 ## Check import order
-- python -m isort . -c -p common -p infection_monkey -p monkey_island
+- python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg
 
 ## Run unit tests
 - cd monkey  # This is our source dir
diff --git a/ci_scripts/README.md b/ci_scripts/README.md
index af555a7dd..09330d298 100644
--- a/ci_scripts/README.md
+++ b/ci_scripts/README.md
@@ -4,4 +4,5 @@ Run this script to validate your code locally and auto fix/format the problems b
 
 # Usage
 
+You've got to manually download swimm for swimm validation.
 run from `infection_monkey` directory: `powershell .\ci_scripts\validate.ps1`
diff --git a/ci_scripts/isort.cfg b/ci_scripts/isort.cfg
index f9a62b0ed..d8651febd 100644
--- a/ci_scripts/isort.cfg
+++ b/ci_scripts/isort.cfg
@@ -1,5 +1,6 @@
-[settings]
-project=common,infection_monkey,monkey_island
-skip=./monkey/common/cloud/scoutsuite,./monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/rule_path_creators_list.py,./monkey/monkey_island/cc/ui
-line-length=120
-wrap-length=120
+[isort]
+
+# Possible options: https://pycqa.github.io/isort/docs/configuration/options/
+
+known_first_party=common,infection_monkey,monkey_island
+skip=monkey/common/cloud/scoutsuite,monkey/monkey_island/cc/services/zero_trust/scoutsuite/data_parsing/rule_path_building/rule_path_creators_list.py,monkey/monkey_island/cc/ui,monkey/common/cloud/scoutsuite
diff --git a/ci_scripts/validate.ps1 b/ci_scripts/validate.ps1
index 98a0ab4dd..d85da6a2a 100644
--- a/ci_scripts/validate.ps1
+++ b/ci_scripts/validate.ps1
@@ -1,4 +1,7 @@
 .\ci_scripts\validation-env\Scripts\activate.ps1
+$ErrorActionPreference = "Stop"
+python -m pip install -r monkey/monkey_island/requirements.txt
+python -m pip install -r monkey/infection_monkey/requirements.txt
 flake8 ./monkey --config ./ci_scripts/flake8_syntax_check.cfg
 flake8 ./monkey --exit-zero --config ./ci_scripts/flake8_linter_check.cfg | Out-File -FilePath .\ci_scripts\flake8_warnings.txt
 Get-Content -Path .\ci_scripts\flake8_warnings.txt
@@ -8,13 +11,29 @@ if ((Get-Item -Path .\ci_scripts\flake8_warnings.txt | Get-Content -Tail 1) -gt
     exit
 }
 python -m isort ./monkey -c --settings-file ./ci_scripts/isort.cfg
-# python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg
-python monkey_island/cc/environment/set_server_config.py testing
+if (!$?) {
+    $confirmation = Read-Host "Isort found errors. Do you want to attmpt to fix them automatically? (y/n)"
+    if ($confirmation -eq 'y') {
+        python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg
+    }
+}
+Push-Location -Path ./monkey
+python ./monkey_island/cc/environment/set_server_config.py testing
 python -m pytest
-Push-Location -Path .\monkey_island\cc\ui
-eslint ./src -c ./.eslintrc
+$lastCommandSucceeded = $?
+python ./monkey_island/cc/environment/set_server_config.py restore
+Pop-Location
 
+if (!$lastCommandSucceeded) {
+    exit
+}
+
+Push-Location -Path .\monkey\monkey_island\cc\ui
+eslint ./src -c ./.eslintrc
 Pop-Location
-Pop-Location
+
+swimm verify
+
+Write-Host "Script finished. Press any key to continue"
 $null = $Host.UI.RawUI.ReadKey('NoEcho,IncludeKeyDown');
 deactivate
diff --git a/monkey/pytest.ini b/monkey/pytest.ini
index 3596bf5f6..9b1766fc2 100644
--- a/monkey/pytest.ini
+++ b/monkey/pytest.ini
@@ -3,5 +3,5 @@ log_cli = 1
 log_cli_level = DEBUG
 log_cli_format = %(asctime)s [%(levelname)s] %(module)s.%(funcName)s.%(lineno)d: %(message)s
 log_cli_date_format=%H:%M:%S
-addopts = -v --capture=sys
+addopts = -v --capture=sys --ignore=common/cloud/scoutsuite
 norecursedirs = node_modules dist

From 6401826ee362e7ddaf2535b51eb997e8d8d47332 Mon Sep 17 00:00:00 2001
From: VakarisZ <vakarisz@yahoo.com>
Date: Fri, 6 Nov 2020 13:18:03 +0200
Subject: [PATCH 4/4] Fixed travis script bug, where flake8 didn't pick up its
 configuration file

---
 .travis.yml                                       |  4 ++--
 ...8_linter_check.cfg => flake8_linter_check.ini} |  0
 ci_scripts/flake8_syntax_check.cfg                |  9 ---------
 ci_scripts/flake8_syntax_check.ini                | 15 +++++++++++++++
 4 files changed, 17 insertions(+), 11 deletions(-)
 rename ci_scripts/{flake8_linter_check.cfg => flake8_linter_check.ini} (100%)
 delete mode 100644 ci_scripts/flake8_syntax_check.cfg
 create mode 100644 ci_scripts/flake8_syntax_check.ini

diff --git a/.travis.yml b/.travis.yml
index 3e4d3104a..9906b2c34 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -60,12 +60,12 @@ before_script:
 
 script:
 # Check Python code
-- flake8 ./monkey --config ./ci_scripts/flake8_syntax_check.config
+- flake8 ./monkey --config=./ci_scripts/flake8_syntax_check.ini
 
 ## Warn about linter issues.
 ### --exit-zero forces Flake8 to use the exit status code 0 even if there are errors, which means this will NOT fail the build.
 ### The output is redirected to a file.
-- flake8 ./monkey --exit-zero --config ./ci_scripts/flake8_linter_check.config > ./ci_scripts/flake8_warnings.txt
+- flake8 ./monkey --exit-zero --config=./ci_scripts/flake8_linter_check.ini > ./ci_scripts/flake8_warnings.txt
 ## Display the linter issues
 - cat ./ci_scripts/flake8_warnings.txt
 ## Make sure that we haven't increased the amount of warnings.
diff --git a/ci_scripts/flake8_linter_check.cfg b/ci_scripts/flake8_linter_check.ini
similarity index 100%
rename from ci_scripts/flake8_linter_check.cfg
rename to ci_scripts/flake8_linter_check.ini
diff --git a/ci_scripts/flake8_syntax_check.cfg b/ci_scripts/flake8_syntax_check.cfg
deleted file mode 100644
index 7171e3db6..000000000
--- a/ci_scripts/flake8_syntax_check.cfg
+++ /dev/null
@@ -1,9 +0,0 @@
-[flake8]
-
-## Check syntax errors and fail the build if any are found.
-exclude = ../monkey/monkey_island/cc/ui,
-          ../monkey/common/cloud
-select = E901,E999,F821,F822,F823
-count = True
-show-source = True
-statistics = True
diff --git a/ci_scripts/flake8_syntax_check.ini b/ci_scripts/flake8_syntax_check.ini
new file mode 100644
index 000000000..969379326
--- /dev/null
+++ b/ci_scripts/flake8_syntax_check.ini
@@ -0,0 +1,15 @@
+[flake8]
+
+## Check syntax errors and fail the build if any are found.
+exclude =
+    ../monkey/monkey_island/cc/ui,
+    ../monkey/common/cloud
+select =
+    E901,
+    E999,
+    F821,
+    F822,
+    F823
+count = True
+show-source = True
+statistics = True