From ecfc59cfd7883a0b7b77cc649e6bd01bf8e44996 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 1 Apr 2021 12:50:49 -0400 Subject: [PATCH 1/9] ci: fail the build if code is not formatted with black --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 509da86ac..8cbdb7ac9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: # Python - pip freeze - pip install -r monkey/monkey_island/requirements.txt # for unit tests -- pip install flake8 pytest pytest-cov dlint isort # for next stages +- pip install black flake8 pytest pytest-cov dlint isort # for next stages - pip install coverage # for code coverage - pip install -r monkey/infection_monkey/requirements.txt # for unit tests - pip install pipdeptree @@ -70,6 +70,9 @@ script: ## Check import order - python -m isort ./monkey --settings-file ./ci_scripts/isort.cfg +## Check that all python is properly formatted. Fail otherwise. +- python -m black --check . + ## Run unit tests and generate coverage data - cd monkey # This is our source dir - python -m pytest --cov=. # Have to use `python -m pytest` instead of `pytest` to add "{$builddir}/monkey/monkey" to sys.path. From 6be346267865239d419fa08cc58058da9e4a67a5 Mon Sep 17 00:00:00 2001 From: Shreya Date: Fri, 2 Apr 2021 16:10:00 +0530 Subject: [PATCH 2/9] Add configuration for black --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..97f50372b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 100 +target-version = ['py37'] From 6ffc527f316f7c895fea4f6a61020b7724b21f30 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Apr 2021 10:20:32 -0400 Subject: [PATCH 3/9] add pre-commit configuration pre-commit (https://pre-commit.com/) is a tool that helps you easily manage pre-commit hooks. We are using this largely for quality control by running tools like flake8 and black. --- .pre-commit-config.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..43e62e2f3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: + - repo: https://github.com/pycqa/isort + rev: 5.8.0 + hooks: + - id: isort + name: isort (python) + - id: isort + name: isort (cython) + types: [cython] + - id: isort + name: isort (pyi) + types: [pyi] + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.0 + hooks: + - id: flake8 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: detect-private-key + - id: end-of-file-fixer + - id: trailing-whitespace From 4966aaf213c0bebccfc686d36806d9ab8ad8e1ce Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Apr 2021 10:22:27 -0400 Subject: [PATCH 4/9] ci: freeze versions of certain packages in travis build Fix the versions of black, flake8, and isort in travis so that the same versions are being used to pass/fail the build as developers are using in their pre-commit hooks --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8cbdb7ac9..f5cf0e5ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: # Python - pip freeze - pip install -r monkey/monkey_island/requirements.txt # for unit tests -- pip install black flake8 pytest pytest-cov dlint isort # for next stages +- pip install black==20.8b1 flake8==3.9.0 pytest pytest-cov dlint isort==5.8.0 # for next stages - pip install coverage # for code coverage - pip install -r monkey/infection_monkey/requirements.txt # for unit tests - pip install pipdeptree From c1d4c7a0d2d68144c805180f04424c76fead5c1f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Apr 2021 12:14:49 -0400 Subject: [PATCH 5/9] docs: add documentation about pre-commit --- deployment_scripts/README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/deployment_scripts/README.md b/deployment_scripts/README.md index 4ee91b5b4..eaa973ff5 100644 --- a/deployment_scripts/README.md +++ b/deployment_scripts/README.md @@ -16,7 +16,7 @@ Invoke-WebRequest https://raw.githubusercontent.com/guardicore/monkey/develop/de This will download our deploy script. It's a good idea to read it quickly before executing it! -After downloading that script, execute it in `powershell`. +After downloading that script, execute it in `powershell`. The first argument is an empty directory (script can create one). The second argument is which branch you want to clone - by default, the script will check out the `develop` branch. Some example usages: @@ -63,3 +63,22 @@ After the `deploy_linux.sh` script completes, you can start the monkey island. cd infection_monkey/monkey ./monkey_island/linux/run.sh ``` + +## Pre-commit hooks + +Both the Linux and Windows deployment scrips will install and configure +[pre-commit](https://pre-commit.com/). Pre-commit is a multi-language package +manager for pre-commit hooks. It will run a set of checks when you attempt to +commit. If your commit does not pass all checks, it will be reformatted and/or +you'll be given a list of errors and warnings that need to be fixed before you +can commit. + +Our CI system runs the same checks when when pull requests are submitted. This +system may report that the build has failed if the pre-commit hooks have not +been run or all issues have not been resolved. + +### Manually installing pre-commit + +To install and configure pre-commit manually, run `pip install --user +pre-commit`. Next, go to the top level directory of this repository and run +`pre-commit install` Now, pre-commit will automatically run whenever you `git commit`. From 575b214c8ed16e3cf0a87fcd6fc50b0883ad727b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 5 Apr 2021 13:32:13 -0400 Subject: [PATCH 6/9] build: setup pre-commit in deploy_linux.sh --- deployment_scripts/deploy_linux.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deployment_scripts/deploy_linux.sh b/deployment_scripts/deploy_linux.sh index 408aa3148..940b763d5 100755 --- a/deployment_scripts/deploy_linux.sh +++ b/deployment_scripts/deploy_linux.sh @@ -23,6 +23,13 @@ log_message() { echo -e "DEPLOYMENT SCRIPT: $1" } +configure_precommit() { + $1 -m pip install --user pre-commit + pushd "$2" + $HOME/.local/bin/pre-commit install + popd +} + if is_root; then log_message "Please don't run this script as root" exit 1 @@ -225,5 +232,7 @@ fi sudo chmod +x "${INFECTION_MONKEY_DIR}/build_linux.sh" +configure_precommit ${python_cmd} ${monkey_home} + log_message "Deployment script finished." exit 0 From a5ce373cbf49f2aaf2bd9a220e964f6c22fcd947 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Tue, 6 Apr 2021 18:01:22 +0300 Subject: [PATCH 7/9] Added pre-commit installation to windows deployment script --- deployment_scripts/deploy_windows.ps1 | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/deployment_scripts/deploy_windows.ps1 b/deployment_scripts/deploy_windows.ps1 index 85a3f0698..bc781b546 100644 --- a/deployment_scripts/deploy_windows.ps1 +++ b/deployment_scripts/deploy_windows.ps1 @@ -9,6 +9,21 @@ param( [Bool] $agents = $true ) + +function Configure-precommit([String] $python_command) +{ + Write-Output "Installing pre-commit and setting up pre-commit hook" + python -m pip install pre-commit + if ($LastExitCode) { + exit + } + pre-commit install + if ($LastExitCode) { + exit + } + Write-Output "Pre-commit successfully installed" +} + function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName, [String] $branch = "develop") { Write-Output "Downloading to $monkey_home" @@ -119,6 +134,8 @@ function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName, $scoutsuiteRequirements = Join-Path -Path $monkey_home -ChildPath $SCOUTSUITE_DIR | Join-Path -ChildPath "\requirements.txt" & python -m pip install --user -r $scoutsuiteRequirements + Configure-precommit + $user_python_dir = cmd.exe /c 'py -m site --user-site' $user_python_dir = Join-Path (Split-Path $user_python_dir) -ChildPath "\Scripts" if (!($ENV:Path | Select-String -SimpleMatch $user_python_dir)) From 1cc2dc24ac198479c44e0b6ad16084eaddb1c301 Mon Sep 17 00:00:00 2001 From: VakarisZ Date: Wed, 7 Apr 2021 14:39:36 +0300 Subject: [PATCH 8/9] Fixed pre-commit to be installer in monkey dir during deployment scripts --- deployment_scripts/deploy_windows.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deployment_scripts/deploy_windows.ps1 b/deployment_scripts/deploy_windows.ps1 index bc781b546..28d34904c 100644 --- a/deployment_scripts/deploy_windows.ps1 +++ b/deployment_scripts/deploy_windows.ps1 @@ -10,9 +10,10 @@ param( $agents = $true ) -function Configure-precommit([String] $python_command) +function Configure-precommit([String] $git_repo_dir) { Write-Output "Installing pre-commit and setting up pre-commit hook" + Push-Location $git_repo_dir python -m pip install pre-commit if ($LastExitCode) { exit @@ -21,6 +22,7 @@ function Configure-precommit([String] $python_command) if ($LastExitCode) { exit } + Pop-Location Write-Output "Pre-commit successfully installed" } @@ -134,7 +136,7 @@ function Deploy-Windows([String] $monkey_home = (Get-Item -Path ".\").FullName, $scoutsuiteRequirements = Join-Path -Path $monkey_home -ChildPath $SCOUTSUITE_DIR | Join-Path -ChildPath "\requirements.txt" & python -m pip install --user -r $scoutsuiteRequirements - Configure-precommit + Configure-precommit($monkey_home) $user_python_dir = cmd.exe /c 'py -m site --user-site' $user_python_dir = Join-Path (Split-Path $user_python_dir) -ChildPath "\Scripts" From 3759c4d07a484df3b07c868c1b5b81990a9d8519 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 7 Apr 2021 08:33:11 -0400 Subject: [PATCH 9/9] build: fix small typo in deployment_scrips/README.md Co-authored-by: Shreya Malviya --- deployment_scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment_scripts/README.md b/deployment_scripts/README.md index eaa973ff5..ff767b33b 100644 --- a/deployment_scripts/README.md +++ b/deployment_scripts/README.md @@ -73,7 +73,7 @@ commit. If your commit does not pass all checks, it will be reformatted and/or you'll be given a list of errors and warnings that need to be fixed before you can commit. -Our CI system runs the same checks when when pull requests are submitted. This +Our CI system runs the same checks when pull requests are submitted. This system may report that the build has failed if the pre-commit hooks have not been run or all issues have not been resolved.