From f9f5861ae38fed2b39ab4c42b20a8fd1832688b7 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 16 Jun 2022 11:45:19 -0700 Subject: [PATCH 001/196] Island: Add marshmallow to dependencies --- monkey/monkey_island/Pipfile | 1 + monkey/monkey_island/Pipfile.lock | 98 +++++++++++++++++++------------ 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/monkey/monkey_island/Pipfile b/monkey/monkey_island/Pipfile index 8ed291b72..8357eeda1 100644 --- a/monkey/monkey_island/Pipfile +++ b/monkey/monkey_island/Pipfile @@ -28,6 +28,7 @@ cffi = "*" # Without explicit install: ModuleNotFoundError: No module named '_c pywin32-ctypes = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requirement on windows pywin32 = {version = "*", sys_platform = "== 'win32'"} # Lock file is not created with sys_platform win32 requirement if not explicitly specified pefile = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requirement on windows +marshmallow = "*" [dev-packages] virtualenv = ">=20.0.26" diff --git a/monkey/monkey_island/Pipfile.lock b/monkey/monkey_island/Pipfile.lock index 6b3af1043..1a8c8ed55 100644 --- a/monkey/monkey_island/Pipfile.lock +++ b/monkey/monkey_island/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "945e5bfe369347efb543a6c8f0ec2bb90df87284d0a65455944479ec2f78a5fa" + "sha256": "c50aee79fc30769d7d6bedd5e9fa749b9b0a1a659a2136977bfd242179bf9fb6" }, "pipfile-spec": 6, "requires": { @@ -72,11 +72,11 @@ }, "certifi": { "hashes": [ - "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", - "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], "markers": "python_version >= '3.6'", - "version": "==2022.5.18.1" + "version": "==2022.6.15" }, "cffi": { "hashes": [ @@ -139,7 +139,7 @@ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==2.0.12" }, "click": { @@ -152,11 +152,11 @@ }, "colorama": { "hashes": [ - "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", - "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da", + "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4" ], "markers": "platform_system == 'Windows'", - "version": "==0.4.4" + "version": "==0.4.5" }, "cryptography": { "hashes": [ @@ -338,7 +338,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==3.3" }, "importlib-metadata": { @@ -435,6 +435,14 @@ "markers": "python_version >= '3.7'", "version": "==2.1.1" }, + "marshmallow": { + "hashes": [ + "sha256:53a1e0ee69f79e1f3e80d17393b25cfc917eda52f859e8183b4af72c3390c1f1", + "sha256:a762c1d8b2bcb0e5c8e964850d03f9f3bffd6a12b626f3c14b9d6b1841999af5" + ], + "index": "pypi", + "version": "==3.16.0" + }, "mongoengine": { "hashes": [ "sha256:6e127f45f71c2bc5e72461ec297a0c20f04c3ee0bf6dd869e336226e325db6ef", @@ -479,6 +487,14 @@ "index": "pypi", "version": "==0.11.0" }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, "pefile": { "hashes": [ "sha256:a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b" @@ -688,6 +704,14 @@ ], "version": "==3.12.3" }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, "pyrsistent": { "hashes": [ "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c", @@ -762,11 +786,11 @@ }, "requests": { "hashes": [ - "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", - "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f", + "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b" ], "index": "pypi", - "version": "==2.27.1" + "version": "==2.28.0" }, "ring": { "hashes": [ @@ -785,11 +809,11 @@ }, "setuptools": { "hashes": [ - "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", - "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + "sha256:5a844ad6e190dccc67d6d7411d119c5152ce01f7c76be4d8a1eaa314501bba77", + "sha256:bf8a748ac98b09d32c9a64a995a6b25921c96cc5743c1efa82763ba80ff54e91" ], "markers": "python_version >= '3.7'", - "version": "==62.3.2" + "version": "==62.4.0" }, "six": { "hashes": [ @@ -928,11 +952,11 @@ }, "babel": { "hashes": [ - "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2", - "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13" + "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51", + "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb" ], "markers": "python_version >= '3.6'", - "version": "==2.10.1" + "version": "==2.10.3" }, "black": { "hashes": [ @@ -965,18 +989,18 @@ }, "certifi": { "hashes": [ - "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", - "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], "markers": "python_version >= '3.6'", - "version": "==2022.5.18.1" + "version": "==2022.6.15" }, "charset-normalizer": { "hashes": [ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==2.0.12" }, "click": { @@ -989,11 +1013,11 @@ }, "colorama": { "hashes": [ - "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", - "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da", + "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4" ], "markers": "platform_system == 'Windows'", - "version": "==0.4.4" + "version": "==0.4.5" }, "coverage": { "hashes": [ @@ -1085,7 +1109,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==3.3" }, "imagesize": { @@ -1291,11 +1315,11 @@ }, "requests": { "hashes": [ - "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", - "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f", + "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b" ], "index": "pypi", - "version": "==2.27.1" + "version": "==2.28.0" }, "requests-mock": { "hashes": [ @@ -1313,11 +1337,11 @@ }, "setuptools": { "hashes": [ - "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", - "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + "sha256:5a844ad6e190dccc67d6d7411d119c5152ce01f7c76be4d8a1eaa314501bba77", + "sha256:bf8a748ac98b09d32c9a64a995a6b25921c96cc5743c1efa82763ba80ff54e91" ], "markers": "python_version >= '3.7'", - "version": "==62.3.2" + "version": "==62.4.0" }, "six": { "hashes": [ @@ -1363,7 +1387,7 @@ "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==1.0.2" }, "sphinxcontrib-devhelp": { @@ -1371,7 +1395,7 @@ "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==1.0.2" }, "sphinxcontrib-htmlhelp": { @@ -1387,7 +1411,7 @@ "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==1.0.1" }, "sphinxcontrib-qthelp": { @@ -1395,7 +1419,7 @@ "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==1.0.3" }, "sphinxcontrib-serializinghtml": { @@ -1403,7 +1427,7 @@ "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==1.1.5" }, "toml": { From 660c1421c033f48a3388bfc78eb10d44daf9a852 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 16 Jun 2022 11:50:05 -0700 Subject: [PATCH 002/196] Agent: Add marshmallow to dependencies --- monkey/infection_monkey/Pipfile | 1 + monkey/infection_monkey/Pipfile.lock | 332 ++++++++++++++++----------- 2 files changed, 196 insertions(+), 137 deletions(-) diff --git a/monkey/infection_monkey/Pipfile b/monkey/infection_monkey/Pipfile index e5e53d39d..14c079054 100644 --- a/monkey/infection_monkey/Pipfile +++ b/monkey/infection_monkey/Pipfile @@ -24,6 +24,7 @@ pywin32-ctypes = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requ pywin32 = {version = "*", sys_platform = "== 'win32'"} # Lock file is not created with sys_platform win32 requirement if not explicitly specified pefile = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requirement on windows paramiko = {editable = true, ref = "2.10.3.dev1", git = "https://github.com/VakarisZ/paramiko.git"} +marshmallow = "*" [dev-packages] ldap3 = "*" diff --git a/monkey/infection_monkey/Pipfile.lock b/monkey/infection_monkey/Pipfile.lock index 879739998..eb7bd1878 100644 --- a/monkey/infection_monkey/Pipfile.lock +++ b/monkey/infection_monkey/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c1c28510b728242624129b39bd9ace9ba2629061bf64226cb43f1f857fc87212" + "sha256": "fc7e8d826d41fa764926d16a224cfd4a583c1a356b2a61af7aa7231256b79255" }, "pipfile-spec": 6, "requires": { @@ -71,26 +71,28 @@ }, "bcrypt": { "hashes": [ - "sha256:56e5da069a76470679f312a7d3d23deb3ac4519991a0361abc11da837087b61d", - "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29", - "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7", - "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34", - "sha256:a0584a92329210fcd75eb8a3250c5a941633f8bfaf2a18f81009b097732839b7", - "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55", - "sha256:b589229207630484aefe5899122fb938a5b017b0f4349f769b8c13e78d99a8fd", - "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6", - "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1", - "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d" + "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521", + "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb", + "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e", + "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26", + "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a", + "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e", + "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa", + "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129", + "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb", + "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40", + "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa" ], "markers": "python_version >= '3.6'", - "version": "==3.2.0" + "version": "==3.2.2" }, "certifi": { "hashes": [ - "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", - "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "version": "==2021.10.8" + "markers": "python_version >= '3.6'", + "version": "==2022.6.15" }, "cffi": { "hashes": [ @@ -160,24 +162,24 @@ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], - "markers": "python_version >= '3'", + "markers": "python_full_version >= '3.5.0'", "version": "==2.0.12" }, "click": { "hashes": [ - "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e", - "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72" + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" ], "markers": "python_version >= '3.7'", - "version": "==8.1.2" + "version": "==8.1.3" }, "colorama": { "hashes": [ - "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", - "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da", + "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==0.4.4" + "version": "==0.4.5" }, "constantly": { "hashes": [ @@ -188,45 +190,47 @@ }, "cryptography": { "hashes": [ - "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b", - "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51", - "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7", - "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d", - "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6", - "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29", - "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9", - "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf", - "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815", - "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf", - "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85", - "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77", - "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86", - "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb", - "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e", - "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0", - "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3", - "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84", - "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2", - "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6" + "sha256:093cb351031656d3ee2f4fa1be579a8c69c754cf874206be1d4cf3b542042804", + "sha256:0cc20f655157d4cfc7bada909dc5cc228211b075ba8407c46467f63597c78178", + "sha256:1b9362d34363f2c71b7853f6251219298124aa4cc2075ae2932e64c91a3e2717", + "sha256:1f3bfbd611db5cb58ca82f3deb35e83af34bb8cf06043fa61500157d50a70982", + "sha256:2bd1096476aaac820426239ab534b636c77d71af66c547b9ddcd76eb9c79e004", + "sha256:31fe38d14d2e5f787e0aecef831457da6cec68e0bb09a35835b0b44ae8b988fe", + "sha256:3b8398b3d0efc420e777c40c16764d6870bcef2eb383df9c6dbb9ffe12c64452", + "sha256:3c81599befb4d4f3d7648ed3217e00d21a9341a9a688ecdd615ff72ffbed7336", + "sha256:419c57d7b63f5ec38b1199a9521d77d7d1754eb97827bbb773162073ccd8c8d4", + "sha256:46f4c544f6557a2fefa7ac8ac7d1b17bf9b647bd20b16decc8fbcab7117fbc15", + "sha256:471e0d70201c069f74c837983189949aa0d24bb2d751b57e26e3761f2f782b8d", + "sha256:59b281eab51e1b6b6afa525af2bd93c16d49358404f814fe2c2410058623928c", + "sha256:731c8abd27693323b348518ed0e0705713a36d79fdbd969ad968fbef0979a7e0", + "sha256:95e590dd70642eb2079d280420a888190aa040ad20f19ec8c6e097e38aa29e06", + "sha256:a68254dd88021f24a68b613d8c51d5c5e74d735878b9e32cc0adf19d1f10aaf9", + "sha256:a7d5137e556cc0ea418dca6186deabe9129cee318618eb1ffecbd35bee55ddc1", + "sha256:aeaba7b5e756ea52c8861c133c596afe93dd716cbcacae23b80bc238202dc023", + "sha256:dc26bb134452081859aa21d4990474ddb7e863aa39e60d1592800a8865a702de", + "sha256:e53258e69874a306fcecb88b7534d61820db8a98655662a3dd2ec7f1afd9132f", + "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181", + "sha256:f224ad253cc9cea7568f49077007d2263efa57396a2f2f78114066fd54b5c68e", + "sha256:f8ec91983e638a9bcd75b39f1396e5c0dc2330cbd9ce4accefe68717e6779e0a" ], "markers": "python_version >= '3.6'", - "version": "==36.0.2" + "version": "==37.0.2" }, "dnspython": { "hashes": [ "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e", "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f" ], - "markers": "python_version >= '3.6' and python_version < '4.0'", + "markers": "python_version >= '3.6' and python_version < '4'", "version": "==2.2.1" }, "flask": { "hashes": [ - "sha256:8a4cf32d904cf5621db9f0c9fbcd7efabf3003f22a04e4d0ce790c7137ec5264", - "sha256:a8c9bd3e558ec99646d177a9739c41df1ded0629480b4c8d2975412f3c9519c8" + "sha256:315ded2ddf8a6281567edb27393010fe3406188bafbfe65a3339d5787d89e477", + "sha256:fad5b446feb0d6db6aec0c3184d16a8c1f6c3e464b511649c8918a9be100b4fe" ], "markers": "python_version >= '3.7'", - "version": "==2.1.1" + "version": "==2.1.2" }, "future": { "hashes": [ @@ -247,23 +251,23 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3'", + "markers": "python_full_version >= '3.5.0'", "version": "==3.3" }, "impacket": { "hashes": [ - "sha256:18d557d387f4914fafa739813b9172bc3f8bd9c036e93bf589a8e0ebb7304bba" + "sha256:b8eb020a2cbb47146669cfe31c64bb2e7d6499d049c493d6418b9716f5c74583" ], "index": "pypi", - "version": "==0.9.24" + "version": "==0.10.0" }, "importlib-metadata": { "hashes": [ - "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6", - "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539" + "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700", + "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec" ], "markers": "python_version < '3.8'", - "version": "==4.11.3" + "version": "==4.11.4" }, "incremental": { "hashes": [ @@ -290,11 +294,11 @@ }, "jinja2": { "hashes": [ - "sha256:539835f51a74a69f41b848a9645dbdc35b4f20a3b601e2d9a7e22947b15ff119", - "sha256:640bed4bb501cbd17194b3cace1dc2126f5b619cf068a726b98192a0fde74ae9" + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], "markers": "python_version >= '3.7'", - "version": "==3.1.1" + "version": "==3.1.2" }, "ldap3": { "hashes": [ @@ -368,6 +372,14 @@ "markers": "python_version >= '3.7'", "version": "==2.1.1" }, + "marshmallow": { + "hashes": [ + "sha256:53a1e0ee69f79e1f3e80d17393b25cfc917eda52f859e8183b4af72c3390c1f1", + "sha256:a762c1d8b2bcb0e5c8e964850d03f9f3bffd6a12b626f3c14b9d6b1841999af5" + ], + "index": "pypi", + "version": "==3.16.0" + }, "minidump": { "hashes": [ "sha256:6a9d2152f76ae633c609e09b48b42f55bd5a6b65f920dbbec756e5d9134a6201", @@ -442,6 +454,14 @@ ], "version": "==1.3.0" }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, "paramiko": { "editable": true, "git": "https://github.com/VakarisZ/paramiko.git", @@ -456,10 +476,11 @@ }, "pefile": { "hashes": [ - "sha256:344a49e40a94e10849f0fe34dddc80f773a12b40675bf2f7be4b8be578bdd94a" + "sha256:a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b" ], + "index": "pypi", "markers": "sys_platform == 'win32'", - "version": "==2021.9.3" + "version": "==2022.5.30" }, "prompt-toolkit": { "hashes": [ @@ -471,41 +492,41 @@ }, "psutil": { "hashes": [ - "sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5", - "sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a", - "sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4", - "sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841", - "sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d", - "sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d", - "sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0", - "sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845", - "sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf", - "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b", - "sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07", - "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618", - "sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2", - "sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd", - "sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666", - "sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce", - "sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3", - "sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d", - "sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25", - "sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492", - "sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b", - "sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d", - "sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2", - "sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203", - "sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2", - "sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94", - "sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9", - "sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64", - "sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56", - "sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3", - "sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c", - "sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3" + "sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685", + "sha256:0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc", + "sha256:0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36", + "sha256:19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1", + "sha256:20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329", + "sha256:28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81", + "sha256:29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de", + "sha256:3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4", + "sha256:32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574", + "sha256:3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237", + "sha256:44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22", + "sha256:4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b", + "sha256:56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0", + "sha256:57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954", + "sha256:58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021", + "sha256:645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537", + "sha256:799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87", + "sha256:79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0", + "sha256:91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc", + "sha256:9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af", + "sha256:944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4", + "sha256:a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453", + "sha256:a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689", + "sha256:abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8", + "sha256:b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680", + "sha256:b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e", + "sha256:c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9", + "sha256:d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b", + "sha256:db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d", + "sha256:e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2", + "sha256:f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5", + "sha256:fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676" ], "index": "pypi", - "version": "==5.9.0" + "version": "==5.9.1" }, "pyasn1": { "hashes": [ @@ -603,11 +624,11 @@ }, "pyinstaller-hooks-contrib": { "hashes": [ - "sha256:9765e68552803327d58f6c5eca970bb245b7cdf073e2f912a2a3cb50360bc2d8", - "sha256:9fa4ca03d058cba676c3cc16005076ce6a529f144c08b87c69998625fbd84e0a" + "sha256:5fdb97dcae177955db7ab27840cba97b89dc0c7f4fd9142bba0f9b8d8df85c48", + "sha256:6675634279cfe9e475580fb310c3d557037baefb065e6cb5a69a124361b926fd" ], "markers": "python_version >= '3.7'", - "version": "==2022.3" + "version": "==2022.7" }, "pymssql": { "hashes": [ @@ -670,11 +691,11 @@ }, "pyparsing": { "hashes": [ - "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954", - "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06" + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" ], "markers": "python_full_version >= '3.6.8'", - "version": "==3.0.8" + "version": "==3.0.9" }, "pypsrp": { "hashes": [ @@ -694,48 +715,66 @@ }, "pysmb": { "hashes": [ - "sha256:298605b8f467ce15b412caaf9af331c135e88fa2172333af14b1b2916361cb6b" + "sha256:3b07db16217465039d0c25694c0705b83663ca82259e209f3566d577536a7395" ], "index": "pypi", - "version": "==1.2.7" + "version": "==1.2.8" }, "pyspnego": { "hashes": [ - "sha256:05438a4e3e1526134bc2d72213417a06a2c3010f5b7271f3122e635e523c3790", - "sha256:12e4da1cbbbd645c0624699a1d99f734161cb9095e9f1fc1c1982ed1b7a44abe", - "sha256:185e0c576cde30d8853d9ea1d69c32cb93e98423934263d6c067bec7adc7dc4f", - "sha256:3361027e7e86de6b784791e09a7b2ba73d06c0be40f027a7be09e45fc92325a5", - "sha256:4971fb166dc9821c98d31d698722d48d0066f1bc63beff8bf3d2a2e60fe507d1", - "sha256:58d352d901baab754f63cb0da790c1f798605eb634f7f922df9bb6822d3de3c5", - "sha256:77b7c75bed737f24989aab453b9b8cd1c1512dfc5bed7a303a1cb1156fd59959", - "sha256:adf2f3e09bc4751c06fab1fedfe734af7f232d79927c753d8981f75a25f791ec", - "sha256:c6993ee6bcfe0036d6246324fcb7975daed858a476bfc7bf1d9334911d3dfca2", - "sha256:e21fc7283caa16761d46bea54e78cbfe3177c21e3b2d17d9ef213edcd86e1250", - "sha256:f05f1a6316a9baeaef243c9420d995c3dc34cfc91841f17db0c793e3fe557728", - "sha256:fe8b2a0d7468d904c61ae63275f8234eb055767aaaba66f6d58d86f47a25aa8e" + "sha256:1fed228edc4b1730844da8237b90489be28c55681cf3934cd04fceb2253e55bf", + "sha256:25fbc90fc6bd16881480316739bab820cc91364765e46340da17f861f89691f1", + "sha256:274b3a2d37e45ad4567bc5754be04b5fefad3f7cdea7d205f739d8a26b5a9189", + "sha256:36db7ec38023a23a545114dfd23825639571f135c72fb4b13a1ed559a0a4d93c", + "sha256:3b1ff3c1d5588b66f8e4ebafa3079a7bf0bdcc6fb144b944c5a101e688a5a280", + "sha256:4b9bda51bd964f40322aa1b33dcfc5d68f23b0680b4b5158175f2e9a04119aa9", + "sha256:5d6d91e35ee63a5de30eb70716bf25274bf16c2c472b046dd21fad60fe63b0b6", + "sha256:7562bc640bf402bb2849f325b0bb41260bd2c0cb06e38b9a8c6f7021b452c873", + "sha256:9c5bdb9f0207e2ce9e5410ee2205bf016755712018534c711ae6c1daff7fa7db", + "sha256:a5c135d20819db3c48f65054d648317f369a61b7b22dc17b9e5ec9c0169541a0", + "sha256:bd95633e7dce69e267579bdbe992fc081a14310236b4e84c3d179b1cf6439ca5", + "sha256:eb41b970dbda0dfe07b1da6fc83fe9f534a66d8dea38c06c0155377697407d9a" ], "markers": "python_version >= '3.6'", - "version": "==0.5.1" + "version": "==0.5.2" }, "pywin32": { - "sys_platform": "== 'win32'", - "version": "==303" + "hashes": [ + "sha256:25746d841201fd9f96b648a248f731c1dec851c9a08b8e33da8b56148e4c65cc", + "sha256:30c53d6ce44c12a316a06c153ea74152d3b1342610f1b99d40ba2795e5af0269", + "sha256:3c7bacf5e24298c86314f03fa20e16558a4e4138fc34615d7de4070c23e65af3", + "sha256:4f32145913a2447736dad62495199a8e280a77a0ca662daa2332acf849f0be48", + "sha256:7ffa0c0fa4ae4077e8b8aa73800540ef8c24530057768c3ac57c609f99a14fd4", + "sha256:94037b5259701988954931333aafd39cf897e990852115656b014ce72e052e96", + "sha256:bb2ea2aa81e96eee6a6b79d87e1d1648d3f8b87f9a64499e0b92b30d141e76df", + "sha256:be253e7b14bc601718f014d2832e4c18a5b023cbe72db826da63df76b77507a1", + "sha256:cbbe34dad39bdbaa2889a424d28752f1b4971939b14b1bb48cbf0182a3bcfc43", + "sha256:d24a3382f013b21aa24a5cfbfad5a2cd9926610c0affde3e8ab5b3d7dbcf4ac9", + "sha256:d3ee45adff48e0551d1aa60d2ec066fec006083b791f5c3527c40cd8aefac71f", + "sha256:de9827c23321dcf43d2f288f09f3b6d772fee11e809015bdae9e69fe13213988", + "sha256:ead865a2e179b30fb717831f73cf4373401fc62fbc3455a0889a7ddac848f83e", + "sha256:f64c0377cf01b61bd5e76c25e1480ca8ab3b73f0c4add50538d332afdf8f69c5" + ], + "index": "pypi", + "markers": "sys_platform == 'win32'", + "version": "==304" }, "pywin32-ctypes": { "hashes": [ "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942", "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98" ], + "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==0.2.0" }, "requests": { "hashes": [ - "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", - "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f", + "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b" ], "index": "pypi", - "version": "==2.27.1" + "version": "==2.28.0" }, "service-identity": { "hashes": [ @@ -746,11 +785,11 @@ }, "setuptools": { "hashes": [ - "sha256:26ead7d1f93efc0f8c804d9fafafbe4a44b179580a7105754b245155f9af05a8", - "sha256:47c7b0c0f8fc10eec4cf1e71c6fdadf8decaa74ffa087e68cd1c20db7ad6a592" + "sha256:5a844ad6e190dccc67d6d7411d119c5152ce01f7c76be4d8a1eaa314501bba77", + "sha256:bf8a748ac98b09d32c9a64a995a6b25921c96cc5743c1efa82763ba80ff54e91" ], "markers": "python_version >= '3.7'", - "version": "==62.1.0" + "version": "==62.4.0" }, "six": { "hashes": [ @@ -770,30 +809,48 @@ }, "twisted": { "extras": [ - + "tls" ], "hashes": [ "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680", "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2" ], - "index": "pypi", + "markers": "python_full_version >= '3.6.7'", "version": "==22.4.0" }, + "twisted-iocpsupport": { + "hashes": [ + "sha256:306becd6e22ab6e8e4f36b6bdafd9c92e867c98a5ce517b27fdd27760ee7ae41", + "sha256:3c61742cb0bc6c1ac117a7e5f422c129832f0c295af49e01d8a6066df8cfc04d", + "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9", + "sha256:7d972cfa8439bdcb35a7be78b7ef86d73b34b808c74be56dfa785c8a93b851bf", + "sha256:81b3abe3527b367da0220482820cb12a16c661672b7bcfcde328902890d63323", + "sha256:851b3735ca7e8102e661872390e3bce88f8901bece95c25a0c8bb9ecb8a23d32", + "sha256:985c06a33f5c0dae92c71a036d1ea63872ee86a21dd9b01e1f287486f15524b4", + "sha256:9dbb8823b49f06d4de52721b47de4d3b3026064ef4788ce62b1a21c57c3fff6f", + "sha256:b435857b9efcbfc12f8c326ef0383f26416272260455bbca2cd8d8eca470c546", + "sha256:b76b4eed9b27fd63ddb0877efdd2d15835fdcb6baa745cb85b66e5d016ac2878", + "sha256:b9fed67cf0f951573f06d560ac2f10f2a4bbdc6697770113a2fc396ea2cb2565", + "sha256:bf4133139d77fc706d8f572e6b7d82871d82ec7ef25d685c2351bdacfb701415" + ], + "markers": "platform_system == 'Windows'", + "version": "==1.0.2" + }, "typing-extensions": { "hashes": [ - "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", - "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" + "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", + "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" ], "index": "pypi", - "version": "==4.1.1" + "version": "==4.2.0" }, "unicrypto": { "hashes": [ - "sha256:69240f260493346861e639697e2fda245f14656c7df20ae6d9e6b1bb36acb29e", - "sha256:822bbf18ca6bc17f98c3029470bd8898e51fcb6a7716f6b1c95ed0bf9e0e4da5" + "sha256:0487f9dd9009c326ee9531a79412ae18ad673425a1c800d64947b96fdeb04cdf", + "sha256:fab49ee41926bb31be49552aa135f7aedc04436b49c8fe326d7b6a823810575e" ], "markers": "python_version >= '3.6'", - "version": "==0.0.5" + "version": "==0.0.8" }, "urllib3": { "hashes": [ @@ -812,27 +869,27 @@ }, "werkzeug": { "hashes": [ - "sha256:3c5493ece8268fecdcdc9c0b112211acd006354723b280d643ec732b6d4063d6", - "sha256:f8e89a20aeabbe8a893c24a461d3ee5dad2123b05cc6abd73ceed01d39c3ae74" + "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6", + "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255" ], "markers": "python_version >= '3.7'", - "version": "==2.1.1" + "version": "==2.1.2" }, "winacl": { "hashes": [ - "sha256:187b4394ef247806f50e1d8320bdb9e33ad1f759d9e61e2e391b97b9adf5f58a", - "sha256:949a66b0f46015c8cf8d9c1bfdb3a5174e70c28ae1b096eb778bc2983ea7ce50" + "sha256:1bac567a9d21300082aa2246bb0f94a591fca8e218e163bab18df0e32eefea06", + "sha256:a0b76a327fd337d5ee707ccff95222e6b6ecaa6d887613a1c3d3437ce0be1d4d" ], "markers": "python_version >= '3.6'", - "version": "==0.1.2" + "version": "==0.1.3" }, "winsspi": { "hashes": [ - "sha256:a2ad9c0f6d70f6e0e0d1f54b8582054c62d8a09f346b5ccaf55da68628ca10e1", - "sha256:a64624a25fc2d3663a2c5376c5291f3c7531e9c8051571de9ca9db8bf25746c2" + "sha256:2f5a8d2c4b9f459144426909e26a74e550512e23b6cf9af52c2a00003c7c3fdb", + "sha256:59b7c7595f91414528cfd80c6cfc77ec6f5e4e28185ebd6418f8368ddc7aca82" ], "markers": "python_version >= '3.6'", - "version": "==0.0.9" + "version": "==0.0.10" }, "winsys-3.x": { "hashes": [ @@ -846,6 +903,7 @@ "sha256:1d6b085e5c445141c475476000b661f60fff1aaa19f76bf82b7abb92e0ff4942", "sha256:b6a6be5711b1b6c8d55bda7a8befd75c48c12b770b9d227d31c1737dbf0d40a6" ], + "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==1.5.1" }, From c79f62e682135ad233f988654d46c6d32021be18 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 14:57:50 -0400 Subject: [PATCH 003/196] Common: Add PluginConfiguration --- monkey/common/agent_configuration | 0 monkey/common/configuration/__init__.py | 1 + .../configuration/agent_configuration.py | 21 +++++++++++++++++++ .../common/test_agent_configuration.py | 12 +++++++++++ vulture_allowlist.py | 6 ++++++ 5 files changed, 40 insertions(+) create mode 100644 monkey/common/agent_configuration create mode 100644 monkey/common/configuration/__init__.py create mode 100644 monkey/common/configuration/agent_configuration.py create mode 100644 monkey/tests/unit_tests/common/test_agent_configuration.py diff --git a/monkey/common/agent_configuration b/monkey/common/agent_configuration new file mode 100644 index 000000000..e69de29bb diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py new file mode 100644 index 000000000..a40b7a97e --- /dev/null +++ b/monkey/common/configuration/__init__.py @@ -0,0 +1 @@ +from .agent_configuration import PluginConfiguration, PluginConfigurationSchema diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py new file mode 100644 index 000000000..fbcc4872d --- /dev/null +++ b/monkey/common/configuration/agent_configuration.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass + +from marshmallow import RAISE, Schema, fields, post_load + + +@dataclass(frozen=True) +class PluginConfiguration: + name: str + options: dict + + +class PluginConfigurationSchema(Schema): + class Meta: + unknown = RAISE + + name = fields.Str() + options = fields.Mapping() + + @post_load + def make_plugin_configuration(self, data, **kwargs): + return PluginConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py new file mode 100644 index 000000000..53555680d --- /dev/null +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -0,0 +1,12 @@ +from common.configuration import PluginConfigurationSchema + + +def test_build_plugin_configuration(): + name = "bond" + options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} + pcs = PluginConfigurationSchema() + + pc = pcs.load({"name": name, "options": options}) + + assert pc.name == name + assert pc.options == options diff --git a/vulture_allowlist.py b/vulture_allowlist.py index ab09127b3..73961d4d7 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -183,6 +183,12 @@ architecture # unused variable (monkey/infection_monkey/exploit/caching_agent_r response_code # unused variable (monkey/monkey_island/cc/services/aws/aws_command_runner.py:26) +# Agent Configuration +Meta # unused class(monkey/common/configuration/agent_configuration.py:13) +unknown # unused variable(monkey/common/configuration/agent_configuration.py:14) +make_plugin_configuration # unused method(monkey/common/configuration/agent_configuration.py:19) + + # TODO DELETE AFTER RESOURCE REFACTORING NetworkMap Arc.dst_machine From bdad41057ca46ead6e1f72881fab8cc305462e50 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 15:45:37 -0400 Subject: [PATCH 004/196] Common: Add CustomPBAConfiguration --- monkey/common/configuration/__init__.py | 7 +++- .../configuration/agent_configuration.py | 22 +++++++++++++ .../common/test_agent_configuration.py | 32 ++++++++++++++++--- vulture_allowlist.py | 1 + 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index a40b7a97e..8d71636a4 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1 +1,6 @@ -from .agent_configuration import PluginConfiguration, PluginConfigurationSchema +from .agent_configuration import ( + PluginConfiguration, + PluginConfigurationSchema, + CustomPBAConfiguration, + CustomPBAConfigurationSchema, +) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index fbcc4872d..f89e27e69 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -3,6 +3,28 @@ from dataclasses import dataclass from marshmallow import RAISE, Schema, fields, post_load +@dataclass(frozen=True) +class CustomPBAConfiguration: + linux_command: str + linux_filename: str + windows_command: str + windows_filename: str + + +class CustomPBAConfigurationSchema(Schema): + class Meta: + unknown = RAISE + + linux_command = fields.Str() + linux_filename = fields.Str() + windows_command = fields.Str() + windows_filename = fields.Str() + + @post_load + def make_custom_pba_configuration(self, data, **kwargs): + return CustomPBAConfiguration(**data) + + @dataclass(frozen=True) class PluginConfiguration: name: str diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 53555680d..38c744b17 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,12 +1,34 @@ -from common.configuration import PluginConfigurationSchema +from common.configuration import CustomPBAConfigurationSchema, PluginConfigurationSchema def test_build_plugin_configuration(): name = "bond" options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} - pcs = PluginConfigurationSchema() + schema = PluginConfigurationSchema() - pc = pcs.load({"name": name, "options": options}) + config = schema.load({"name": name, "options": options}) - assert pc.name == name - assert pc.options == options + assert config.name == name + assert config.options == options + + +def test_custom_pba_configuration_schema(): + linux_command = "a" + linux_filename = "b" + windows_command = "c" + windows_filename = "d" + schema = CustomPBAConfigurationSchema() + + config = schema.load( + { + "linux_command": linux_command, + "linux_filename": linux_filename, + "windows_command": windows_command, + "windows_filename": windows_filename, + } + ) + + assert config.linux_command == linux_command + assert config.linux_filename == linux_filename + assert config.windows_command == windows_command + assert config.windows_filename == windows_filename diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 73961d4d7..9e3522f86 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -187,6 +187,7 @@ response_code # unused variable (monkey/monkey_island/cc/services/aws/aws_comma Meta # unused class(monkey/common/configuration/agent_configuration.py:13) unknown # unused variable(monkey/common/configuration/agent_configuration.py:14) make_plugin_configuration # unused method(monkey/common/configuration/agent_configuration.py:19) +make_custom_pba_configuration # unused method(monkey/common/configuration/agent_configuration.py:34) # TODO DELETE AFTER RESOURCE REFACTORING From 88dbf4feb7d5f383f416e7f9c38737f1b7e2ea3b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 15:47:52 -0400 Subject: [PATCH 005/196] Common: Use Dict instead of dict in PluginConfiguration --- monkey/common/configuration/agent_configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index f89e27e69..b52b7cbc0 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Dict from marshmallow import RAISE, Schema, fields, post_load @@ -28,7 +29,7 @@ class CustomPBAConfigurationSchema(Schema): @dataclass(frozen=True) class PluginConfiguration: name: str - options: dict + options: Dict class PluginConfigurationSchema(Schema): From 9d73252ff5bb947b651e09c65090d297090b94c7 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 15:53:34 -0400 Subject: [PATCH 006/196] Common: Add ExploiterConfiguration --- monkey/common/configuration/__init__.py | 2 ++ .../configuration/agent_configuration.py | 22 ++++++++++++++++++- .../common/test_agent_configuration.py | 19 +++++++++++++++- vulture_allowlist.py | 1 + 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 8d71636a4..ed2b3cbc9 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -3,4 +3,6 @@ from .agent_configuration import ( PluginConfigurationSchema, CustomPBAConfiguration, CustomPBAConfigurationSchema, + ExploiterConfiguration, + ExploiterConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index b52b7cbc0..95592ecb7 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict +from typing import Dict, List from marshmallow import RAISE, Schema, fields, post_load @@ -42,3 +42,23 @@ class PluginConfigurationSchema(Schema): @post_load def make_plugin_configuration(self, data, **kwargs): return PluginConfiguration(**data) + + +@dataclass(frozen=True) +class ExploiterConfiguration: + name: str + options: Dict + supported_os: List[str] + + +class ExploiterConfigurationSchema(Schema): + class Meta: + unknown = RAISE + + name = fields.Str() + options = fields.Mapping() + supported_os = fields.List(fields.Str()) + + @post_load + def make_exploiter_configuration(self, data, **kwargs): + return ExploiterConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 38c744b17..7854a48ea 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,4 +1,8 @@ -from common.configuration import CustomPBAConfigurationSchema, PluginConfigurationSchema +from common.configuration import ( + CustomPBAConfigurationSchema, + ExploiterConfigurationSchema, + PluginConfigurationSchema, +) def test_build_plugin_configuration(): @@ -32,3 +36,16 @@ def test_custom_pba_configuration_schema(): assert config.linux_filename == linux_filename assert config.windows_command == windows_command assert config.windows_filename == windows_filename + + +def test_exploiter_configuration_schema(): + name = "bond" + options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} + supported_os = ["linux", "windows"] + schema = ExploiterConfigurationSchema() + + config = schema.load({"name": name, "options": options, "supported_os": supported_os}) + + assert config.name == name + assert config.options == options + assert config.supported_os == supported_os diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 9e3522f86..dec1aa7dd 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -188,6 +188,7 @@ Meta # unused class(monkey/common/configuration/agent_configuration.py:13) unknown # unused variable(monkey/common/configuration/agent_configuration.py:14) make_plugin_configuration # unused method(monkey/common/configuration/agent_configuration.py:19) make_custom_pba_configuration # unused method(monkey/common/configuration/agent_configuration.py:34) +make_exploiter_configuration # unused method(monkey/common/configuration/agent_configuration.py:62) # TODO DELETE AFTER RESOURCE REFACTORING From 39e18b9c73641239b4cd4cbbd261f583d62574e4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 19:31:52 -0400 Subject: [PATCH 007/196] Common: Add OperatingSystems enum Enough is enough! I have had it with these monkey-fighting strings on this Monday to Friday plane! Everybody strap in. Seriously, it's time to stop using strings all over the place to identify the OS. An Enum is a better, more strongly-typed solution. --- monkey/common/__init__.py | 1 + monkey/common/operating_systems.py | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 monkey/common/operating_systems.py diff --git a/monkey/common/__init__.py b/monkey/common/__init__.py index 9c6f01004..c10545c7f 100644 --- a/monkey/common/__init__.py +++ b/monkey/common/__init__.py @@ -2,3 +2,4 @@ Used for a common things between agent and island """ from .di_container import DIContainer, UnregisteredTypeError +from .operating_systems import OperatingSystems diff --git a/monkey/common/operating_systems.py b/monkey/common/operating_systems.py new file mode 100644 index 000000000..67f67da81 --- /dev/null +++ b/monkey/common/operating_systems.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class OperatingSystems(Enum): + LINUX = "linux" + WINDOWS = "windows" From 526fe24aa66d9948fe41cbb8d100af231ff4db50 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 16 Jun 2022 12:57:57 -0700 Subject: [PATCH 008/196] Island: Add marshmallow-enum to dependencies --- monkey/monkey_island/Pipfile | 1 + monkey/monkey_island/Pipfile.lock | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/monkey/monkey_island/Pipfile b/monkey/monkey_island/Pipfile index 8357eeda1..fbc8b16ae 100644 --- a/monkey/monkey_island/Pipfile +++ b/monkey/monkey_island/Pipfile @@ -29,6 +29,7 @@ pywin32-ctypes = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requ pywin32 = {version = "*", sys_platform = "== 'win32'"} # Lock file is not created with sys_platform win32 requirement if not explicitly specified pefile = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requirement on windows marshmallow = "*" +marshmallow-enum = "*" [dev-packages] virtualenv = ">=20.0.26" diff --git a/monkey/monkey_island/Pipfile.lock b/monkey/monkey_island/Pipfile.lock index 1a8c8ed55..c3e59c1d5 100644 --- a/monkey/monkey_island/Pipfile.lock +++ b/monkey/monkey_island/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c50aee79fc30769d7d6bedd5e9fa749b9b0a1a659a2136977bfd242179bf9fb6" + "sha256": "04efa1f593acdfcdc48b7089108921a46421acbacec80d8a664ec674b221dd4b" }, "pipfile-spec": 6, "requires": { @@ -139,7 +139,7 @@ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==2.0.12" }, "click": { @@ -338,7 +338,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==3.3" }, "importlib-metadata": { @@ -443,6 +443,14 @@ "index": "pypi", "version": "==3.16.0" }, + "marshmallow-enum": { + "hashes": [ + "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58", + "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072" + ], + "index": "pypi", + "version": "==1.5.1" + }, "mongoengine": { "hashes": [ "sha256:6e127f45f71c2bc5e72461ec297a0c20f04c3ee0bf6dd869e336226e325db6ef", @@ -1000,7 +1008,7 @@ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==2.0.12" }, "click": { @@ -1109,7 +1117,7 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3.5'", + "markers": "python_full_version >= '3.5.0'", "version": "==3.3" }, "imagesize": { @@ -1387,7 +1395,7 @@ "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-devhelp": { @@ -1395,7 +1403,7 @@ "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-htmlhelp": { @@ -1411,7 +1419,7 @@ "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==1.0.1" }, "sphinxcontrib-qthelp": { @@ -1419,7 +1427,7 @@ "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "sphinxcontrib-serializinghtml": { @@ -1427,7 +1435,7 @@ "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952" ], - "markers": "python_full_version >= '3.5.0'", + "markers": "python_version >= '3.5'", "version": "==1.1.5" }, "toml": { From 737aacf0bde43babfc8ba926f6b7b7bfee673275 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 16 Jun 2022 13:03:41 -0700 Subject: [PATCH 009/196] Agent: Add marshmallow-enum to dependencies --- monkey/infection_monkey/Pipfile | 1 + monkey/infection_monkey/Pipfile.lock | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/Pipfile b/monkey/infection_monkey/Pipfile index 14c079054..7df872e2d 100644 --- a/monkey/infection_monkey/Pipfile +++ b/monkey/infection_monkey/Pipfile @@ -25,6 +25,7 @@ pywin32 = {version = "*", sys_platform = "== 'win32'"} # Lock file is not create pefile = {version = "*", sys_platform = "== 'win32'"} # Pyinstaller requirement on windows paramiko = {editable = true, ref = "2.10.3.dev1", git = "https://github.com/VakarisZ/paramiko.git"} marshmallow = "*" +marshmallow-enum = "*" [dev-packages] ldap3 = "*" diff --git a/monkey/infection_monkey/Pipfile.lock b/monkey/infection_monkey/Pipfile.lock index eb7bd1878..48f0e9b87 100644 --- a/monkey/infection_monkey/Pipfile.lock +++ b/monkey/infection_monkey/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fc7e8d826d41fa764926d16a224cfd4a583c1a356b2a61af7aa7231256b79255" + "sha256": "202bc4667be3a990bdf39be19ce2ddbacc0b3189810d89f75464ad361c9142b2" }, "pipfile-spec": 6, "requires": { @@ -221,7 +221,7 @@ "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e", "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f" ], - "markers": "python_version >= '3.6' and python_version < '4'", + "markers": "python_version >= '3.6' and python_version < '4.0'", "version": "==2.2.1" }, "flask": { @@ -380,6 +380,14 @@ "index": "pypi", "version": "==3.16.0" }, + "marshmallow-enum": { + "hashes": [ + "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58", + "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072" + ], + "index": "pypi", + "version": "==1.5.1" + }, "minidump": { "hashes": [ "sha256:6a9d2152f76ae633c609e09b48b42f55bd5a6b65f920dbbec756e5d9134a6201", From 70e8bca1ea70c20546887bb65bc7b159ba6f1713 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 19:52:12 -0400 Subject: [PATCH 010/196] Common: Use OperatingSystems enum in ExploiterConfigurationSchema --- monkey/common/configuration/agent_configuration.py | 7 +++++-- .../unit_tests/common/test_agent_configuration.py | 7 +++++-- vulture_allowlist.py | 12 +++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 95592ecb7..6ac8f9a2a 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -2,6 +2,9 @@ from dataclasses import dataclass from typing import Dict, List from marshmallow import RAISE, Schema, fields, post_load +from marshmallow_enum import EnumField + +from common import OperatingSystems @dataclass(frozen=True) @@ -48,7 +51,7 @@ class PluginConfigurationSchema(Schema): class ExploiterConfiguration: name: str options: Dict - supported_os: List[str] + supported_os: List[OperatingSystems] class ExploiterConfigurationSchema(Schema): @@ -57,7 +60,7 @@ class ExploiterConfigurationSchema(Schema): name = fields.Str() options = fields.Mapping() - supported_os = fields.List(fields.Str()) + supported_os = fields.List(EnumField(OperatingSystems)) @post_load def make_exploiter_configuration(self, data, **kwargs): diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 7854a48ea..6fd5ac290 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,3 +1,4 @@ +from common import OperatingSystems from common.configuration import ( CustomPBAConfigurationSchema, ExploiterConfigurationSchema, @@ -41,10 +42,12 @@ def test_custom_pba_configuration_schema(): def test_exploiter_configuration_schema(): name = "bond" options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} - supported_os = ["linux", "windows"] + supported_os = [OperatingSystems.LINUX, OperatingSystems.WINDOWS] schema = ExploiterConfigurationSchema() - config = schema.load({"name": name, "options": options, "supported_os": supported_os}) + config = schema.load( + {"name": name, "options": options, "supported_os": [os_.name for os_ in supported_os]} + ) assert config.name == name assert config.options == options diff --git a/vulture_allowlist.py b/vulture_allowlist.py index dec1aa7dd..7e1c4056a 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -184,11 +184,13 @@ architecture # unused variable (monkey/infection_monkey/exploit/caching_agent_r response_code # unused variable (monkey/monkey_island/cc/services/aws/aws_command_runner.py:26) # Agent Configuration -Meta # unused class(monkey/common/configuration/agent_configuration.py:13) -unknown # unused variable(monkey/common/configuration/agent_configuration.py:14) -make_plugin_configuration # unused method(monkey/common/configuration/agent_configuration.py:19) -make_custom_pba_configuration # unused method(monkey/common/configuration/agent_configuration.py:34) -make_exploiter_configuration # unused method(monkey/common/configuration/agent_configuration.py:62) +Meta # unused class (monkey/common/configuration/agent_configuration.py:13) +unknown # unused variable (monkey/common/configuration/agent_configuration.py:14) +make_plugin_configuration # unused method (monkey/common/configuration/agent_configuration.py:19) +make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:34) +make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:62) +LINUX # unused variable (monkey/common/operating_systems.py:5) +WINDOWS # unused variable (monkey/common/operating_systems.py:6) # TODO DELETE AFTER RESOURCE REFACTORING From afd3160c2f37cc76f30a0c1927d84a06ac19dfa0 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 20:00:23 -0400 Subject: [PATCH 011/196] Common: Add ExploitationOptionsConfiguration --- monkey/common/configuration/__init__.py | 2 ++ monkey/common/configuration/agent_configuration.py | 13 +++++++++++++ .../unit_tests/common/test_agent_configuration.py | 10 ++++++++++ vulture_allowlist.py | 1 + 4 files changed, 26 insertions(+) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index ed2b3cbc9..f9b876cdf 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -3,6 +3,8 @@ from .agent_configuration import ( PluginConfigurationSchema, CustomPBAConfiguration, CustomPBAConfigurationSchema, + ExploitationOptionsConfiguration, + ExploitationOptionsConfigurationSchema, ExploiterConfiguration, ExploiterConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 6ac8f9a2a..5bb1feba4 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -47,6 +47,19 @@ class PluginConfigurationSchema(Schema): return PluginConfiguration(**data) +@dataclass(frozen=True) +class ExploitationOptionsConfiguration: + http_ports: List[int] + + +class ExploitationOptionsConfigurationSchema(Schema): + http_ports = fields.List(fields.Int()) + + @post_load + def make_exploitation_options_configuration(self, data, **kwargs): + return ExploitationOptionsConfiguration(**data) + + @dataclass(frozen=True) class ExploiterConfiguration: name: str diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 6fd5ac290..21c0c8575 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,6 +1,7 @@ from common import OperatingSystems from common.configuration import ( CustomPBAConfigurationSchema, + ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, PluginConfigurationSchema, ) @@ -39,6 +40,15 @@ def test_custom_pba_configuration_schema(): assert config.windows_filename == windows_filename +def test_exploitation_options_configuration_schema(): + ports = [1, 2, 3] + schema = ExploitationOptionsConfigurationSchema() + + config = schema.load({"http_ports": ports}) + + assert config.http_ports == ports + + def test_exploiter_configuration_schema(): name = "bond" options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 7e1c4056a..bff9d7991 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -188,6 +188,7 @@ Meta # unused class (monkey/common/configuration/agent_configuration.py:13) unknown # unused variable (monkey/common/configuration/agent_configuration.py:14) make_plugin_configuration # unused method (monkey/common/configuration/agent_configuration.py:19) make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:34) +make_exploitation_options_configuration # unused method (monkey/common/configuration/agent_configuration.py:58) make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:62) LINUX # unused variable (monkey/common/operating_systems.py:5) WINDOWS # unused variable (monkey/common/operating_systems.py:6) From db9d57a526d548aa44756c3ae5ffb87756a9d125 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 20:01:28 -0400 Subject: [PATCH 012/196] Common: Remove `class Meta` from marshmallow schemas `RAISE` is the default behavior, so there's no need to copy/paste this all over the place. --- monkey/common/configuration/agent_configuration.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 5bb1feba4..bb7e08bbc 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Dict, List -from marshmallow import RAISE, Schema, fields, post_load +from marshmallow import Schema, fields, post_load from marshmallow_enum import EnumField from common import OperatingSystems @@ -16,9 +16,6 @@ class CustomPBAConfiguration: class CustomPBAConfigurationSchema(Schema): - class Meta: - unknown = RAISE - linux_command = fields.Str() linux_filename = fields.Str() windows_command = fields.Str() @@ -36,9 +33,6 @@ class PluginConfiguration: class PluginConfigurationSchema(Schema): - class Meta: - unknown = RAISE - name = fields.Str() options = fields.Mapping() @@ -68,9 +62,6 @@ class ExploiterConfiguration: class ExploiterConfigurationSchema(Schema): - class Meta: - unknown = RAISE - name = fields.Str() options = fields.Mapping() supported_os = fields.List(EnumField(OperatingSystems)) From 4065bc23fb61ad5d90fb3cb7c90f35242d2ee2a8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 16 Jun 2022 20:17:22 -0400 Subject: [PATCH 013/196] Common: Add ExploitationConfiguration --- monkey/common/configuration/__init__.py | 2 ++ .../configuration/agent_configuration.py | 17 ++++++++++ .../common/test_agent_configuration.py | 33 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index f9b876cdf..c22370603 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -7,4 +7,6 @@ from .agent_configuration import ( ExploitationOptionsConfigurationSchema, ExploiterConfiguration, ExploiterConfigurationSchema, + ExploitationConfiguration, + ExploitationConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index bb7e08bbc..e961a82f3 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -69,3 +69,20 @@ class ExploiterConfigurationSchema(Schema): @post_load def make_exploiter_configuration(self, data, **kwargs): return ExploiterConfiguration(**data) + + +@dataclass(frozen=True) +class ExploitationConfiguration: + options: ExploitationOptionsConfiguration + brute_force: List[ExploiterConfiguration] + vulnerability: List[ExploiterConfiguration] + + +class ExploitationConfigurationSchema(Schema): + options = fields.Nested(ExploitationOptionsConfigurationSchema) + brute_force = fields.List(fields.Nested(ExploiterConfigurationSchema)) + vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) + + @post_load + def make_exploitation_options_configuration(self, data, **kwargs): + return ExploitationConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 21c0c8575..cee26fe5b 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,6 +1,8 @@ from common import OperatingSystems from common.configuration import ( CustomPBAConfigurationSchema, + ExploitationConfiguration, + ExploitationConfigurationSchema, ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, PluginConfigurationSchema, @@ -62,3 +64,34 @@ def test_exploiter_configuration_schema(): assert config.name == name assert config.options == options assert config.supported_os == supported_os + + +def test_exploitation_configuration(): + ports = [1, 2, 3] + brute_force = [ + {"name": "ex1", "options": {}, "supported_os": ["LINUX"]}, + { + "name": "ex2", + "options": {"smb_download_timeout": 10}, + "supported_os": ["LINUX", "WINDOWS"], + }, + ] + vulnerability = [ + { + "name": "ex3", + "options": {"smb_download_timeout": 10}, + "supported_os": ["WINDOWS"], + }, + ] + exploitation_config = { + "options": {"http_ports": ports}, + "brute_force": brute_force, + "vulnerability": vulnerability, + } + schema = ExploitationConfigurationSchema() + + config = schema.load(exploitation_config) + config_dict = schema.dump(config) + + assert isinstance(config, ExploitationConfiguration) + assert config_dict == exploitation_config From 969f916cf594c2411c42ce512f826b140d88b8b0 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 17 Jun 2022 15:07:34 +0300 Subject: [PATCH 014/196] Common: Add ScanTargetConfiguration dataclass to agent_configuration.py --- monkey/common/configuration/agent_configuration.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index e961a82f3..ab00d7ae1 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -86,3 +86,11 @@ class ExploitationConfigurationSchema(Schema): @post_load def make_exploitation_options_configuration(self, data, **kwargs): return ExploitationConfiguration(**data) + + +@dataclass(frozen=True) +class ScanTargetConfiguration: + blocked_ips: List[str] + inaccessible_subnets: List[str] + local_network_scan: bool + subnets: List[str] From af761ea48fd60decd0475f79bc857dea06249583 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 17 Jun 2022 15:08:50 +0300 Subject: [PATCH 015/196] Common: Add ICMPScanConfiguration dataclass to agent_configuration.py --- monkey/common/configuration/agent_configuration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index ab00d7ae1..2c7605691 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -94,3 +94,8 @@ class ScanTargetConfiguration: inaccessible_subnets: List[str] local_network_scan: bool subnets: List[str] + + +@dataclass(frozen=True) +class ICMPScanConfiguration: + timeout_ms: int From 2e561181ea90ccb9399fb183fe9c9aeece97ec59 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 17 Jun 2022 15:09:25 +0300 Subject: [PATCH 016/196] Common: Add TCPScanConfiguration dataclass to agent_configuration.py --- monkey/common/configuration/agent_configuration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 2c7605691..7d1f64b4c 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -99,3 +99,9 @@ class ScanTargetConfiguration: @dataclass(frozen=True) class ICMPScanConfiguration: timeout_ms: int + + +@dataclass(frozen=True) +class TCPScanConfiguration: + timeout_ms: int + ports: List[int] From 0b810f5d56b4ecbec614c09eef1f443ed8d6edd6 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:14:35 -0400 Subject: [PATCH 017/196] Common: Add TCPScanConfigurationSchema --- monkey/common/configuration/__init__.py | 2 ++ monkey/common/configuration/agent_configuration.py | 9 +++++++++ .../unit_tests/common/test_agent_configuration.py | 12 ++++++++++++ vulture_allowlist.py | 1 + 4 files changed, 24 insertions(+) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index c22370603..9b5cdfb64 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -9,4 +9,6 @@ from .agent_configuration import ( ExploiterConfigurationSchema, ExploitationConfiguration, ExploitationConfigurationSchema, + TCPScanConfiguration, + TCPScanConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 7d1f64b4c..42f7dfcac 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -105,3 +105,12 @@ class ICMPScanConfiguration: class TCPScanConfiguration: timeout_ms: int ports: List[int] + + +class TCPScanConfigurationSchema(Schema): + timeout_ms = fields.Int() + ports = fields.List(fields.Int()) + + @post_load + def make_tcp_scan_configuration(self, data, **kwargs): + return TCPScanConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index cee26fe5b..ad4e52790 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -6,6 +6,7 @@ from common.configuration import ( ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, PluginConfigurationSchema, + TCPScanConfigurationSchema, ) @@ -95,3 +96,14 @@ def test_exploitation_configuration(): assert isinstance(config, ExploitationConfiguration) assert config_dict == exploitation_config + + +def test_tcp_scan_configuration_schema(): + timeout_ms = 2525 + ports = [8080, 443] + schema = TCPScanConfigurationSchema() + + config = schema.load({"timeout_ms": timeout_ms, "ports": ports}) + + assert config.timeout_ms == timeout_ms + assert config.ports == ports diff --git a/vulture_allowlist.py b/vulture_allowlist.py index bff9d7991..c38586f72 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -190,6 +190,7 @@ make_plugin_configuration # unused method (monkey/common/configuration/agent_co make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:34) make_exploitation_options_configuration # unused method (monkey/common/configuration/agent_configuration.py:58) make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:62) +make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:114) LINUX # unused variable (monkey/common/operating_systems.py:5) WINDOWS # unused variable (monkey/common/operating_systems.py:6) From ee8e949a0db3a1f2c845647edad696c4f802850e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:20:26 -0400 Subject: [PATCH 018/196] Common: Add ICMPScanConfigurationSchema --- monkey/common/configuration/__init__.py | 2 ++ monkey/common/configuration/agent_configuration.py | 8 ++++++++ .../unit_tests/common/test_agent_configuration.py | 10 ++++++++++ 3 files changed, 20 insertions(+) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 9b5cdfb64..c36144499 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -9,6 +9,8 @@ from .agent_configuration import ( ExploiterConfigurationSchema, ExploitationConfiguration, ExploitationConfigurationSchema, + ICMPScanConfiguration, + ICMPScanConfigurationSchema, TCPScanConfiguration, TCPScanConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 42f7dfcac..550d58d01 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -101,6 +101,14 @@ class ICMPScanConfiguration: timeout_ms: int +class ICMPScanConfigurationSchema(Schema): + timeout_ms = fields.Int() + + @post_load + def make_icmp_scan_configuration(self, data, **kwargs): + return ICMPScanConfiguration(**data) + + @dataclass(frozen=True) class TCPScanConfiguration: timeout_ms: int diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index ad4e52790..d6c1a3a18 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -5,6 +5,7 @@ from common.configuration import ( ExploitationConfigurationSchema, ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, + ICMPScanConfigurationSchema, PluginConfigurationSchema, TCPScanConfigurationSchema, ) @@ -98,6 +99,15 @@ def test_exploitation_configuration(): assert config_dict == exploitation_config +def test_icmp_scan_configuration_schema(): + timeout_ms = 2525 + schema = ICMPScanConfigurationSchema() + + config = schema.load({"timeout_ms": timeout_ms}) + + assert config.timeout_ms == timeout_ms + + def test_tcp_scan_configuration_schema(): timeout_ms = 2525 ports = [8080, 443] From 7e9c481992296deef9b5bc2b8bf7a3cee9c81b76 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:22:14 -0400 Subject: [PATCH 019/196] Common: Rename marshmallow post_load methods to be protected --- monkey/common/configuration/agent_configuration.py | 14 +++++++------- vulture_allowlist.py | 11 ++++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 550d58d01..aa716a7a6 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -22,7 +22,7 @@ class CustomPBAConfigurationSchema(Schema): windows_filename = fields.Str() @post_load - def make_custom_pba_configuration(self, data, **kwargs): + def _make_custom_pba_configuration(self, data, **kwargs): return CustomPBAConfiguration(**data) @@ -37,7 +37,7 @@ class PluginConfigurationSchema(Schema): options = fields.Mapping() @post_load - def make_plugin_configuration(self, data, **kwargs): + def _make_plugin_configuration(self, data, **kwargs): return PluginConfiguration(**data) @@ -50,7 +50,7 @@ class ExploitationOptionsConfigurationSchema(Schema): http_ports = fields.List(fields.Int()) @post_load - def make_exploitation_options_configuration(self, data, **kwargs): + def _make_exploitation_options_configuration(self, data, **kwargs): return ExploitationOptionsConfiguration(**data) @@ -67,7 +67,7 @@ class ExploiterConfigurationSchema(Schema): supported_os = fields.List(EnumField(OperatingSystems)) @post_load - def make_exploiter_configuration(self, data, **kwargs): + def _make_exploiter_configuration(self, data, **kwargs): return ExploiterConfiguration(**data) @@ -84,7 +84,7 @@ class ExploitationConfigurationSchema(Schema): vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) @post_load - def make_exploitation_options_configuration(self, data, **kwargs): + def _make_exploitation_options_configuration(self, data, **kwargs): return ExploitationConfiguration(**data) @@ -105,7 +105,7 @@ class ICMPScanConfigurationSchema(Schema): timeout_ms = fields.Int() @post_load - def make_icmp_scan_configuration(self, data, **kwargs): + def _make_icmp_scan_configuration(self, data, **kwargs): return ICMPScanConfiguration(**data) @@ -120,5 +120,5 @@ class TCPScanConfigurationSchema(Schema): ports = fields.List(fields.Int()) @post_load - def make_tcp_scan_configuration(self, data, **kwargs): + def _make_tcp_scan_configuration(self, data, **kwargs): return TCPScanConfiguration(**data) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index c38586f72..fcdefe8a0 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -186,11 +186,12 @@ response_code # unused variable (monkey/monkey_island/cc/services/aws/aws_comma # Agent Configuration Meta # unused class (monkey/common/configuration/agent_configuration.py:13) unknown # unused variable (monkey/common/configuration/agent_configuration.py:14) -make_plugin_configuration # unused method (monkey/common/configuration/agent_configuration.py:19) -make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:34) -make_exploitation_options_configuration # unused method (monkey/common/configuration/agent_configuration.py:58) -make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:62) -make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:114) +_make_plugin_configuration # unused method (monkey/common/configuration/agent_configuration.py:19) +_make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:24) +_make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:69) +_make_exploitation_options_configuration # unused method (monkey/common/configuration/agent_configuration.py:86) +_make_icmp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:107) +_make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:122) LINUX # unused variable (monkey/common/operating_systems.py:5) WINDOWS # unused variable (monkey/common/operating_systems.py:6) From 3c879f444d4c5a556882777b4a477fdf6f111777 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:30:42 -0400 Subject: [PATCH 020/196] Common: Add ScanTargetConfigurationSchema --- monkey/common/configuration/__init__.py | 2 ++ .../configuration/agent_configuration.py | 11 ++++++++++ .../common/test_agent_configuration.py | 22 +++++++++++++++++++ vulture_allowlist.py | 1 + 4 files changed, 36 insertions(+) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index c36144499..393bba020 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -13,4 +13,6 @@ from .agent_configuration import ( ICMPScanConfigurationSchema, TCPScanConfiguration, TCPScanConfigurationSchema, + ScanTargetConfiguration, + ScanTargetConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index aa716a7a6..a34b93d94 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -96,6 +96,17 @@ class ScanTargetConfiguration: subnets: List[str] +class ScanTargetConfigurationSchema(Schema): + blocked_ips = fields.List(fields.Str()) + inaccessible_subnets = fields.List(fields.Str()) + local_network_scan = fields.Bool() + subnets = fields.List(fields.Str()) + + @post_load + def _make_scan_target_configuration(self, data, **kwargs): + return ScanTargetConfiguration(**data) + + @dataclass(frozen=True) class ICMPScanConfiguration: timeout_ms: int diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index d6c1a3a18..12697021d 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -7,6 +7,7 @@ from common.configuration import ( ExploiterConfigurationSchema, ICMPScanConfigurationSchema, PluginConfigurationSchema, + ScanTargetConfigurationSchema, TCPScanConfigurationSchema, ) @@ -99,6 +100,27 @@ def test_exploitation_configuration(): assert config_dict == exploitation_config +def test_scan_target_configuration(): + blocked_ips = ["10.0.0.1", "192.168.1.1"] + inaccessible_subnets = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] + local_network_scan = True + subnets = ["10.0.0.2", "10.0.0.2/16"] + scan_target_config = { + "blocked_ips": blocked_ips, + "inaccessible_subnets": inaccessible_subnets, + "local_network_scan": local_network_scan, + "subnets": subnets, + } + schema = ScanTargetConfigurationSchema() + + config = schema.load(scan_target_config) + + assert config.blocked_ips == blocked_ips + assert config.inaccessible_subnets == inaccessible_subnets + assert config.local_network_scan == local_network_scan + assert config.subnets == subnets + + def test_icmp_scan_configuration_schema(): timeout_ms = 2525 schema = ICMPScanConfigurationSchema() diff --git a/vulture_allowlist.py b/vulture_allowlist.py index fcdefe8a0..0082925c6 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -190,6 +190,7 @@ _make_plugin_configuration # unused method (monkey/common/configuration/agent_c _make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:24) _make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:69) _make_exploitation_options_configuration # unused method (monkey/common/configuration/agent_configuration.py:86) +_make_scan_target_configuration # unused method (monkey/common/configuration/agent_configuration.py:105) _make_icmp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:107) _make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:122) LINUX # unused variable (monkey/common/operating_systems.py:5) From 2c4069ae1b568d3b86b758d8eb5115a07f41173e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:31:04 -0400 Subject: [PATCH 021/196] Project: Remove Meta, unknown from vulture_allowlist.py These are no longer needed after db9d57a526d548a. --- vulture_allowlist.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 0082925c6..770b7960f 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -184,8 +184,6 @@ architecture # unused variable (monkey/infection_monkey/exploit/caching_agent_r response_code # unused variable (monkey/monkey_island/cc/services/aws/aws_command_runner.py:26) # Agent Configuration -Meta # unused class (monkey/common/configuration/agent_configuration.py:13) -unknown # unused variable (monkey/common/configuration/agent_configuration.py:14) _make_plugin_configuration # unused method (monkey/common/configuration/agent_configuration.py:19) _make_custom_pba_configuration # unused method (monkey/common/configuration/agent_configuration.py:24) _make_exploiter_configuration # unused method (monkey/common/configuration/agent_configuration.py:69) From 5845bb73afb049e6521112e0402dcd6bcb6d51e3 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:33:43 -0400 Subject: [PATCH 022/196] Common: Reorder scan and exploit configuration classes --- monkey/common/configuration/__init__.py | 12 +-- .../configuration/agent_configuration.py | 84 +++++++++---------- .../common/test_agent_configuration.py | 82 +++++++++--------- 3 files changed, 89 insertions(+), 89 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 393bba020..4b43ef699 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -3,16 +3,16 @@ from .agent_configuration import ( PluginConfigurationSchema, CustomPBAConfiguration, CustomPBAConfigurationSchema, - ExploitationOptionsConfiguration, - ExploitationOptionsConfigurationSchema, - ExploiterConfiguration, - ExploiterConfigurationSchema, - ExploitationConfiguration, - ExploitationConfigurationSchema, ICMPScanConfiguration, ICMPScanConfigurationSchema, TCPScanConfiguration, TCPScanConfigurationSchema, ScanTargetConfiguration, ScanTargetConfigurationSchema, + ExploitationOptionsConfiguration, + ExploitationOptionsConfigurationSchema, + ExploiterConfiguration, + ExploiterConfigurationSchema, + ExploitationConfiguration, + ExploitationConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index a34b93d94..5de701934 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -46,48 +46,6 @@ class ExploitationOptionsConfiguration: http_ports: List[int] -class ExploitationOptionsConfigurationSchema(Schema): - http_ports = fields.List(fields.Int()) - - @post_load - def _make_exploitation_options_configuration(self, data, **kwargs): - return ExploitationOptionsConfiguration(**data) - - -@dataclass(frozen=True) -class ExploiterConfiguration: - name: str - options: Dict - supported_os: List[OperatingSystems] - - -class ExploiterConfigurationSchema(Schema): - name = fields.Str() - options = fields.Mapping() - supported_os = fields.List(EnumField(OperatingSystems)) - - @post_load - def _make_exploiter_configuration(self, data, **kwargs): - return ExploiterConfiguration(**data) - - -@dataclass(frozen=True) -class ExploitationConfiguration: - options: ExploitationOptionsConfiguration - brute_force: List[ExploiterConfiguration] - vulnerability: List[ExploiterConfiguration] - - -class ExploitationConfigurationSchema(Schema): - options = fields.Nested(ExploitationOptionsConfigurationSchema) - brute_force = fields.List(fields.Nested(ExploiterConfigurationSchema)) - vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) - - @post_load - def _make_exploitation_options_configuration(self, data, **kwargs): - return ExploitationConfiguration(**data) - - @dataclass(frozen=True) class ScanTargetConfiguration: blocked_ips: List[str] @@ -133,3 +91,45 @@ class TCPScanConfigurationSchema(Schema): @post_load def _make_tcp_scan_configuration(self, data, **kwargs): return TCPScanConfiguration(**data) + + +class ExploitationOptionsConfigurationSchema(Schema): + http_ports = fields.List(fields.Int()) + + @post_load + def _make_exploitation_options_configuration(self, data, **kwargs): + return ExploitationOptionsConfiguration(**data) + + +@dataclass(frozen=True) +class ExploiterConfiguration: + name: str + options: Dict + supported_os: List[OperatingSystems] + + +class ExploiterConfigurationSchema(Schema): + name = fields.Str() + options = fields.Mapping() + supported_os = fields.List(EnumField(OperatingSystems)) + + @post_load + def _make_exploiter_configuration(self, data, **kwargs): + return ExploiterConfiguration(**data) + + +@dataclass(frozen=True) +class ExploitationConfiguration: + options: ExploitationOptionsConfiguration + brute_force: List[ExploiterConfiguration] + vulnerability: List[ExploiterConfiguration] + + +class ExploitationConfigurationSchema(Schema): + options = fields.Nested(ExploitationOptionsConfigurationSchema) + brute_force = fields.List(fields.Nested(ExploiterConfigurationSchema)) + vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) + + @post_load + def _make_exploitation_options_configuration(self, data, **kwargs): + return ExploitationConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 12697021d..745e19222 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -45,6 +45,47 @@ def test_custom_pba_configuration_schema(): assert config.windows_filename == windows_filename +def test_scan_target_configuration(): + blocked_ips = ["10.0.0.1", "192.168.1.1"] + inaccessible_subnets = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] + local_network_scan = True + subnets = ["10.0.0.2", "10.0.0.2/16"] + scan_target_config = { + "blocked_ips": blocked_ips, + "inaccessible_subnets": inaccessible_subnets, + "local_network_scan": local_network_scan, + "subnets": subnets, + } + schema = ScanTargetConfigurationSchema() + + config = schema.load(scan_target_config) + + assert config.blocked_ips == blocked_ips + assert config.inaccessible_subnets == inaccessible_subnets + assert config.local_network_scan == local_network_scan + assert config.subnets == subnets + + +def test_icmp_scan_configuration_schema(): + timeout_ms = 2525 + schema = ICMPScanConfigurationSchema() + + config = schema.load({"timeout_ms": timeout_ms}) + + assert config.timeout_ms == timeout_ms + + +def test_tcp_scan_configuration_schema(): + timeout_ms = 2525 + ports = [8080, 443] + schema = TCPScanConfigurationSchema() + + config = schema.load({"timeout_ms": timeout_ms, "ports": ports}) + + assert config.timeout_ms == timeout_ms + assert config.ports == ports + + def test_exploitation_options_configuration_schema(): ports = [1, 2, 3] schema = ExploitationOptionsConfigurationSchema() @@ -98,44 +139,3 @@ def test_exploitation_configuration(): assert isinstance(config, ExploitationConfiguration) assert config_dict == exploitation_config - - -def test_scan_target_configuration(): - blocked_ips = ["10.0.0.1", "192.168.1.1"] - inaccessible_subnets = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] - local_network_scan = True - subnets = ["10.0.0.2", "10.0.0.2/16"] - scan_target_config = { - "blocked_ips": blocked_ips, - "inaccessible_subnets": inaccessible_subnets, - "local_network_scan": local_network_scan, - "subnets": subnets, - } - schema = ScanTargetConfigurationSchema() - - config = schema.load(scan_target_config) - - assert config.blocked_ips == blocked_ips - assert config.inaccessible_subnets == inaccessible_subnets - assert config.local_network_scan == local_network_scan - assert config.subnets == subnets - - -def test_icmp_scan_configuration_schema(): - timeout_ms = 2525 - schema = ICMPScanConfigurationSchema() - - config = schema.load({"timeout_ms": timeout_ms}) - - assert config.timeout_ms == timeout_ms - - -def test_tcp_scan_configuration_schema(): - timeout_ms = 2525 - ports = [8080, 443] - schema = TCPScanConfigurationSchema() - - config = schema.load({"timeout_ms": timeout_ms, "ports": ports}) - - assert config.timeout_ms == timeout_ms - assert config.ports == ports From 9bbf5c8ae72c719121b66f4fa452986af0941ac8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 08:50:48 -0400 Subject: [PATCH 023/196] Common: Add NetworkScanConfiguration --- monkey/common/configuration/__init__.py | 2 + .../configuration/agent_configuration.py | 19 +++++ .../common/test_agent_configuration.py | 79 +++++++++++++------ vulture_allowlist.py | 1 + 4 files changed, 78 insertions(+), 23 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 4b43ef699..1c9e6e2da 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -9,6 +9,8 @@ from .agent_configuration import ( TCPScanConfigurationSchema, ScanTargetConfiguration, ScanTargetConfigurationSchema, + NetworkScanConfiguration, + NetworkScanConfigurationSchema, ExploitationOptionsConfiguration, ExploitationOptionsConfigurationSchema, ExploiterConfiguration, diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 5de701934..dda656419 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -93,6 +93,25 @@ class TCPScanConfigurationSchema(Schema): return TCPScanConfiguration(**data) +@dataclass(frozen=True) +class NetworkScanConfiguration: + tcp: TCPScanConfiguration + icmp: ICMPScanConfiguration + fingerprinters: List[PluginConfiguration] + targets: ScanTargetConfiguration + + +class NetworkScanConfigurationSchema(Schema): + tcp = fields.Nested(TCPScanConfigurationSchema) + icmp = fields.Nested(ICMPScanConfigurationSchema) + fingerprinters = fields.List(fields.Nested(PluginConfigurationSchema)) + targets = fields.Nested(ScanTargetConfigurationSchema) + + @post_load + def _make_network_scan_configuration(self, data, **kwargs): + return NetworkScanConfiguration(**data) + + class ExploitationOptionsConfigurationSchema(Schema): http_ports = fields.List(fields.Int()) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 745e19222..02d3d4eac 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -6,6 +6,7 @@ from common.configuration import ( ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, ICMPScanConfigurationSchema, + NetworkScanConfigurationSchema, PluginConfigurationSchema, ScanTargetConfigurationSchema, TCPScanConfigurationSchema, @@ -45,45 +46,77 @@ def test_custom_pba_configuration_schema(): assert config.windows_filename == windows_filename +BLOCKED_IPS = ["10.0.0.1", "192.168.1.1"] +INACCESSIBLE_SUBNETS = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] +LOCAL_NETWORK_SCAN = True +SUBNETS = ["10.0.0.2", "10.0.0.2/16"] +SCAN_TARGET_CONFIGURATION = { + "blocked_ips": BLOCKED_IPS, + "inaccessible_subnets": INACCESSIBLE_SUBNETS, + "local_network_scan": LOCAL_NETWORK_SCAN, + "subnets": SUBNETS, +} + + def test_scan_target_configuration(): - blocked_ips = ["10.0.0.1", "192.168.1.1"] - inaccessible_subnets = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] - local_network_scan = True - subnets = ["10.0.0.2", "10.0.0.2/16"] - scan_target_config = { - "blocked_ips": blocked_ips, - "inaccessible_subnets": inaccessible_subnets, - "local_network_scan": local_network_scan, - "subnets": subnets, - } schema = ScanTargetConfigurationSchema() - config = schema.load(scan_target_config) + config = schema.load(SCAN_TARGET_CONFIGURATION) - assert config.blocked_ips == blocked_ips - assert config.inaccessible_subnets == inaccessible_subnets - assert config.local_network_scan == local_network_scan - assert config.subnets == subnets + assert config.blocked_ips == BLOCKED_IPS + assert config.inaccessible_subnets == INACCESSIBLE_SUBNETS + assert config.local_network_scan == LOCAL_NETWORK_SCAN + assert config.subnets == SUBNETS + + +TIMEOUT_MS = 2525 +ICMP_CONFIGURATION = {"timeout_ms": TIMEOUT_MS} def test_icmp_scan_configuration_schema(): - timeout_ms = 2525 schema = ICMPScanConfigurationSchema() - config = schema.load({"timeout_ms": timeout_ms}) + config = schema.load(ICMP_CONFIGURATION) - assert config.timeout_ms == timeout_ms + assert config.timeout_ms == TIMEOUT_MS + + +TIMEOUT_MS = 2525 +PORTS = [8080, 443] + +TCP_SCAN_CONFIGURATION = {"timeout_ms": TIMEOUT_MS, "ports": PORTS} def test_tcp_scan_configuration_schema(): - timeout_ms = 2525 - ports = [8080, 443] schema = TCPScanConfigurationSchema() - config = schema.load({"timeout_ms": timeout_ms, "ports": ports}) + config = schema.load(TCP_SCAN_CONFIGURATION) - assert config.timeout_ms == timeout_ms - assert config.ports == ports + assert config.timeout_ms == TIMEOUT_MS + assert config.ports == PORTS + + +def test_network_scan_configuration(): + fingerprinters = [{"name": "mssql", "options": {}}] + network_scan_configuration = { + "tcp": TCP_SCAN_CONFIGURATION, + "icmp": ICMP_CONFIGURATION, + "fingerprinters": fingerprinters, + "targets": SCAN_TARGET_CONFIGURATION, + } + schema = NetworkScanConfigurationSchema() + + config = schema.load(network_scan_configuration) + + assert config.tcp.ports == TCP_SCAN_CONFIGURATION["ports"] + assert config.tcp.timeout_ms == TCP_SCAN_CONFIGURATION["timeout_ms"] + assert config.icmp.timeout_ms == ICMP_CONFIGURATION["timeout_ms"] + assert config.fingerprinters[0].name == fingerprinters[0]["name"] + assert config.fingerprinters[0].options == fingerprinters[0]["options"] + assert config.targets.blocked_ips == BLOCKED_IPS + assert config.targets.inaccessible_subnets == INACCESSIBLE_SUBNETS + assert config.targets.local_network_scan == LOCAL_NETWORK_SCAN + assert config.targets.subnets == SUBNETS def test_exploitation_options_configuration_schema(): diff --git a/vulture_allowlist.py b/vulture_allowlist.py index 770b7960f..ef63dab97 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -191,6 +191,7 @@ _make_exploitation_options_configuration # unused method (monkey/common/configu _make_scan_target_configuration # unused method (monkey/common/configuration/agent_configuration.py:105) _make_icmp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:107) _make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:122) +_make_network_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:110) LINUX # unused variable (monkey/common/operating_systems.py:5) WINDOWS # unused variable (monkey/common/operating_systems.py:6) From a41b2e3ea42e285925536a8207e1e86d20fc792e Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 09:04:00 -0400 Subject: [PATCH 024/196] Common: Add PropagationConfiguration --- monkey/common/configuration/__init__.py | 2 + .../configuration/agent_configuration.py | 15 +++ .../common/test_agent_configuration.py | 91 ++++++++++++------- vulture_allowlist.py | 1 + 4 files changed, 76 insertions(+), 33 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 1c9e6e2da..d49080618 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -17,4 +17,6 @@ from .agent_configuration import ( ExploiterConfigurationSchema, ExploitationConfiguration, ExploitationConfigurationSchema, + PropagationConfiguration, + PropagationConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index dda656419..f7b0c950e 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -152,3 +152,18 @@ class ExploitationConfigurationSchema(Schema): @post_load def _make_exploitation_options_configuration(self, data, **kwargs): return ExploitationConfiguration(**data) + + +@dataclass(frozen=True) +class PropagationConfiguration: + network_scan: NetworkScanConfiguration + exploitation: ExploitationConfiguration + + +class PropagationConfigurationSchema(Schema): + network_scan = fields.Nested(NetworkScanConfigurationSchema) + exploitation = fields.Nested(ExploitationConfigurationSchema) + + @post_load + def _make_propagation_configuration(self, data, **kwargs): + return PropagationConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 02d3d4eac..2c10e646b 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -6,8 +6,11 @@ from common.configuration import ( ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, ICMPScanConfigurationSchema, + NetworkScanConfiguration, NetworkScanConfigurationSchema, PluginConfigurationSchema, + PropagationConfiguration, + PropagationConfigurationSchema, ScanTargetConfigurationSchema, TCPScanConfigurationSchema, ) @@ -96,23 +99,25 @@ def test_tcp_scan_configuration_schema(): assert config.ports == PORTS +FINGERPRINTERS = [{"name": "mssql", "options": {}}] +NETWORK_SCAN_CONFIGURATION = { + "tcp": TCP_SCAN_CONFIGURATION, + "icmp": ICMP_CONFIGURATION, + "fingerprinters": FINGERPRINTERS, + "targets": SCAN_TARGET_CONFIGURATION, +} + + def test_network_scan_configuration(): - fingerprinters = [{"name": "mssql", "options": {}}] - network_scan_configuration = { - "tcp": TCP_SCAN_CONFIGURATION, - "icmp": ICMP_CONFIGURATION, - "fingerprinters": fingerprinters, - "targets": SCAN_TARGET_CONFIGURATION, - } schema = NetworkScanConfigurationSchema() - config = schema.load(network_scan_configuration) + config = schema.load(NETWORK_SCAN_CONFIGURATION) assert config.tcp.ports == TCP_SCAN_CONFIGURATION["ports"] assert config.tcp.timeout_ms == TCP_SCAN_CONFIGURATION["timeout_ms"] assert config.icmp.timeout_ms == ICMP_CONFIGURATION["timeout_ms"] - assert config.fingerprinters[0].name == fingerprinters[0]["name"] - assert config.fingerprinters[0].options == fingerprinters[0]["options"] + assert config.fingerprinters[0].name == FINGERPRINTERS[0]["name"] + assert config.fingerprinters[0].options == FINGERPRINTERS[0]["options"] assert config.targets.blocked_ips == BLOCKED_IPS assert config.targets.inaccessible_subnets == INACCESSIBLE_SUBNETS assert config.targets.local_network_scan == LOCAL_NETWORK_SCAN @@ -143,32 +148,52 @@ def test_exploiter_configuration_schema(): assert config.supported_os == supported_os +BRUTE_FORCE = [ + {"name": "ex1", "options": {}, "supported_os": ["LINUX"]}, + { + "name": "ex2", + "options": {"smb_download_timeout": 10}, + "supported_os": ["LINUX", "WINDOWS"], + }, +] +VULNERABILITY = [ + { + "name": "ex3", + "options": {"smb_download_timeout": 10}, + "supported_os": ["WINDOWS"], + }, +] +EXPLOITATION_CONFIGURATION = { + "options": {"http_ports": PORTS}, + "brute_force": BRUTE_FORCE, + "vulnerability": VULNERABILITY, +} + + def test_exploitation_configuration(): - ports = [1, 2, 3] - brute_force = [ - {"name": "ex1", "options": {}, "supported_os": ["LINUX"]}, - { - "name": "ex2", - "options": {"smb_download_timeout": 10}, - "supported_os": ["LINUX", "WINDOWS"], - }, - ] - vulnerability = [ - { - "name": "ex3", - "options": {"smb_download_timeout": 10}, - "supported_os": ["WINDOWS"], - }, - ] - exploitation_config = { - "options": {"http_ports": ports}, - "brute_force": brute_force, - "vulnerability": vulnerability, - } schema = ExploitationConfigurationSchema() - config = schema.load(exploitation_config) + config = schema.load(EXPLOITATION_CONFIGURATION) config_dict = schema.dump(config) assert isinstance(config, ExploitationConfiguration) - assert config_dict == exploitation_config + assert config_dict == EXPLOITATION_CONFIGURATION + + +PROPAGATION_CONFIGURATION = { + "network_scan": NETWORK_SCAN_CONFIGURATION, + "exploitation": EXPLOITATION_CONFIGURATION, +} + + +def test_propagation_configuration(): + schema = PropagationConfigurationSchema() + + config = schema.load(PROPAGATION_CONFIGURATION) + config_dict = schema.dump(config) + + assert isinstance(config, PropagationConfiguration) + assert isinstance(config.network_scan, NetworkScanConfiguration) + assert isinstance(config.exploitation, ExploitationConfiguration) + + assert config_dict == PROPAGATION_CONFIGURATION diff --git a/vulture_allowlist.py b/vulture_allowlist.py index ef63dab97..a05814c54 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -192,6 +192,7 @@ _make_scan_target_configuration # unused method (monkey/common/configuration/ag _make_icmp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:107) _make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:122) _make_network_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:110) +_make_propagation_configuration # unused method (monkey/common/configuration/agent_configuration.py:167) LINUX # unused variable (monkey/common/operating_systems.py:5) WINDOWS # unused variable (monkey/common/operating_systems.py:6) From bd7ea7fdb10548cdcb0564c5fbe721ef17fcc4b8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 09:07:13 -0400 Subject: [PATCH 025/196] Common: Add maximum_depth to PropagationConfiguration --- monkey/common/configuration/agent_configuration.py | 2 ++ monkey/tests/unit_tests/common/test_agent_configuration.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index f7b0c950e..2f4dfce2f 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -156,11 +156,13 @@ class ExploitationConfigurationSchema(Schema): @dataclass(frozen=True) class PropagationConfiguration: + maximum_depth: int network_scan: NetworkScanConfiguration exploitation: ExploitationConfiguration class PropagationConfigurationSchema(Schema): + maximum_depth = fields.Int() network_scan = fields.Nested(NetworkScanConfigurationSchema) exploitation = fields.Nested(ExploitationConfigurationSchema) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 2c10e646b..fb34b5211 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -181,6 +181,7 @@ def test_exploitation_configuration(): PROPAGATION_CONFIGURATION = { + "maximum_depth": 5, "network_scan": NETWORK_SCAN_CONFIGURATION, "exploitation": EXPLOITATION_CONFIGURATION, } @@ -195,5 +196,5 @@ def test_propagation_configuration(): assert isinstance(config, PropagationConfiguration) assert isinstance(config.network_scan, NetworkScanConfiguration) assert isinstance(config.exploitation, ExploitationConfiguration) - + assert config.maximum_depth == 5 assert config_dict == PROPAGATION_CONFIGURATION From 7039ccf708825130fb23b3f6fd728ab089ab1bdc Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 09:14:22 -0400 Subject: [PATCH 026/196] Common: Switch configuration timeouts from ms to floating-point seconds --- .../common/configuration/agent_configuration.py | 8 ++++---- .../unit_tests/common/test_agent_configuration.py | 15 +++++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 2f4dfce2f..222ff707c 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -67,11 +67,11 @@ class ScanTargetConfigurationSchema(Schema): @dataclass(frozen=True) class ICMPScanConfiguration: - timeout_ms: int + timeout: float class ICMPScanConfigurationSchema(Schema): - timeout_ms = fields.Int() + timeout = fields.Float() @post_load def _make_icmp_scan_configuration(self, data, **kwargs): @@ -80,12 +80,12 @@ class ICMPScanConfigurationSchema(Schema): @dataclass(frozen=True) class TCPScanConfiguration: - timeout_ms: int + timeout: float ports: List[int] class TCPScanConfigurationSchema(Schema): - timeout_ms = fields.Int() + timeout = fields.Float() ports = fields.List(fields.Int()) @post_load diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index fb34b5211..da52ab12a 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -72,8 +72,8 @@ def test_scan_target_configuration(): assert config.subnets == SUBNETS -TIMEOUT_MS = 2525 -ICMP_CONFIGURATION = {"timeout_ms": TIMEOUT_MS} +TIMEOUT = 2.525 +ICMP_CONFIGURATION = {"timeout": TIMEOUT} def test_icmp_scan_configuration_schema(): @@ -81,13 +81,12 @@ def test_icmp_scan_configuration_schema(): config = schema.load(ICMP_CONFIGURATION) - assert config.timeout_ms == TIMEOUT_MS + assert config.timeout == TIMEOUT -TIMEOUT_MS = 2525 PORTS = [8080, 443] -TCP_SCAN_CONFIGURATION = {"timeout_ms": TIMEOUT_MS, "ports": PORTS} +TCP_SCAN_CONFIGURATION = {"timeout": TIMEOUT, "ports": PORTS} def test_tcp_scan_configuration_schema(): @@ -95,7 +94,7 @@ def test_tcp_scan_configuration_schema(): config = schema.load(TCP_SCAN_CONFIGURATION) - assert config.timeout_ms == TIMEOUT_MS + assert config.timeout == TIMEOUT assert config.ports == PORTS @@ -114,8 +113,8 @@ def test_network_scan_configuration(): config = schema.load(NETWORK_SCAN_CONFIGURATION) assert config.tcp.ports == TCP_SCAN_CONFIGURATION["ports"] - assert config.tcp.timeout_ms == TCP_SCAN_CONFIGURATION["timeout_ms"] - assert config.icmp.timeout_ms == ICMP_CONFIGURATION["timeout_ms"] + assert config.tcp.timeout == TCP_SCAN_CONFIGURATION["timeout"] + assert config.icmp.timeout == ICMP_CONFIGURATION["timeout"] assert config.fingerprinters[0].name == FINGERPRINTERS[0]["name"] assert config.fingerprinters[0].options == FINGERPRINTERS[0]["options"] assert config.targets.blocked_ips == BLOCKED_IPS From e0ae109368321fec5635cb1dee05d621676e3887 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 09:27:51 -0400 Subject: [PATCH 027/196] Common: Add AgentConfiguration --- monkey/common/configuration/__init__.py | 2 + .../configuration/agent_configuration.py | 23 ++++++ .../common/test_agent_configuration.py | 73 +++++++++++++------ vulture_allowlist.py | 1 + 4 files changed, 78 insertions(+), 21 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index d49080618..973e6e28b 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -19,4 +19,6 @@ from .agent_configuration import ( ExploitationConfigurationSchema, PropagationConfiguration, PropagationConfigurationSchema, + AgentConfiguration, + AgentConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 222ff707c..40e82d5c3 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -169,3 +169,26 @@ class PropagationConfigurationSchema(Schema): @post_load def _make_propagation_configuration(self, data, **kwargs): return PropagationConfiguration(**data) + + +@dataclass(frozen=True) +class AgentConfiguration: + keep_tunnel_open_time: float + custom_pbas: CustomPBAConfiguration + post_breach_actions: List[PluginConfiguration] + credential_collectors: List[PluginConfiguration] + payloads: List[PluginConfiguration] + propagation: PropagationConfiguration + + +class AgentConfigurationSchema(Schema): + keep_tunnel_open_time = fields.Float() + custom_pbas = fields.Nested(CustomPBAConfigurationSchema) + post_breach_actions = fields.List(fields.Nested(PluginConfigurationSchema)) + credential_collectors = fields.List(fields.Nested(PluginConfigurationSchema)) + payloads = fields.List(fields.Nested(PluginConfigurationSchema)) + propagation = fields.Nested(PropagationConfigurationSchema) + + @post_load + def _make_agent_configuration(self, data, **kwargs): + return AgentConfiguration(**data) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index da52ab12a..6954407bd 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,5 +1,8 @@ from common import OperatingSystems from common.configuration import ( + AgentConfiguration, + AgentConfigurationSchema, + CustomPBAConfiguration, CustomPBAConfigurationSchema, ExploitationConfiguration, ExploitationConfigurationSchema, @@ -8,6 +11,7 @@ from common.configuration import ( ICMPScanConfigurationSchema, NetworkScanConfiguration, NetworkScanConfigurationSchema, + PluginConfiguration, PluginConfigurationSchema, PropagationConfiguration, PropagationConfigurationSchema, @@ -15,38 +19,41 @@ from common.configuration import ( TCPScanConfigurationSchema, ) +NAME = "bond" +OPTIONS = {"gun": "Walther PPK", "car": "Aston Martin DB5"} +PLUGIN_CONFIGURATION = {"name": NAME, "options": OPTIONS} + def test_build_plugin_configuration(): - name = "bond" - options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} schema = PluginConfigurationSchema() - config = schema.load({"name": name, "options": options}) + config = schema.load(PLUGIN_CONFIGURATION) - assert config.name == name - assert config.options == options + assert config.name == NAME + assert config.options == OPTIONS + + +LINUX_COMMAND = "a" +LINUX_FILENAME = "b" +WINDOWS_COMMAND = "c" +WINDOWS_FILENAME = "d" +CUSTOM_PBA_CONFIGURATION = { + "linux_command": LINUX_COMMAND, + "linux_filename": LINUX_FILENAME, + "windows_command": WINDOWS_COMMAND, + "windows_filename": WINDOWS_FILENAME, +} def test_custom_pba_configuration_schema(): - linux_command = "a" - linux_filename = "b" - windows_command = "c" - windows_filename = "d" schema = CustomPBAConfigurationSchema() - config = schema.load( - { - "linux_command": linux_command, - "linux_filename": linux_filename, - "windows_command": windows_command, - "windows_filename": windows_filename, - } - ) + config = schema.load(CUSTOM_PBA_CONFIGURATION) - assert config.linux_command == linux_command - assert config.linux_filename == linux_filename - assert config.windows_command == windows_command - assert config.windows_filename == windows_filename + assert config.linux_command == LINUX_COMMAND + assert config.linux_filename == LINUX_FILENAME + assert config.windows_command == WINDOWS_COMMAND + assert config.windows_filename == WINDOWS_FILENAME BLOCKED_IPS = ["10.0.0.1", "192.168.1.1"] @@ -197,3 +204,27 @@ def test_propagation_configuration(): assert isinstance(config.exploitation, ExploitationConfiguration) assert config.maximum_depth == 5 assert config_dict == PROPAGATION_CONFIGURATION + + +def test_agent_configuration(): + agent_configuration = { + "keep_tunnel_open_time": 30, + "custom_pbas": CUSTOM_PBA_CONFIGURATION, + "post_breach_actions": [PLUGIN_CONFIGURATION], + "credential_collectors": [PLUGIN_CONFIGURATION], + "payloads": [PLUGIN_CONFIGURATION], + "propagation": PROPAGATION_CONFIGURATION, + } + schema = AgentConfigurationSchema() + + config = schema.load(agent_configuration) + config_dict = schema.dump(config) + + assert isinstance(config, AgentConfiguration) + assert config.keep_tunnel_open_time == 30 + assert isinstance(config.custom_pbas, CustomPBAConfiguration) + assert isinstance(config.post_breach_actions[0], PluginConfiguration) + assert isinstance(config.credential_collectors[0], PluginConfiguration) + assert isinstance(config.payloads[0], PluginConfiguration) + assert isinstance(config.propagation, PropagationConfiguration) + assert config_dict == agent_configuration diff --git a/vulture_allowlist.py b/vulture_allowlist.py index a05814c54..3a8ad98c1 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -193,6 +193,7 @@ _make_icmp_scan_configuration # unused method (monkey/common/configuration/agen _make_tcp_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:122) _make_network_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:110) _make_propagation_configuration # unused method (monkey/common/configuration/agent_configuration.py:167) +_make_agent_configuration # unused method (monkey/common/configuration/agent_configuration.py:192) LINUX # unused variable (monkey/common/operating_systems.py:5) WINDOWS # unused variable (monkey/common/operating_systems.py:6) From f8855d290d466131b181fe6ac98f86110e8ca025 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 17 Jun 2022 16:38:52 +0300 Subject: [PATCH 028/196] Agent: Remove WormConfiguration and cleanup related infrastructure --- monkey/infection_monkey/config.py | 3 --- monkey/infection_monkey/control.py | 38 +++--------------------------- monkey/infection_monkey/main.py | 5 ---- monkey/infection_monkey/monkey.py | 9 +++++-- 4 files changed, 10 insertions(+), 45 deletions(-) diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index 8e783dbf5..1b7b7f60f 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -64,6 +64,3 @@ class Configuration(object): max_depth = None keep_tunnel_open_time = 30 - - -WormConfiguration = Configuration() diff --git a/monkey/infection_monkey/control.py b/monkey/infection_monkey/control.py index 985778b7c..8d1e48a22 100644 --- a/monkey/infection_monkey/control.py +++ b/monkey/infection_monkey/control.py @@ -1,7 +1,6 @@ import json import logging import platform -from pprint import pformat from socket import gethostname from typing import Mapping, Optional @@ -10,7 +9,7 @@ from requests.exceptions import ConnectionError import infection_monkey.tunnel as tunnel from common.common_consts.timeouts import LONG_REQUEST_TIMEOUT, MEDIUM_REQUEST_TIMEOUT -from infection_monkey.config import GUID, WormConfiguration +from infection_monkey.config import GUID from infection_monkey.network.info import get_host_subnets, local_ips from infection_monkey.transport.http import HTTPConnectProxy from infection_monkey.transport.tcp import TcpProxy @@ -151,38 +150,7 @@ class ControlClient: except Exception as exc: logger.warning(f"Error connecting to control server {self.server_address}: {exc}") - def load_control_config(self): - if not self.server_address: - return - try: - reply = requests.get( # noqa: DUO123 - f"https://{self.server_address}/api/agent/", - verify=False, - proxies=self.proxies, - timeout=MEDIUM_REQUEST_TIMEOUT, - ) - - except Exception as exc: - logger.warning(f"Error connecting to control server {self.server_address}: {exc}") - return - - try: - WormConfiguration.from_kv(reply.json().get("config")) - formatted_config = pformat( - WormConfiguration.hide_sensitive_info(WormConfiguration.as_dict()) - ) - logger.info(f"New configuration was loaded from server:\n{formatted_config}") - except Exception as exc: - # we don't continue with default conf here because it might be dangerous - logger.error( - "Error parsing JSON reply from control server %s (%s): %s", - self.server_address, - reply._content, - exc, - ) - raise Exception("Couldn't load from from server's configuration, aborting. %s" % exc) - - def create_control_tunnel(self): + def create_control_tunnel(self, keep_tunnel_open_time: int): if not self.server_address: return None @@ -200,7 +168,7 @@ class ControlClient: return tunnel.MonkeyTunnel( proxy_class, - keep_tunnel_open_time=WormConfiguration.keep_tunnel_open_time, + keep_tunnel_open_time=keep_tunnel_open_time, target_addr=target_addr, target_port=target_port, ) diff --git a/monkey/infection_monkey/main.py b/monkey/infection_monkey/main.py index 3c71c9721..0102503ca 100644 --- a/monkey/infection_monkey/main.py +++ b/monkey/infection_monkey/main.py @@ -5,13 +5,11 @@ import os import sys import traceback from multiprocessing import freeze_support -from pprint import pformat # dummy import for pyinstaller # noinspection PyUnresolvedReferences import infection_monkey.post_breach # noqa: F401 from common.version import get_version -from infection_monkey.config import WormConfiguration from infection_monkey.dropper import MonkeyDrops from infection_monkey.model import DROPPER_ARG, MONKEY_ARG from infection_monkey.monkey import InfectionMonkey @@ -57,9 +55,6 @@ def main(): mode_args, mode_specific_args = arg_parser.parse_known_args() mode = mode_args.mode - formatted_config = pformat(WormConfiguration.hide_sensitive_info(WormConfiguration.as_dict())) - print(f"Loaded Configuration:\n{formatted_config}") - try: if MONKEY_ARG == mode: log_path = get_agent_log_path() diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index d7a051193..a0fe5f009 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -149,7 +149,6 @@ class InfectionMonkey: raise Exception(f"Monkey couldn't find server with {self._opts.tunnel} default tunnel.") self._control_client.wakeup(parent=self._opts.parent) - self._control_client.load_control_config() def _current_server_is_set(self) -> bool: if self._control_client.find_server(default_tunnel=self._opts.tunnel): @@ -165,7 +164,13 @@ class InfectionMonkey: if firewall.is_enabled(): firewall.add_firewall_rule() - self._monkey_inbound_tunnel = self._control_client.create_control_tunnel() + control_channel = ControlChannel( + self._control_client.server_address, GUID, self._control_client.proxies + ) + keep_tunnel_open_time = control_channel.get_config()["config"]["keep_tunnel_open_time"] + self._monkey_inbound_tunnel = self._control_client.create_control_tunnel( + keep_tunnel_open_time + ) if self._monkey_inbound_tunnel and self._propagation_enabled(): self._monkey_inbound_tunnel.start() From 291b82c28ddd707f00747a4e10dbd159a5ebbbda Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 10:20:49 -0400 Subject: [PATCH 029/196] Common: Separate agent sub configurations and schemas --- monkey/common/configuration/__init__.py | 26 ++- .../configuration/agent_configuration.py | 178 ++---------------- .../agent_sub_configuration_schemas.py | 114 +++++++++++ .../configuration/agent_sub_configurations.py | 71 +++++++ 4 files changed, 212 insertions(+), 177 deletions(-) create mode 100644 monkey/common/configuration/agent_sub_configuration_schemas.py create mode 100644 monkey/common/configuration/agent_sub_configurations.py diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 973e6e28b..4b35d4184 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1,24 +1,30 @@ -from .agent_configuration import ( +from .agent_sub_configurations import ( PluginConfiguration, - PluginConfigurationSchema, CustomPBAConfiguration, - CustomPBAConfigurationSchema, ICMPScanConfiguration, - ICMPScanConfigurationSchema, TCPScanConfiguration, - TCPScanConfigurationSchema, ScanTargetConfiguration, - ScanTargetConfigurationSchema, NetworkScanConfiguration, - NetworkScanConfigurationSchema, ExploitationOptionsConfiguration, - ExploitationOptionsConfigurationSchema, ExploiterConfiguration, - ExploiterConfigurationSchema, ExploitationConfiguration, - ExploitationConfigurationSchema, PropagationConfiguration, +) + +from .agent_sub_configuration_schemas import ( + PluginConfigurationSchema, + CustomPBAConfigurationSchema, + ICMPScanConfigurationSchema, + TCPScanConfigurationSchema, + ScanTargetConfigurationSchema, + NetworkScanConfigurationSchema, + ExploitationOptionsConfigurationSchema, + ExploiterConfigurationSchema, + ExploitationConfigurationSchema, PropagationConfigurationSchema, +) + +from .agent_configuration import ( AgentConfiguration, AgentConfigurationSchema, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 40e82d5c3..15b338fe0 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -1,174 +1,18 @@ from dataclasses import dataclass -from typing import Dict, List +from typing import List from marshmallow import Schema, fields, post_load -from marshmallow_enum import EnumField -from common import OperatingSystems - - -@dataclass(frozen=True) -class CustomPBAConfiguration: - linux_command: str - linux_filename: str - windows_command: str - windows_filename: str - - -class CustomPBAConfigurationSchema(Schema): - linux_command = fields.Str() - linux_filename = fields.Str() - windows_command = fields.Str() - windows_filename = fields.Str() - - @post_load - def _make_custom_pba_configuration(self, data, **kwargs): - return CustomPBAConfiguration(**data) - - -@dataclass(frozen=True) -class PluginConfiguration: - name: str - options: Dict - - -class PluginConfigurationSchema(Schema): - name = fields.Str() - options = fields.Mapping() - - @post_load - def _make_plugin_configuration(self, data, **kwargs): - return PluginConfiguration(**data) - - -@dataclass(frozen=True) -class ExploitationOptionsConfiguration: - http_ports: List[int] - - -@dataclass(frozen=True) -class ScanTargetConfiguration: - blocked_ips: List[str] - inaccessible_subnets: List[str] - local_network_scan: bool - subnets: List[str] - - -class ScanTargetConfigurationSchema(Schema): - blocked_ips = fields.List(fields.Str()) - inaccessible_subnets = fields.List(fields.Str()) - local_network_scan = fields.Bool() - subnets = fields.List(fields.Str()) - - @post_load - def _make_scan_target_configuration(self, data, **kwargs): - return ScanTargetConfiguration(**data) - - -@dataclass(frozen=True) -class ICMPScanConfiguration: - timeout: float - - -class ICMPScanConfigurationSchema(Schema): - timeout = fields.Float() - - @post_load - def _make_icmp_scan_configuration(self, data, **kwargs): - return ICMPScanConfiguration(**data) - - -@dataclass(frozen=True) -class TCPScanConfiguration: - timeout: float - ports: List[int] - - -class TCPScanConfigurationSchema(Schema): - timeout = fields.Float() - ports = fields.List(fields.Int()) - - @post_load - def _make_tcp_scan_configuration(self, data, **kwargs): - return TCPScanConfiguration(**data) - - -@dataclass(frozen=True) -class NetworkScanConfiguration: - tcp: TCPScanConfiguration - icmp: ICMPScanConfiguration - fingerprinters: List[PluginConfiguration] - targets: ScanTargetConfiguration - - -class NetworkScanConfigurationSchema(Schema): - tcp = fields.Nested(TCPScanConfigurationSchema) - icmp = fields.Nested(ICMPScanConfigurationSchema) - fingerprinters = fields.List(fields.Nested(PluginConfigurationSchema)) - targets = fields.Nested(ScanTargetConfigurationSchema) - - @post_load - def _make_network_scan_configuration(self, data, **kwargs): - return NetworkScanConfiguration(**data) - - -class ExploitationOptionsConfigurationSchema(Schema): - http_ports = fields.List(fields.Int()) - - @post_load - def _make_exploitation_options_configuration(self, data, **kwargs): - return ExploitationOptionsConfiguration(**data) - - -@dataclass(frozen=True) -class ExploiterConfiguration: - name: str - options: Dict - supported_os: List[OperatingSystems] - - -class ExploiterConfigurationSchema(Schema): - name = fields.Str() - options = fields.Mapping() - supported_os = fields.List(EnumField(OperatingSystems)) - - @post_load - def _make_exploiter_configuration(self, data, **kwargs): - return ExploiterConfiguration(**data) - - -@dataclass(frozen=True) -class ExploitationConfiguration: - options: ExploitationOptionsConfiguration - brute_force: List[ExploiterConfiguration] - vulnerability: List[ExploiterConfiguration] - - -class ExploitationConfigurationSchema(Schema): - options = fields.Nested(ExploitationOptionsConfigurationSchema) - brute_force = fields.List(fields.Nested(ExploiterConfigurationSchema)) - vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) - - @post_load - def _make_exploitation_options_configuration(self, data, **kwargs): - return ExploitationConfiguration(**data) - - -@dataclass(frozen=True) -class PropagationConfiguration: - maximum_depth: int - network_scan: NetworkScanConfiguration - exploitation: ExploitationConfiguration - - -class PropagationConfigurationSchema(Schema): - maximum_depth = fields.Int() - network_scan = fields.Nested(NetworkScanConfigurationSchema) - exploitation = fields.Nested(ExploitationConfigurationSchema) - - @post_load - def _make_propagation_configuration(self, data, **kwargs): - return PropagationConfiguration(**data) +from .agent_sub_configuration_schemas import ( + CustomPBAConfigurationSchema, + PluginConfigurationSchema, + PropagationConfigurationSchema, +) +from .agent_sub_configurations import ( + CustomPBAConfiguration, + PluginConfiguration, + PropagationConfiguration, +) @dataclass(frozen=True) diff --git a/monkey/common/configuration/agent_sub_configuration_schemas.py b/monkey/common/configuration/agent_sub_configuration_schemas.py new file mode 100644 index 000000000..ceec0af24 --- /dev/null +++ b/monkey/common/configuration/agent_sub_configuration_schemas.py @@ -0,0 +1,114 @@ +from marshmallow import Schema, fields, post_load +from marshmallow_enum import EnumField + +from common import OperatingSystems + +from .agent_sub_configurations import ( + CustomPBAConfiguration, + ExploitationConfiguration, + ExploitationOptionsConfiguration, + ExploiterConfiguration, + ICMPScanConfiguration, + NetworkScanConfiguration, + PluginConfiguration, + PropagationConfiguration, + ScanTargetConfiguration, + TCPScanConfiguration, +) + + +class CustomPBAConfigurationSchema(Schema): + linux_command = fields.Str() + linux_filename = fields.Str() + windows_command = fields.Str() + windows_filename = fields.Str() + + @post_load + def _make_custom_pba_configuration(self, data, **kwargs): + return CustomPBAConfiguration(**data) + + +class PluginConfigurationSchema(Schema): + name = fields.Str() + options = fields.Mapping() + + @post_load + def _make_plugin_configuration(self, data, **kwargs): + return PluginConfiguration(**data) + + +class ScanTargetConfigurationSchema(Schema): + blocked_ips = fields.List(fields.Str()) + inaccessible_subnets = fields.List(fields.Str()) + local_network_scan = fields.Bool() + subnets = fields.List(fields.Str()) + + @post_load + def _make_scan_target_configuration(self, data, **kwargs): + return ScanTargetConfiguration(**data) + + +class ICMPScanConfigurationSchema(Schema): + timeout = fields.Float() + + @post_load + def _make_icmp_scan_configuration(self, data, **kwargs): + return ICMPScanConfiguration(**data) + + +class TCPScanConfigurationSchema(Schema): + timeout = fields.Float() + ports = fields.List(fields.Int()) + + @post_load + def _make_tcp_scan_configuration(self, data, **kwargs): + return TCPScanConfiguration(**data) + + +class NetworkScanConfigurationSchema(Schema): + tcp = fields.Nested(TCPScanConfigurationSchema) + icmp = fields.Nested(ICMPScanConfigurationSchema) + fingerprinters = fields.List(fields.Nested(PluginConfigurationSchema)) + targets = fields.Nested(ScanTargetConfigurationSchema) + + @post_load + def _make_network_scan_configuration(self, data, **kwargs): + return NetworkScanConfiguration(**data) + + +class ExploitationOptionsConfigurationSchema(Schema): + http_ports = fields.List(fields.Int()) + + @post_load + def _make_exploitation_options_configuration(self, data, **kwargs): + return ExploitationOptionsConfiguration(**data) + + +class ExploiterConfigurationSchema(Schema): + name = fields.Str() + options = fields.Mapping() + supported_os = fields.List(EnumField(OperatingSystems)) + + @post_load + def _make_exploiter_configuration(self, data, **kwargs): + return ExploiterConfiguration(**data) + + +class ExploitationConfigurationSchema(Schema): + options = fields.Nested(ExploitationOptionsConfigurationSchema) + brute_force = fields.List(fields.Nested(ExploiterConfigurationSchema)) + vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) + + @post_load + def _make_exploitation_options_configuration(self, data, **kwargs): + return ExploitationConfiguration(**data) + + +class PropagationConfigurationSchema(Schema): + maximum_depth = fields.Int() + network_scan = fields.Nested(NetworkScanConfigurationSchema) + exploitation = fields.Nested(ExploitationConfigurationSchema) + + @post_load + def _make_propagation_configuration(self, data, **kwargs): + return PropagationConfiguration(**data) diff --git a/monkey/common/configuration/agent_sub_configurations.py b/monkey/common/configuration/agent_sub_configurations.py new file mode 100644 index 000000000..560542e1d --- /dev/null +++ b/monkey/common/configuration/agent_sub_configurations.py @@ -0,0 +1,71 @@ +from dataclasses import dataclass +from typing import Dict, List + +from common import OperatingSystems + + +@dataclass(frozen=True) +class CustomPBAConfiguration: + linux_command: str + linux_filename: str + windows_command: str + windows_filename: str + + +@dataclass(frozen=True) +class PluginConfiguration: + name: str + options: Dict + + +@dataclass(frozen=True) +class ScanTargetConfiguration: + blocked_ips: List[str] + inaccessible_subnets: List[str] + local_network_scan: bool + subnets: List[str] + + +@dataclass(frozen=True) +class ICMPScanConfiguration: + timeout: float + + +@dataclass(frozen=True) +class TCPScanConfiguration: + timeout: float + ports: List[int] + + +@dataclass(frozen=True) +class NetworkScanConfiguration: + tcp: TCPScanConfiguration + icmp: ICMPScanConfiguration + fingerprinters: List[PluginConfiguration] + targets: ScanTargetConfiguration + + +@dataclass(frozen=True) +class ExploitationOptionsConfiguration: + http_ports: List[int] + + +@dataclass(frozen=True) +class ExploiterConfiguration: + name: str + options: Dict + supported_os: List[OperatingSystems] + + +@dataclass(frozen=True) +class ExploitationConfiguration: + options: ExploitationOptionsConfiguration + brute_force: List[ExploiterConfiguration] + vulnerability: List[ExploiterConfiguration] + + +@dataclass(frozen=True) +class PropagationConfiguration: + maximum_depth: int + network_scan: NetworkScanConfiguration + exploitation: ExploitationConfiguration From ed39d155bfaf5c843c9a9b54d44e20139864f773 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 10:23:43 -0400 Subject: [PATCH 030/196] Common: Remove sub-configuration and sub-schemas from configuration --- monkey/common/configuration/__init__.py | 26 ------------------- .../common/test_agent_configuration.py | 17 ++++++------ 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 4b35d4184..59186d56c 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1,29 +1,3 @@ -from .agent_sub_configurations import ( - PluginConfiguration, - CustomPBAConfiguration, - ICMPScanConfiguration, - TCPScanConfiguration, - ScanTargetConfiguration, - NetworkScanConfiguration, - ExploitationOptionsConfiguration, - ExploiterConfiguration, - ExploitationConfiguration, - PropagationConfiguration, -) - -from .agent_sub_configuration_schemas import ( - PluginConfigurationSchema, - CustomPBAConfigurationSchema, - ICMPScanConfigurationSchema, - TCPScanConfigurationSchema, - ScanTargetConfigurationSchema, - NetworkScanConfigurationSchema, - ExploitationOptionsConfigurationSchema, - ExploiterConfigurationSchema, - ExploitationConfigurationSchema, - PropagationConfigurationSchema, -) - from .agent_configuration import ( AgentConfiguration, AgentConfigurationSchema, diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 6954407bd..512b46bc4 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,23 +1,24 @@ from common import OperatingSystems -from common.configuration import ( - AgentConfiguration, - AgentConfigurationSchema, - CustomPBAConfiguration, +from common.configuration import AgentConfiguration, AgentConfigurationSchema +from common.configuration.agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, - ExploitationConfiguration, ExploitationConfigurationSchema, ExploitationOptionsConfigurationSchema, ExploiterConfigurationSchema, ICMPScanConfigurationSchema, - NetworkScanConfiguration, NetworkScanConfigurationSchema, - PluginConfiguration, PluginConfigurationSchema, - PropagationConfiguration, PropagationConfigurationSchema, ScanTargetConfigurationSchema, TCPScanConfigurationSchema, ) +from common.configuration.agent_sub_configurations import ( + CustomPBAConfiguration, + ExploitationConfiguration, + NetworkScanConfiguration, + PluginConfiguration, + PropagationConfiguration, +) NAME = "bond" OPTIONS = {"gun": "Walther PPK", "car": "Aston Martin DB5"} From 84db00b72809bcab2ec06e622ffce84cabe96393 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:05:10 -0400 Subject: [PATCH 031/196] Island: Use AgentConfiguration in IConfigRepository --- monkey/monkey_island/cc/repository/i_config_repository.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/repository/i_config_repository.py b/monkey/monkey_island/cc/repository/i_config_repository.py index 70e59a89f..c9967ad45 100644 --- a/monkey/monkey_island/cc/repository/i_config_repository.py +++ b/monkey/monkey_island/cc/repository/i_config_repository.py @@ -1,5 +1,7 @@ from abc import ABC -from typing import Any, Mapping, Sequence +from typing import Any, Sequence + +from common.configuration import AgentConfiguration class IConfigRepository(ABC): @@ -10,10 +12,10 @@ class IConfigRepository(ABC): # This returns the current config # TODO investigate if encryption should be here or where # TODO potentially should be a DTO as well, but it's structure is defined in schema already - def get_config(self) -> Mapping: + def get_config(self) -> AgentConfiguration: pass - def set_config(self, config: dict): + def set_config(self, config: AgentConfiguration): pass # Used when only a subset of config is submitted, for example only PBAFiles From 62056175a18c1045fc12d15baf79a718b76986ad Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:05:55 -0400 Subject: [PATCH 032/196] Island: Remove unneeded methods in IConfigRepository Also remove stale comments. No encryption is needed here because configuration does not contain anything sensitive. --- .../cc/repository/i_config_repository.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/monkey/monkey_island/cc/repository/i_config_repository.py b/monkey/monkey_island/cc/repository/i_config_repository.py index c9967ad45..ce6935b87 100644 --- a/monkey/monkey_island/cc/repository/i_config_repository.py +++ b/monkey/monkey_island/cc/repository/i_config_repository.py @@ -1,32 +1,11 @@ from abc import ABC -from typing import Any, Sequence from common.configuration import AgentConfiguration class IConfigRepository(ABC): - - # Config - ############################################### - - # This returns the current config - # TODO investigate if encryption should be here or where - # TODO potentially should be a DTO as well, but it's structure is defined in schema already def get_config(self) -> AgentConfiguration: pass def set_config(self, config: AgentConfiguration): pass - - # Used when only a subset of config is submitted, for example only PBAFiles - # Used by passing keys, like ['monkey', 'post_breach_actions', 'linux_filename'] - # Using a list is less ambiguous IMO, than using . notation - def set_config_field(self, key_list: Sequence[str], value: Any): - pass - - # Used when only a subset of config is needed, for example only PBAFiles - # Used by passing keys, like ['monkey', 'post_breach_actions', 'linux_filename'] - # Using a list is less ambiguous IMO, than using . notation - # TODO Still in doubt about encryption, this should probably be determined automatically - def get_config_field(self, key_list: Sequence[str]) -> Any: - pass From ae0c440603bbc116bbec16692e5a9ab8723b2d95 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:08:45 -0400 Subject: [PATCH 033/196] Island: Add documentation for IConfigRepository --- .../cc/repository/i_config_repository.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/i_config_repository.py b/monkey/monkey_island/cc/repository/i_config_repository.py index ce6935b87..def95e228 100644 --- a/monkey/monkey_island/cc/repository/i_config_repository.py +++ b/monkey/monkey_island/cc/repository/i_config_repository.py @@ -4,8 +4,21 @@ from common.configuration import AgentConfiguration class IConfigRepository(ABC): + """ + A repository used to store and retrieve the agent configuration. + """ def get_config(self) -> AgentConfiguration: + """ + Retrieve the agent configuration from the repository + + :return: The agent configuration + """ pass - def set_config(self, config: AgentConfiguration): + def set_config(self, agent_config: AgentConfiguration): + """ + Store the agent configuration in the repository + + :param agent_config: The agent configuration to store in the repository + """ pass From 59e29456c03abeddd9ca2e1bea17b216c95f4184 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:09:33 -0400 Subject: [PATCH 034/196] Island: Rename i_config_repository.py -> i_configuration_repository.py --- .../{i_config_repository.py => i_configuration_repository.py} | 1 + 1 file changed, 1 insertion(+) rename monkey/monkey_island/cc/repository/{i_config_repository.py => i_configuration_repository.py} (99%) diff --git a/monkey/monkey_island/cc/repository/i_config_repository.py b/monkey/monkey_island/cc/repository/i_configuration_repository.py similarity index 99% rename from monkey/monkey_island/cc/repository/i_config_repository.py rename to monkey/monkey_island/cc/repository/i_configuration_repository.py index def95e228..8d06fc5c7 100644 --- a/monkey/monkey_island/cc/repository/i_config_repository.py +++ b/monkey/monkey_island/cc/repository/i_configuration_repository.py @@ -7,6 +7,7 @@ class IConfigRepository(ABC): """ A repository used to store and retrieve the agent configuration. """ + def get_config(self) -> AgentConfiguration: """ Retrieve the agent configuration from the repository From 91476a7a066d3d9d71b4822030ee3ef6ad39d25f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:11:14 -0400 Subject: [PATCH 035/196] Island: Protract config -> configuration in IConfigurationRepository --- .../cc/repository/i_configuration_repository.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/repository/i_configuration_repository.py b/monkey/monkey_island/cc/repository/i_configuration_repository.py index 8d06fc5c7..58a97bce9 100644 --- a/monkey/monkey_island/cc/repository/i_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_configuration_repository.py @@ -3,12 +3,12 @@ from abc import ABC from common.configuration import AgentConfiguration -class IConfigRepository(ABC): +class IConfigurationRepository(ABC): """ A repository used to store and retrieve the agent configuration. """ - def get_config(self) -> AgentConfiguration: + def get_configuration(self) -> AgentConfiguration: """ Retrieve the agent configuration from the repository @@ -16,10 +16,10 @@ class IConfigRepository(ABC): """ pass - def set_config(self, agent_config: AgentConfiguration): + def set_configuration(self, agent_configuration: AgentConfiguration): """ Store the agent configuration in the repository - :param agent_config: The agent configuration to store in the repository + :param agent_configuration: The agent configuration to store in the repository """ pass From ace3eb8718475d093bd40462c748782e2beb9904 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:16:44 -0400 Subject: [PATCH 036/196] UT: Extract SingleFileRepository from test_pba_file_upload.py --- monkey/tests/common/SingleFileRepository.py | 23 ++++++++++++++++ monkey/tests/monkey_island/__init__.py | 1 + .../monkey_island/single_file_repository.py | 23 ++++++++++++++++ .../cc/resources/test_pba_file_upload.py | 27 +++---------------- 4 files changed, 50 insertions(+), 24 deletions(-) create mode 100644 monkey/tests/common/SingleFileRepository.py create mode 100644 monkey/tests/monkey_island/__init__.py create mode 100644 monkey/tests/monkey_island/single_file_repository.py diff --git a/monkey/tests/common/SingleFileRepository.py b/monkey/tests/common/SingleFileRepository.py new file mode 100644 index 000000000..d00ee9ae9 --- /dev/null +++ b/monkey/tests/common/SingleFileRepository.py @@ -0,0 +1,23 @@ +import io +from typing import BinaryIO + +from monkey_island.cc.repository import FileRetrievalError, IFileRepository + + +class SingleFileRepository(IFileRepository): + def __init__(self): + self._file = None + + def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): + self._file = io.BytesIO(file_contents.read()) + + def open_file(self, unsafe_file_name: str) -> BinaryIO: + if self._file is None: + raise FileRetrievalError() + return self._file + + def delete_file(self, unsafe_file_name: str): + self._file = None + + def delete_all_files(self): + self.delete_file("") diff --git a/monkey/tests/monkey_island/__init__.py b/monkey/tests/monkey_island/__init__.py new file mode 100644 index 000000000..7bd9a314d --- /dev/null +++ b/monkey/tests/monkey_island/__init__.py @@ -0,0 +1 @@ +from .single_file_repository import SingleFileRepository diff --git a/monkey/tests/monkey_island/single_file_repository.py b/monkey/tests/monkey_island/single_file_repository.py new file mode 100644 index 000000000..d00ee9ae9 --- /dev/null +++ b/monkey/tests/monkey_island/single_file_repository.py @@ -0,0 +1,23 @@ +import io +from typing import BinaryIO + +from monkey_island.cc.repository import FileRetrievalError, IFileRepository + + +class SingleFileRepository(IFileRepository): + def __init__(self): + self._file = None + + def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): + self._file = io.BytesIO(file_contents.read()) + + def open_file(self, unsafe_file_name: str) -> BinaryIO: + if self._file is None: + raise FileRetrievalError() + return self._file + + def delete_file(self, unsafe_file_name: str): + self._file = None + + def delete_all_files(self): + self.delete_file("") diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py index 2642c0758..50da88915 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py @@ -1,12 +1,10 @@ -import io -from typing import BinaryIO - import pytest from tests.common import StubDIContainer +from tests.monkey_island import SingleFileRepository from tests.unit_tests.monkey_island.conftest import get_url_for_resource from tests.utils import raise_ -from monkey_island.cc.repository import FileRetrievalError, IFileRepository +from monkey_island.cc.repository import IFileRepository from monkey_island.cc.resources.pba_file_upload import LINUX_PBA_TYPE, WINDOWS_PBA_TYPE, FileUpload TEST_FILE_CONTENTS = b"m0nk3y" @@ -40,28 +38,9 @@ def mock_get_config_value(monkeypatch): ) -class MockFileRepository(IFileRepository): - def __init__(self): - self._file = None - - def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): - self._file = io.BytesIO(file_contents.read()) - - def open_file(self, unsafe_file_name: str) -> BinaryIO: - if self._file is None: - raise FileRetrievalError() - return self._file - - def delete_file(self, unsafe_file_name: str): - self._file = None - - def delete_all_files(self): - self.delete_file("") - - @pytest.fixture def file_repository(): - return MockFileRepository() + return SingleFileRepository() @pytest.fixture From 7cb7f7ab5a62debb3e9e6b52b769fd017a33a5ec Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:33:01 -0400 Subject: [PATCH 037/196] UT: Extract agent configuration from test_agent_configuration.py --- .../common/example_agent_configuration.py | 75 +++++++++++ .../common/test_agent_configuration.py | 116 +++++------------- 2 files changed, 104 insertions(+), 87 deletions(-) create mode 100644 monkey/tests/common/example_agent_configuration.py diff --git a/monkey/tests/common/example_agent_configuration.py b/monkey/tests/common/example_agent_configuration.py new file mode 100644 index 000000000..640771df4 --- /dev/null +++ b/monkey/tests/common/example_agent_configuration.py @@ -0,0 +1,75 @@ +PLUGIN_NAME = "bond" +PLUGIN_OPTIONS = {"gun": "Walther PPK", "car": "Aston Martin DB5"} +PLUGIN_CONFIGURATION = {"name": PLUGIN_NAME, "options": PLUGIN_OPTIONS} + +LINUX_COMMAND = "a" +LINUX_FILENAME = "b" +WINDOWS_COMMAND = "c" +WINDOWS_FILENAME = "d" +CUSTOM_PBA_CONFIGURATION = { + "linux_command": LINUX_COMMAND, + "linux_filename": LINUX_FILENAME, + "windows_command": WINDOWS_COMMAND, + "windows_filename": WINDOWS_FILENAME, +} + +BLOCKED_IPS = ["10.0.0.1", "192.168.1.1"] +INACCESSIBLE_SUBNETS = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] +LOCAL_NETWORK_SCAN = True +SUBNETS = ["10.0.0.2", "10.0.0.2/16"] +SCAN_TARGET_CONFIGURATION = { + "blocked_ips": BLOCKED_IPS, + "inaccessible_subnets": INACCESSIBLE_SUBNETS, + "local_network_scan": LOCAL_NETWORK_SCAN, + "subnets": SUBNETS, +} + +TIMEOUT = 2.525 +ICMP_CONFIGURATION = {"timeout": TIMEOUT} + +PORTS = [8080, 443] +TCP_SCAN_CONFIGURATION = {"timeout": TIMEOUT, "ports": PORTS} + +FINGERPRINTERS = [{"name": "mssql", "options": {}}] +NETWORK_SCAN_CONFIGURATION = { + "tcp": TCP_SCAN_CONFIGURATION, + "icmp": ICMP_CONFIGURATION, + "fingerprinters": FINGERPRINTERS, + "targets": SCAN_TARGET_CONFIGURATION, +} + +BRUTE_FORCE = [ + {"name": "ex1", "options": {}, "supported_os": ["LINUX"]}, + { + "name": "ex2", + "options": {"smb_download_timeout": 10}, + "supported_os": ["LINUX", "WINDOWS"], + }, +] +VULNERABILITY = [ + { + "name": "ex3", + "options": {"smb_download_timeout": 10}, + "supported_os": ["WINDOWS"], + }, +] +EXPLOITATION_CONFIGURATION = { + "options": {"http_ports": PORTS}, + "brute_force": BRUTE_FORCE, + "vulnerability": VULNERABILITY, +} + +PROPAGATION_CONFIGURATION = { + "maximum_depth": 5, + "network_scan": NETWORK_SCAN_CONFIGURATION, + "exploitation": EXPLOITATION_CONFIGURATION, +} + +AGENT_CONFIGURATION = { + "keep_tunnel_open_time": 30, + "custom_pbas": CUSTOM_PBA_CONFIGURATION, + "post_breach_actions": [PLUGIN_CONFIGURATION], + "credential_collectors": [PLUGIN_CONFIGURATION], + "payloads": [PLUGIN_CONFIGURATION], + "propagation": PROPAGATION_CONFIGURATION, +} diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 512b46bc4..9609a2d3a 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -1,3 +1,28 @@ +from tests.common.example_agent_configuration import ( + AGENT_CONFIGURATION, + BLOCKED_IPS, + CUSTOM_PBA_CONFIGURATION, + EXPLOITATION_CONFIGURATION, + FINGERPRINTERS, + ICMP_CONFIGURATION, + INACCESSIBLE_SUBNETS, + LINUX_COMMAND, + LINUX_FILENAME, + LOCAL_NETWORK_SCAN, + NETWORK_SCAN_CONFIGURATION, + PLUGIN_CONFIGURATION, + PLUGIN_NAME, + PLUGIN_OPTIONS, + PORTS, + PROPAGATION_CONFIGURATION, + SCAN_TARGET_CONFIGURATION, + SUBNETS, + TCP_SCAN_CONFIGURATION, + TIMEOUT, + WINDOWS_COMMAND, + WINDOWS_FILENAME, +) + from common import OperatingSystems from common.configuration import AgentConfiguration, AgentConfigurationSchema from common.configuration.agent_sub_configuration_schemas import ( @@ -20,30 +45,14 @@ from common.configuration.agent_sub_configurations import ( PropagationConfiguration, ) -NAME = "bond" -OPTIONS = {"gun": "Walther PPK", "car": "Aston Martin DB5"} -PLUGIN_CONFIGURATION = {"name": NAME, "options": OPTIONS} - def test_build_plugin_configuration(): schema = PluginConfigurationSchema() config = schema.load(PLUGIN_CONFIGURATION) - assert config.name == NAME - assert config.options == OPTIONS - - -LINUX_COMMAND = "a" -LINUX_FILENAME = "b" -WINDOWS_COMMAND = "c" -WINDOWS_FILENAME = "d" -CUSTOM_PBA_CONFIGURATION = { - "linux_command": LINUX_COMMAND, - "linux_filename": LINUX_FILENAME, - "windows_command": WINDOWS_COMMAND, - "windows_filename": WINDOWS_FILENAME, -} + assert config.name == PLUGIN_NAME + assert config.options == PLUGIN_OPTIONS def test_custom_pba_configuration_schema(): @@ -57,18 +66,6 @@ def test_custom_pba_configuration_schema(): assert config.windows_filename == WINDOWS_FILENAME -BLOCKED_IPS = ["10.0.0.1", "192.168.1.1"] -INACCESSIBLE_SUBNETS = ["172.0.0.0/24", "172.2.2.0/24", "192.168.56.0/24"] -LOCAL_NETWORK_SCAN = True -SUBNETS = ["10.0.0.2", "10.0.0.2/16"] -SCAN_TARGET_CONFIGURATION = { - "blocked_ips": BLOCKED_IPS, - "inaccessible_subnets": INACCESSIBLE_SUBNETS, - "local_network_scan": LOCAL_NETWORK_SCAN, - "subnets": SUBNETS, -} - - def test_scan_target_configuration(): schema = ScanTargetConfigurationSchema() @@ -80,10 +77,6 @@ def test_scan_target_configuration(): assert config.subnets == SUBNETS -TIMEOUT = 2.525 -ICMP_CONFIGURATION = {"timeout": TIMEOUT} - - def test_icmp_scan_configuration_schema(): schema = ICMPScanConfigurationSchema() @@ -92,11 +85,6 @@ def test_icmp_scan_configuration_schema(): assert config.timeout == TIMEOUT -PORTS = [8080, 443] - -TCP_SCAN_CONFIGURATION = {"timeout": TIMEOUT, "ports": PORTS} - - def test_tcp_scan_configuration_schema(): schema = TCPScanConfigurationSchema() @@ -106,15 +94,6 @@ def test_tcp_scan_configuration_schema(): assert config.ports == PORTS -FINGERPRINTERS = [{"name": "mssql", "options": {}}] -NETWORK_SCAN_CONFIGURATION = { - "tcp": TCP_SCAN_CONFIGURATION, - "icmp": ICMP_CONFIGURATION, - "fingerprinters": FINGERPRINTERS, - "targets": SCAN_TARGET_CONFIGURATION, -} - - def test_network_scan_configuration(): schema = NetworkScanConfigurationSchema() @@ -155,28 +134,6 @@ def test_exploiter_configuration_schema(): assert config.supported_os == supported_os -BRUTE_FORCE = [ - {"name": "ex1", "options": {}, "supported_os": ["LINUX"]}, - { - "name": "ex2", - "options": {"smb_download_timeout": 10}, - "supported_os": ["LINUX", "WINDOWS"], - }, -] -VULNERABILITY = [ - { - "name": "ex3", - "options": {"smb_download_timeout": 10}, - "supported_os": ["WINDOWS"], - }, -] -EXPLOITATION_CONFIGURATION = { - "options": {"http_ports": PORTS}, - "brute_force": BRUTE_FORCE, - "vulnerability": VULNERABILITY, -} - - def test_exploitation_configuration(): schema = ExploitationConfigurationSchema() @@ -187,13 +144,6 @@ def test_exploitation_configuration(): assert config_dict == EXPLOITATION_CONFIGURATION -PROPAGATION_CONFIGURATION = { - "maximum_depth": 5, - "network_scan": NETWORK_SCAN_CONFIGURATION, - "exploitation": EXPLOITATION_CONFIGURATION, -} - - def test_propagation_configuration(): schema = PropagationConfigurationSchema() @@ -208,17 +158,9 @@ def test_propagation_configuration(): def test_agent_configuration(): - agent_configuration = { - "keep_tunnel_open_time": 30, - "custom_pbas": CUSTOM_PBA_CONFIGURATION, - "post_breach_actions": [PLUGIN_CONFIGURATION], - "credential_collectors": [PLUGIN_CONFIGURATION], - "payloads": [PLUGIN_CONFIGURATION], - "propagation": PROPAGATION_CONFIGURATION, - } schema = AgentConfigurationSchema() - config = schema.load(agent_configuration) + config = schema.load(AGENT_CONFIGURATION) config_dict = schema.dump(config) assert isinstance(config, AgentConfiguration) @@ -228,4 +170,4 @@ def test_agent_configuration(): assert isinstance(config.credential_collectors[0], PluginConfiguration) assert isinstance(config.payloads[0], PluginConfiguration) assert isinstance(config.propagation, PropagationConfiguration) - assert config_dict == agent_configuration + assert config_dict == AGENT_CONFIGURATION From a75041e93ff1dca5e8a53e2771411d79672c6f0f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:36:50 -0400 Subject: [PATCH 038/196] Island: Rename IConfigurationRepository IAgentConfigurationRepository --- ...ration_repository.py => i_agent_configuration_repository.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename monkey/monkey_island/cc/repository/{i_configuration_repository.py => i_agent_configuration_repository.py} (93%) diff --git a/monkey/monkey_island/cc/repository/i_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py similarity index 93% rename from monkey/monkey_island/cc/repository/i_configuration_repository.py rename to monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index 58a97bce9..05817b80d 100644 --- a/monkey/monkey_island/cc/repository/i_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -3,7 +3,7 @@ from abc import ABC from common.configuration import AgentConfiguration -class IConfigurationRepository(ABC): +class IAgentConfigurationRepository(ABC): """ A repository used to store and retrieve the agent configuration. """ From 5ebdb60ea4297f28ecbfceb05bf6ca8465c552d7 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 11:41:29 -0400 Subject: [PATCH 039/196] Island: Add @abstractmethod in IAgentConfigurationRepository --- .../cc/repository/i_agent_configuration_repository.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index 05817b80d..4129d119c 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from common.configuration import AgentConfiguration @@ -8,6 +8,7 @@ class IAgentConfigurationRepository(ABC): A repository used to store and retrieve the agent configuration. """ + @abstractmethod def get_configuration(self) -> AgentConfiguration: """ Retrieve the agent configuration from the repository @@ -16,6 +17,7 @@ class IAgentConfigurationRepository(ABC): """ pass + @abstractmethod def set_configuration(self, agent_configuration: AgentConfiguration): """ Store the agent configuration in the repository From bdce5f84a6dbf43e5b8b38cf72ba7d30c1d8b61c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:09:30 -0400 Subject: [PATCH 040/196] Island: Add FileAgentConfigurationRepository --- .../monkey_island/cc/repository/__init__.py | 2 ++ .../file_agent_configuration_repository.py | 25 +++++++++++++++++++ ...est_file_agent_configuration_repository.py | 16 ++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 monkey/monkey_island/cc/repository/file_agent_configuration_repository.py create mode 100644 monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index 885d467be..ffe00e736 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -1,3 +1,5 @@ from .file_storage import FileRetrievalError, IFileRepository, LocalStorageFileRepository from .i_agent_binary_repository import IAgentBinaryRepository, AgentRetrievalError from .agent_binary_repository import AgentBinaryRepository +from .i_agent_configuration_repository import IAgentConfigurationRepository +from .file_agent_configuration_repository import FileAgentConfigurationRepository diff --git a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py new file mode 100644 index 000000000..e9724aa4b --- /dev/null +++ b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py @@ -0,0 +1,25 @@ +import io + +from common.configuration import AgentConfiguration, AgentConfigurationSchema +from monkey_island.cc.repository import IAgentConfigurationRepository, IFileRepository + +AGENT_CONFIGURATION_FILE_NAME = "agent_configuration.json" + + +class FileAgentConfigurationRepository(IAgentConfigurationRepository): + def __init__(self, file_repository: IFileRepository): + self._file_repository = file_repository + self._schema = AgentConfigurationSchema() + + def get_configuration(self) -> AgentConfiguration: + with self._file_repository.open_file(AGENT_CONFIGURATION_FILE_NAME) as f: + configuration_json = f.read().decode() + + return self._schema.loads(configuration_json) + + def set_configuration(self, agent_configuration: AgentConfiguration): + configuration_json = self._schema.dumps(agent_configuration) + + self._file_repository.save_file( + AGENT_CONFIGURATION_FILE_NAME, io.BytesIO(configuration_json.encode()) + ) diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py new file mode 100644 index 000000000..bfb894913 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -0,0 +1,16 @@ +from tests.common.example_agent_configuration import AGENT_CONFIGURATION +from tests.monkey_island import SingleFileRepository + +from common.configuration import AgentConfigurationSchema +from monkey_island.cc.repository import FileAgentConfigurationRepository + + +def test_store_agent_config(): + repository = FileAgentConfigurationRepository(SingleFileRepository()) + schema = AgentConfigurationSchema() + agent_configuration = schema.load(AGENT_CONFIGURATION) + + repository.set_configuration(agent_configuration) + retrieved_agent_configuration = repository.get_configuration() + + assert retrieved_agent_configuration == agent_configuration From 73ead3bb261944fa0430e462c711d0543541676b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:10:43 -0400 Subject: [PATCH 041/196] Island: Rename set_configuration() -> store_configuration() --- .../cc/repository/file_agent_configuration_repository.py | 2 +- .../cc/repository/i_agent_configuration_repository.py | 2 +- .../cc/repository/test_file_agent_configuration_repository.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py index e9724aa4b..2cd2c8c93 100644 --- a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py @@ -17,7 +17,7 @@ class FileAgentConfigurationRepository(IAgentConfigurationRepository): return self._schema.loads(configuration_json) - def set_configuration(self, agent_configuration: AgentConfiguration): + def store_configuration(self, agent_configuration: AgentConfiguration): configuration_json = self._schema.dumps(agent_configuration) self._file_repository.save_file( diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index 4129d119c..f91895c8b 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -18,7 +18,7 @@ class IAgentConfigurationRepository(ABC): pass @abstractmethod - def set_configuration(self, agent_configuration: AgentConfiguration): + def store_configuration(self, agent_configuration: AgentConfiguration): """ Store the agent configuration in the repository diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py index bfb894913..941dd2586 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -10,7 +10,7 @@ def test_store_agent_config(): schema = AgentConfigurationSchema() agent_configuration = schema.load(AGENT_CONFIGURATION) - repository.set_configuration(agent_configuration) + repository.store_configuration(agent_configuration) retrieved_agent_configuration = repository.get_configuration() assert retrieved_agent_configuration == agent_configuration From 30065952a5cdbaa794cdf2648e6705cf83b475ed Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:13:18 -0400 Subject: [PATCH 042/196] Island: Add RetrievalError --- monkey/monkey_island/cc/repository/__init__.py | 1 + monkey/monkey_island/cc/repository/errors.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 monkey/monkey_island/cc/repository/errors.py diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index ffe00e736..0ff63b842 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -3,3 +3,4 @@ from .i_agent_binary_repository import IAgentBinaryRepository, AgentRetrievalErr from .agent_binary_repository import AgentBinaryRepository from .i_agent_configuration_repository import IAgentConfigurationRepository from .file_agent_configuration_repository import FileAgentConfigurationRepository +from .errors import RetrievalError diff --git a/monkey/monkey_island/cc/repository/errors.py b/monkey/monkey_island/cc/repository/errors.py new file mode 100644 index 000000000..b7beb36fe --- /dev/null +++ b/monkey/monkey_island/cc/repository/errors.py @@ -0,0 +1,2 @@ +class RetrievalError(RuntimeError): + pass From 922cb8cea97f8744a47615c76e3afb33f4d54a6b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:14:30 -0400 Subject: [PATCH 043/196] Island: Add RetrievalError --- monkey/monkey_island/cc/repository/__init__.py | 2 +- .../cc/repository/file_storage/i_file_repository.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index 0ff63b842..cc89c5c57 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -1,6 +1,6 @@ +from .errors import RetrievalError from .file_storage import FileRetrievalError, IFileRepository, LocalStorageFileRepository from .i_agent_binary_repository import IAgentBinaryRepository, AgentRetrievalError from .agent_binary_repository import AgentBinaryRepository from .i_agent_configuration_repository import IAgentConfigurationRepository from .file_agent_configuration_repository import FileAgentConfigurationRepository -from .errors import RetrievalError diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index a0865b797..2531fd711 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -1,8 +1,10 @@ import abc from typing import BinaryIO +from monkey_island.cc.repository import RetrievalError -class FileRetrievalError(RuntimeError): + +class FileRetrievalError(RetrievalError): pass From 9a216e9c3ddb9bd2f668d68a2746aea283b4b05d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:15:55 -0400 Subject: [PATCH 044/196] Island: Add "raises" to docstring in get_configuration() --- .../cc/repository/i_agent_configuration_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index f91895c8b..d99934ad9 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -14,6 +14,7 @@ class IAgentConfigurationRepository(ABC): Retrieve the agent configuration from the repository :return: The agent configuration + :raises RetrievalError: if the configuration can not be retrieved """ pass From a3e3e3e3245672e57773277d0a3f49ae75a8f0d9 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:48:54 -0400 Subject: [PATCH 045/196] Common: Add DEFAULT_AGENT_CONFIGURATION --- monkey/common/configuration/__init__.py | 1 + .../configuration/agent_configuration.py | 218 ++++++++++++++++++ .../common/test_agent_configuration.py | 14 +- 3 files changed, 232 insertions(+), 1 deletion(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 59186d56c..29f8f1e2c 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1,4 +1,5 @@ from .agent_configuration import ( AgentConfiguration, AgentConfigurationSchema, + DEFAULT_AGENT_CONFIGURATION, ) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 15b338fe0..a4db46f2c 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -14,6 +14,224 @@ from .agent_sub_configurations import ( PropagationConfiguration, ) +DEFAULT_AGENT_CONFIGURATION = """{ + "keep_tunnel_open_time": 30, + "post_breach_actions": [ + { + "name": "CommunicateAsBackdoorUser", + "options": {} + }, + { + "name": "ModifyShellStartupFiles", + "options": {} + }, + { + "name": "HiddenFiles", + "options": {} + }, + { + "name": "TrapCommand", + "options": {} + }, + { + "name": "ChangeSetuidSetgid", + "options": {} + }, + { + "name": "ScheduleJobs", + "options": {} + }, + { + "name": "Timestomping", + "options": {} + }, + { + "name": "AccountDiscovery", + "options": {} + }, + { + "name": "ProcessListCollection", + "options": {} + } + ], + "credential_collectors": [ + { + "name": "MimikatzCollector", + "options": {} + }, + { + "name": "SSHCollector", + "options": {} + } + ], + "payloads": [ + { + "name": "ransomware", + "options": { + "encryption": { + "enabled": true, + "directories": { + "linux_target_dir": "", + "windows_target_dir": "" + } + }, + "other_behaviors": { + "readme": true + } + } + } + ], + "custom_pbas": { + "linux_command": "", + "linux_filename": "", + "windows_command": "", + "windows_filename": "" + }, + "propagation": { + "maximum_depth": 2, + "network_scan": { + "tcp": { + "timeout": 3000, + "ports": [ + 22, + 80, + 135, + 443, + 445, + 2222, + 3306, + 3389, + 5985, + 5986, + 7001, + 8008, + 8080, + 8088, + 8983, + 9200, + 9600 + ] + }, + "icmp": { + "timeout": 1000 + }, + "fingerprinters": [ + { + "name": "elastic", + "options": {} + }, + { + "name": "http", + "options": { + "http_ports": [ + 80, + 443, + 7001, + 8008, + 8080, + 8983, + 9200, + 9600 + ] + } + }, + { + "name": "mssql", + "options": {} + }, + { + "name": "smb", + "options": {} + }, + { + "name": "ssh", + "options": {} + } + ], + "targets": { + "blocked_ips": [], + "inaccessible_subnets": [], + "local_network_scan": true, + "subnets": [] + } + }, + "exploitation": { + "options": { + "http_ports": [ + 80, + 443, + 7001, + 8008, + 8080, + 8983, + 9200, + 9600 + ] + }, + "brute_force": [ + { + "name": "MSSQLExploiter", + "options": {}, + "supported_os": [ + "WINDOWS" + ] + }, + { + "name": "PowerShellExploiter", + "options": {}, + "supported_os": [ + "WINDOWS" + ] + }, + { + "name": "SSHExploiter", + "options": {}, + "supported_os": [ + "LINUX" + ] + }, + { + "name": "SmbExploiter", + "options": { + "smb_download_timeout": 30 + }, + "supported_os": [ + "WINDOWS" + ] + }, + { + "name": "WmiExploiter", + "options": { + "smb_download_timeout": 30 + }, + "supported_os": [ + "WINDOWS" + ] + } + ], + "vulnerability": [ + { + "name": "HadoopExploiter", + "options": {}, + "supported_os": [ + "LINUX", + "WINDOWS" + ] + }, + { + "name": "Log4ShellExploiter", + "options": {}, + "supported_os": [ + "LINUX", + "WINDOWS" + ] + } + ] + } + } + } +""" + @dataclass(frozen=True) class AgentConfiguration: diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 9609a2d3a..733e2e708 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -24,7 +24,11 @@ from tests.common.example_agent_configuration import ( ) from common import OperatingSystems -from common.configuration import AgentConfiguration, AgentConfigurationSchema +from common.configuration import ( + DEFAULT_AGENT_CONFIGURATION, + AgentConfiguration, + AgentConfigurationSchema, +) from common.configuration.agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, ExploitationConfigurationSchema, @@ -171,3 +175,11 @@ def test_agent_configuration(): assert isinstance(config.payloads[0], PluginConfiguration) assert isinstance(config.propagation, PropagationConfiguration) assert config_dict == AGENT_CONFIGURATION + + +def test_default_agent_configuration(): + schema = AgentConfigurationSchema() + + config = schema.loads(DEFAULT_AGENT_CONFIGURATION) + + assert isinstance(config, AgentConfiguration) From 45168b5ba77a04a29e4b7aa3e238a147bf026f2f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 12:55:28 -0400 Subject: [PATCH 046/196] Island: Return default configuration from get_configuration() --- .../file_agent_configuration_repository.py | 23 +++++++++++++++---- .../i_agent_configuration_repository.py | 3 ++- ...est_file_agent_configuration_repository.py | 12 +++++++++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py index 2cd2c8c93..fcccd49a1 100644 --- a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py @@ -1,7 +1,15 @@ import io -from common.configuration import AgentConfiguration, AgentConfigurationSchema -from monkey_island.cc.repository import IAgentConfigurationRepository, IFileRepository +from common.configuration import ( + DEFAULT_AGENT_CONFIGURATION, + AgentConfiguration, + AgentConfigurationSchema, +) +from monkey_island.cc.repository import ( + IAgentConfigurationRepository, + IFileRepository, + RetrievalError, +) AGENT_CONFIGURATION_FILE_NAME = "agent_configuration.json" @@ -12,10 +20,15 @@ class FileAgentConfigurationRepository(IAgentConfigurationRepository): self._schema = AgentConfigurationSchema() def get_configuration(self) -> AgentConfiguration: - with self._file_repository.open_file(AGENT_CONFIGURATION_FILE_NAME) as f: - configuration_json = f.read().decode() + try: + with self._file_repository.open_file(AGENT_CONFIGURATION_FILE_NAME) as f: + configuration_json = f.read().decode() - return self._schema.loads(configuration_json) + return self._schema.loads(configuration_json) + # TODO: Handle FileRetrievalError vs FileNotFoundError + # https://github.com/guardicore/monkey/blob/e8001d8cf76340e42bf17ff62523bd2d85fc4841/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py#L47-L50 + except RetrievalError: + return self._schema.loads(DEFAULT_AGENT_CONFIGURATION) def store_configuration(self, agent_configuration: AgentConfiguration): configuration_json = self._schema.dumps(agent_configuration) diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index d99934ad9..105d26fe9 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -13,7 +13,8 @@ class IAgentConfigurationRepository(ABC): """ Retrieve the agent configuration from the repository - :return: The agent configuration + :return: The agent configuration as retrieved from the repository, or the default + configuration if none could be retrieved. :raises RetrievalError: if the configuration can not be retrieved """ pass diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py index 941dd2586..7ad066623 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -1,7 +1,7 @@ from tests.common.example_agent_configuration import AGENT_CONFIGURATION from tests.monkey_island import SingleFileRepository -from common.configuration import AgentConfigurationSchema +from common.configuration import DEFAULT_AGENT_CONFIGURATION, AgentConfigurationSchema from monkey_island.cc.repository import FileAgentConfigurationRepository @@ -14,3 +14,13 @@ def test_store_agent_config(): retrieved_agent_configuration = repository.get_configuration() assert retrieved_agent_configuration == agent_configuration + + +def test_get_default_agent_config(): + repository = FileAgentConfigurationRepository(SingleFileRepository()) + schema = AgentConfigurationSchema() + default_agent_configuration = schema.loads(DEFAULT_AGENT_CONFIGURATION) + + retrieved_agent_configuration = repository.get_configuration() + + assert retrieved_agent_configuration == default_agent_configuration From 764bc1559bf37da23dbebbeaf0cd11d70a5cab46 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 13:08:22 -0400 Subject: [PATCH 047/196] Island: Add IAgentConfigurationRepository to the DIContainer --- monkey/monkey_island/cc/services/initialize.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/services/initialize.py b/monkey/monkey_island/cc/services/initialize.py index a5d283cb3..74d447c90 100644 --- a/monkey/monkey_island/cc/services/initialize.py +++ b/monkey/monkey_island/cc/services/initialize.py @@ -7,7 +7,9 @@ from common.utils.file_utils import get_binary_io_sha256_hash from monkey_island.cc.repository import ( AgentBinaryRepository, AgentRetrievalError, + FileAgentConfigurationRepository, IAgentBinaryRepository, + IAgentConfigurationRepository, IFileRepository, LocalStorageFileRepository, ) @@ -31,11 +33,14 @@ def initialize_services(data_dir: Path) -> DIContainer: container.register_instance(AWSInstance, AWSInstance()) container.register_instance( - IFileRepository, LocalStorageFileRepository(data_dir / "custom_pbas") + IFileRepository, LocalStorageFileRepository(data_dir / "runtime_data") ) container.register_instance(AWSService, container.resolve(AWSService)) container.register_instance(IAgentBinaryRepository, _build_agent_binary_repository()) container.register_instance(LocalMonkeyRunService, container.resolve(LocalMonkeyRunService)) + container.register_instance( + IAgentConfigurationRepository, container.resolve(FileAgentConfigurationRepository) + ) # This is temporary until we get DI all worked out. PostBreachFilesService.initialize(container.resolve(IFileRepository)) From e730695407856305b2c06cb7572fcb5f75ab51c8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 13:11:30 -0400 Subject: [PATCH 048/196] Agent: Remove disused Configuration class --- monkey/infection_monkey/config.py | 55 ------------------------------- 1 file changed, 55 deletions(-) diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index 1b7b7f60f..5c48a4b1e 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -1,5 +1,4 @@ import uuid -from abc import ABCMeta GUID = str(uuid.getnode()) @@ -10,57 +9,3 @@ SENSITIVE_FIELDS = [ ] LOCAL_CONFIG_VARS = ["name", "id", "max_depth"] HIDDEN_FIELD_REPLACEMENT_CONTENT = "hidden" - - -class Configuration(object): - def from_kv(self, formatted_data): - for key, value in list(formatted_data.items()): - if key.startswith("_"): - continue - if key in LOCAL_CONFIG_VARS: - continue - if hasattr(self, key): - setattr(self, key, value) - if not self.max_depth: - self.max_depth = self.depth - - @staticmethod - def hide_sensitive_info(config_dict): - for field in SENSITIVE_FIELDS: - config_dict[field] = HIDDEN_FIELD_REPLACEMENT_CONTENT - return config_dict - - def as_dict(self): - result = {} - for key in dir(Configuration): - if key.startswith("_"): - continue - try: - value = getattr(self, key) - except AttributeError: - continue - - val_type = type(value) - - if callable(value): - continue - - if val_type in (type, ABCMeta): - value = value.__name__ - elif val_type is tuple or val_type is list: - if len(value) != 0 and type(value[0]) in (type, ABCMeta): - value = val_type([x.__name__ for x in value]) - - result[key] = value - - return result - - ########################### - # monkey config - ########################### - - # depth of propagation - depth = 2 - max_depth = None - - keep_tunnel_open_time = 30 From d3a03d265317026006b3b5c25130a204d71935a8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 13:11:47 -0400 Subject: [PATCH 049/196] Agent: Remove disused constants --- monkey/infection_monkey/config.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index 5c48a4b1e..bcf95ee72 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -1,11 +1,3 @@ import uuid GUID = str(uuid.getnode()) - -SENSITIVE_FIELDS = [ - "exploit_password_list", - "exploit_user_list", - "exploit_ssh_keys", -] -LOCAL_CONFIG_VARS = ["name", "id", "max_depth"] -HIDDEN_FIELD_REPLACEMENT_CONTENT = "hidden" From e4a2a04765e20d4d7b0481cf5e11e4bd169d6bd8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 19:14:38 -0400 Subject: [PATCH 050/196] UT: Remove duplicate SingleFileRepository This was added by mistake. Remove it! --- monkey/tests/common/SingleFileRepository.py | 23 --------------------- 1 file changed, 23 deletions(-) delete mode 100644 monkey/tests/common/SingleFileRepository.py diff --git a/monkey/tests/common/SingleFileRepository.py b/monkey/tests/common/SingleFileRepository.py deleted file mode 100644 index d00ee9ae9..000000000 --- a/monkey/tests/common/SingleFileRepository.py +++ /dev/null @@ -1,23 +0,0 @@ -import io -from typing import BinaryIO - -from monkey_island.cc.repository import FileRetrievalError, IFileRepository - - -class SingleFileRepository(IFileRepository): - def __init__(self): - self._file = None - - def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): - self._file = io.BytesIO(file_contents.read()) - - def open_file(self, unsafe_file_name: str) -> BinaryIO: - if self._file is None: - raise FileRetrievalError() - return self._file - - def delete_file(self, unsafe_file_name: str): - self._file = None - - def delete_all_files(self): - self.delete_file("") From 58ea11ae9fbb101164de79ac5928796acb922be2 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 19:16:19 -0400 Subject: [PATCH 051/196] Common: Move DEFAULT_AGENT_CONFIGURATION to its own file --- monkey/common/configuration/__init__.py | 2 +- .../configuration/agent_configuration.py | 218 ------------------ .../default_agent_configuration.py | 217 +++++++++++++++++ 3 files changed, 218 insertions(+), 219 deletions(-) create mode 100644 monkey/common/configuration/default_agent_configuration.py diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 29f8f1e2c..7a9131cc7 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1,5 +1,5 @@ from .agent_configuration import ( AgentConfiguration, AgentConfigurationSchema, - DEFAULT_AGENT_CONFIGURATION, ) +from .default_agent_configuration import DEFAULT_AGENT_CONFIGURATION diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index a4db46f2c..15b338fe0 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -14,224 +14,6 @@ from .agent_sub_configurations import ( PropagationConfiguration, ) -DEFAULT_AGENT_CONFIGURATION = """{ - "keep_tunnel_open_time": 30, - "post_breach_actions": [ - { - "name": "CommunicateAsBackdoorUser", - "options": {} - }, - { - "name": "ModifyShellStartupFiles", - "options": {} - }, - { - "name": "HiddenFiles", - "options": {} - }, - { - "name": "TrapCommand", - "options": {} - }, - { - "name": "ChangeSetuidSetgid", - "options": {} - }, - { - "name": "ScheduleJobs", - "options": {} - }, - { - "name": "Timestomping", - "options": {} - }, - { - "name": "AccountDiscovery", - "options": {} - }, - { - "name": "ProcessListCollection", - "options": {} - } - ], - "credential_collectors": [ - { - "name": "MimikatzCollector", - "options": {} - }, - { - "name": "SSHCollector", - "options": {} - } - ], - "payloads": [ - { - "name": "ransomware", - "options": { - "encryption": { - "enabled": true, - "directories": { - "linux_target_dir": "", - "windows_target_dir": "" - } - }, - "other_behaviors": { - "readme": true - } - } - } - ], - "custom_pbas": { - "linux_command": "", - "linux_filename": "", - "windows_command": "", - "windows_filename": "" - }, - "propagation": { - "maximum_depth": 2, - "network_scan": { - "tcp": { - "timeout": 3000, - "ports": [ - 22, - 80, - 135, - 443, - 445, - 2222, - 3306, - 3389, - 5985, - 5986, - 7001, - 8008, - 8080, - 8088, - 8983, - 9200, - 9600 - ] - }, - "icmp": { - "timeout": 1000 - }, - "fingerprinters": [ - { - "name": "elastic", - "options": {} - }, - { - "name": "http", - "options": { - "http_ports": [ - 80, - 443, - 7001, - 8008, - 8080, - 8983, - 9200, - 9600 - ] - } - }, - { - "name": "mssql", - "options": {} - }, - { - "name": "smb", - "options": {} - }, - { - "name": "ssh", - "options": {} - } - ], - "targets": { - "blocked_ips": [], - "inaccessible_subnets": [], - "local_network_scan": true, - "subnets": [] - } - }, - "exploitation": { - "options": { - "http_ports": [ - 80, - 443, - 7001, - 8008, - 8080, - 8983, - 9200, - 9600 - ] - }, - "brute_force": [ - { - "name": "MSSQLExploiter", - "options": {}, - "supported_os": [ - "WINDOWS" - ] - }, - { - "name": "PowerShellExploiter", - "options": {}, - "supported_os": [ - "WINDOWS" - ] - }, - { - "name": "SSHExploiter", - "options": {}, - "supported_os": [ - "LINUX" - ] - }, - { - "name": "SmbExploiter", - "options": { - "smb_download_timeout": 30 - }, - "supported_os": [ - "WINDOWS" - ] - }, - { - "name": "WmiExploiter", - "options": { - "smb_download_timeout": 30 - }, - "supported_os": [ - "WINDOWS" - ] - } - ], - "vulnerability": [ - { - "name": "HadoopExploiter", - "options": {}, - "supported_os": [ - "LINUX", - "WINDOWS" - ] - }, - { - "name": "Log4ShellExploiter", - "options": {}, - "supported_os": [ - "LINUX", - "WINDOWS" - ] - } - ] - } - } - } -""" - @dataclass(frozen=True) class AgentConfiguration: diff --git a/monkey/common/configuration/default_agent_configuration.py b/monkey/common/configuration/default_agent_configuration.py new file mode 100644 index 000000000..933b723ea --- /dev/null +++ b/monkey/common/configuration/default_agent_configuration.py @@ -0,0 +1,217 @@ +DEFAULT_AGENT_CONFIGURATION = """{ + "keep_tunnel_open_time": 30, + "post_breach_actions": [ + { + "name": "CommunicateAsBackdoorUser", + "options": {} + }, + { + "name": "ModifyShellStartupFiles", + "options": {} + }, + { + "name": "HiddenFiles", + "options": {} + }, + { + "name": "TrapCommand", + "options": {} + }, + { + "name": "ChangeSetuidSetgid", + "options": {} + }, + { + "name": "ScheduleJobs", + "options": {} + }, + { + "name": "Timestomping", + "options": {} + }, + { + "name": "AccountDiscovery", + "options": {} + }, + { + "name": "ProcessListCollection", + "options": {} + } + ], + "credential_collectors": [ + { + "name": "MimikatzCollector", + "options": {} + }, + { + "name": "SSHCollector", + "options": {} + } + ], + "payloads": [ + { + "name": "ransomware", + "options": { + "encryption": { + "enabled": true, + "directories": { + "linux_target_dir": "", + "windows_target_dir": "" + } + }, + "other_behaviors": { + "readme": true + } + } + } + ], + "custom_pbas": { + "linux_command": "", + "linux_filename": "", + "windows_command": "", + "windows_filename": "" + }, + "propagation": { + "maximum_depth": 2, + "network_scan": { + "tcp": { + "timeout": 3000, + "ports": [ + 22, + 80, + 135, + 443, + 445, + 2222, + 3306, + 3389, + 5985, + 5986, + 7001, + 8008, + 8080, + 8088, + 8983, + 9200, + 9600 + ] + }, + "icmp": { + "timeout": 1000 + }, + "fingerprinters": [ + { + "name": "elastic", + "options": {} + }, + { + "name": "http", + "options": { + "http_ports": [ + 80, + 443, + 7001, + 8008, + 8080, + 8983, + 9200, + 9600 + ] + } + }, + { + "name": "mssql", + "options": {} + }, + { + "name": "smb", + "options": {} + }, + { + "name": "ssh", + "options": {} + } + ], + "targets": { + "blocked_ips": [], + "inaccessible_subnets": [], + "local_network_scan": true, + "subnets": [] + } + }, + "exploitation": { + "options": { + "http_ports": [ + 80, + 443, + 7001, + 8008, + 8080, + 8983, + 9200, + 9600 + ] + }, + "brute_force": [ + { + "name": "MSSQLExploiter", + "options": {}, + "supported_os": [ + "WINDOWS" + ] + }, + { + "name": "PowerShellExploiter", + "options": {}, + "supported_os": [ + "WINDOWS" + ] + }, + { + "name": "SSHExploiter", + "options": {}, + "supported_os": [ + "LINUX" + ] + }, + { + "name": "SmbExploiter", + "options": { + "smb_download_timeout": 30 + }, + "supported_os": [ + "WINDOWS" + ] + }, + { + "name": "WmiExploiter", + "options": { + "smb_download_timeout": 30 + }, + "supported_os": [ + "WINDOWS" + ] + } + ], + "vulnerability": [ + { + "name": "HadoopExploiter", + "options": {}, + "supported_os": [ + "LINUX", + "WINDOWS" + ] + }, + { + "name": "Log4ShellExploiter", + "options": {}, + "supported_os": [ + "LINUX", + "WINDOWS" + ] + } + ] + } + } + } +""" From cb7dae28bff3cf9f2afd31cd00da8e008f37488c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 19:55:47 -0400 Subject: [PATCH 052/196] Island: Add a docstring for RetrievalError --- monkey/monkey_island/cc/repository/errors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monkey/monkey_island/cc/repository/errors.py b/monkey/monkey_island/cc/repository/errors.py index b7beb36fe..b9d5b7e47 100644 --- a/monkey/monkey_island/cc/repository/errors.py +++ b/monkey/monkey_island/cc/repository/errors.py @@ -1,2 +1,5 @@ class RetrievalError(RuntimeError): + """ + Raised when a repository encounters an error while attempting to retrieve data. + """ pass From b99ad70774026151132075d5fc95721ee92d5590 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 17 Jun 2022 19:56:09 -0400 Subject: [PATCH 053/196] Island: Add StorageError --- monkey/monkey_island/cc/repository/errors.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/monkey/monkey_island/cc/repository/errors.py b/monkey/monkey_island/cc/repository/errors.py index b9d5b7e47..586f23cb2 100644 --- a/monkey/monkey_island/cc/repository/errors.py +++ b/monkey/monkey_island/cc/repository/errors.py @@ -2,4 +2,13 @@ class RetrievalError(RuntimeError): """ Raised when a repository encounters an error while attempting to retrieve data. """ + + pass + + +class StorageError(RuntimeError): + """ + Raised when a repository encounters an error while attempting to store data. + """ + pass From d393a0b3c632b03420d0f96c868d8eed37cf09c1 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Fri, 17 Jun 2022 18:05:26 +0300 Subject: [PATCH 054/196] Agent: Change credential collectors, payloads and pbas in flat config Flat config changes are made in order for config object to be serializable --- monkey/monkey_island/cc/services/config.py | 18 +++++++--- .../monkey_configs/flat_config.json | 1 + .../monkey_island/cc/services/test_config.py | 33 +++++++++---------- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 82346258f..3b466cd31 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -357,6 +357,7 @@ class ConfigService: ConfigService._format_payloads_from_flat_config(config) ConfigService._format_pbas_from_flat_config(config) ConfigService._format_propagation_from_flat_config(config) + ConfigService._format_credential_collectors(config) # Ok, I'll admit this is just sort of jammed in here. But this code is going away very soon. del config["HTTP_PORTS"] @@ -376,9 +377,18 @@ class ConfigService: for field in fields_to_remove: config.pop(field, None) + @staticmethod + def _format_credential_collectors(config: Dict): + collectors = [ + {"name": collector, "options": {}} for collector in config["credential_collectors"] + ] + config["credential_collectors"] = collectors + @staticmethod def _format_payloads_from_flat_config(config: Dict): - config.setdefault("payloads", {})["ransomware"] = config["ransomware"] + config.setdefault("payloads", []).append( + {"name": "ransomware", "options": config["ransomware"]} + ) config.pop("ransomware", None) @staticmethod @@ -388,9 +398,9 @@ class ConfigService: flat_windows_command_field = "custom_PBA_windows_cmd" flat_windows_filename_field = "PBA_windows_filename" - formatted_pbas_config = {} - for pba in config.get("post_breach_actions", []): - formatted_pbas_config[pba] = {} + formatted_pbas_config = [ + {"name": pba, "options": {}} for pba in config.get("post_breach_actions", []) + ] config["custom_pbas"] = { "linux_command": config.get(flat_linux_command_field, ""), diff --git a/monkey/tests/data_for_tests/monkey_configs/flat_config.json b/monkey/tests/data_for_tests/monkey_configs/flat_config.json index 33bf50da1..441c176ed 100644 --- a/monkey/tests/data_for_tests/monkey_configs/flat_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/flat_config.json @@ -27,6 +27,7 @@ "private_key": "my_private_key" } ], + "credential_collectors": ["MimikatzCollector", "SSHCollector"], "exploit_user_list": [ "Administrator", "root", diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index 9e01b8365..108659ddf 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -25,34 +25,33 @@ def test_format_config_for_agent__credentials_removed(): def test_format_config_for_agent__ransomware_payload(): expected_ransomware_options = { - "ransomware": { - "encryption": { - "enabled": True, - "directories": { - "linux_target_dir": "/tmp/ransomware-target", - "windows_target_dir": "C:\\windows\\temp\\ransomware-target", - }, + "encryption": { + "enabled": True, + "directories": { + "linux_target_dir": "/tmp/ransomware-target", + "windows_target_dir": "C:\\windows\\temp\\ransomware-target", }, - "other_behaviors": {"readme": True}, - } + }, + "other_behaviors": {"readme": True}, } flat_monkey_config = ConfigService.format_flat_config_for_agent() assert "payloads" in flat_monkey_config - assert flat_monkey_config["payloads"] == expected_ransomware_options + assert flat_monkey_config["payloads"][0]["name"] == "ransomware" + assert flat_monkey_config["payloads"][0]["options"] == expected_ransomware_options assert "ransomware" not in flat_monkey_config def test_format_config_for_agent__pbas(): - expected_pbas_config = { - "CommunicateAsBackdoorUser": {}, - "ModifyShellStartupFiles": {}, - "ScheduleJobs": {}, - "Timestomping": {}, - "AccountDiscovery": {}, - } + expected_pbas_config = [ + {"name": "CommunicateAsBackdoorUser", "options": {}}, + {"name": "ModifyShellStartupFiles", "options": {}}, + {"name": "ScheduleJobs", "options": {}}, + {"name": "Timestomping", "options": {}}, + {"name": "AccountDiscovery", "options": {}}, + ] flat_monkey_config = ConfigService.format_flat_config_for_agent() assert "post_breach_actions" in flat_monkey_config From 83dd4334b22fe6398ce1f573243bcb4dc578dffa Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Sun, 19 Jun 2022 21:54:54 -0700 Subject: [PATCH 055/196] UT: Fix flat_config.json to match current config schema --- monkey/tests/data_for_tests/monkey_configs/flat_config.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/monkey/tests/data_for_tests/monkey_configs/flat_config.json b/monkey/tests/data_for_tests/monkey_configs/flat_config.json index 441c176ed..42568404a 100644 --- a/monkey/tests/data_for_tests/monkey_configs/flat_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/flat_config.json @@ -9,7 +9,6 @@ ], "PBA_linux_filename": "test.sh", "PBA_windows_filename": "test.ps1", - "alive": true, "blocked_ips": ["192.168.1.1", "192.168.1.100"], "custom_PBA_linux_cmd": "bash test.sh", "custom_PBA_windows_cmd": "powershell test.ps1", @@ -54,7 +53,6 @@ "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], "keep_tunnel_open_time": 60, "local_network_scan": true, - "max_depth": null, "ping_scan_timeout": 1000, "post_breach_actions": [ "CommunicateAsBackdoorUser", @@ -76,9 +74,6 @@ } }, "subnet_scan_list": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], - "system_info_collector_classes": [ - "MimikatzCollector" - ], "tcp_scan_timeout": 3000, "tcp_target_ports": [ 22, From f9a7989f5e150cb51852b402a45210d2523432ba Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Sun, 19 Jun 2022 22:07:47 -0700 Subject: [PATCH 056/196] Island: Fix 'propagation' field of config flattening --- monkey/monkey_island/cc/services/config.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 3b466cd31..aca2217e3 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -418,24 +418,24 @@ class ConfigService: @staticmethod def _format_propagation_from_flat_config(config: Dict): - formatted_propagation_config = {"network_scan": {}, "targets": {}} + formatted_propagation_config = {"network_scan": {}, "maximum_depth": {}, "exploiters": {}} formatted_propagation_config[ "network_scan" ] = ConfigService._format_network_scan_from_flat_config(config) - formatted_propagation_config["targets"] = ConfigService._format_targets_from_flat_config( - config - ) formatted_propagation_config[ "exploiters" ] = ConfigService._format_exploiters_from_flat_config(config) + formatted_propagation_config["maximum_depth"] = config["depth"] + del config["depth"] + config["propagation"] = formatted_propagation_config @staticmethod def _format_network_scan_from_flat_config(config: Dict) -> Dict[str, Any]: - formatted_network_scan_config = {"tcp": {}, "icmp": {}, "fingerprinters": []} + formatted_network_scan_config = {"tcp": {}, "icmp": {}, "fingerprinters": [], "targets": {}} formatted_network_scan_config["tcp"] = ConfigService._format_tcp_scan_from_flat_config( config @@ -447,6 +447,10 @@ class ConfigService: "fingerprinters" ] = ConfigService._format_fingerprinters_from_flat_config(config) + formatted_network_scan_config["targets"] = ConfigService._format_targets_from_flat_config( + config + ) + return formatted_network_scan_config @staticmethod @@ -457,7 +461,7 @@ class ConfigService: formatted_tcp_scan_config = {} - formatted_tcp_scan_config["timeout_ms"] = config[flat_tcp_timeout_field] + formatted_tcp_scan_config["timeout"] = config[flat_tcp_timeout_field] ports = ConfigService._union_tcp_and_http_ports( config[flat_tcp_ports_field], config[flat_http_ports_field] @@ -481,7 +485,7 @@ class ConfigService: flat_ping_timeout_field = "ping_scan_timeout" formatted_icmp_scan_config = {} - formatted_icmp_scan_config["timeout_ms"] = config[flat_ping_timeout_field] + formatted_icmp_scan_config["timeout"] = config[flat_ping_timeout_field] config.pop(flat_ping_timeout_field, None) @@ -529,7 +533,7 @@ class ConfigService: formatted_scan_targets_config[flat_local_network_scan_field] = config[ flat_local_network_scan_field ] - formatted_scan_targets_config[flat_subnet_scan_list_field] = config[ + formatted_scan_targets_config["subnets"] = config[ flat_subnet_scan_list_field ] From ba3af5a9c293a5e4b375426af7ee42a2389305aa Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Sun, 19 Jun 2022 22:13:54 -0700 Subject: [PATCH 057/196] Island: Fix 'exploitation' field of config flattening --- monkey/monkey_island/cc/services/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index aca2217e3..b47c0edd5 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -418,14 +418,14 @@ class ConfigService: @staticmethod def _format_propagation_from_flat_config(config: Dict): - formatted_propagation_config = {"network_scan": {}, "maximum_depth": {}, "exploiters": {}} + formatted_propagation_config = {"network_scan": {}, "maximum_depth": {}, "exploitation": {}} formatted_propagation_config[ "network_scan" ] = ConfigService._format_network_scan_from_flat_config(config) formatted_propagation_config[ - "exploiters" + "exploitation" ] = ConfigService._format_exploiters_from_flat_config(config) formatted_propagation_config["maximum_depth"] = config["depth"] From ab23b3c9cb3024994ee0bb02144fc9e71bbab0f4 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Sun, 19 Jun 2022 23:11:00 -0700 Subject: [PATCH 058/196] Island: Fix exploiters' `supported_os`'s capitalisation in config flattening --- monkey/monkey_island/cc/services/config.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index b47c0edd5..684801ac2 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -600,14 +600,14 @@ class ConfigService: formatted_config: Dict, ) -> Dict[str, List[Dict[str, Any]]]: supported_os = { - "HadoopExploiter": ["linux", "windows"], - "Log4ShellExploiter": ["linux", "windows"], - "MSSQLExploiter": ["windows"], - "PowerShellExploiter": ["windows"], - "SSHExploiter": ["linux"], - "SmbExploiter": ["windows"], - "WmiExploiter": ["windows"], - "ZerologonExploiter": ["windows"], + "HadoopExploiter": ["LINUX", "WINDOWS"], + "Log4ShellExploiter": ["LINUX", "WINDOWS"], + "MSSQLExploiter": ["WINDOWS"], + "PowerShellExploiter": ["WINDOWS"], + "SSHExploiter": ["LINUX"], + "SmbExploiter": ["WINDOWS"], + "WmiExploiter": ["WINDOWS"], + "ZerologonExploiter": ["WINDOWS"], } new_config = copy.deepcopy(formatted_config) for exploiter in chain(new_config["brute_force"], new_config["vulnerability"]): From 02dcee8bfceee04f3d37267c79909d4a9465b96c Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Sun, 19 Jun 2022 23:35:35 -0700 Subject: [PATCH 059/196] UT: Modify tests to pass with config flattening changes --- .../monkey_island/cc/services/test_config.py | 67 ++++++++++--------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index 108659ddf..b70a6bd3b 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -92,32 +92,14 @@ def test_format_config_for_agent__propagation(): flat_monkey_config = ConfigService.format_flat_config_for_agent() assert "propagation" in flat_monkey_config - assert "targets" in flat_monkey_config["propagation"] assert "network_scan" in flat_monkey_config["propagation"] - assert "exploiters" in flat_monkey_config["propagation"] - - -def test_format_config_for_agent__propagation_targets(): - expected_targets = { - "blocked_ips": ["192.168.1.1", "192.168.1.100"], - "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], - "local_network_scan": True, - "subnet_scan_list": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], - } - - flat_monkey_config = ConfigService.format_flat_config_for_agent() - - assert flat_monkey_config["propagation"]["targets"] == expected_targets - assert "blocked_ips" not in flat_monkey_config - assert "inaccessible_subnets" not in flat_monkey_config - assert "local_network_scan" not in flat_monkey_config - assert "subnet_scan_list" not in flat_monkey_config + assert "exploitation" in flat_monkey_config["propagation"] def test_format_config_for_agent__network_scan(): expected_network_scan_config = { "tcp": { - "timeout_ms": 3000, + "timeout": 3000, "ports": [ 22, 80, @@ -135,7 +117,13 @@ def test_format_config_for_agent__network_scan(): ], }, "icmp": { - "timeout_ms": 1000, + "timeout": 1000, + }, + "targets": { + "blocked_ips": ["192.168.1.1", "192.168.1.100"], + "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], + "local_network_scan": True, + "subnets": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], }, "fingerprinters": [ {"name": "elastic", "options": {}}, @@ -160,36 +148,53 @@ def test_format_config_for_agent__network_scan(): assert "finger_classes" not in flat_monkey_config +def test_format_config_for_agent__propagation_network_scan_targets(): + expected_targets = { + "blocked_ips": ["192.168.1.1", "192.168.1.100"], + "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], + "local_network_scan": True, + "subnets": ["192.168.1.50", "192.168.56.0/24", "10.0.33.0/30"], + } + + flat_monkey_config = ConfigService.format_flat_config_for_agent() + + assert flat_monkey_config["propagation"]["network_scan"]["targets"] == expected_targets + assert "blocked_ips" not in flat_monkey_config + assert "inaccessible_subnets" not in flat_monkey_config + assert "local_network_scan" not in flat_monkey_config + assert "subnet_scan_list" not in flat_monkey_config + + def test_format_config_for_agent__exploiters(): expected_exploiters_config = { "options": { "http_ports": [80, 443, 7001, 8008, 8080, 9200], }, "brute_force": [ - {"name": "MSSQLExploiter", "supported_os": ["windows"], "options": {}}, - {"name": "PowerShellExploiter", "supported_os": ["windows"], "options": {}}, - {"name": "SSHExploiter", "supported_os": ["linux"], "options": {}}, + {"name": "MSSQLExploiter", "supported_os": ["WINDOWS"], "options": {}}, + {"name": "PowerShellExploiter", "supported_os": ["WINDOWS"], "options": {}}, + {"name": "SSHExploiter", "supported_os": ["LINUX"], "options": {}}, { "name": "SmbExploiter", - "supported_os": ["windows"], + "supported_os": ["WINDOWS"], "options": {"smb_download_timeout": 30}, }, { "name": "WmiExploiter", - "supported_os": ["windows"], + "supported_os": ["WINDOWS"], "options": {"smb_download_timeout": 30}, }, ], "vulnerability": [ - {"name": "HadoopExploiter", "supported_os": ["linux", "windows"], "options": {}}, - {"name": "Log4ShellExploiter", "supported_os": ["linux", "windows"], "options": {}}, - {"name": "ZerologonExploiter", "supported_os": ["windows"], "options": {}}, + {"name": "HadoopExploiter", "supported_os": ["LINUX", "WINDOWS"], "options": {}}, + {"name": "Log4ShellExploiter", "supported_os": ["LINUX", "WINDOWS"], "options": {}}, + {"name": "ZerologonExploiter", "supported_os": ["WINDOWS"], "options": {}}, ], } flat_monkey_config = ConfigService.format_flat_config_for_agent() assert "propagation" in flat_monkey_config - assert "exploiters" in flat_monkey_config["propagation"] + assert "exploitation" in flat_monkey_config["propagation"] - assert flat_monkey_config["propagation"]["exploiters"] == expected_exploiters_config + assert flat_monkey_config["propagation"]["exploitation"] == expected_exploiters_config assert "exploiter_classes" not in flat_monkey_config From c0f0d35f0b531be606b5c230504466cdabd3a07b Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 20 Jun 2022 12:18:03 +0300 Subject: [PATCH 060/196] Island: Use OperatingSystems enum in config.py --- monkey/monkey_island/cc/services/config.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 684801ac2..86541c33d 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List from jsonschema import Draft4Validator, validators +from common import OperatingSystems from common.config_value_paths import ( LM_HASH_LIST_PATH, NTLM_HASH_LIST_PATH, @@ -533,9 +534,7 @@ class ConfigService: formatted_scan_targets_config[flat_local_network_scan_field] = config[ flat_local_network_scan_field ] - formatted_scan_targets_config["subnets"] = config[ - flat_subnet_scan_list_field - ] + formatted_scan_targets_config["subnets"] = config[flat_subnet_scan_list_field] config.pop(flat_blocked_ips_field, None) config.pop(flat_inaccessible_subnets_field, None) @@ -600,14 +599,14 @@ class ConfigService: formatted_config: Dict, ) -> Dict[str, List[Dict[str, Any]]]: supported_os = { - "HadoopExploiter": ["LINUX", "WINDOWS"], - "Log4ShellExploiter": ["LINUX", "WINDOWS"], - "MSSQLExploiter": ["WINDOWS"], - "PowerShellExploiter": ["WINDOWS"], - "SSHExploiter": ["LINUX"], - "SmbExploiter": ["WINDOWS"], - "WmiExploiter": ["WINDOWS"], - "ZerologonExploiter": ["WINDOWS"], + "HadoopExploiter": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], + "Log4ShellExploiter": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], + "MSSQLExploiter": [OperatingSystems.WINDOWS], + "PowerShellExploiter": [OperatingSystems.WINDOWS], + "SSHExploiter": [OperatingSystems.LINUX], + "SmbExploiter": [OperatingSystems.WINDOWS], + "WmiExploiter": [OperatingSystems.WINDOWS], + "ZerologonExploiter": [OperatingSystems.WINDOWS], } new_config = copy.deepcopy(formatted_config) for exploiter in chain(new_config["brute_force"], new_config["vulnerability"]): From 0474e2a5f79f90b95c8de765ba2b2aeb8548afdc Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 20 Jun 2022 12:27:01 +0300 Subject: [PATCH 061/196] Island: Change json encoding to encode Enums to name string Enum objects couldn't get encoded, so for each enum we had to decide whether the name or the value would be used to represent that enum value. Changing the encoding to name allows us to use enum object on the island without having to worry about encoding. --- .../cc/services/representations.py | 25 +++++++++++++------ .../cc/services/test_representations.py | 9 +++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/monkey/monkey_island/cc/services/representations.py b/monkey/monkey_island/cc/services/representations.py index 0193fae0d..8a1e849a6 100644 --- a/monkey/monkey_island/cc/services/representations.py +++ b/monkey/monkey_island/cc/services/representations.py @@ -1,4 +1,5 @@ from datetime import datetime +from enum import Enum import bson from bson.json_util import dumps @@ -11,19 +12,27 @@ def normalize_obj(obj): del obj["_id"] for key, value in list(obj.items()): - if isinstance(value, bson.objectid.ObjectId): - obj[key] = str(value) - if isinstance(value, datetime): - obj[key] = str(value) - if isinstance(value, dict): - obj[key] = normalize_obj(value) if isinstance(value, list): for i in range(0, len(value)): - if isinstance(value[i], dict): - value[i] = normalize_obj(value[i]) + obj[key][i] = _normalize_value(value[i]) + else: + obj[key] = _normalize_value(value) return obj +def _normalize_value(value): + if type(value) == dict: + return normalize_obj(value) + if isinstance(value, bson.objectid.ObjectId): + return str(value) + if isinstance(value, datetime): + return str(value) + if issubclass(type(value), Enum): + return value.name + else: + return value + + def output_json(obj, code, headers=None): obj = normalize_obj(obj) resp = make_response(dumps(obj), code) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_representations.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_representations.py index c088c3dce..e40e4470f 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_representations.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_representations.py @@ -1,4 +1,5 @@ from datetime import datetime +from enum import Enum from unittest import TestCase import bson @@ -44,3 +45,11 @@ class TestRepresentations(TestCase): } ), ) + + def test_normalize__enum(self): + class BogusEnum(Enum): + bogus_val = "Bogus" + + my_obj = {"something": "something", "my_enum": BogusEnum.bogus_val} + + assert {"something": "something", "my_enum": "bogus_val"} == normalize_obj(my_obj) From 3757e3318026318e994e6bace5b65c06a39d0fc3 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 20 Jun 2022 12:30:25 +0300 Subject: [PATCH 062/196] UT: Use OperatingSystems enum instead of strings --- .../monkey_island/cc/services/test_config.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index b70a6bd3b..df866e388 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -1,5 +1,6 @@ import pytest +from common import OperatingSystems from monkey_island.cc.services.config import ConfigService # If tests fail because config path is changed, sync with @@ -171,24 +172,40 @@ def test_format_config_for_agent__exploiters(): "http_ports": [80, 443, 7001, 8008, 8080, 9200], }, "brute_force": [ - {"name": "MSSQLExploiter", "supported_os": ["WINDOWS"], "options": {}}, - {"name": "PowerShellExploiter", "supported_os": ["WINDOWS"], "options": {}}, - {"name": "SSHExploiter", "supported_os": ["LINUX"], "options": {}}, + {"name": "MSSQLExploiter", "supported_os": [OperatingSystems.WINDOWS], "options": {}}, + { + "name": "PowerShellExploiter", + "supported_os": [OperatingSystems.WINDOWS], + "options": {}, + }, + {"name": "SSHExploiter", "supported_os": [OperatingSystems.LINUX], "options": {}}, { "name": "SmbExploiter", - "supported_os": ["WINDOWS"], + "supported_os": [OperatingSystems.WINDOWS], "options": {"smb_download_timeout": 30}, }, { "name": "WmiExploiter", - "supported_os": ["WINDOWS"], + "supported_os": [OperatingSystems.WINDOWS], "options": {"smb_download_timeout": 30}, }, ], "vulnerability": [ - {"name": "HadoopExploiter", "supported_os": ["LINUX", "WINDOWS"], "options": {}}, - {"name": "Log4ShellExploiter", "supported_os": ["LINUX", "WINDOWS"], "options": {}}, - {"name": "ZerologonExploiter", "supported_os": ["WINDOWS"], "options": {}}, + { + "name": "HadoopExploiter", + "supported_os": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], + "options": {}, + }, + { + "name": "Log4ShellExploiter", + "supported_os": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], + "options": {}, + }, + { + "name": "ZerologonExploiter", + "supported_os": [OperatingSystems.WINDOWS], + "options": {}, + }, ], } flat_monkey_config = ConfigService.format_flat_config_for_agent() From 3cb678ad3274620de549ec16f2b8759f59a8291c Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Mon, 20 Jun 2022 14:55:26 +0200 Subject: [PATCH 063/196] Island: Raise distinct errors when openning a file IFileRepository now distincts between file not found and a file that could not be retrieved --- .../repository/file_storage/i_file_repository.py | 3 ++- .../file_storage/local_storage_file_repository.py | 12 +++++++----- .../test_local_storage_file_repository.py | 14 +++++++++++++- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index 2531fd711..5069b36d7 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -30,7 +30,8 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that identifies the file to be opened :return: A file-like object providing access to the file's contents - :raises FileRetrievalError: if the file cannot be opened + :raises FileRetrievalError: if the file cannot be retrieved + :raises FileNotFoundError: if the file cannot be found """ pass diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index 4202496ad..a23b25228 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -1,3 +1,4 @@ +import errno import logging import shutil from pathlib import Path @@ -44,11 +45,12 @@ class LocalStorageFileRepository(IFileRepository): logger.debug(f"Opening {safe_file_path}") return open(safe_file_path, "rb") except OSError as err: - # TODO: The interface should make a destinction between file not found and an error when - # retrieving a file that should exist. The built-in `FileNotFoundError` is not - # sufficient because it inherits from `OSError` and the interface does not - # guarantee that the file is stored on the local file system. - raise FileRetrievalError(f"Failed to retrieve file {safe_file_path}: {err}") from err + if err.errno == errno.ENOENT: + raise FileNotFoundError(f"No file {safe_file_path} found: {err}") from err + else: + raise FileRetrievalError( + f"Failed to retrieve file {safe_file_path}: {err}" + ) from err def delete_file(self, unsafe_file_name: str): safe_file_path = self._get_safe_file_path(unsafe_file_name) diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py index cf506ffd0..804b03c79 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py @@ -1,5 +1,7 @@ +import errno import io from pathlib import Path +from unittest.mock import Mock, patch import pytest from tests.monkey_island.utils import assert_linux_permissions, assert_windows_permissions @@ -131,5 +133,15 @@ def test_remove_nonexistant_file(tmp_path): def test_open_nonexistant_file(tmp_path): fss = LocalStorageFileRepository(tmp_path) - with pytest.raises(FileRetrievalError): + with pytest.raises(FileNotFoundError): fss.open_file("nonexistant_file.txt") + + +def test_open_locked_file(tmp_path, monkeypatch): + fss = LocalStorageFileRepository(tmp_path) + + with patch( + "builtins.open", Mock(side_effect=OSError(errno.EIO, "File could not be retrieved")) + ): + with pytest.raises(FileRetrievalError): + fss.open_file("locked_file.txt") From c424262f12b841175f68a9d64f33e2a3e629cc9f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 08:22:47 -0400 Subject: [PATCH 064/196] Island: Improve description of return value for get_configuration() --- .../cc/repository/i_agent_configuration_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index 105d26fe9..de7aa4ef2 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -14,7 +14,7 @@ class IAgentConfigurationRepository(ABC): Retrieve the agent configuration from the repository :return: The agent configuration as retrieved from the repository, or the default - configuration if none could be retrieved. + configuration if the repository is empty :raises RetrievalError: if the configuration can not be retrieved """ pass From cd34cd5eae4d2fb22958ff20a1172ba6d55b5e87 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 08:49:36 -0400 Subject: [PATCH 065/196] Island: Add repository.FileNotFoundError --- .../monkey_island/cc/repository/__init__.py | 2 +- .../cc/repository/agent_binary_repository.py | 7 +++++-- .../cc/repository/file_storage/__init__.py | 2 +- .../file_storage/i_file_repository.py | 4 ++-- .../local_storage_file_repository.py | 20 ++++++++++--------- .../cc/resources/pba_file_download.py | 13 +++++++----- .../cc/resources/pba_file_upload.py | 13 +++++++----- .../monkey_island/single_file_repository.py | 5 +++-- .../test_local_storage_file_repository.py | 12 +++++------ .../cc/resources/test_pba_file_download.py | 4 ++-- 10 files changed, 46 insertions(+), 36 deletions(-) diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index cc89c5c57..a44e7b1ad 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -1,5 +1,5 @@ from .errors import RetrievalError -from .file_storage import FileRetrievalError, IFileRepository, LocalStorageFileRepository +from .file_storage import FileNotFoundError, IFileRepository, LocalStorageFileRepository from .i_agent_binary_repository import IAgentBinaryRepository, AgentRetrievalError from .agent_binary_repository import AgentBinaryRepository from .i_agent_configuration_repository import IAgentConfigurationRepository diff --git a/monkey/monkey_island/cc/repository/agent_binary_repository.py b/monkey/monkey_island/cc/repository/agent_binary_repository.py index 4c1334f71..a09d88868 100644 --- a/monkey/monkey_island/cc/repository/agent_binary_repository.py +++ b/monkey/monkey_island/cc/repository/agent_binary_repository.py @@ -1,6 +1,8 @@ from typing import BinaryIO -from . import AgentRetrievalError, FileRetrievalError, IAgentBinaryRepository, IFileRepository +from monkey_island.cc import repository + +from . import AgentRetrievalError, IAgentBinaryRepository, IFileRepository LINUX_AGENT_FILE_NAME = "monkey-linux-64" WINDOWS_AGENT_FILE_NAME = "monkey-windows-64.exe" @@ -20,7 +22,8 @@ class AgentBinaryRepository(IAgentBinaryRepository): try: agent_binary = self._file_repository.open_file(filename) return agent_binary - except FileRetrievalError as err: + # TODO: Reevaluate this + except repository.FileNotFoundError as err: raise AgentRetrievalError( f"An error occurred while retrieving the {filename}" f" agent binary from {self._file_repository}: {err}" diff --git a/monkey/monkey_island/cc/repository/file_storage/__init__.py b/monkey/monkey_island/cc/repository/file_storage/__init__.py index 30b59b5d9..14227bc66 100644 --- a/monkey/monkey_island/cc/repository/file_storage/__init__.py +++ b/monkey/monkey_island/cc/repository/file_storage/__init__.py @@ -1,2 +1,2 @@ -from .i_file_repository import IFileRepository, FileRetrievalError +from .i_file_repository import IFileRepository, FileNotFoundError from .local_storage_file_repository import LocalStorageFileRepository diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index 5069b36d7..681b3d7cf 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -4,7 +4,7 @@ from typing import BinaryIO from monkey_island.cc.repository import RetrievalError -class FileRetrievalError(RetrievalError): +class FileNotFoundError(RetrievalError): pass @@ -30,8 +30,8 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that identifies the file to be opened :return: A file-like object providing access to the file's contents - :raises FileRetrievalError: if the file cannot be retrieved :raises FileNotFoundError: if the file cannot be found + :raises RetrievalError: if the file cannot be retrieved """ pass diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index a23b25228..cb18c6ba8 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -1,13 +1,13 @@ -import errno import logging import shutil from pathlib import Path from typing import BinaryIO from common.utils.file_utils import get_all_regular_files_in_directory +from monkey_island.cc.repository import RetrievalError from monkey_island.cc.server_utils.file_utils import create_secure_directory -from . import FileRetrievalError, IFileRepository +from . import IFileRepository, i_file_repository logger = logging.getLogger(__name__) @@ -44,13 +44,15 @@ class LocalStorageFileRepository(IFileRepository): try: logger.debug(f"Opening {safe_file_path}") return open(safe_file_path, "rb") - except OSError as err: - if err.errno == errno.ENOENT: - raise FileNotFoundError(f"No file {safe_file_path} found: {err}") from err - else: - raise FileRetrievalError( - f"Failed to retrieve file {safe_file_path}: {err}" - ) from err + except FileNotFoundError as err: + # Wrap Python's FileNotFound error, which is-an OSError, in repository.FileNotFoundError + raise i_file_repository.FileNotFoundError( + f'The requested file "{unsafe_file_name}" does not exist: {err}' + ) + except Exception as err: + raise RetrievalError( + f'Error retrieving file "{unsafe_file_name}" from the repository: {err}' + ) def delete_file(self, unsafe_file_name: str): safe_file_path = self._get_safe_file_path(unsafe_file_name) diff --git a/monkey/monkey_island/cc/resources/pba_file_download.py b/monkey/monkey_island/cc/resources/pba_file_download.py index 85395d5e3..3b7e92047 100644 --- a/monkey/monkey_island/cc/resources/pba_file_download.py +++ b/monkey/monkey_island/cc/resources/pba_file_download.py @@ -2,7 +2,8 @@ import logging from flask import make_response, send_file -from monkey_island.cc.repository import FileRetrievalError, IFileRepository +from monkey_island.cc import repository +from monkey_island.cc.repository import IFileRepository from monkey_island.cc.resources.AbstractResource import AbstractResource logger = logging.getLogger(__file__) @@ -24,7 +25,9 @@ class PBAFileDownload(AbstractResource): # `send_file()` handles the closing of the open file. return send_file(file, mimetype="application/octet-stream") - except FileRetrievalError as err: - error_msg = f"Failed to open file {filename}: {err}" - logger.error(error_msg) - return make_response({"error": error_msg}, 404) + except repository.FileNotFoundError as err: + # TODO: Do we need to log? Or will flask handle it when we `make_response()`? + logger.error(str(err)) + return make_response({"error": str(err)}, 404) + + # TODO: Add unit tests that test 404 vs 500 errors diff --git a/monkey/monkey_island/cc/resources/pba_file_upload.py b/monkey/monkey_island/cc/resources/pba_file_upload.py index 9d5cce8b1..833086998 100644 --- a/monkey/monkey_island/cc/resources/pba_file_upload.py +++ b/monkey/monkey_island/cc/resources/pba_file_upload.py @@ -5,7 +5,8 @@ from flask import Response, make_response, request, send_file from werkzeug.utils import secure_filename as sanitize_filename from common.config_value_paths import PBA_LINUX_FILENAME_PATH, PBA_WINDOWS_FILENAME_PATH -from monkey_island.cc.repository import FileRetrievalError, IFileRepository +from monkey_island.cc import repository +from monkey_island.cc.repository import IFileRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required from monkey_island.cc.services.config import ConfigService @@ -57,10 +58,12 @@ class FileUpload(AbstractResource): # `send_file()` handles the closing of the open file. return send_file(file, mimetype="application/octet-stream") - except FileRetrievalError as err: - error_msg = f"Failed to open file {filename}: {err}" - logger.error(error_msg) - return make_response({"error": error_msg}, 404) + except repository.FileNotFoundError as err: + # TODO: Do we need to log? Or will flask handle it when we `make_response()`? + logger.error(str(err)) + return make_response({"error": str(err)}, 404) + + # TODO: Add unit tests that test 404 vs 500 errors @jwt_required def post(self, target_os): diff --git a/monkey/tests/monkey_island/single_file_repository.py b/monkey/tests/monkey_island/single_file_repository.py index d00ee9ae9..462969acb 100644 --- a/monkey/tests/monkey_island/single_file_repository.py +++ b/monkey/tests/monkey_island/single_file_repository.py @@ -1,7 +1,8 @@ import io from typing import BinaryIO -from monkey_island.cc.repository import FileRetrievalError, IFileRepository +from monkey_island.cc import repository +from monkey_island.cc.repository import IFileRepository class SingleFileRepository(IFileRepository): @@ -13,7 +14,7 @@ class SingleFileRepository(IFileRepository): def open_file(self, unsafe_file_name: str) -> BinaryIO: if self._file is None: - raise FileRetrievalError() + raise repository.FileNotFoundError() return self._file def delete_file(self, unsafe_file_name: str): diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py index 804b03c79..64541bcbe 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_local_storage_file_repository.py @@ -1,4 +1,3 @@ -import errno import io from pathlib import Path from unittest.mock import Mock, patch @@ -6,7 +5,8 @@ from unittest.mock import Mock, patch import pytest from tests.monkey_island.utils import assert_linux_permissions, assert_windows_permissions -from monkey_island.cc.repository import FileRetrievalError, LocalStorageFileRepository +from monkey_island.cc import repository +from monkey_island.cc.repository import LocalStorageFileRepository from monkey_island.cc.server_utils.file_utils import is_windows_os @@ -133,15 +133,13 @@ def test_remove_nonexistant_file(tmp_path): def test_open_nonexistant_file(tmp_path): fss = LocalStorageFileRepository(tmp_path) - with pytest.raises(FileNotFoundError): + with pytest.raises(repository.FileNotFoundError): fss.open_file("nonexistant_file.txt") def test_open_locked_file(tmp_path, monkeypatch): fss = LocalStorageFileRepository(tmp_path) - with patch( - "builtins.open", Mock(side_effect=OSError(errno.EIO, "File could not be retrieved")) - ): - with pytest.raises(FileRetrievalError): + with patch("builtins.open", Mock(side_effect=Exception())): + with pytest.raises(repository.RetrievalError): fss.open_file("locked_file.txt") diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py index cf2a109b9..caf86edab 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py @@ -5,7 +5,7 @@ import pytest from tests.common import StubDIContainer from tests.unit_tests.monkey_island.conftest import get_url_for_resource -from monkey_island.cc.repository import FileRetrievalError, IFileRepository +from monkey_island.cc.repository import FileNotFoundError, IFileRepository from monkey_island.cc.resources.pba_file_download import PBAFileDownload FILE_NAME = "test_file" @@ -21,7 +21,7 @@ class MockFileRepository(IFileRepository): def open_file(self, unsafe_file_name: str) -> BinaryIO: if unsafe_file_name != FILE_NAME: - raise FileRetrievalError() + raise FileNotFoundError() return self._file From 44795531b850e419f1b6496bc1fca03aa945688e Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Tue, 21 Jun 2022 15:04:34 +0200 Subject: [PATCH 066/196] Island: Remove logging TODOs for pba_file upload/download Resources should log the errors --- monkey/monkey_island/cc/resources/pba_file_download.py | 1 - monkey/monkey_island/cc/resources/pba_file_upload.py | 1 - 2 files changed, 2 deletions(-) diff --git a/monkey/monkey_island/cc/resources/pba_file_download.py b/monkey/monkey_island/cc/resources/pba_file_download.py index 3b7e92047..39acb5d2c 100644 --- a/monkey/monkey_island/cc/resources/pba_file_download.py +++ b/monkey/monkey_island/cc/resources/pba_file_download.py @@ -26,7 +26,6 @@ class PBAFileDownload(AbstractResource): # `send_file()` handles the closing of the open file. return send_file(file, mimetype="application/octet-stream") except repository.FileNotFoundError as err: - # TODO: Do we need to log? Or will flask handle it when we `make_response()`? logger.error(str(err)) return make_response({"error": str(err)}, 404) diff --git a/monkey/monkey_island/cc/resources/pba_file_upload.py b/monkey/monkey_island/cc/resources/pba_file_upload.py index 833086998..5f46273c4 100644 --- a/monkey/monkey_island/cc/resources/pba_file_upload.py +++ b/monkey/monkey_island/cc/resources/pba_file_upload.py @@ -59,7 +59,6 @@ class FileUpload(AbstractResource): # `send_file()` handles the closing of the open file. return send_file(file, mimetype="application/octet-stream") except repository.FileNotFoundError as err: - # TODO: Do we need to log? Or will flask handle it when we `make_response()`? logger.error(str(err)) return make_response({"error": str(err)}, 404) From bcc5265a998da21f09967b4736c4f7bdd63fdf13 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 08:58:21 -0400 Subject: [PATCH 067/196] UT: Add test_file_download_endpoint_500() for PBAFileDownload --- .../cc/resources/test_pba_file_download.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py index caf86edab..067dcd102 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py @@ -5,7 +5,8 @@ import pytest from tests.common import StubDIContainer from tests.unit_tests.monkey_island.conftest import get_url_for_resource -from monkey_island.cc.repository import FileNotFoundError, IFileRepository +from monkey_island.cc import repository +from monkey_island.cc.repository import IFileRepository, RetrievalError from monkey_island.cc.resources.pba_file_download import PBAFileDownload FILE_NAME = "test_file" @@ -21,7 +22,7 @@ class MockFileRepository(IFileRepository): def open_file(self, unsafe_file_name: str) -> BinaryIO: if unsafe_file_name != FILE_NAME: - raise FileNotFoundError() + raise repository.FileNotFoundError() return self._file @@ -56,3 +57,25 @@ def test_file_download_endpoint_404(tmp_path, flask_client): resp = flask_client.get(download_url) assert resp.status_code == 404 + + +class OpenErrorFileRepository(MockFileRepository): + def open_file(self, unsafe_file_name: str) -> BinaryIO: + raise RetrievalError("Error retrieving file") + + +@pytest.fixture +def open_error_flask_client(build_flask_client): + container = StubDIContainer() + container.register(IFileRepository, OpenErrorFileRepository) + + with build_flask_client(container) as flask_client: + yield flask_client + + +def test_file_download_endpoint_500(tmp_path, open_error_flask_client): + download_url = get_url_for_resource(PBAFileDownload, filename="test") + + resp = open_error_flask_client.get(download_url) + + assert resp.status_code == 500 From bf2f58aacedd3e8895f647eae9f3d89b56b84fef Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 08:59:31 -0400 Subject: [PATCH 068/196] UT: Add __init__.py --- monkey/tests/unit_tests/monkey_island/cc/resources/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/resources/__init__.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/__init__.py b/monkey/tests/unit_tests/monkey_island/cc/resources/__init__.py new file mode 100644 index 000000000..e69de29bb From f973c9d6e970d16396eae464da36780d46d290c7 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:05:42 -0400 Subject: [PATCH 069/196] UT: Extract MockFileRepository into its own module --- .../cc/resources/mock_file_repository.py | 28 +++++++++++++++++++ .../cc/resources/test_pba_file_download.py | 25 +---------------- 2 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/resources/mock_file_repository.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/mock_file_repository.py b/monkey/tests/unit_tests/monkey_island/cc/resources/mock_file_repository.py new file mode 100644 index 000000000..782c9838b --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/mock_file_repository.py @@ -0,0 +1,28 @@ +import io +from typing import BinaryIO + +from monkey_island.cc import repository +from monkey_island.cc.repository import IFileRepository + +FILE_NAME = "test_file" +FILE_CONTENTS = b"HelloWorld!" + + +class MockFileRepository(IFileRepository): + def __init__(self): + self._file = io.BytesIO(FILE_CONTENTS) + + def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): + pass + + def open_file(self, unsafe_file_name: str) -> BinaryIO: + if unsafe_file_name != FILE_NAME: + raise repository.FileNotFoundError() + + return self._file + + def delete_file(self, unsafe_file_name: str): + pass + + def delete_all_files(self): + pass diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py index 067dcd102..885f69609 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py @@ -1,36 +1,13 @@ -import io from typing import BinaryIO import pytest from tests.common import StubDIContainer from tests.unit_tests.monkey_island.conftest import get_url_for_resource -from monkey_island.cc import repository from monkey_island.cc.repository import IFileRepository, RetrievalError from monkey_island.cc.resources.pba_file_download import PBAFileDownload -FILE_NAME = "test_file" -FILE_CONTENTS = b"HelloWorld!" - - -class MockFileRepository(IFileRepository): - def __init__(self): - self._file = io.BytesIO(FILE_CONTENTS) - - def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): - pass - - def open_file(self, unsafe_file_name: str) -> BinaryIO: - if unsafe_file_name != FILE_NAME: - raise repository.FileNotFoundError() - - return self._file - - def delete_file(self, unsafe_file_name: str): - pass - - def delete_all_files(self): - pass +from .mock_file_repository import FILE_CONTENTS, FILE_NAME, MockFileRepository @pytest.fixture From 8939ca21066cf429808e8cbf3432eb684fe5475f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:08:32 -0400 Subject: [PATCH 070/196] UT: Extract open_error_flask_client into conftest.py --- .../monkey_island/cc/resources/conftest.py | 19 +++++++++++++++++++ .../cc/resources/test_pba_file_download.py | 18 +----------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py index a40766d5e..532282e8b 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py @@ -1,12 +1,17 @@ +from typing import BinaryIO from unittest.mock import MagicMock import flask_jwt_extended import pytest +from tests.common import StubDIContainer from tests.unit_tests.monkey_island.conftest import init_mock_app import monkey_island.cc.app import monkey_island.cc.resources.auth.auth import monkey_island.cc.resources.island_mode +from monkey_island.cc.repository import IFileRepository, RetrievalError + +from .mock_file_repository import MockFileRepository @pytest.fixture @@ -38,3 +43,17 @@ def get_mock_app(container): flask_jwt_extended.JWTManager(app) return app + + +class OpenErrorFileRepository(MockFileRepository): + def open_file(self, unsafe_file_name: str) -> BinaryIO: + raise RetrievalError("Error retrieving file") + + +@pytest.fixture +def open_error_flask_client(build_flask_client): + container = StubDIContainer() + container.register(IFileRepository, OpenErrorFileRepository) + + with build_flask_client(container) as flask_client: + yield flask_client diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py index 885f69609..eb189b4e7 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py @@ -1,10 +1,8 @@ -from typing import BinaryIO - import pytest from tests.common import StubDIContainer from tests.unit_tests.monkey_island.conftest import get_url_for_resource -from monkey_island.cc.repository import IFileRepository, RetrievalError +from monkey_island.cc.repository import IFileRepository from monkey_island.cc.resources.pba_file_download import PBAFileDownload from .mock_file_repository import FILE_CONTENTS, FILE_NAME, MockFileRepository @@ -36,20 +34,6 @@ def test_file_download_endpoint_404(tmp_path, flask_client): assert resp.status_code == 404 -class OpenErrorFileRepository(MockFileRepository): - def open_file(self, unsafe_file_name: str) -> BinaryIO: - raise RetrievalError("Error retrieving file") - - -@pytest.fixture -def open_error_flask_client(build_flask_client): - container = StubDIContainer() - container.register(IFileRepository, OpenErrorFileRepository) - - with build_flask_client(container) as flask_client: - yield flask_client - - def test_file_download_endpoint_500(tmp_path, open_error_flask_client): download_url = get_url_for_resource(PBAFileDownload, filename="test") From 63a2527f3f233ac7aff3c5c1b317b2c916270cde Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:10:40 -0400 Subject: [PATCH 071/196] UT: Add test_file_download_endpoint_500() for PBAFileUpload --- .../monkey_island/cc/resources/test_pba_file_upload.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py index 50da88915..e4cb5a487 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_upload.py @@ -98,6 +98,15 @@ def test_pba_file_upload_get__file_not_found(flask_client, pba_os, mock_get_conf assert resp.status_code == 404 +@pytest.mark.parametrize("pba_os", [LINUX_PBA_TYPE, WINDOWS_PBA_TYPE]) +def test_file_download_endpoint_500(open_error_flask_client, pba_os): + url = get_url_for_resource(FileUpload, target_os=pba_os, filename="bobug_mogus.py") + + resp = open_error_flask_client.get(url) + + assert resp.status_code == 500 + + @pytest.mark.parametrize("pba_os", [LINUX_PBA_TYPE, WINDOWS_PBA_TYPE]) def test_pba_file_upload_endpoint( flask_client, pba_os, mock_get_config_value, mock_set_config_value From 22b22c5f0a7ce5a24d45bb1df00b62808e450670 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:11:54 -0400 Subject: [PATCH 072/196] Island: Remove testing TODOs from PBAFile{Download,Upload} --- monkey/monkey_island/cc/resources/pba_file_download.py | 2 -- monkey/monkey_island/cc/resources/pba_file_upload.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/monkey/monkey_island/cc/resources/pba_file_download.py b/monkey/monkey_island/cc/resources/pba_file_download.py index 39acb5d2c..d8359049a 100644 --- a/monkey/monkey_island/cc/resources/pba_file_download.py +++ b/monkey/monkey_island/cc/resources/pba_file_download.py @@ -28,5 +28,3 @@ class PBAFileDownload(AbstractResource): except repository.FileNotFoundError as err: logger.error(str(err)) return make_response({"error": str(err)}, 404) - - # TODO: Add unit tests that test 404 vs 500 errors diff --git a/monkey/monkey_island/cc/resources/pba_file_upload.py b/monkey/monkey_island/cc/resources/pba_file_upload.py index 5f46273c4..277804a72 100644 --- a/monkey/monkey_island/cc/resources/pba_file_upload.py +++ b/monkey/monkey_island/cc/resources/pba_file_upload.py @@ -62,8 +62,6 @@ class FileUpload(AbstractResource): logger.error(str(err)) return make_response({"error": str(err)}, 404) - # TODO: Add unit tests that test 404 vs 500 errors - @jwt_required def post(self, target_os): """ From 47df2575459f2e9a1392d6f8d0f44dbf97ae1ee4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:18:42 -0400 Subject: [PATCH 073/196] Island: Raise RetrievalError from IAgentBinaryRepository --- monkey/monkey_island/cc/repository/__init__.py | 2 +- .../cc/repository/agent_binary_repository.py | 9 +++------ .../cc/repository/i_agent_binary_repository.py | 6 ++---- monkey/monkey_island/cc/resources/agent_binaries.py | 8 ++++---- monkey/monkey_island/cc/services/initialize.py | 4 ++-- monkey/monkey_island/cc/services/run_local_monkey.py | 4 ++-- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index a44e7b1ad..8fadb9ccf 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -1,6 +1,6 @@ from .errors import RetrievalError from .file_storage import FileNotFoundError, IFileRepository, LocalStorageFileRepository -from .i_agent_binary_repository import IAgentBinaryRepository, AgentRetrievalError +from .i_agent_binary_repository import IAgentBinaryRepository from .agent_binary_repository import AgentBinaryRepository from .i_agent_configuration_repository import IAgentConfigurationRepository from .file_agent_configuration_repository import FileAgentConfigurationRepository diff --git a/monkey/monkey_island/cc/repository/agent_binary_repository.py b/monkey/monkey_island/cc/repository/agent_binary_repository.py index a09d88868..9b753ce00 100644 --- a/monkey/monkey_island/cc/repository/agent_binary_repository.py +++ b/monkey/monkey_island/cc/repository/agent_binary_repository.py @@ -1,8 +1,6 @@ from typing import BinaryIO -from monkey_island.cc import repository - -from . import AgentRetrievalError, IAgentBinaryRepository, IFileRepository +from . import IAgentBinaryRepository, IFileRepository, RetrievalError LINUX_AGENT_FILE_NAME = "monkey-linux-64" WINDOWS_AGENT_FILE_NAME = "monkey-windows-64.exe" @@ -22,9 +20,8 @@ class AgentBinaryRepository(IAgentBinaryRepository): try: agent_binary = self._file_repository.open_file(filename) return agent_binary - # TODO: Reevaluate this - except repository.FileNotFoundError as err: - raise AgentRetrievalError( + except Exception as err: + raise RetrievalError( f"An error occurred while retrieving the {filename}" f" agent binary from {self._file_repository}: {err}" ) diff --git a/monkey/monkey_island/cc/repository/i_agent_binary_repository.py b/monkey/monkey_island/cc/repository/i_agent_binary_repository.py index 56cf2f96b..6c9abcc40 100644 --- a/monkey/monkey_island/cc/repository/i_agent_binary_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_binary_repository.py @@ -2,10 +2,6 @@ import abc from typing import BinaryIO -class AgentRetrievalError(IOError): - pass - - class IAgentBinaryRepository(metaclass=abc.ABCMeta): """ A repository that retrieves the agent binaries @@ -17,6 +13,7 @@ class IAgentBinaryRepository(metaclass=abc.ABCMeta): Retrieve linux agent binary :return: A file-like object that represents the linux agent binary + :raises RetrievalError: If the agent binary could not be retrieved """ @abc.abstractmethod @@ -25,4 +22,5 @@ class IAgentBinaryRepository(metaclass=abc.ABCMeta): Retrieve windows agent binary :return: A file-like object that represents the windows agent binary + :raises RetrievalError: If the agent binary could not be retrieved """ diff --git a/monkey/monkey_island/cc/resources/agent_binaries.py b/monkey/monkey_island/cc/resources/agent_binaries.py index 8d960b6a4..0d746b932 100644 --- a/monkey/monkey_island/cc/resources/agent_binaries.py +++ b/monkey/monkey_island/cc/resources/agent_binaries.py @@ -2,7 +2,7 @@ import logging from flask import make_response, send_file -from monkey_island.cc.repository import AgentRetrievalError, IAgentBinaryRepository +from monkey_island.cc.repository import IAgentBinaryRepository, RetrievalError from monkey_island.cc.resources.AbstractResource import AbstractResource logger = logging.getLogger(__name__) @@ -31,10 +31,10 @@ class AgentBinaries(AbstractResource): file = agent_binaries[os]() return send_file(file, mimetype="application/octet-stream") - except AgentRetrievalError as err: - logger.error(err) - return make_response({"error": str(err)}, 500) except KeyError as err: error_msg = f'No Agents are available for unsupported operating system "{os}": {err}' logger.error(error_msg) return make_response({"error": error_msg}, 404) + except RetrievalError as err: + logger.error(err) + return make_response({"error": str(err)}, 500) diff --git a/monkey/monkey_island/cc/services/initialize.py b/monkey/monkey_island/cc/services/initialize.py index 74d447c90..4b03daf7d 100644 --- a/monkey/monkey_island/cc/services/initialize.py +++ b/monkey/monkey_island/cc/services/initialize.py @@ -6,12 +6,12 @@ from common.aws import AWSInstance from common.utils.file_utils import get_binary_io_sha256_hash from monkey_island.cc.repository import ( AgentBinaryRepository, - AgentRetrievalError, FileAgentConfigurationRepository, IAgentBinaryRepository, IAgentConfigurationRepository, IFileRepository, LocalStorageFileRepository, + RetrievalError, ) from monkey_island.cc.server_utils.consts import MONKEY_ISLAND_ABS_PATH from monkey_island.cc.services import AWSService @@ -76,7 +76,7 @@ def _log_agent_binary_hashes(agent_binary_repository: IAgentBinaryRepository): agent_binary = get_agent_binary() binary_sha256_hash = get_binary_io_sha256_hash(agent_binary) agent_hashes[os] = binary_sha256_hash - except AgentRetrievalError as err: + except RetrievalError as err: logger.error(f"No agent available for {os}: {err}") for os, binary_sha256_hash in agent_hashes.items(): diff --git a/monkey/monkey_island/cc/services/run_local_monkey.py b/monkey/monkey_island/cc/services/run_local_monkey.py index a54e2c6b9..a56642e2c 100644 --- a/monkey/monkey_island/cc/services/run_local_monkey.py +++ b/monkey/monkey_island/cc/services/run_local_monkey.py @@ -5,7 +5,7 @@ import subprocess from pathlib import Path from shutil import copyfileobj -from monkey_island.cc.repository import AgentRetrievalError, IAgentBinaryRepository +from monkey_island.cc.repository import IAgentBinaryRepository, RetrievalError from monkey_island.cc.server_utils.consts import ISLAND_PORT from monkey_island.cc.services.utils.network_utils import local_ip_addresses @@ -29,7 +29,7 @@ class LocalMonkeyRunService: } agent_binary = agents[platform.system().lower()]() - except AgentRetrievalError as err: + except RetrievalError as err: logger.error( f"No Agent can be retrieved for the specified operating system" f'"{operating_system}"' From 4de9f3cb6d2181abe3f32ed6bdb04769be1f3829 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:35:23 -0400 Subject: [PATCH 074/196] UT: Move OpenErrorFileRepository into tests/monkey_island --- monkey/tests/monkey_island/__init__.py | 2 ++ .../mock_file_repository.py | 0 .../tests/monkey_island/open_error_file_repository.py | 10 ++++++++++ .../unit_tests/monkey_island/cc/resources/conftest.py | 11 ++--------- .../cc/resources/test_pba_file_download.py | 3 +-- 5 files changed, 15 insertions(+), 11 deletions(-) rename monkey/tests/{unit_tests/monkey_island/cc/resources => monkey_island}/mock_file_repository.py (100%) create mode 100644 monkey/tests/monkey_island/open_error_file_repository.py diff --git a/monkey/tests/monkey_island/__init__.py b/monkey/tests/monkey_island/__init__.py index 7bd9a314d..aed4d07f3 100644 --- a/monkey/tests/monkey_island/__init__.py +++ b/monkey/tests/monkey_island/__init__.py @@ -1 +1,3 @@ from .single_file_repository import SingleFileRepository +from .mock_file_repository import MockFileRepository, FILE_CONTENTS, FILE_NAME +from .open_error_file_repository import OpenErrorFileRepository diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/mock_file_repository.py b/monkey/tests/monkey_island/mock_file_repository.py similarity index 100% rename from monkey/tests/unit_tests/monkey_island/cc/resources/mock_file_repository.py rename to monkey/tests/monkey_island/mock_file_repository.py diff --git a/monkey/tests/monkey_island/open_error_file_repository.py b/monkey/tests/monkey_island/open_error_file_repository.py new file mode 100644 index 000000000..c13559613 --- /dev/null +++ b/monkey/tests/monkey_island/open_error_file_repository.py @@ -0,0 +1,10 @@ +from typing import BinaryIO + +from monkey_island.cc.repository import RetrievalError + +from . import MockFileRepository + + +class OpenErrorFileRepository(MockFileRepository): + def open_file(self, unsafe_file_name: str) -> BinaryIO: + raise RetrievalError("Error retrieving file") diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py b/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py index 532282e8b..26fe24821 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/conftest.py @@ -1,17 +1,15 @@ -from typing import BinaryIO from unittest.mock import MagicMock import flask_jwt_extended import pytest from tests.common import StubDIContainer +from tests.monkey_island import OpenErrorFileRepository from tests.unit_tests.monkey_island.conftest import init_mock_app import monkey_island.cc.app import monkey_island.cc.resources.auth.auth import monkey_island.cc.resources.island_mode -from monkey_island.cc.repository import IFileRepository, RetrievalError - -from .mock_file_repository import MockFileRepository +from monkey_island.cc.repository import IFileRepository @pytest.fixture @@ -45,11 +43,6 @@ def get_mock_app(container): return app -class OpenErrorFileRepository(MockFileRepository): - def open_file(self, unsafe_file_name: str) -> BinaryIO: - raise RetrievalError("Error retrieving file") - - @pytest.fixture def open_error_flask_client(build_flask_client): container = StubDIContainer() diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py index eb189b4e7..31ae0309a 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_pba_file_download.py @@ -1,12 +1,11 @@ import pytest from tests.common import StubDIContainer +from tests.monkey_island import FILE_CONTENTS, FILE_NAME, MockFileRepository from tests.unit_tests.monkey_island.conftest import get_url_for_resource from monkey_island.cc.repository import IFileRepository from monkey_island.cc.resources.pba_file_download import PBAFileDownload -from .mock_file_repository import FILE_CONTENTS, FILE_NAME, MockFileRepository - @pytest.fixture def flask_client(build_flask_client): From c008db4cf2857779eb0f062748da1e313d3c98d6 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 09:40:00 -0400 Subject: [PATCH 075/196] Island: Raise RetrievalError in FileAgentConfigurationRepository --- .../file_agent_configuration_repository.py | 7 ++++--- .../test_file_agent_configuration_repository.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py index fcccd49a1..89fc57960 100644 --- a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py @@ -5,6 +5,7 @@ from common.configuration import ( AgentConfiguration, AgentConfigurationSchema, ) +from monkey_island.cc import repository from monkey_island.cc.repository import ( IAgentConfigurationRepository, IFileRepository, @@ -25,10 +26,10 @@ class FileAgentConfigurationRepository(IAgentConfigurationRepository): configuration_json = f.read().decode() return self._schema.loads(configuration_json) - # TODO: Handle FileRetrievalError vs FileNotFoundError - # https://github.com/guardicore/monkey/blob/e8001d8cf76340e42bf17ff62523bd2d85fc4841/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py#L47-L50 - except RetrievalError: + except repository.FileNotFoundError: return self._schema.loads(DEFAULT_AGENT_CONFIGURATION) + except Exception as err: + raise RetrievalError(f"Error retrieving the agent configuration: {err}") def store_configuration(self, agent_configuration: AgentConfiguration): configuration_json = self._schema.dumps(agent_configuration) diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py index 7ad066623..93a80dea6 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -1,8 +1,9 @@ +import pytest from tests.common.example_agent_configuration import AGENT_CONFIGURATION -from tests.monkey_island import SingleFileRepository +from tests.monkey_island import OpenErrorFileRepository, SingleFileRepository from common.configuration import DEFAULT_AGENT_CONFIGURATION, AgentConfigurationSchema -from monkey_island.cc.repository import FileAgentConfigurationRepository +from monkey_island.cc.repository import FileAgentConfigurationRepository, RetrievalError def test_store_agent_config(): @@ -24,3 +25,10 @@ def test_get_default_agent_config(): retrieved_agent_configuration = repository.get_configuration() assert retrieved_agent_configuration == default_agent_configuration + + +def test_get_agent_config_retrieval_error(): + repository = FileAgentConfigurationRepository(OpenErrorFileRepository()) + + with pytest.raises(RetrievalError): + repository.get_configuration() From 3bd977ed551cd5a17cf41ec7f30b53b537e463fb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 10:34:27 -0400 Subject: [PATCH 076/196] Island: Improve IFileRepository.open_file() docstring --- .../cc/repository/file_storage/i_file_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index 681b3d7cf..bb9e345dc 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -30,8 +30,8 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that identifies the file to be opened :return: A file-like object providing access to the file's contents - :raises FileNotFoundError: if the file cannot be found - :raises RetrievalError: if the file cannot be retrieved + :raises FileNotFoundError: if the file does not exist + :raises RetrievalError: if the file exists but cannot be retrieved """ pass From 838a2e742cfb11ace9b832096824e6d8d89466e3 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 10:36:56 -0400 Subject: [PATCH 077/196] Island: Change can -> could in get_configuration() --- .../cc/repository/i_agent_configuration_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index de7aa4ef2..1e85fadf5 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -15,7 +15,7 @@ class IAgentConfigurationRepository(ABC): :return: The agent configuration as retrieved from the repository, or the default configuration if the repository is empty - :raises RetrievalError: if the configuration can not be retrieved + :raises RetrievalError: if the configuration could not be retrieved """ pass From 5c97d6bdbf035f2b568050c67b421c01d39f8e62 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 11:00:13 -0400 Subject: [PATCH 078/196] UT: Add tests for AgentBinaryRepository --- .../test_agent_binary_repository.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/repository/test_agent_binary_repository.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_agent_binary_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_agent_binary_repository.py new file mode 100644 index 000000000..585867b2e --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_agent_binary_repository.py @@ -0,0 +1,31 @@ +from unittest.mock import MagicMock + +import pytest + +from monkey_island.cc.repository import AgentBinaryRepository, IFileRepository, RetrievalError + +LINUX_AGENT_BINARY = b"linux_binary" +WINDOWS_AGENT_BINARY = b"windows_binary" + + +@pytest.fixture +def error_mock_file_repository(): + mock_file_repository = MagicMock(wraps=IFileRepository) + mock_file_repository.open_file = MagicMock(side_effect=OSError) + + return mock_file_repository + + +@pytest.fixture +def agent_binary_repository(error_mock_file_repository): + return AgentBinaryRepository(error_mock_file_repository) + + +def test_get_linux_binary_retrieval_error(agent_binary_repository): + with pytest.raises(RetrievalError): + agent_binary_repository.get_linux_binary() + + +def test_get_windows_binary_retrieval_error(agent_binary_repository): + with pytest.raises(RetrievalError): + agent_binary_repository.get_windows_binary() From f60c2f1dde008cfa3ba8d3a50431647c469c3334 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:16:00 -0400 Subject: [PATCH 079/196] Island: Add ValueError to IFileRepository docstring --- .../cc/repository/file_storage/i_file_repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index bb9e345dc..237cc2c0d 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -20,6 +20,7 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that will identify the file :param file_contents: The data to be stored in the file + :raises ValueError: If the file name is an attempted directory traversal """ pass @@ -32,6 +33,7 @@ class IFileRepository(metaclass=abc.ABCMeta): :return: A file-like object providing access to the file's contents :raises FileNotFoundError: if the file does not exist :raises RetrievalError: if the file exists but cannot be retrieved + :raises ValueError: If the file name is an attempted directory traversal """ pass @@ -44,6 +46,7 @@ class IFileRepository(metaclass=abc.ABCMeta): idempotent and will succeed if the file to be deleted does not exist. :param unsafe_file_name: An unsanitized file name that identifies the file to be deleted + :raises ValueError: If the file name is an attempted directory traversal """ pass From 3446dbf0aa6611715cfc185fefe67a578a76402d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:18:35 -0400 Subject: [PATCH 080/196] Island: Export StorageError from monkey_island.cc.repository --- monkey/monkey_island/cc/repository/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index 8fadb9ccf..3a8c552e3 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -1,4 +1,4 @@ -from .errors import RetrievalError +from .errors import RetrievalError, StorageError from .file_storage import FileNotFoundError, IFileRepository, LocalStorageFileRepository from .i_agent_binary_repository import IAgentBinaryRepository from .agent_binary_repository import AgentBinaryRepository From 63404c7bed2c0f3ffed701e92f747b3808059cc0 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:19:56 -0400 Subject: [PATCH 081/196] Island: Raise StorageError in LocalStorageFileRepository.save_file() --- .../cc/repository/file_storage/i_file_repository.py | 1 + .../file_storage/local_storage_file_repository.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index 237cc2c0d..73d3b4759 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -21,6 +21,7 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that will identify the file :param file_contents: The data to be stored in the file :raises ValueError: If the file name is an attempted directory traversal + :raises StorageError: If an error was encountered while attempting to store the file """ pass diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index cb18c6ba8..a766746ae 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import BinaryIO from common.utils.file_utils import get_all_regular_files_in_directory -from monkey_island.cc.repository import RetrievalError +from monkey_island.cc.repository import RetrievalError, StorageError from monkey_island.cc.server_utils.file_utils import create_secure_directory from . import IFileRepository, i_file_repository @@ -35,8 +35,11 @@ class LocalStorageFileRepository(IFileRepository): safe_file_path = self._get_safe_file_path(unsafe_file_name) logger.debug(f"Saving file contents to {safe_file_path}") - with open(safe_file_path, "wb") as dest: - shutil.copyfileobj(file_contents, dest) + try: + with open(safe_file_path, "wb") as dest: + shutil.copyfileobj(file_contents, dest) + except Exception as err: + raise StorageError(f"Error while attempting to store {unsafe_file_name}: {err}") def open_file(self, unsafe_file_name: str) -> BinaryIO: safe_file_path = self._get_safe_file_path(unsafe_file_name) From cd1481e4fe8f70f651c0c0ce52a75643d7dfd8e3 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:22:29 -0400 Subject: [PATCH 082/196] Island: Add monkey_island.cc.repository.RemovalError --- monkey/monkey_island/cc/repository/__init__.py | 2 +- monkey/monkey_island/cc/repository/errors.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/__init__.py b/monkey/monkey_island/cc/repository/__init__.py index 3a8c552e3..52e6f6873 100644 --- a/monkey/monkey_island/cc/repository/__init__.py +++ b/monkey/monkey_island/cc/repository/__init__.py @@ -1,4 +1,4 @@ -from .errors import RetrievalError, StorageError +from .errors import RemovalError, RetrievalError, StorageError from .file_storage import FileNotFoundError, IFileRepository, LocalStorageFileRepository from .i_agent_binary_repository import IAgentBinaryRepository from .agent_binary_repository import AgentBinaryRepository diff --git a/monkey/monkey_island/cc/repository/errors.py b/monkey/monkey_island/cc/repository/errors.py index 586f23cb2..aeb9fa23c 100644 --- a/monkey/monkey_island/cc/repository/errors.py +++ b/monkey/monkey_island/cc/repository/errors.py @@ -1,3 +1,11 @@ +class RemovalError(RuntimeError): + """ + Raised when a repository encounters an error while attempting to remove data. + """ + + pass + + class RetrievalError(RuntimeError): """ Raised when a repository encounters an error while attempting to retrieve data. From da1339e41069a87d939637aa12895a93a662f504 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:24:22 -0400 Subject: [PATCH 083/196] Island: Raise RemovalError in LocalStorageFileRepository.delete_file() --- .../cc/repository/file_storage/i_file_repository.py | 1 + .../repository/file_storage/local_storage_file_repository.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index 73d3b4759..cfebe5c05 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -48,6 +48,7 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that identifies the file to be deleted :raises ValueError: If the file name is an attempted directory traversal + :raises RemovalError: If an error was encountered while attempting to remove a file """ pass diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index a766746ae..d18e029f4 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import BinaryIO from common.utils.file_utils import get_all_regular_files_in_directory -from monkey_island.cc.repository import RetrievalError, StorageError +from monkey_island.cc.repository import RemovalError, RetrievalError, StorageError from monkey_island.cc.server_utils.file_utils import create_secure_directory from . import IFileRepository, i_file_repository @@ -66,6 +66,8 @@ class LocalStorageFileRepository(IFileRepository): except FileNotFoundError: # This method is idempotent. pass + except Exception as err: + raise RemovalError(f"Error while attempting to remove {unsafe_file_name}: {err}") def _get_safe_file_path(self, unsafe_file_name: str): # Remove any path information from the file name. From 21be5fc2be5d89ee2543b58e13843c21469e62dc Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:28:44 -0400 Subject: [PATCH 084/196] Island: raise RemovalError in Local...FileRepository.delete_all_files() --- .../cc/repository/file_storage/i_file_repository.py | 2 ++ .../file_storage/local_storage_file_repository.py | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index cfebe5c05..c0897aa1a 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -56,5 +56,7 @@ class IFileRepository(metaclass=abc.ABCMeta): def delete_all_files(self): """ Delete all files that have been stored using this service. + + :raises RemovalError: If an error was encountered while attempting to remove a file """ pass diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index d18e029f4..564582c76 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -82,6 +82,9 @@ class LocalStorageFileRepository(IFileRepository): return safe_file_path def delete_all_files(self): - for file in get_all_regular_files_in_directory(self._storage_directory): - logger.debug(f"Deleting {file}") - file.unlink() + try: + for file in get_all_regular_files_in_directory(self._storage_directory): + logger.debug(f"Deleting {file}") + file.unlink() + except Exception as err: + raise RemovalError(f"Error while attempting to clear the repository: {err}") From 40b192accc14bc684aa5ea78f0897b2f3f9246e0 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 13:46:15 -0400 Subject: [PATCH 085/196] Island: Wrap directory traversal errors in repository errors A given `IFileRepository` may have no concept of directories. `LocalStorageFileRepository` should wrap the ValueErrors raised to prevent directory traversal in repository errors. --- .../repository/file_storage/i_file_repository.py | 3 --- .../file_storage/local_storage_file_repository.py | 14 +++++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py index c0897aa1a..baa52fdda 100644 --- a/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/i_file_repository.py @@ -20,7 +20,6 @@ class IFileRepository(metaclass=abc.ABCMeta): :param unsafe_file_name: An unsanitized file name that will identify the file :param file_contents: The data to be stored in the file - :raises ValueError: If the file name is an attempted directory traversal :raises StorageError: If an error was encountered while attempting to store the file """ pass @@ -34,7 +33,6 @@ class IFileRepository(metaclass=abc.ABCMeta): :return: A file-like object providing access to the file's contents :raises FileNotFoundError: if the file does not exist :raises RetrievalError: if the file exists but cannot be retrieved - :raises ValueError: If the file name is an attempted directory traversal """ pass @@ -47,7 +45,6 @@ class IFileRepository(metaclass=abc.ABCMeta): idempotent and will succeed if the file to be deleted does not exist. :param unsafe_file_name: An unsanitized file name that identifies the file to be deleted - :raises ValueError: If the file name is an attempted directory traversal :raises RemovalError: If an error was encountered while attempting to remove a file """ pass diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index 564582c76..09fd14453 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -32,19 +32,19 @@ class LocalStorageFileRepository(IFileRepository): self._storage_directory = storage_directory def save_file(self, unsafe_file_name: str, file_contents: BinaryIO): - safe_file_path = self._get_safe_file_path(unsafe_file_name) - - logger.debug(f"Saving file contents to {safe_file_path}") try: + safe_file_path = self._get_safe_file_path(unsafe_file_name) + + logger.debug(f"Saving file contents to {safe_file_path}") with open(safe_file_path, "wb") as dest: shutil.copyfileobj(file_contents, dest) except Exception as err: raise StorageError(f"Error while attempting to store {unsafe_file_name}: {err}") def open_file(self, unsafe_file_name: str) -> BinaryIO: - safe_file_path = self._get_safe_file_path(unsafe_file_name) - try: + safe_file_path = self._get_safe_file_path(unsafe_file_name) + logger.debug(f"Opening {safe_file_path}") return open(safe_file_path, "rb") except FileNotFoundError as err: @@ -58,9 +58,9 @@ class LocalStorageFileRepository(IFileRepository): ) def delete_file(self, unsafe_file_name: str): - safe_file_path = self._get_safe_file_path(unsafe_file_name) - try: + safe_file_path = self._get_safe_file_path(unsafe_file_name) + logger.debug(f"Deleting {safe_file_path}") safe_file_path.unlink() except FileNotFoundError: From f9775f5b5407c7233d3f3902434e9792b5d78cdb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Tue, 21 Jun 2022 14:02:19 -0400 Subject: [PATCH 086/196] Island: Improve directory traversal error message --- .../repository/file_storage/local_storage_file_repository.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py index 09fd14453..4c7662b0c 100644 --- a/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py +++ b/monkey/monkey_island/cc/repository/file_storage/local_storage_file_repository.py @@ -76,7 +76,9 @@ class LocalStorageFileRepository(IFileRepository): # This is a paranoid check to avoid directory traversal attacks. if self._storage_directory.resolve() not in safe_file_path.parents: - raise ValueError(f"The file named {unsafe_file_name} can not be safely retrieved") + raise ValueError( + f'The file named "{unsafe_file_name}" cannot be safely retrieved or written' + ) logger.debug(f"Untrusted file name {unsafe_file_name} sanitized: {safe_file_path}") return safe_file_path From 8e875143acc8f7804a79398a11950dccf86ab910 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 22 Jun 2022 07:31:05 -0400 Subject: [PATCH 087/196] Project: Add `monkey/` to PYTHONPATH when running pytest Without explicitly setting the python path, conftest.py can not find common.*, monkey_island.*, or infection_monkey.*. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 10415750f..5f8e07b67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ log_cli_date_format = "%H:%M:%S" addopts = "-v --capture=sys tests/unit_tests" norecursedirs = "node_modules dist" markers = ["slow: mark test as slow"] +pythonpath = "./monkey" [tool.vulture] exclude = ["monkey/monkey_island/cc/ui/", "monkey/tests/", "monkey/monkey_island/docs/"] From 1f00a13649d784d80dc89d9685799c23bc41df82 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 22 Jun 2022 07:35:06 -0400 Subject: [PATCH 088/196] Island: Pass default agent configuration to repository constructor --- monkey/common/configuration/__init__.py | 5 ++++- .../default_agent_configuration.py | 9 +++++++- .../file_agent_configuration_repository.py | 13 +++++------ .../monkey_island/cc/services/initialize.py | 4 ++++ .../common/test_agent_configuration.py | 4 ++-- monkey/tests/unit_tests/conftest.py | 7 ++++++ ...est_file_agent_configuration_repository.py | 22 ++++++++++--------- 7 files changed, 43 insertions(+), 21 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 7a9131cc7..2246e27f2 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -2,4 +2,7 @@ from .agent_configuration import ( AgentConfiguration, AgentConfigurationSchema, ) -from .default_agent_configuration import DEFAULT_AGENT_CONFIGURATION +from .default_agent_configuration import ( + DEFAULT_AGENT_CONFIGURATION_JSON, + build_default_agent_configuration, +) diff --git a/monkey/common/configuration/default_agent_configuration.py b/monkey/common/configuration/default_agent_configuration.py index 933b723ea..8c909f15d 100644 --- a/monkey/common/configuration/default_agent_configuration.py +++ b/monkey/common/configuration/default_agent_configuration.py @@ -1,4 +1,6 @@ -DEFAULT_AGENT_CONFIGURATION = """{ +from . import AgentConfiguration, AgentConfigurationSchema + +DEFAULT_AGENT_CONFIGURATION_JSON = """{ "keep_tunnel_open_time": 30, "post_breach_actions": [ { @@ -215,3 +217,8 @@ DEFAULT_AGENT_CONFIGURATION = """{ } } """ + + +def build_default_agent_configuration() -> AgentConfiguration: + schema = AgentConfigurationSchema() + return schema.loads(DEFAULT_AGENT_CONFIGURATION_JSON) diff --git a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py index 89fc57960..b63ee817c 100644 --- a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py @@ -1,10 +1,6 @@ import io -from common.configuration import ( - DEFAULT_AGENT_CONFIGURATION, - AgentConfiguration, - AgentConfigurationSchema, -) +from common.configuration import AgentConfiguration, AgentConfigurationSchema from monkey_island.cc import repository from monkey_island.cc.repository import ( IAgentConfigurationRepository, @@ -16,7 +12,10 @@ AGENT_CONFIGURATION_FILE_NAME = "agent_configuration.json" class FileAgentConfigurationRepository(IAgentConfigurationRepository): - def __init__(self, file_repository: IFileRepository): + def __init__( + self, default_agent_configuration: AgentConfiguration, file_repository: IFileRepository + ): + self._default_agent_configuration = default_agent_configuration self._file_repository = file_repository self._schema = AgentConfigurationSchema() @@ -27,7 +26,7 @@ class FileAgentConfigurationRepository(IAgentConfigurationRepository): return self._schema.loads(configuration_json) except repository.FileNotFoundError: - return self._schema.loads(DEFAULT_AGENT_CONFIGURATION) + return self._default_agent_configuration except Exception as err: raise RetrievalError(f"Error retrieving the agent configuration: {err}") diff --git a/monkey/monkey_island/cc/services/initialize.py b/monkey/monkey_island/cc/services/initialize.py index 4b03daf7d..52343bbf5 100644 --- a/monkey/monkey_island/cc/services/initialize.py +++ b/monkey/monkey_island/cc/services/initialize.py @@ -3,6 +3,7 @@ from pathlib import Path from common import DIContainer from common.aws import AWSInstance +from common.configuration import AgentConfiguration, build_default_agent_configuration from common.utils.file_utils import get_binary_io_sha256_hash from monkey_island.cc.repository import ( AgentBinaryRepository, @@ -30,6 +31,9 @@ def initialize_services(data_dir: Path) -> DIContainer: container = DIContainer() container.register_convention(Path, "data_dir", data_dir) + container.register_convention( + AgentConfiguration, "default_agent_configuration", build_default_agent_configuration() + ) container.register_instance(AWSInstance, AWSInstance()) container.register_instance( diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 733e2e708..3383ab1b7 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -25,7 +25,7 @@ from tests.common.example_agent_configuration import ( from common import OperatingSystems from common.configuration import ( - DEFAULT_AGENT_CONFIGURATION, + DEFAULT_AGENT_CONFIGURATION_JSON, AgentConfiguration, AgentConfigurationSchema, ) @@ -180,6 +180,6 @@ def test_agent_configuration(): def test_default_agent_configuration(): schema = AgentConfigurationSchema() - config = schema.loads(DEFAULT_AGENT_CONFIGURATION) + config = schema.loads(DEFAULT_AGENT_CONFIGURATION_JSON) assert isinstance(config, AgentConfiguration) diff --git a/monkey/tests/unit_tests/conftest.py b/monkey/tests/unit_tests/conftest.py index 5b83a7e69..2f00d3d16 100644 --- a/monkey/tests/unit_tests/conftest.py +++ b/monkey/tests/unit_tests/conftest.py @@ -6,6 +6,8 @@ from typing import Callable, Dict import pytest from _pytest.monkeypatch import MonkeyPatch +from common.configuration import AgentConfiguration, build_default_agent_configuration + MONKEY_BASE_PATH = str(Path(__file__).parent.parent.parent) sys.path.insert(0, MONKEY_BASE_PATH) @@ -54,3 +56,8 @@ def load_monkey_config(data_for_tests_dir) -> Callable[[str], Dict]: return json.loads(open(config_path, "r").read()) return inner + + +@pytest.fixture +def default_agent_configuration() -> AgentConfiguration: + return build_default_agent_configuration() diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py index 93a80dea6..4ab111606 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -2,12 +2,16 @@ import pytest from tests.common.example_agent_configuration import AGENT_CONFIGURATION from tests.monkey_island import OpenErrorFileRepository, SingleFileRepository -from common.configuration import DEFAULT_AGENT_CONFIGURATION, AgentConfigurationSchema +from common.configuration import AgentConfigurationSchema from monkey_island.cc.repository import FileAgentConfigurationRepository, RetrievalError -def test_store_agent_config(): - repository = FileAgentConfigurationRepository(SingleFileRepository()) +@pytest.fixture +def repository(default_agent_configuration): + return FileAgentConfigurationRepository(default_agent_configuration, SingleFileRepository()) + + +def test_store_agent_config(repository): schema = AgentConfigurationSchema() agent_configuration = schema.load(AGENT_CONFIGURATION) @@ -17,18 +21,16 @@ def test_store_agent_config(): assert retrieved_agent_configuration == agent_configuration -def test_get_default_agent_config(): - repository = FileAgentConfigurationRepository(SingleFileRepository()) - schema = AgentConfigurationSchema() - default_agent_configuration = schema.loads(DEFAULT_AGENT_CONFIGURATION) - +def test_get_default_agent_config(repository, default_agent_configuration): retrieved_agent_configuration = repository.get_configuration() assert retrieved_agent_configuration == default_agent_configuration -def test_get_agent_config_retrieval_error(): - repository = FileAgentConfigurationRepository(OpenErrorFileRepository()) +def test_get_agent_config_retrieval_error(default_agent_configuration): + repository = FileAgentConfigurationRepository( + default_agent_configuration, OpenErrorFileRepository() + ) with pytest.raises(RetrievalError): repository.get_configuration() From 7a28ba4c4d818998973a7339b041273de7ecd404 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Mon, 20 Jun 2022 15:25:36 -0700 Subject: [PATCH 089/196] Island: Create new resource `Configuration` --- .../cc/resources/configuration.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 monkey/monkey_island/cc/resources/configuration.py diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py new file mode 100644 index 000000000..79f1d2a0c --- /dev/null +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -0,0 +1,18 @@ +from monkey_island.cc.resources.AbstractResource import AbstractResource +from monkey_island.cc.resources.request_authentication import jwt_required + + +class Configuration(AbstractResource): + urls = ["/api/configuration"] + + @jwt_required + def get(self): + pass + + @jwt_required + def post(self): + pass + + @jwt_required + def patch(self): # reset the config here? + pass From 5dd27eeea535e6250f241a8fb3d22816124630d4 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Mon, 20 Jun 2022 19:50:38 -0700 Subject: [PATCH 090/196] Island: Add definition for GET in new Configuration resource --- .../cc/resources/configuration.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 79f1d2a0c..055fe5db1 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -1,13 +1,26 @@ +from enum import Enum +from flask import jsonify, request + from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required +from monkey_island.cc.repository import FileAgentConfigurationRepository + +class ConfigurationTypeEnum(Enum): + ISLAND = "island" + AGENT = "agent" class Configuration(AbstractResource): - urls = ["/api/configuration"] + urls = ["/api/configuration/"] @jwt_required - def get(self): - pass + def get(self, configuration_type: str): + # we probably still need this because of credential fields, HTTP ports, etc in the config? + if configuration_type == ConfigurationTypeEnum.ISLAND: + pass + elif configuration_type == ConfigurationTypeEnum.AGENT: + configuration = FileAgentConfigurationRepository.get_configuration() + return jsonify(configuration=configuration) @jwt_required def post(self): From 138480f0217a66438c68944c1efddaab8e1cc72e Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Mon, 20 Jun 2022 23:01:12 -0700 Subject: [PATCH 091/196] Island: Roughly implement POST in new Configuration resource + lots of questions --- .../cc/resources/configuration.py | 77 +++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 055fe5db1..dfd7ae70c 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -1,21 +1,49 @@ +import json +from dataclasses import dataclass from enum import Enum +from itertools import chain +from typing import Mapping + from flask import jsonify, request +from common.configuration.agent_configuration import AgentConfigurationSchema +from common.utils.exceptions import InvalidConfigurationError +from monkey_island.cc.repository import FileAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required -from monkey_island.cc.repository import FileAgentConfigurationRepository + class ConfigurationTypeEnum(Enum): ISLAND = "island" AGENT = "agent" +class ImportStatuses: + UNSAFE_OPTION_VERIFICATION_REQUIRED = "unsafe_options_verification_required" + INVALID_CONFIGURATION = "invalid_configuration" + INVALID_CREDENTIALS = "invalid_credentials" + IMPORTED = "imported" + + +@dataclass +class ResponseContents: + import_status: str = ImportStatuses.IMPORTED + message: str = "" + status_code: int = 200 + config: str = "" + config_schema: str = "" + + def form_response(self): + return self.__dict__ + + class Configuration(AbstractResource): urls = ["/api/configuration/"] @jwt_required def get(self, configuration_type: str): - # we probably still need this because of credential fields, HTTP ports, etc in the config? + # Q: we probably still need this because of credential fields, HTTP ports, etc in the + # config? if configuration_type == ConfigurationTypeEnum.ISLAND: pass elif configuration_type == ConfigurationTypeEnum.AGENT: @@ -23,9 +51,48 @@ class Configuration(AbstractResource): return jsonify(configuration=configuration) @jwt_required - def post(self): - pass + def post(self, configuration_type: str): + request_contents = json.loads(request.data) + configuration_json = json.loads(request_contents["config"]) + Configuration._remove_metadata_from_config(configuration_json) + + try: + # Q: encryption is moving to the frontend; also check this in the frontend? + if request_contents["unsafeOptionsVerified"]: + schema = AgentConfigurationSchema() + # Q: in what format/schema are we getting the config from the Island? + # Q: when does flattening the config go away? + configuration_object = schema.loads(configuration_json) + FileAgentConfigurationRepository.store_configuration( + configuration_object + ) # check error handling + return ResponseContents().form_response() + else: + return ResponseContents( + config=json.dumps(configuration_json), + # Q: do we still need a separate config schema like this? + # config_schema=ConfigService.get_config_schema(), + import_status=ImportStatuses.UNSAFE_OPTION_VERIFICATION_REQUIRED, + ).form_response() + except InvalidConfigurationError: + return ResponseContents( + import_status=ImportStatuses.INVALID_CONFIGURATION, + message="Invalid configuration supplied. " + "Maybe the format is outdated or the file has been corrupted.", + status_code=400, + ).form_response() + + @staticmethod + # Q: why is this really needed? besides the fact that it just doesn't belong in the config + # which is being saved in mongo? if nothing, can't we just wait to change the exploiters + # to plugins? + def _remove_metadata_from_config(configuration_json: Mapping): + for exploiter in chain( + configuration_json["propagation"]["exploitation"]["brute_force"], + configuration_json["propagation"]["exploitation"]["vulnerability"], + ): + del exploiter["supported_os"] @jwt_required - def patch(self): # reset the config here? + def patch(self): # Q: reset the configuration here? or does that make more sense in delete? pass From 5f253e79b306c81207f5a8ac4ab289eb41c6d11a Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Tue, 21 Jun 2022 12:50:52 +0200 Subject: [PATCH 092/196] Island: Init Configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index dfd7ae70c..417db6832 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -8,7 +8,7 @@ from flask import jsonify, request from common.configuration.agent_configuration import AgentConfigurationSchema from common.utils.exceptions import InvalidConfigurationError -from monkey_island.cc.repository import FileAgentConfigurationRepository +from monkey_island.cc.repository import IAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required @@ -40,6 +40,9 @@ class ResponseContents: class Configuration(AbstractResource): urls = ["/api/configuration/"] + def __init__(self, file_agent_configuration_repository: IAgentConfigurationRepository): + self._file_agent_configuration_repository = file_agent_configuration_repository + @jwt_required def get(self, configuration_type: str): # Q: we probably still need this because of credential fields, HTTP ports, etc in the @@ -47,7 +50,7 @@ class Configuration(AbstractResource): if configuration_type == ConfigurationTypeEnum.ISLAND: pass elif configuration_type == ConfigurationTypeEnum.AGENT: - configuration = FileAgentConfigurationRepository.get_configuration() + configuration = self._file_agent_configuration_repository.get_configuration() return jsonify(configuration=configuration) @jwt_required @@ -63,7 +66,7 @@ class Configuration(AbstractResource): # Q: in what format/schema are we getting the config from the Island? # Q: when does flattening the config go away? configuration_object = schema.loads(configuration_json) - FileAgentConfigurationRepository.store_configuration( + self._file_agent_configuration_repository.store_configuration( configuration_object ) # check error handling return ResponseContents().form_response() From 74bc55e077132862f4995ac20f7736eed3664e5a Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 14:31:23 -0700 Subject: [PATCH 093/196] Island: Remove config type logic from new configuration resource --- .../cc/resources/configuration.py | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 417db6832..b611eaec6 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -1,6 +1,5 @@ import json from dataclasses import dataclass -from enum import Enum from itertools import chain from typing import Mapping @@ -13,11 +12,6 @@ from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required -class ConfigurationTypeEnum(Enum): - ISLAND = "island" - AGENT = "agent" - - class ImportStatuses: UNSAFE_OPTION_VERIFICATION_REQUIRED = "unsafe_options_verification_required" INVALID_CONFIGURATION = "invalid_configuration" @@ -38,23 +32,18 @@ class ResponseContents: class Configuration(AbstractResource): - urls = ["/api/configuration/"] + urls = ["/api/configuration"] def __init__(self, file_agent_configuration_repository: IAgentConfigurationRepository): self._file_agent_configuration_repository = file_agent_configuration_repository @jwt_required - def get(self, configuration_type: str): - # Q: we probably still need this because of credential fields, HTTP ports, etc in the - # config? - if configuration_type == ConfigurationTypeEnum.ISLAND: - pass - elif configuration_type == ConfigurationTypeEnum.AGENT: - configuration = self._file_agent_configuration_repository.get_configuration() - return jsonify(configuration=configuration) + def get(self): + configuration = self._file_agent_configuration_repository.get_configuration() + return jsonify(configuration=configuration) @jwt_required - def post(self, configuration_type: str): + def post(self): request_contents = json.loads(request.data) configuration_json = json.loads(request_contents["config"]) Configuration._remove_metadata_from_config(configuration_json) From 0d8cc713d2502601f1173bbe85d27f51c71cf94b Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 14:35:42 -0700 Subject: [PATCH 094/196] Island: Rename `file_agent_configuration_repository` to `agent_configuration_repository` in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index b611eaec6..0eab79bf9 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -34,12 +34,12 @@ class ResponseContents: class Configuration(AbstractResource): urls = ["/api/configuration"] - def __init__(self, file_agent_configuration_repository: IAgentConfigurationRepository): - self._file_agent_configuration_repository = file_agent_configuration_repository + def __init__(self, agent_configuration_repository: IAgentConfigurationRepository): + self._agent_configuration_repository = agent_configuration_repository @jwt_required def get(self): - configuration = self._file_agent_configuration_repository.get_configuration() + configuration = self._agent_configuration_repository.get_configuration() return jsonify(configuration=configuration) @jwt_required @@ -55,7 +55,7 @@ class Configuration(AbstractResource): # Q: in what format/schema are we getting the config from the Island? # Q: when does flattening the config go away? configuration_object = schema.loads(configuration_json) - self._file_agent_configuration_repository.store_configuration( + self._agent_configuration_repository.store_configuration( configuration_object ) # check error handling return ResponseContents().form_response() From 8c14423c4ea5efb7c81a7e1ee270f34c70fbeaa8 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 14:41:55 -0700 Subject: [PATCH 095/196] Island: Prepend 'agent' to everything having 'configuration' in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 0eab79bf9..710db13aa 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -31,8 +31,8 @@ class ResponseContents: return self.__dict__ -class Configuration(AbstractResource): - urls = ["/api/configuration"] +class AgentConfiguration(AbstractResource): + urls = ["/api/agent-configuration"] def __init__(self, agent_configuration_repository: IAgentConfigurationRepository): self._agent_configuration_repository = agent_configuration_repository @@ -46,7 +46,7 @@ class Configuration(AbstractResource): def post(self): request_contents = json.loads(request.data) configuration_json = json.loads(request_contents["config"]) - Configuration._remove_metadata_from_config(configuration_json) + AgentConfiguration._remove_metadata_from_config(configuration_json) try: # Q: encryption is moving to the frontend; also check this in the frontend? From 15615e08c47e619a8d2b0adab0adcb7711fb227d Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 14:51:40 -0700 Subject: [PATCH 096/196] Island: Get rid of unsafe config options' check in new configuration resource --- .../cc/resources/configuration.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 710db13aa..24185eee9 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -49,24 +49,13 @@ class AgentConfiguration(AbstractResource): AgentConfiguration._remove_metadata_from_config(configuration_json) try: - # Q: encryption is moving to the frontend; also check this in the frontend? - if request_contents["unsafeOptionsVerified"]: - schema = AgentConfigurationSchema() - # Q: in what format/schema are we getting the config from the Island? - # Q: when does flattening the config go away? - configuration_object = schema.loads(configuration_json) - self._agent_configuration_repository.store_configuration( - configuration_object - ) # check error handling - return ResponseContents().form_response() - else: - return ResponseContents( - config=json.dumps(configuration_json), - # Q: do we still need a separate config schema like this? - # config_schema=ConfigService.get_config_schema(), - import_status=ImportStatuses.UNSAFE_OPTION_VERIFICATION_REQUIRED, - ).form_response() - except InvalidConfigurationError: + schema = AgentConfigurationSchema() + configuration_object = schema.loads(configuration_json) + self._agent_configuration_repository.store_configuration( + configuration_object + ) # check error handling + return ResponseContents().form_response() + except InvalidConfigurationError: # don't need this probably either; if invalid, schema should raise error (catch marshmallow exception and return 400) return ResponseContents( import_status=ImportStatuses.INVALID_CONFIGURATION, message="Invalid configuration supplied. " From 00e38391b48175d35b1c88a86b0e45a9b60898cf Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 14:58:08 -0700 Subject: [PATCH 097/196] Island: Catch appropriate `marshmallow` error when loading config which could be invalid, in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 24185eee9..84c28e337 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -3,19 +3,17 @@ from dataclasses import dataclass from itertools import chain from typing import Mapping +import marshmallow from flask import jsonify, request from common.configuration.agent_configuration import AgentConfigurationSchema -from common.utils.exceptions import InvalidConfigurationError from monkey_island.cc.repository import IAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required class ImportStatuses: - UNSAFE_OPTION_VERIFICATION_REQUIRED = "unsafe_options_verification_required" INVALID_CONFIGURATION = "invalid_configuration" - INVALID_CREDENTIALS = "invalid_credentials" IMPORTED = "imported" @@ -51,11 +49,9 @@ class AgentConfiguration(AbstractResource): try: schema = AgentConfigurationSchema() configuration_object = schema.loads(configuration_json) - self._agent_configuration_repository.store_configuration( - configuration_object - ) # check error handling + self._agent_configuration_repository.store_configuration(configuration_object) return ResponseContents().form_response() - except InvalidConfigurationError: # don't need this probably either; if invalid, schema should raise error (catch marshmallow exception and return 400) + except marshmallow.exceptions.ValidationError: return ResponseContents( import_status=ImportStatuses.INVALID_CONFIGURATION, message="Invalid configuration supplied. " From 63d5330386b1340cc9bc347955cbd233b341ab84 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 15:07:32 -0700 Subject: [PATCH 098/196] Island: Remove unneeded patch function in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 84c28e337..ae9b86688 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -69,7 +69,3 @@ class AgentConfiguration(AbstractResource): configuration_json["propagation"]["exploitation"]["vulnerability"], ): del exploiter["supported_os"] - - @jwt_required - def patch(self): # Q: reset the configuration here? or does that make more sense in delete? - pass From 32fe7c6a4b6ebfaf0e44aa471eab2582c9414208 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 15:09:16 -0700 Subject: [PATCH 099/196] Island: Remove unneeded fields from `ResponseContents` in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index ae9b86688..2ec3c095a 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -22,8 +22,6 @@ class ResponseContents: import_status: str = ImportStatuses.IMPORTED message: str = "" status_code: int = 200 - config: str = "" - config_schema: str = "" def form_response(self): return self.__dict__ From 6b45d62d813066ab690b954b439f9e38392284cb Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 15:31:38 -0700 Subject: [PATCH 100/196] Island: Fix logic to remove metadata from config in new configuration resource's POST --- monkey/monkey_island/cc/resources/configuration.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 2ec3c095a..2ed567561 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -58,12 +58,10 @@ class AgentConfiguration(AbstractResource): ).form_response() @staticmethod - # Q: why is this really needed? besides the fact that it just doesn't belong in the config - # which is being saved in mongo? if nothing, can't we just wait to change the exploiters - # to plugins? def _remove_metadata_from_config(configuration_json: Mapping): for exploiter in chain( configuration_json["propagation"]["exploitation"]["brute_force"], configuration_json["propagation"]["exploitation"]["vulnerability"], ): - del exploiter["supported_os"] + if "supported_os" in exploiter: + del exploiter["supported_os"] From 922495785c19f3624a15a608f8fb3ddf13aa0fc0 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 15:55:39 -0700 Subject: [PATCH 101/196] Island: Create class variable for agent config schema in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 2ed567561..5ec163ded 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -32,6 +32,7 @@ class AgentConfiguration(AbstractResource): def __init__(self, agent_configuration_repository: IAgentConfigurationRepository): self._agent_configuration_repository = agent_configuration_repository + self._schema = AgentConfigurationSchema() @jwt_required def get(self): @@ -45,8 +46,7 @@ class AgentConfiguration(AbstractResource): AgentConfiguration._remove_metadata_from_config(configuration_json) try: - schema = AgentConfigurationSchema() - configuration_object = schema.loads(configuration_json) + configuration_object = self._schema.loads(configuration_json) self._agent_configuration_repository.store_configuration(configuration_object) return ResponseContents().form_response() except marshmallow.exceptions.ValidationError: From d861def86cfea2860c6770e27feea6c11e6822e8 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Tue, 21 Jun 2022 15:57:58 -0700 Subject: [PATCH 102/196] Island: Add logic to add metadata to config in new configuration resource's GET --- .../monkey_island/cc/resources/configuration.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 5ec163ded..7b8929ab9 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -7,6 +7,7 @@ import marshmallow from flask import jsonify, request from common.configuration.agent_configuration import AgentConfigurationSchema +from common.configuration.default_agent_configuration import DEFAULT_AGENT_CONFIGURATION from monkey_island.cc.repository import IAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required @@ -37,7 +38,21 @@ class AgentConfiguration(AbstractResource): @jwt_required def get(self): configuration = self._agent_configuration_repository.get_configuration() - return jsonify(configuration=configuration) + configuration_json = self._schema.dumps(configuration) + # for when the agent requests the config; it needs the exploiters' metadata + # Q: but this'll cause issues when the frontend requests the config? + AgentConfiguration._add_metadata_to_config(configuration_json) + return jsonify(configuration_json=configuration_json) + + @staticmethod + def _add_metadata_to_config(configuration_json): + for exploiter_type in ["brute_force", "vulnerability"]: + for exploiter in configuration_json["propagation"]["exploitation"][exploiter_type]: + # there has to be a better way to do this + if "supported_os" not in exploiter: + exploiter["supported_os"] = DEFAULT_AGENT_CONFIGURATION["propagation"][ + "exploitation" + ][exploiter_type]["supported_os"] @jwt_required def post(self): From ec710d9e5f4b7f72b0252e5f8f9579c9243f3153 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 07:48:34 -0700 Subject: [PATCH 103/196] Island: Get rid of ResponseContents and ImportStatuses in new configuration resource --- .../cc/resources/configuration.py | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 7b8929ab9..725dbe580 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -1,10 +1,9 @@ import json -from dataclasses import dataclass from itertools import chain from typing import Mapping import marshmallow -from flask import jsonify, request +from flask import jsonify, make_response, request from common.configuration.agent_configuration import AgentConfigurationSchema from common.configuration.default_agent_configuration import DEFAULT_AGENT_CONFIGURATION @@ -13,21 +12,6 @@ from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required -class ImportStatuses: - INVALID_CONFIGURATION = "invalid_configuration" - IMPORTED = "imported" - - -@dataclass -class ResponseContents: - import_status: str = ImportStatuses.IMPORTED - message: str = "" - status_code: int = 200 - - def form_response(self): - return self.__dict__ - - class AgentConfiguration(AbstractResource): urls = ["/api/agent-configuration"] @@ -63,14 +47,15 @@ class AgentConfiguration(AbstractResource): try: configuration_object = self._schema.loads(configuration_json) self._agent_configuration_repository.store_configuration(configuration_object) - return ResponseContents().form_response() + return make_response({}, 200) except marshmallow.exceptions.ValidationError: - return ResponseContents( - import_status=ImportStatuses.INVALID_CONFIGURATION, - message="Invalid configuration supplied. " - "Maybe the format is outdated or the file has been corrupted.", - status_code=400, - ).form_response() + return make_response( + { + "message": "Invalid configuration supplied. " + "Maybe the format is outdated or the file has been corrupted." + }, + 400, + ) @staticmethod def _remove_metadata_from_config(configuration_json: Mapping): From 142eed72ac5a75302eaa701ec796c271a94046c9 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 08:12:09 -0700 Subject: [PATCH 104/196] Island: Remove logic to remove/add config metadata in new configuration resource --- .../cc/resources/configuration.py | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 725dbe580..0a889a6da 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -1,12 +1,9 @@ import json -from itertools import chain -from typing import Mapping import marshmallow from flask import jsonify, make_response, request from common.configuration.agent_configuration import AgentConfigurationSchema -from common.configuration.default_agent_configuration import DEFAULT_AGENT_CONFIGURATION from monkey_island.cc.repository import IAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required @@ -23,26 +20,12 @@ class AgentConfiguration(AbstractResource): def get(self): configuration = self._agent_configuration_repository.get_configuration() configuration_json = self._schema.dumps(configuration) - # for when the agent requests the config; it needs the exploiters' metadata - # Q: but this'll cause issues when the frontend requests the config? - AgentConfiguration._add_metadata_to_config(configuration_json) return jsonify(configuration_json=configuration_json) - @staticmethod - def _add_metadata_to_config(configuration_json): - for exploiter_type in ["brute_force", "vulnerability"]: - for exploiter in configuration_json["propagation"]["exploitation"][exploiter_type]: - # there has to be a better way to do this - if "supported_os" not in exploiter: - exploiter["supported_os"] = DEFAULT_AGENT_CONFIGURATION["propagation"][ - "exploitation" - ][exploiter_type]["supported_os"] - @jwt_required def post(self): request_contents = json.loads(request.data) configuration_json = json.loads(request_contents["config"]) - AgentConfiguration._remove_metadata_from_config(configuration_json) try: configuration_object = self._schema.loads(configuration_json) @@ -56,12 +39,3 @@ class AgentConfiguration(AbstractResource): }, 400, ) - - @staticmethod - def _remove_metadata_from_config(configuration_json: Mapping): - for exploiter in chain( - configuration_json["propagation"]["exploitation"]["brute_force"], - configuration_json["propagation"]["exploitation"]["vulnerability"], - ): - if "supported_os" in exploiter: - del exploiter["supported_os"] From 03037b566233864906e8bbcef49fc7e8bd83dd4b Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 08:28:57 -0700 Subject: [PATCH 105/196] Common: Remove `supported_os` field for exploiters in configuration --- .../agent_sub_configuration_schemas.py | 4 -- .../configuration/agent_sub_configurations.py | 3 -- .../default_agent_configuration.py | 44 ++++++------------- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/monkey/common/configuration/agent_sub_configuration_schemas.py b/monkey/common/configuration/agent_sub_configuration_schemas.py index ceec0af24..4d2ee2d8e 100644 --- a/monkey/common/configuration/agent_sub_configuration_schemas.py +++ b/monkey/common/configuration/agent_sub_configuration_schemas.py @@ -1,7 +1,4 @@ from marshmallow import Schema, fields, post_load -from marshmallow_enum import EnumField - -from common import OperatingSystems from .agent_sub_configurations import ( CustomPBAConfiguration, @@ -87,7 +84,6 @@ class ExploitationOptionsConfigurationSchema(Schema): class ExploiterConfigurationSchema(Schema): name = fields.Str() options = fields.Mapping() - supported_os = fields.List(EnumField(OperatingSystems)) @post_load def _make_exploiter_configuration(self, data, **kwargs): diff --git a/monkey/common/configuration/agent_sub_configurations.py b/monkey/common/configuration/agent_sub_configurations.py index 560542e1d..c4a0c704c 100644 --- a/monkey/common/configuration/agent_sub_configurations.py +++ b/monkey/common/configuration/agent_sub_configurations.py @@ -1,8 +1,6 @@ from dataclasses import dataclass from typing import Dict, List -from common import OperatingSystems - @dataclass(frozen=True) class CustomPBAConfiguration: @@ -54,7 +52,6 @@ class ExploitationOptionsConfiguration: class ExploiterConfiguration: name: str options: Dict - supported_os: List[OperatingSystems] @dataclass(frozen=True) diff --git a/monkey/common/configuration/default_agent_configuration.py b/monkey/common/configuration/default_agent_configuration.py index 8c909f15d..c83169566 100644 --- a/monkey/common/configuration/default_agent_configuration.py +++ b/monkey/common/configuration/default_agent_configuration.py @@ -157,60 +157,44 @@ DEFAULT_AGENT_CONFIGURATION_JSON = """{ "brute_force": [ { "name": "MSSQLExploiter", - "options": {}, - "supported_os": [ - "WINDOWS" - ] + "options": {} + }, { "name": "PowerShellExploiter", - "options": {}, - "supported_os": [ - "WINDOWS" - ] + "options": {} + }, { "name": "SSHExploiter", - "options": {}, - "supported_os": [ - "LINUX" - ] + "options": {} + }, { "name": "SmbExploiter", "options": { "smb_download_timeout": 30 - }, - "supported_os": [ - "WINDOWS" - ] + } + }, { "name": "WmiExploiter", "options": { "smb_download_timeout": 30 - }, - "supported_os": [ - "WINDOWS" - ] + } + } ], "vulnerability": [ { "name": "HadoopExploiter", - "options": {}, - "supported_os": [ - "LINUX", - "WINDOWS" - ] + "options": {} + }, { "name": "Log4ShellExploiter", - "options": {}, - "supported_os": [ - "LINUX", - "WINDOWS" - ] + "options": {} + } ] } From 26ece213a23ec8b5f09103943bf350723b6cc1b2 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 08:30:44 -0700 Subject: [PATCH 106/196] Island: Remove logic to add `supported_os` for exploiters to configuration --- monkey/monkey_island/cc/services/config.py | 24 +--------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 86541c33d..69db1c2e1 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -3,12 +3,10 @@ import copy import functools import logging import re -from itertools import chain from typing import Any, Dict, List from jsonschema import Draft4Validator, validators -from common import OperatingSystems from common.config_value_paths import ( LM_HASH_LIST_PATH, NTLM_HASH_LIST_PATH, @@ -580,7 +578,7 @@ class ConfigService: formatted_exploiters_config = ConfigService._add_smb_download_timeout_to_exploiters( formatted_exploiters_config ) - return ConfigService._add_supported_os_to_exploiters(formatted_exploiters_config) + return formatted_exploiters_config @staticmethod def _add_smb_download_timeout_to_exploiters( @@ -593,23 +591,3 @@ class ConfigService: exploiter["options"]["smb_download_timeout"] = SMB_DOWNLOAD_TIMEOUT return new_config - - @staticmethod - def _add_supported_os_to_exploiters( - formatted_config: Dict, - ) -> Dict[str, List[Dict[str, Any]]]: - supported_os = { - "HadoopExploiter": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], - "Log4ShellExploiter": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], - "MSSQLExploiter": [OperatingSystems.WINDOWS], - "PowerShellExploiter": [OperatingSystems.WINDOWS], - "SSHExploiter": [OperatingSystems.LINUX], - "SmbExploiter": [OperatingSystems.WINDOWS], - "WmiExploiter": [OperatingSystems.WINDOWS], - "ZerologonExploiter": [OperatingSystems.WINDOWS], - } - new_config = copy.deepcopy(formatted_config) - for exploiter in chain(new_config["brute_force"], new_config["vulnerability"]): - exploiter["supported_os"] = supported_os.get(exploiter["name"], []) - - return new_config From fd41d9179e9c279a61f2dfa5df377f5c59fb3f20 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 08:34:55 -0700 Subject: [PATCH 107/196] Agent: Add `SUPPORTED_OS` dict for exploiters and change checking logic in master --- monkey/infection_monkey/master/exploiter.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/exploiter.py b/monkey/infection_monkey/master/exploiter.py index b9eada5b6..8f99cba7b 100644 --- a/monkey/infection_monkey/master/exploiter.py +++ b/monkey/infection_monkey/master/exploiter.py @@ -7,6 +7,7 @@ from queue import Queue from threading import Event from typing import Callable, Dict, List, Mapping +from common import OperatingSystems from infection_monkey.custom_types import PropagationCredentials from infection_monkey.i_puppet import ExploiterResultData, IPuppet from infection_monkey.model import VictimHost @@ -20,6 +21,18 @@ ExploiterName = str Callback = Callable[[ExploiterName, VictimHost, ExploiterResultData], None] +SUPPORTED_OS = { + "HadoopExploiter": [OperatingSystems.LINUX.value, OperatingSystems.WINDOWS.value], + "Log4ShellExploiter": [OperatingSystems.LINUX.value, OperatingSystems.WINDOWS.value], + "MSSQLExploiter": [OperatingSystems.WINDOWS.value], + "PowerShellExploiter": [OperatingSystems.WINDOWS.value], + "SSHExploiter": [OperatingSystems.LINUX.value], + "SmbExploiter": [OperatingSystems.WINDOWS.value], + "WmiExploiter": [OperatingSystems.WINDOWS.value], + "ZerologonExploiter": [OperatingSystems.WINDOWS.value], +} + + class Exploiter: def __init__( self, @@ -118,7 +131,7 @@ class Exploiter: victim_os = victim_host.os.get("type") # We want to try all exploiters if the victim's OS is unknown - if victim_os is not None and victim_os not in exploiter["supported_os"]: + if victim_os is not None and victim_os not in SUPPORTED_OS[exploiter_name]: logger.debug( f"Skipping {exploiter_name} because it does not support " f"the victim's OS ({victim_os})" From e25eb194a1a00b117384a931e124edc28ce446f0 Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 08:41:11 -0700 Subject: [PATCH 108/196] UT: Remove `supported_os` for exploiters from all tests --- .../tests/common/example_agent_configuration.py | 4 +--- .../monkey_configs/automated_master_config.json | 16 ++++++++-------- .../common/test_agent_configuration.py | 7 +------ .../infection_monkey/master/test_exploiter.py | 8 ++++---- .../monkey_island/cc/services/test_config.py | 11 ++--------- 5 files changed, 16 insertions(+), 30 deletions(-) diff --git a/monkey/tests/common/example_agent_configuration.py b/monkey/tests/common/example_agent_configuration.py index 640771df4..25a1dbd5e 100644 --- a/monkey/tests/common/example_agent_configuration.py +++ b/monkey/tests/common/example_agent_configuration.py @@ -39,18 +39,16 @@ NETWORK_SCAN_CONFIGURATION = { } BRUTE_FORCE = [ - {"name": "ex1", "options": {}, "supported_os": ["LINUX"]}, + {"name": "ex1", "options": {}}, { "name": "ex2", "options": {"smb_download_timeout": 10}, - "supported_os": ["LINUX", "WINDOWS"], }, ] VULNERABILITY = [ { "name": "ex3", "options": {"smb_download_timeout": 10}, - "supported_os": ["WINDOWS"], }, ] EXPLOITATION_CONFIGURATION = { diff --git a/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json b/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json index 7fcc2285d..c0c035b9d 100644 --- a/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json +++ b/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json @@ -47,16 +47,16 @@ "exploiters": { "options": {}, "brute_force": [ - {"name": "MSSQLExploiter", "supported_os": ["windows"], "options": {}}, - {"name": "PowerShellExploiter", "supported_os": ["windows"], "options": {}}, - {"name": "SmbExploiter", "supported_os": ["windows"], "options": {}}, - {"name": "SSHExploiter", "supported_os": ["linux"], "options": {}}, - {"name": "WmiExploiter", "supported_os": ["windows"], "options": {}} + {"name": "MSSQLExploiter", "options": {}}, + {"name": "PowerShellExploiter", "options": {}}, + {"name": "SmbExploiter", "options": {}}, + {"name": "SSHExploiter", "options": {}}, + {"name": "WmiExploiter", "options": {}} ], "vulnerability": [ - {"name": "HadoopExploiter", "supported_os": ["linux", "windows"], "options": {}}, - {"name": "ShellShockExploiter", "supported_os": ["linux"], "options": {}}, - {"name": "ZerologonExploiter", "supported_os": ["windows"], "options": {}} + {"name": "HadoopExploiter", "options": {}}, + {"name": "ShellShockExploiter", "options": {}}, + {"name": "ZerologonExploiter", "options": {}} ] } }, diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/test_agent_configuration.py index 3383ab1b7..7ea80cfc5 100644 --- a/monkey/tests/unit_tests/common/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/test_agent_configuration.py @@ -23,7 +23,6 @@ from tests.common.example_agent_configuration import ( WINDOWS_FILENAME, ) -from common import OperatingSystems from common.configuration import ( DEFAULT_AGENT_CONFIGURATION_JSON, AgentConfiguration, @@ -126,16 +125,12 @@ def test_exploitation_options_configuration_schema(): def test_exploiter_configuration_schema(): name = "bond" options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} - supported_os = [OperatingSystems.LINUX, OperatingSystems.WINDOWS] schema = ExploiterConfigurationSchema() - config = schema.load( - {"name": name, "options": options, "supported_os": [os_.name for os_ in supported_os]} - ) + config = schema.load({"name": name, "options": options}) assert config.name == name assert config.options == options - assert config.supported_os == supported_os def test_exploitation_configuration(): diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py index 3c76c903f..42b64821f 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py @@ -38,12 +38,12 @@ def exploiter_config(): return { "options": {"dropper_path_linux": "/tmp/monkey"}, "brute_force": [ - {"name": "HadoopExploiter", "supported_os": ["windows"], "options": {"timeout": 10}}, - {"name": "SSHExploiter", "supported_os": ["linux"], "options": {}}, - {"name": "WmiExploiter", "supported_os": ["windows"], "options": {"timeout": 10}}, + {"name": "HadoopExploiter", "options": {"timeout": 10}}, + {"name": "SSHExploiter", "options": {}}, + {"name": "WmiExploiter", "options": {"timeout": 10}}, ], "vulnerability": [ - {"name": "ZerologonExploiter", "supported_os": ["windows"], "options": {}}, + {"name": "ZerologonExploiter", "options": {}}, ], } diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index df866e388..85f3f4823 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -1,6 +1,5 @@ import pytest -from common import OperatingSystems from monkey_island.cc.services.config import ConfigService # If tests fail because config path is changed, sync with @@ -172,38 +171,32 @@ def test_format_config_for_agent__exploiters(): "http_ports": [80, 443, 7001, 8008, 8080, 9200], }, "brute_force": [ - {"name": "MSSQLExploiter", "supported_os": [OperatingSystems.WINDOWS], "options": {}}, + {"name": "MSSQLExploiter", "options": {}}, { "name": "PowerShellExploiter", - "supported_os": [OperatingSystems.WINDOWS], "options": {}, }, - {"name": "SSHExploiter", "supported_os": [OperatingSystems.LINUX], "options": {}}, + {"name": "SSHExploiter", "options": {}}, { "name": "SmbExploiter", - "supported_os": [OperatingSystems.WINDOWS], "options": {"smb_download_timeout": 30}, }, { "name": "WmiExploiter", - "supported_os": [OperatingSystems.WINDOWS], "options": {"smb_download_timeout": 30}, }, ], "vulnerability": [ { "name": "HadoopExploiter", - "supported_os": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], "options": {}, }, { "name": "Log4ShellExploiter", - "supported_os": [OperatingSystems.LINUX, OperatingSystems.WINDOWS], "options": {}, }, { "name": "ZerologonExploiter", - "supported_os": [OperatingSystems.WINDOWS], "options": {}, }, ], From 104c7ac210af96ea64f7f1c6f21f028d2e32e91d Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 08:51:21 -0700 Subject: [PATCH 109/196] Island: Fix function call to load config in new configuration resource --- monkey/monkey_island/cc/resources/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/configuration.py index 0a889a6da..6e4a00826 100644 --- a/monkey/monkey_island/cc/resources/configuration.py +++ b/monkey/monkey_island/cc/resources/configuration.py @@ -28,7 +28,7 @@ class AgentConfiguration(AbstractResource): configuration_json = json.loads(request_contents["config"]) try: - configuration_object = self._schema.loads(configuration_json) + configuration_object = self._schema.load(configuration_json) self._agent_configuration_repository.store_configuration(configuration_object) return make_response({}, 200) except marshmallow.exceptions.ValidationError: From 1fc0eae4802846a755058351630d8adc6420644b Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 09:35:46 -0700 Subject: [PATCH 110/196] UT: Change import order in conftest.py --- monkey/tests/unit_tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/tests/unit_tests/conftest.py b/monkey/tests/unit_tests/conftest.py index 2f00d3d16..3634e52b9 100644 --- a/monkey/tests/unit_tests/conftest.py +++ b/monkey/tests/unit_tests/conftest.py @@ -6,11 +6,11 @@ from typing import Callable, Dict import pytest from _pytest.monkeypatch import MonkeyPatch -from common.configuration import AgentConfiguration, build_default_agent_configuration - MONKEY_BASE_PATH = str(Path(__file__).parent.parent.parent) sys.path.insert(0, MONKEY_BASE_PATH) +from common.configuration import AgentConfiguration, build_default_agent_configuration # noqa: E402 + @pytest.fixture(scope="session") def data_for_tests_dir(pytestconfig): From 4c1c8044cd8d2bd5c352073d5ee744653d6cdf49 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Wed, 22 Jun 2022 17:55:13 +0300 Subject: [PATCH 111/196] Agent: Use operating systems consts --- monkey/infection_monkey/exploit/hadoop.py | 6 +++--- monkey/infection_monkey/exploit/log4shell.py | 9 +++++---- monkey/infection_monkey/exploit/tools/helpers.py | 2 +- monkey/infection_monkey/exploit/web_rce.py | 16 +++++++++------- monkey/infection_monkey/model/host.py | 5 +++++ .../network_scanning/ping_scanner.py | 5 +++-- .../network_scanning/smb_fingerprinter.py | 5 +++-- .../network_scanning/ssh_fingerprinter.py | 3 ++- 8 files changed, 31 insertions(+), 20 deletions(-) diff --git a/monkey/infection_monkey/exploit/hadoop.py b/monkey/infection_monkey/exploit/hadoop.py index 8bafa6969..5cf30f23c 100644 --- a/monkey/infection_monkey/exploit/hadoop.py +++ b/monkey/infection_monkey/exploit/hadoop.py @@ -105,10 +105,10 @@ class HadoopExploiter(WebRCE): def _build_command(self, path, http_path): # Build command to execute monkey_cmd = build_monkey_commandline(self.host, self.current_depth + 1) - if "linux" in self.host.os["type"]: - base_command = HADOOP_LINUX_COMMAND - else: + if self.host.is_windows(): base_command = HADOOP_WINDOWS_COMMAND + else: + base_command = HADOOP_LINUX_COMMAND return base_command % { "monkey_path": path, diff --git a/monkey/infection_monkey/exploit/log4shell.py b/monkey/infection_monkey/exploit/log4shell.py index 077c7c865..ffbcdd0d6 100644 --- a/monkey/infection_monkey/exploit/log4shell.py +++ b/monkey/infection_monkey/exploit/log4shell.py @@ -2,6 +2,7 @@ import logging import time from pathlib import PurePath +from common import OperatingSystems from common.common_consts.timeouts import LONG_REQUEST_TIMEOUT, MEDIUM_REQUEST_TIMEOUT from common.utils import Timer from infection_monkey.exploit.log4shell_utils import ( @@ -115,10 +116,10 @@ class Log4ShellExploiter(WebRCE): def _build_command(self, path: PurePath, http_path) -> str: # Build command to execute monkey_cmd = build_monkey_commandline(self.host, self.current_depth + 1, location=path) - if "linux" in self.host.os["type"]: - base_command = LOG4SHELL_LINUX_COMMAND - else: + if self.host.is_windows(): base_command = LOG4SHELL_WINDOWS_COMMAND + else: + base_command = LOG4SHELL_LINUX_COMMAND return base_command % { "monkey_path": path, @@ -128,7 +129,7 @@ class Log4ShellExploiter(WebRCE): } def _build_java_class(self, exploit_command: str) -> bytes: - if "linux" in self.host.os["type"]: + if OperatingSystems.LINUX in self.host.os["type"]: return build_exploit_bytecode(exploit_command, LINUX_EXPLOIT_TEMPLATE_PATH) else: return build_exploit_bytecode(exploit_command, WINDOWS_EXPLOIT_TEMPLATE_PATH) diff --git a/monkey/infection_monkey/exploit/tools/helpers.py b/monkey/infection_monkey/exploit/tools/helpers.py index 0ce1c474e..e268fe4c3 100644 --- a/monkey/infection_monkey/exploit/tools/helpers.py +++ b/monkey/infection_monkey/exploit/tools/helpers.py @@ -15,7 +15,7 @@ AGENT_BINARY_PATH_WIN64 = PureWindowsPath(r"C:\Windows\temp\monkey64.exe") def get_agent_dst_path(host: VictimHost) -> PurePath: - if host.os["type"] == "windows": + if host.is_windows(): path = PureWindowsPath(AGENT_BINARY_PATH_WIN64) else: path = PurePosixPath(AGENT_BINARY_PATH_LINUX) diff --git a/monkey/infection_monkey/exploit/web_rce.py b/monkey/infection_monkey/exploit/web_rce.py index 99438a0a7..34f245fca 100644 --- a/monkey/infection_monkey/exploit/web_rce.py +++ b/monkey/infection_monkey/exploit/web_rce.py @@ -3,6 +3,7 @@ from abc import abstractmethod from posixpath import join from typing import List, Tuple +from common import OperatingSystems from common.utils.attack_utils import BITS_UPLOAD_STRING, ScanStatus from infection_monkey.exploit.HostExploiter import HostExploiter from infection_monkey.exploit.tools.http_tools import HTTPTools @@ -162,10 +163,10 @@ class WebRCE(HostExploiter): def get_command(self, path, http_path, commands): try: - if "linux" in self.host.os["type"]: - command = commands["linux"] - else: + if self.host.is_windows(): command = commands["windows"] + else: + command = commands["linux"] # Format command command = command % {"monkey_path": path, "http_path": http_path} except KeyError: @@ -326,7 +327,7 @@ class WebRCE(HostExploiter): :return: response, False if failed and True if permission change is not needed """ logger.info("Changing monkey's permissions") - if "windows" in self.host.os["type"]: + if self.host.is_windows(): logger.info("Permission change not required for windows") return True if not command: @@ -411,13 +412,14 @@ class WebRCE(HostExploiter): :return: Default monkey's destination path for corresponding host or False if failed. """ if not self.host.os.get("type") or ( - self.host.os["type"] != "linux" and self.host.os["type"] != "windows" + self.host.os["type"] != OperatingSystems.LINUX + and self.host.os["type"] != OperatingSystems.LINUX ): logger.error("Target's OS was either unidentified or not supported. Aborting") return False - if self.host.os["type"] == "linux": + if self.host.os["type"] == OperatingSystems.LINUX: return DROPPER_TARGET_PATH_LINUX - if self.host.os["type"] == "windows": + if self.host.os["type"] == OperatingSystems.WINDOWS: return DROPPER_TARGET_PATH_WIN64 def get_target_url(self): diff --git a/monkey/infection_monkey/model/host.py b/monkey/infection_monkey/model/host.py index 95cc85810..6a1295e58 100644 --- a/monkey/infection_monkey/model/host.py +++ b/monkey/infection_monkey/model/host.py @@ -1,5 +1,7 @@ from typing import Optional +from common import OperatingSystems + class VictimHost(object): def __init__(self, ip_addr: str, domain_name: str = ""): @@ -14,6 +16,9 @@ class VictimHost(object): def as_dict(self): return self.__dict__ + def is_windows(self) -> bool: + return OperatingSystems.WINDOWS in self.os["type"] + def __hash__(self): return hash(self.ip_addr) diff --git a/monkey/infection_monkey/network_scanning/ping_scanner.py b/monkey/infection_monkey/network_scanning/ping_scanner.py index 16fb2df96..cddf4bdd4 100644 --- a/monkey/infection_monkey/network_scanning/ping_scanner.py +++ b/monkey/infection_monkey/network_scanning/ping_scanner.py @@ -5,6 +5,7 @@ import re import subprocess import sys +from common import OperatingSystems from infection_monkey.i_puppet import PingScanData from infection_monkey.utils.environment import is_windows_os @@ -79,9 +80,9 @@ def _process_ping_command_output(ping_command_output: str) -> PingScanData: operating_system = None if ttl <= LINUX_TTL: - operating_system = "linux" + operating_system = OperatingSystems.LINUX else: # as far we we know, could also be OSX/BSD, but lets handle that when it comes up. - operating_system = "windows" + operating_system = OperatingSystems.WINDOWS return PingScanData(True, operating_system) diff --git a/monkey/infection_monkey/network_scanning/smb_fingerprinter.py b/monkey/infection_monkey/network_scanning/smb_fingerprinter.py index d47ce224e..438e13db0 100644 --- a/monkey/infection_monkey/network_scanning/smb_fingerprinter.py +++ b/monkey/infection_monkey/network_scanning/smb_fingerprinter.py @@ -5,6 +5,7 @@ from typing import Dict from odict import odict +from common import OperatingSystems from infection_monkey.i_puppet import ( FingerprintData, IFingerprinter, @@ -193,9 +194,9 @@ class SMBFingerprinter(IFingerprinter): logger.debug(f'os_version: "{os_version}", service_client: "{service_client}"') if os_version.lower() != "unix": - os_type = "windows" + os_type = OperatingSystems.WINDOWS else: - os_type = "linux" + os_type = OperatingSystems.LINUX smb_service["name"] = service_client diff --git a/monkey/infection_monkey/network_scanning/ssh_fingerprinter.py b/monkey/infection_monkey/network_scanning/ssh_fingerprinter.py index 32aa20ad9..86eb8f420 100644 --- a/monkey/infection_monkey/network_scanning/ssh_fingerprinter.py +++ b/monkey/infection_monkey/network_scanning/ssh_fingerprinter.py @@ -1,6 +1,7 @@ import re from typing import Dict, Optional, Tuple +from common import OperatingSystems from infection_monkey.i_puppet import FingerprintData, IFingerprinter, PingScanData, PortScanData SSH_REGEX = r"SSH-\d\.\d-OpenSSH" @@ -40,6 +41,6 @@ class SSHFingerprinter(IFingerprinter): for dist in LINUX_DIST_SSH: if banner.lower().find(dist) != -1: os_version = banner.split(" ").pop().strip() - os = "linux" + os = OperatingSystems.LINUX return os, os_version From 77f8be523cfb1270591d1f2cf57453059268ca49 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Thu, 23 Jun 2022 11:34:11 +0300 Subject: [PATCH 112/196] UT: Use operating systems consts --- .../infection_monkey/master/mock_puppet.py | 35 ++++++++++++++----- .../infection_monkey/master/test_exploiter.py | 25 +++++++++---- .../network_scanning/test_ping_scanner.py | 5 +-- .../test_ssh_fingerprinter.py | 5 +-- 4 files changed, 51 insertions(+), 19 deletions(-) diff --git a/monkey/tests/unit_tests/infection_monkey/master/mock_puppet.py b/monkey/tests/unit_tests/infection_monkey/master/mock_puppet.py index 295c68ceb..4d77bd0b3 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/mock_puppet.py +++ b/monkey/tests/unit_tests/infection_monkey/master/mock_puppet.py @@ -2,6 +2,7 @@ import logging import threading from typing import Dict, Iterable, List, Sequence +from common import OperatingSystems from infection_monkey.credential_collectors import LMHash, Password, SSHKeypair, Username from infection_monkey.i_puppet import ( Credentials, @@ -182,19 +183,23 @@ class MockPuppet(IPuppet): "vulnerable_ports": [22], "executed_cmds": [], } - os_windows = "windows" - os_linux = "linux" successful_exploiters = { DOT_1: { "ZerologonExploiter": ExploiterResultData( - False, False, False, os_windows, {}, [], "Zerologon failed" + False, False, False, OperatingSystems.WINDOWS, {}, [], "Zerologon failed" ), "SSHExploiter": ExploiterResultData( - False, False, False, os_linux, info_ssh, attempts, "Failed exploiting" + False, + False, + False, + OperatingSystems.LINUX, + info_ssh, + attempts, + "Failed exploiting", ), "WmiExploiter": ExploiterResultData( - True, True, False, os_windows, info_wmi, attempts, None + True, True, False, OperatingSystems.WINDOWS, info_wmi, attempts, None ), }, DOT_3: { @@ -202,16 +207,22 @@ class MockPuppet(IPuppet): False, False, False, - os_windows, + OperatingSystems.WINDOWS, info_wmi, attempts, "PowerShell Exploiter Failed", ), "SSHExploiter": ExploiterResultData( - False, False, False, os_linux, info_ssh, attempts, "Failed exploiting" + False, + False, + False, + OperatingSystems.LINUX, + info_ssh, + attempts, + "Failed exploiting", ), "ZerologonExploiter": ExploiterResultData( - True, False, False, os_windows, {}, [], None + True, False, False, OperatingSystems.WINDOWS, {}, [], None ), }, } @@ -220,7 +231,13 @@ class MockPuppet(IPuppet): return successful_exploiters[host.ip_addr][name] except KeyError: return ExploiterResultData( - False, False, False, os_linux, {}, [], f"{name} failed for host {host}" + False, + False, + False, + OperatingSystems.LINUX, + {}, + [], + f"{name} failed for host {host}", ) def run_payload(self, name: str, options: Dict, interrupt: threading.Event): diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py index 3c76c903f..c804b60e5 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock import pytest from tests.unit_tests.infection_monkey.master.mock_puppet import MockPuppet +from common import OperatingSystems from infection_monkey.master import Exploiter from infection_monkey.model import VictimHost @@ -38,12 +39,24 @@ def exploiter_config(): return { "options": {"dropper_path_linux": "/tmp/monkey"}, "brute_force": [ - {"name": "HadoopExploiter", "supported_os": ["windows"], "options": {"timeout": 10}}, - {"name": "SSHExploiter", "supported_os": ["linux"], "options": {}}, - {"name": "WmiExploiter", "supported_os": ["windows"], "options": {"timeout": 10}}, + { + "name": "HadoopExploiter", + "supported_os": [OperatingSystems.WINDOWS], + "options": {"timeout": 10}, + }, + {"name": "SSHExploiter", "supported_os": [OperatingSystems.LINUX], "options": {}}, + { + "name": "WmiExploiter", + "supported_os": [OperatingSystems.WINDOWS], + "options": {"timeout": 10}, + }, ], "vulnerability": [ - {"name": "ZerologonExploiter", "supported_os": ["windows"], "options": {}}, + { + "name": "ZerologonExploiter", + "supported_os": [OperatingSystems.WINDOWS], + "options": {}, + }, ], } @@ -160,7 +173,7 @@ def test_exploiter_raises_exception(callback, hosts, hosts_to_exploit, run_explo def test_windows_exploiters_run_on_windows_host(callback, hosts, hosts_to_exploit, run_exploiters): host = VictimHost("10.0.0.1") - host.os["type"] = "windows" + host.os["type"] = OperatingSystems.WINDOWS q = enqueue_hosts([host]) run_exploiters(MockPuppet(), 1, q) @@ -172,7 +185,7 @@ def test_windows_exploiters_run_on_windows_host(callback, hosts, hosts_to_exploi def test_linux_exploiters_run_on_linux_host(callback, hosts, hosts_to_exploit, run_exploiters): host = VictimHost("10.0.0.1") - host.os["type"] = "linux" + host.os["type"] = OperatingSystems.LINUX q = enqueue_hosts([host]) run_exploiters(MockPuppet(), 1, q) diff --git a/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ping_scanner.py b/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ping_scanner.py index 88c9dbeca..9e2891e3f 100644 --- a/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ping_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ping_scanner.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock import pytest +from common import OperatingSystems from infection_monkey.network_scanning import ping from infection_monkey.network_scanning.ping_scanner import EMPTY_PING_SCAN @@ -91,7 +92,7 @@ def test_linux_ping_success(patch_subprocess_running_ping_with_ping_output): result = ping("192.168.1.1", 1.0) assert result.response_received - assert result.os == "linux" + assert result.os == OperatingSystems.LINUX @pytest.mark.usefixtures("set_os_linux") @@ -109,7 +110,7 @@ def test_windows_ping_success(patch_subprocess_running_ping_with_ping_output): result = ping("192.168.1.1", 1.0) assert result.response_received - assert result.os == "windows" + assert result.os == OperatingSystems.WINDOWS @pytest.mark.usefixtures("set_os_windows") diff --git a/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ssh_fingerprinter.py b/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ssh_fingerprinter.py index 69c8eb580..09d0705ef 100644 --- a/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ssh_fingerprinter.py +++ b/monkey/tests/unit_tests/infection_monkey/network_scanning/test_ssh_fingerprinter.py @@ -1,5 +1,6 @@ import pytest +from common import OperatingSystems from infection_monkey.i_puppet import FingerprintData, PortScanData, PortStatus from infection_monkey.network_scanning.ssh_fingerprinter import SSHFingerprinter @@ -56,7 +57,7 @@ def test_ssh_os(ssh_fingerprinter): results = ssh_fingerprinter.get_host_fingerprint("127.0.0.1", None, port_scan_data, None) assert results == FingerprintData( - "linux", + OperatingSystems.LINUX, "Ubuntu-4ubuntu0.2", { "tcp-22": { @@ -78,7 +79,7 @@ def test_multiple_os(ssh_fingerprinter): results = ssh_fingerprinter.get_host_fingerprint("127.0.0.1", None, port_scan_data, None) assert results == FingerprintData( - "linux", + OperatingSystems.LINUX, "Debian", { "tcp-22": { From 717801e9ad49e4440557746a077abad50df7b6d6 Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Thu, 23 Jun 2022 11:44:13 +0000 Subject: [PATCH 113/196] Agent: Add telemetry json encoder Telemetry json encoder is needed to encode OperatingSystems enums when sending back telemetries --- monkey/infection_monkey/telemetry/base_telem.py | 3 ++- monkey/infection_monkey/telemetry/telem_encoder.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 monkey/infection_monkey/telemetry/telem_encoder.py diff --git a/monkey/infection_monkey/telemetry/base_telem.py b/monkey/infection_monkey/telemetry/base_telem.py index 18e8d2158..32b5110f2 100644 --- a/monkey/infection_monkey/telemetry/base_telem.py +++ b/monkey/infection_monkey/telemetry/base_telem.py @@ -4,6 +4,7 @@ import logging from infection_monkey.control import ControlClient from infection_monkey.telemetry.i_telem import ITelem +from infection_monkey.telemetry.telem_encoder import TelemetryJSONEncoder logger = logging.getLogger(__name__) LOGGED_DATA_LENGTH = 300 # How many characters of telemetry data will be logged @@ -39,7 +40,7 @@ class BaseTelem(ITelem, metaclass=abc.ABCMeta): @property def json_encoder(self): - return json.JSONEncoder + return TelemetryJSONEncoder def _log_telem_sending(self, serialized_data: str, log_data=True): logger.debug(f"Sending {self.telem_category} telemetry.") diff --git a/monkey/infection_monkey/telemetry/telem_encoder.py b/monkey/infection_monkey/telemetry/telem_encoder.py new file mode 100644 index 000000000..019569107 --- /dev/null +++ b/monkey/infection_monkey/telemetry/telem_encoder.py @@ -0,0 +1,10 @@ +import json + +from common import OperatingSystems + + +class TelemetryJSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, OperatingSystems): + return obj.name + return json.JSONEncoder.default(self, obj) From 68c27969d3a3521259687f2c3b6c4ea8cfa6f6b7 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Thu, 23 Jun 2022 16:30:38 +0300 Subject: [PATCH 114/196] Agent: Fix a bug in incompatible os check web_rce.py checks for incompatible OS on the victim. Bug was that it checked linux twice instead of linux and windows --- monkey/infection_monkey/exploit/web_rce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/exploit/web_rce.py b/monkey/infection_monkey/exploit/web_rce.py index 34f245fca..45e4b58ee 100644 --- a/monkey/infection_monkey/exploit/web_rce.py +++ b/monkey/infection_monkey/exploit/web_rce.py @@ -413,7 +413,7 @@ class WebRCE(HostExploiter): """ if not self.host.os.get("type") or ( self.host.os["type"] != OperatingSystems.LINUX - and self.host.os["type"] != OperatingSystems.LINUX + and self.host.os["type"] != OperatingSystems.WINDOWS ): logger.error("Target's OS was either unidentified or not supported. Aborting") return False From 452028f221034722bedb635d3afc80e3dae13f0b Mon Sep 17 00:00:00 2001 From: Shreya Malviya Date: Wed, 22 Jun 2022 09:27:47 -0700 Subject: [PATCH 115/196] UT: Replace Hadoop with MSSQL in test data Previously, in the UT data, Hadoop had only windows in the "supported_os" field in the config. Now that that field is stripped out from the config, the supported OSes are picked up from the main code (from `SUPPORTED_OS` in the master's `Exploiter` class) which has both winodws and linux for Hadoop. This caused the tests to fail. This commit changes the UT data to include the MSSQL exploiter (windows only) instead of the Hadoop exploiter. The tests pass now. --- .../unit_tests/infection_monkey/master/test_exploiter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py index 42b64821f..f48dc74d4 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py @@ -38,7 +38,7 @@ def exploiter_config(): return { "options": {"dropper_path_linux": "/tmp/monkey"}, "brute_force": [ - {"name": "HadoopExploiter", "options": {"timeout": 10}}, + {"name": "MSSQLExploiter", "options": {"timeout": 10}}, {"name": "SSHExploiter", "options": {}}, {"name": "WmiExploiter", "options": {"timeout": 10}}, ], @@ -105,11 +105,11 @@ def test_exploiter(callback, hosts, hosts_to_exploit, run_exploiters): host_exploit_combos = get_host_exploit_combos_from_call_args_list(callback.call_args_list) assert ("ZerologonExploiter", hosts[0]) in host_exploit_combos - assert ("HadoopExploiter", hosts[0]) in host_exploit_combos + assert ("MSSQLExploiter", hosts[0]) in host_exploit_combos assert ("SSHExploiter", hosts[0]) in host_exploit_combos assert ("WmiExploiter", hosts[0]) in host_exploit_combos assert ("ZerologonExploiter", hosts[1]) in host_exploit_combos - assert ("HadoopExploiter", hosts[1]) in host_exploit_combos + assert ("MSSQLExploiter", hosts[1]) in host_exploit_combos assert ("WmiExploiter", hosts[1]) in host_exploit_combos assert ("SSHExploiter", hosts[1]) in host_exploit_combos @@ -196,6 +196,6 @@ def test_all_exploiters_run_on_unknown_host(callback, hosts, hosts_to_exploit, r host_exploit_combos = get_host_exploit_combos_from_call_args_list(callback.call_args_list) assert ("ZerologonExploiter", hosts[0]) in host_exploit_combos - assert ("HadoopExploiter", hosts[0]) in host_exploit_combos + assert ("MSSQLExploiter", hosts[0]) in host_exploit_combos assert ("SSHExploiter", host) in host_exploit_combos assert ("WmiExploiter", hosts[0]) in host_exploit_combos From 48fab89e11a20e0440a2253c1e735d10888bf810 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 09:50:24 +0200 Subject: [PATCH 116/196] Island: Rename configuration.py to agent_configuration.py Per convention it must match class name --- .../cc/resources/{configuration.py => agent_configuration.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename monkey/monkey_island/cc/resources/{configuration.py => agent_configuration.py} (100%) diff --git a/monkey/monkey_island/cc/resources/configuration.py b/monkey/monkey_island/cc/resources/agent_configuration.py similarity index 100% rename from monkey/monkey_island/cc/resources/configuration.py rename to monkey/monkey_island/cc/resources/agent_configuration.py From 39e4180dfe4daec53a292bf6342df2a6f43a2700 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 14:26:14 +0200 Subject: [PATCH 117/196] Island: Use make_response in GET agent_configuration --- .../cc/resources/agent_configuration.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/monkey/monkey_island/cc/resources/agent_configuration.py b/monkey/monkey_island/cc/resources/agent_configuration.py index 6e4a00826..ed0258471 100644 --- a/monkey/monkey_island/cc/resources/agent_configuration.py +++ b/monkey/monkey_island/cc/resources/agent_configuration.py @@ -1,7 +1,7 @@ import json import marshmallow -from flask import jsonify, make_response, request +from flask import make_response, request from common.configuration.agent_configuration import AgentConfigurationSchema from monkey_island.cc.repository import IAgentConfigurationRepository @@ -20,22 +20,19 @@ class AgentConfiguration(AbstractResource): def get(self): configuration = self._agent_configuration_repository.get_configuration() configuration_json = self._schema.dumps(configuration) - return jsonify(configuration_json=configuration_json) + return make_response(configuration_json, 200) @jwt_required def post(self): - request_contents = json.loads(request.data) - configuration_json = json.loads(request_contents["config"]) try: - configuration_object = self._schema.load(configuration_json) + request_contents = json.loads(request.data) + + configuration_object = self._schema.load(request_contents) self._agent_configuration_repository.store_configuration(configuration_object) return make_response({}, 200) - except marshmallow.exceptions.ValidationError: + except (marshmallow.exceptions.ValidationError, json.JSONDecodeError) as err: return make_response( - { - "message": "Invalid configuration supplied. " - "Maybe the format is outdated or the file has been corrupted." - }, + {"message": f"Invalid configuration supplied: {err}"}, 400, ) From 891794d927210ddaf2bf56bc23a84a7e16784a65 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 14:27:58 +0200 Subject: [PATCH 118/196] Island: Add AgentConfiguration resource to app.py --- monkey/monkey_island/cc/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monkey/monkey_island/cc/app.py b/monkey/monkey_island/cc/app.py index 9db06d486..e2253b36c 100644 --- a/monkey/monkey_island/cc/app.py +++ b/monkey/monkey_island/cc/app.py @@ -12,6 +12,7 @@ from common import DIContainer from monkey_island.cc.database import database, mongo from monkey_island.cc.resources import AgentBinaries, RemoteRun from monkey_island.cc.resources.AbstractResource import AbstractResource +from monkey_island.cc.resources.agent_configuration import AgentConfiguration from monkey_island.cc.resources.agent_controls import StopAgentCheck, StopAllAgents from monkey_island.cc.resources.attack.attack_report import AttackReport from monkey_island.cc.resources.auth.auth import Authenticate, init_jwt @@ -155,6 +156,7 @@ def init_api_resources(api: FlaskDIWrapper): api.add_resource(IslandConfiguration) api.add_resource(ConfigurationExport) api.add_resource(ConfigurationImport) + api.add_resource(AgentConfiguration) api.add_resource(AgentBinaries) api.add_resource(NetMap) api.add_resource(Edge) From b35832b9dd960742da320a10618d831100fdeef5 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 14:29:30 +0200 Subject: [PATCH 119/196] UT: Add InMemoryFileAgentConfigurationRepository --- monkey/tests/monkey_island/__init__.py | 1 + ..._memory_file_agent_configuration_repository.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py diff --git a/monkey/tests/monkey_island/__init__.py b/monkey/tests/monkey_island/__init__.py index aed4d07f3..8f8ae7404 100644 --- a/monkey/tests/monkey_island/__init__.py +++ b/monkey/tests/monkey_island/__init__.py @@ -1,3 +1,4 @@ from .single_file_repository import SingleFileRepository from .mock_file_repository import MockFileRepository, FILE_CONTENTS, FILE_NAME from .open_error_file_repository import OpenErrorFileRepository +from .in_memory_file_agent_configuration_repository import InMemoryFileAgentConfigurationRepository diff --git a/monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py b/monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py new file mode 100644 index 000000000..955e78878 --- /dev/null +++ b/monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py @@ -0,0 +1,15 @@ +from tests.common.example_agent_configuration import AGENT_CONFIGURATION + +from common.configuration.agent_configuration import AgentConfigurationSchema +from monkey_island.cc.repository import IAgentConfigurationRepository + + +class InMemoryFileAgentConfigurationRepository(IAgentConfigurationRepository): + def __init__(self): + self._configuration = AgentConfigurationSchema().load(AGENT_CONFIGURATION) + + def get_configuration(self): + return self._configuration + + def store_configuration(self, agent_configuration): + self._configuration = agent_configuration From d7329ea83942587c2cb49b04b3796c1318c2802f Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 14:31:51 +0200 Subject: [PATCH 120/196] UT: Add tests for AgentConfiguration resource --- .../cc/resources/test_agent_configuration.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py new file mode 100644 index 000000000..5cac629e0 --- /dev/null +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py @@ -0,0 +1,50 @@ +import json + +import pytest +from tests.common import StubDIContainer +from tests.common.example_agent_configuration import AGENT_CONFIGURATION +from tests.monkey_island import InMemoryFileAgentConfigurationRepository +from tests.unit_tests.monkey_island.conftest import get_url_for_resource + +from monkey_island.cc.repository import IAgentConfigurationRepository +from monkey_island.cc.resources.agent_configuration import AgentConfiguration + + +@pytest.fixture +def flask_client(build_flask_client): + container = StubDIContainer() + + container.register(IAgentConfigurationRepository, InMemoryFileAgentConfigurationRepository) + + with build_flask_client(container) as flask_client: + yield flask_client + + +def test_agent_configuration_endpoint(flask_client): + agent_configuration_url = get_url_for_resource(AgentConfiguration) + + flask_client.post( + agent_configuration_url, data=json.dumps(AGENT_CONFIGURATION), follow_redirects=True + ) + resp = flask_client.get(agent_configuration_url) + + assert resp.status_code == 200 + assert json.loads(resp.data) == AGENT_CONFIGURATION + + +def test_agent_configuration_invalid_config(flask_client): + agent_configuration_url = get_url_for_resource(AgentConfiguration) + + resp = flask_client.post( + agent_configuration_url, data=json.dumps({"invalid_config": "invalid_stuff"}) + ) + + assert resp.status_code == 400 + + +def test_agent_configuration_invalid_json(flask_client): + agent_configuration_url = get_url_for_resource(AgentConfiguration) + + resp = flask_client.post(agent_configuration_url, data="InvalidJson!") + + assert resp.status_code == 400 From 42c4803376e2f6792ae33bbd4fab2fe4b23331b3 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 15:56:41 +0200 Subject: [PATCH 121/196] Island: Use schema.loads in AgentConfiguration POST method --- monkey/monkey_island/cc/resources/agent_configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monkey/monkey_island/cc/resources/agent_configuration.py b/monkey/monkey_island/cc/resources/agent_configuration.py index ed0258471..f0ad73cb8 100644 --- a/monkey/monkey_island/cc/resources/agent_configuration.py +++ b/monkey/monkey_island/cc/resources/agent_configuration.py @@ -26,9 +26,7 @@ class AgentConfiguration(AbstractResource): def post(self): try: - request_contents = json.loads(request.data) - - configuration_object = self._schema.load(request_contents) + configuration_object = self._schema.loads(request.data) self._agent_configuration_repository.store_configuration(configuration_object) return make_response({}, 200) except (marshmallow.exceptions.ValidationError, json.JSONDecodeError) as err: From 1ae3bd4b4f826909ae2e0883301b0975838d95eb Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Thu, 23 Jun 2022 17:15:50 +0200 Subject: [PATCH 122/196] UT: Rename InMemoryFileAgentConfigurationRepository to InMemoryAgentConfigurationRepository --- monkey/tests/monkey_island/__init__.py | 2 +- ...ository.py => in_memory_agent_configuration_repository.py} | 2 +- .../monkey_island/cc/resources/test_agent_configuration.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename monkey/tests/monkey_island/{in_memory_file_agent_configuration_repository.py => in_memory_agent_configuration_repository.py} (86%) diff --git a/monkey/tests/monkey_island/__init__.py b/monkey/tests/monkey_island/__init__.py index 8f8ae7404..8f3654a6e 100644 --- a/monkey/tests/monkey_island/__init__.py +++ b/monkey/tests/monkey_island/__init__.py @@ -1,4 +1,4 @@ from .single_file_repository import SingleFileRepository from .mock_file_repository import MockFileRepository, FILE_CONTENTS, FILE_NAME from .open_error_file_repository import OpenErrorFileRepository -from .in_memory_file_agent_configuration_repository import InMemoryFileAgentConfigurationRepository +from .in_memory_agent_configuration_repository import InMemoryAgentConfigurationRepository diff --git a/monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py b/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py similarity index 86% rename from monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py rename to monkey/tests/monkey_island/in_memory_agent_configuration_repository.py index 955e78878..e737d645c 100644 --- a/monkey/tests/monkey_island/in_memory_file_agent_configuration_repository.py +++ b/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py @@ -4,7 +4,7 @@ from common.configuration.agent_configuration import AgentConfigurationSchema from monkey_island.cc.repository import IAgentConfigurationRepository -class InMemoryFileAgentConfigurationRepository(IAgentConfigurationRepository): +class InMemoryAgentConfigurationRepository(IAgentConfigurationRepository): def __init__(self): self._configuration = AgentConfigurationSchema().load(AGENT_CONFIGURATION) diff --git a/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py b/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py index 5cac629e0..874500b81 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py +++ b/monkey/tests/unit_tests/monkey_island/cc/resources/test_agent_configuration.py @@ -3,7 +3,7 @@ import json import pytest from tests.common import StubDIContainer from tests.common.example_agent_configuration import AGENT_CONFIGURATION -from tests.monkey_island import InMemoryFileAgentConfigurationRepository +from tests.monkey_island import InMemoryAgentConfigurationRepository from tests.unit_tests.monkey_island.conftest import get_url_for_resource from monkey_island.cc.repository import IAgentConfigurationRepository @@ -14,7 +14,7 @@ from monkey_island.cc.resources.agent_configuration import AgentConfiguration def flask_client(build_flask_client): container = StubDIContainer() - container.register(IAgentConfigurationRepository, InMemoryFileAgentConfigurationRepository) + container.register(IAgentConfigurationRepository, InMemoryAgentConfigurationRepository) with build_flask_client(container) as flask_client: yield flask_client From 77804caab5ce53e2cb7d8a0fbe9f20aa581fe046 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 20 Jun 2022 16:29:53 +0300 Subject: [PATCH 123/196] Agent: Add from_dict method to AgentConfiguration Creating AgentConfiguration object from dictionary makes sense because it doesn't couple the configuration to any specific serialization methods. Also, the json sent from the island doesn't match the config structure because it stores config in a dict under "config" key. --- monkey/common/configuration/agent_configuration.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 15b338fe0..a636e3d95 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -24,6 +24,10 @@ class AgentConfiguration: payloads: List[PluginConfiguration] propagation: PropagationConfiguration + @staticmethod + def from_dict(_dict: dict): + return AgentConfigurationSchema().load(_dict) + class AgentConfigurationSchema(Schema): keep_tunnel_open_time = fields.Float() From d8ac441c5963cba8f5335ab29dc1ed6461e1fed8 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Tue, 21 Jun 2022 15:49:33 +0300 Subject: [PATCH 124/196] Agent: Fix configuration retrieval in _run_simulation --- monkey/infection_monkey/master/automated_master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index f79fd5f12..909d36de6 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -141,7 +141,7 @@ class AutomatedMaster(IMaster): try: config = AutomatedMaster._try_communicate_with_island( self._control_channel.get_config, CHECK_FOR_CONFIG_COUNT - )["config"] + ) except IslandCommunicationError as e: logger.error(f"An error occurred while fetching configuration: {e}") return From e83995d96284e7bb77d5883ffc0f562b5d73a004 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Tue, 21 Jun 2022 15:50:37 +0300 Subject: [PATCH 125/196] UT: Add a new fixture for config object --- .../monkey_configs/default_config.py | 127 ++++++++++++++++++ monkey/tests/unit_tests/conftest.py | 3 + 2 files changed, 130 insertions(+) create mode 100644 monkey/tests/data_for_tests/monkey_configs/default_config.py diff --git a/monkey/tests/data_for_tests/monkey_configs/default_config.py b/monkey/tests/data_for_tests/monkey_configs/default_config.py new file mode 100644 index 000000000..84d1f54e2 --- /dev/null +++ b/monkey/tests/data_for_tests/monkey_configs/default_config.py @@ -0,0 +1,127 @@ +from common import OperatingSystems +from common.configuration import AgentConfigurationSchema + +flat_config = { + "keep_tunnel_open_time": 30, + "post_breach_actions": [ + {"name": "CommunicateAsBackdoorUser", "options": {}}, + {"name": "ModifyShellStartupFiles", "options": {}}, + {"name": "HiddenFiles", "options": {}}, + {"name": "TrapCommand", "options": {}}, + {"name": "ChangeSetuidSetgid", "options": {}}, + {"name": "ScheduleJobs", "options": {}}, + {"name": "Timestomping", "options": {}}, + {"name": "AccountDiscovery", "options": {}}, + {"name": "ProcessListCollection", "options": {}}, + ], + "credential_collectors": [ + {"name": "MimikatzCollector", "options": {}}, + {"name": "SSHCollector", "options": {}}, + ], + "payloads": [ + { + "name": "ransomware", + "options": { + "encryption": { + "enabled": True, + "directories": {"linux_target_dir": "", "windows_target_dir": ""}, + }, + "other_behaviors": {"readme": True}, + }, + } + ], + "custom_pbas": { + "linux_command": "", + "linux_filename": "", + "windows_command": "", + "windows_filename": "", + }, + "propagation": { + "network_scan": { + "tcp": { + "timeout": 3000, + "ports": [ + 22, + 80, + 135, + 443, + 445, + 2222, + 3306, + 3389, + 5985, + 5986, + 7001, + 8008, + 8080, + 8088, + 8983, + 9200, + 9600, + ], + }, + "icmp": {"timeout": 1000}, + "fingerprinters": [ + {"name": "elastic", "options": {}}, + { + "name": "http", + "options": {"http_ports": [80, 443, 7001, 8008, 8080, 8983, 9200, 9600]}, + }, + {"name": "mssql", "options": {}}, + {"name": "smb", "options": {}}, + {"name": "ssh", "options": {}}, + ], + "targets": { + "blocked_ips": [], + "inaccessible_subnets": [], + "local_network_scan": True, + "subnets": [], + }, + }, + "maximum_depth": 2, + "exploitation": { + "options": {"http_ports": [80, 443, 7001, 8008, 8080, 8983, 9200, 9600]}, + "brute_force": [ + { + "name": "MSSQLExploiter", + "options": {}, + "supported_os": [OperatingSystems.WINDOWS.name], + }, + { + "name": "PowerShellExploiter", + "options": {}, + "supported_os": [OperatingSystems.WINDOWS.name], + }, + { + "name": "SSHExploiter", + "options": {}, + "supported_os": [OperatingSystems.LINUX.name], + }, + { + "name": "SmbExploiter", + "options": {"smb_download_timeout": 30}, + "supported_os": [OperatingSystems.WINDOWS.name], + }, + { + "name": "WmiExploiter", + "options": {"smb_download_timeout": 30}, + "supported_os": [OperatingSystems.WINDOWS.name], + }, + ], + "vulnerability": [ + { + "name": "HadoopExploiter", + "options": {}, + "supported_os": [OperatingSystems.LINUX.name, OperatingSystems.WINDOWS.name], + }, + { + "name": "Log4ShellExploiter", + "options": {}, + "supported_os": [OperatingSystems.LINUX.name, OperatingSystems.WINDOWS.name], + }, + ], + }, + }, +} + +DEFAULT_CONFIG = AgentConfigurationSchema().load(flat_config) diff --git a/monkey/tests/unit_tests/conftest.py b/monkey/tests/unit_tests/conftest.py index 3634e52b9..c54ccce23 100644 --- a/monkey/tests/unit_tests/conftest.py +++ b/monkey/tests/unit_tests/conftest.py @@ -5,6 +5,9 @@ from typing import Callable, Dict import pytest from _pytest.monkeypatch import MonkeyPatch +from tests.data_for_tests.monkey_configs.default_config import DEFAULT_CONFIG + +from common.configuration import AgentConfiguration MONKEY_BASE_PATH = str(Path(__file__).parent.parent.parent) sys.path.insert(0, MONKEY_BASE_PATH) From 6b406ef68685aec5d5a0b5ecb411a415356d808d Mon Sep 17 00:00:00 2001 From: vakarisz Date: Tue, 21 Jun 2022 15:57:57 +0300 Subject: [PATCH 126/196] Agent: Change configuration to object in control channel --- monkey/infection_monkey/i_control_channel.py | 8 +++++--- monkey/infection_monkey/master/control_channel.py | 5 +++-- monkey/infection_monkey/monkey.py | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/monkey/infection_monkey/i_control_channel.py b/monkey/infection_monkey/i_control_channel.py index 33539417c..b90308018 100644 --- a/monkey/infection_monkey/i_control_channel.py +++ b/monkey/infection_monkey/i_control_channel.py @@ -1,5 +1,7 @@ import abc +from common.configuration import AgentConfiguration + class IControlChannel(metaclass=abc.ABCMeta): @abc.abstractmethod @@ -11,10 +13,10 @@ class IControlChannel(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def get_config(self) -> dict: + def get_config(self) -> AgentConfiguration: """ - :return: A dictionary containing Agent Configuration - :rtype: dict + :return: An AgentConfiguration object + :rtype: AgentConfiguration """ pass diff --git a/monkey/infection_monkey/master/control_channel.py b/monkey/infection_monkey/master/control_channel.py index c93af43e0..8567a301f 100644 --- a/monkey/infection_monkey/master/control_channel.py +++ b/monkey/infection_monkey/master/control_channel.py @@ -5,6 +5,7 @@ from typing import Mapping import requests from common.common_consts.timeouts import SHORT_REQUEST_TIMEOUT +from common.configuration import AgentConfiguration from infection_monkey.custom_types import PropagationCredentials from infection_monkey.i_control_channel import IControlChannel, IslandCommunicationError @@ -47,7 +48,7 @@ class ControlChannel(IControlChannel): ) as e: raise IslandCommunicationError(e) - def get_config(self) -> dict: + def get_config(self) -> AgentConfiguration: try: response = requests.get( # noqa: DUO123 f"https://{self._control_channel_server}/api/agent", @@ -57,7 +58,7 @@ class ControlChannel(IControlChannel): ) response.raise_for_status() - return json.loads(response.content.decode()) + return AgentConfiguration.from_dict(json.loads(response.text)["config"]) except ( json.JSONDecodeError, requests.exceptions.ConnectionError, diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index 466de6664..afcf460b2 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -172,8 +172,9 @@ class InfectionMonkey: ) config = control_channel.get_config() + keep_tunnel_open_time = control_channel.get_config().keep_tunnel_open_time self._monkey_inbound_tunnel = self._control_client.create_control_tunnel( - config["config"]["keep_tunnel_open_time"] + keep_tunnel_open_time ) if self._monkey_inbound_tunnel and should_propagate(config, self._current_depth): self._inbound_tunnel_opened = True From ffe8c3451b9bb1a01f0b8d94afedbc81809dacea Mon Sep 17 00:00:00 2001 From: vakarisz Date: Tue, 21 Jun 2022 16:15:54 +0300 Subject: [PATCH 127/196] Agent: Change scanners to use the config object --- monkey/common/configuration/__init__.py | 12 +++ .../master/automated_master.py | 2 +- monkey/infection_monkey/master/ip_scanner.py | 31 ++++--- monkey/infection_monkey/master/propagator.py | 29 +++--- .../master/test_ip_scanner.py | 51 +++++----- .../master/test_propagator.py | 92 ++++++++++--------- 6 files changed, 130 insertions(+), 87 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 2246e27f2..107e5a491 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -2,6 +2,18 @@ from .agent_configuration import ( AgentConfiguration, AgentConfigurationSchema, ) +from .agent_sub_configurations import ( + CustomPBAConfiguration, + PluginConfiguration, + ScanTargetConfiguration, + ICMPScanConfiguration, + TCPScanConfiguration, + NetworkScanConfiguration, + ExploitationOptionsConfiguration, + ExploiterConfiguration, + ExploitationConfiguration, + PropagationConfiguration, +) from .default_agent_configuration import ( DEFAULT_AGENT_CONFIGURATION_JSON, build_default_agent_configuration, diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 909d36de6..573299629 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -174,7 +174,7 @@ class AutomatedMaster(IMaster): logger.info(f"Current depth is {current_depth}") if should_propagate(self._control_channel.get_config(), self._current_depth): - self._propagator.propagate(config["propagation"], current_depth, self._stop) + self._propagator.propagate(config.propagation, current_depth, self._stop) else: logger.info("Skipping propagation: maximum depth reached") diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 8c0ea5caa..14391765f 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -3,8 +3,13 @@ import queue import threading from queue import Queue from threading import Event -from typing import Any, Callable, Dict, List +from typing import Callable, Dict, List +from common.configuration.agent_sub_configurations import ( + NetworkScanConfiguration, + PluginConfiguration, + ScanTargetConfiguration, +) from infection_monkey.i_puppet import ( FingerprintData, IPuppet, @@ -30,7 +35,7 @@ class IPScanner: def scan( self, addresses_to_scan: List[NetworkAddress], - options: Dict, + options: ScanTargetConfiguration, results_callback: Callback, stop: Event, ): @@ -49,12 +54,16 @@ class IPScanner: ) def _scan_addresses( - self, addresses: Queue, options: Dict, results_callback: Callback, stop: Event + self, + addresses: Queue, + options: NetworkScanConfiguration, + results_callback: Callback, + stop: Event, ): - logger.debug(f"Starting scan thread -- Thread ID: {threading.get_ident()}") - icmp_timeout = options["icmp"]["timeout_ms"] / 1000 - tcp_timeout = options["tcp"]["timeout_ms"] / 1000 - tcp_ports = options["tcp"]["ports"] + logger.debug(f"Starting scan .read -- Thread ID: {threading.get_ident()}") + icmp_timeout = options.icmp.timeout + tcp_timeout = options.tcp.timeout + tcp_ports = options.tcp.ports try: while not stop.is_set(): @@ -66,7 +75,7 @@ class IPScanner: fingerprint_data = {} if IPScanner.port_scan_found_open_port(port_scan_data): - fingerprinters = options["fingerprinters"] + fingerprinters = options.fingerprinters fingerprint_data = self._run_fingerprinters( address.ip, fingerprinters, ping_scan_data, port_scan_data, stop ) @@ -90,7 +99,7 @@ class IPScanner: def _run_fingerprinters( self, ip: str, - fingerprinters: List[Dict[str, Any]], + fingerprinters: List[PluginConfiguration], ping_scan_data: PingScanData, port_scan_data: Dict[int, PortScanData], stop: Event, @@ -98,8 +107,8 @@ class IPScanner: fingerprint_data = {} for f in interruptible_iter(fingerprinters, stop): - fingerprint_data[f["name"]] = self._puppet.fingerprint( - f["name"], ip, ping_scan_data, port_scan_data, f["options"] + fingerprint_data[f.name] = self._puppet.fingerprint( + f.name, ip, ping_scan_data, port_scan_data, f.options ) return fingerprint_data diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index be4d6caf2..a74cf7c86 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -3,6 +3,8 @@ from queue import Queue from threading import Event from typing import Dict, List +from common.configuration import PropagationConfiguration,\ + NetworkScanConfiguration, ScanTargetConfiguration from infection_monkey.i_puppet import ( ExploiterResultData, FingerprintData, @@ -39,14 +41,18 @@ class Propagator: self._local_network_interfaces = local_network_interfaces self._hosts_to_exploit = None - def propagate(self, propagation_config: Dict, current_depth: int, stop: Event): + def propagate( + self, propagation_config: PropagationConfiguration, current_depth: int, stop: Event + ): logger.info("Attempting to propagate") network_scan_completed = Event() self._hosts_to_exploit = Queue() scan_thread = create_daemon_thread( - target=self._scan_network, name="PropagatorScanThread", args=(propagation_config, stop) + target=self._scan_network, + name="PropagatorScanThread", + args=(propagation_config.network_scan, stop), ) exploit_thread = create_daemon_thread( target=self._exploit_hosts, @@ -64,22 +70,21 @@ class Propagator: logger.info("Finished attempting to propagate") - def _scan_network(self, propagation_config: Dict, stop: Event): + def _scan_network(self, scan_config: NetworkScanConfiguration, stop: Event): logger.info("Starting network scan") - target_config = propagation_config["targets"] - scan_config = propagation_config["network_scan"] - - addresses_to_scan = self._compile_scan_target_list(target_config) + addresses_to_scan = self._compile_scan_target_list(scan_config.targets) self._ip_scanner.scan(addresses_to_scan, scan_config, self._process_scan_results, stop) logger.info("Finished network scan") - def _compile_scan_target_list(self, target_config: Dict) -> List[NetworkAddress]: - ranges_to_scan = target_config["subnet_scan_list"] - inaccessible_subnets = target_config["inaccessible_subnets"] - blocklisted_ips = target_config["blocked_ips"] - enable_local_network_scan = target_config["local_network_scan"] + def _compile_scan_target_list( + self, target_config: ScanTargetConfiguration + ) -> List[NetworkAddress]: + ranges_to_scan = target_config.subnets + inaccessible_subnets = target_config.inaccessible_subnets + blocklisted_ips = target_config.blocked_ips + enable_local_network_scan = target_config.local_network_scan return compile_scan_target_list( self._local_network_interfaces, diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py index 9fafdaee2..069f931b9 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py @@ -5,6 +5,12 @@ from unittest.mock import MagicMock import pytest from tests.unit_tests.infection_monkey.master.mock_puppet import MockPuppet +from common.configuration.agent_sub_configurations import ( + ICMPScanConfiguration, + NetworkScanConfiguration, + PluginConfiguration, + TCPScanConfiguration, +) from infection_monkey.i_puppet import FingerprintData, PortScanData, PortStatus from infection_monkey.master import IPScanner from infection_monkey.network import NetworkAddress @@ -14,28 +20,31 @@ LINUX_OS = "linux" @pytest.fixture -def scan_config(): - return { - "tcp": { - "timeout_ms": 3000, - "ports": [ - 22, - 445, - 3389, - 443, - 8008, - 3306, - ], - }, - "icmp": { - "timeout_ms": 1000, - }, - "fingerprinters": [ - {"name": "HTTPFinger", "options": {}}, - {"name": "SMBFinger", "options": {}}, - {"name": "SSHFinger", "options": {}}, +def scan_config(default_agent_config): + tcp_config = TCPScanConfiguration( + timeout=3, + ports=[ + 22, + 445, + 3389, + 443, + 8008, + 3306, ], - } + ) + icmp_config = ICMPScanConfiguration(timeout=1) + fingerprinter_config = [ + PluginConfiguration(name="HTTPFinger", options={}), + PluginConfiguration(name="SMBFinger", options={}), + PluginConfiguration(name="SSHFinger", options={}), + ] + scan_config = NetworkScanConfiguration( + tcp_config, + icmp_config, + fingerprinter_config, + default_agent_config.propagation.network_scan.targets, + ) + return scan_config @pytest.fixture diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py index 3746e65eb..a2581de60 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py @@ -3,6 +3,11 @@ from unittest.mock import MagicMock import pytest +from common.configuration.agent_sub_configurations import ( + NetworkScanConfiguration, + PropagationConfiguration, + ScanTargetConfiguration, +) from infection_monkey.i_puppet import ( ExploiterResultData, FingerprintData, @@ -135,24 +140,35 @@ class StubExploiter: pass -def test_scan_result_processing(telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory): +def get_propagation_config(default_agent_config, scan_target_config: ScanTargetConfiguration): + network_scan = NetworkScanConfiguration( + default_agent_config.propagation.network_scan.tcp, + default_agent_config.propagation.network_scan.icmp, + default_agent_config.propagation.network_scan.fingerprinters, + scan_target_config, + ) + propagation_config = PropagationConfiguration( + default_agent_config.propagation.maximum_depth, + network_scan, + default_agent_config.propagation.exploitation, + ) + return propagation_config + + +def test_scan_result_processing( + telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_config +): p = Propagator( telemetry_messenger_spy, mock_ip_scanner, StubExploiter(), mock_victim_host_factory, [] ) - p.propagate( - { - "targets": { - "subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"], - "local_network_scan": False, - "inaccessible_subnets": [], - "blocked_ips": [], - }, - "network_scan": {}, # This is empty since MockIPscanner ignores it - "exploiters": {}, # This is empty since StubExploiter ignores it - }, - 1, - Event(), + targets = ScanTargetConfiguration( + blocked_ips=[], + inaccessible_subnets=[], + local_network_scan=False, + subnets=["10.0.0.1", "10.0.0.2", "10.0.0.3"], ) + propagation_config = get_propagation_config(default_agent_config, targets) + p.propagate(propagation_config, 1, Event()) assert len(telemetry_messenger_spy.telemetries) == 3 @@ -237,25 +253,20 @@ class MockExploiter: def test_exploiter_result_processing( - telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory + telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_config ): p = Propagator( telemetry_messenger_spy, mock_ip_scanner, MockExploiter(), mock_victim_host_factory, [] ) - p.propagate( - { - "targets": { - "subnet_scan_list": ["10.0.0.1", "10.0.0.2", "10.0.0.3"], - "local_network_scan": False, - "inaccessible_subnets": [], - "blocked_ips": [], - }, - "network_scan": {}, # This is empty since MockIPscanner ignores it - "exploiters": {}, # This is empty since MockExploiter ignores it - }, - 1, - Event(), + + targets = ScanTargetConfiguration( + blocked_ips=[], + inaccessible_subnets=[], + local_network_scan=False, + subnets=["10.0.0.1", "10.0.0.2", "10.0.0.3"], ) + propagation_config = get_propagation_config(default_agent_config, targets) + p.propagate(propagation_config, 1, Event()) exploit_telems = [t for t in telemetry_messenger_spy.telemetries if isinstance(t, ExploitTelem)] assert len(exploit_telems) == 4 @@ -278,7 +289,9 @@ def test_exploiter_result_processing( assert data["propagation_result"] -def test_scan_target_generation(telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory): +def test_scan_target_generation( + telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_config +): local_network_interfaces = [NetworkInterface("10.0.0.9", "/29")] p = Propagator( telemetry_messenger_spy, @@ -287,20 +300,15 @@ def test_scan_target_generation(telemetry_messenger_spy, mock_ip_scanner, mock_v mock_victim_host_factory, local_network_interfaces, ) - p.propagate( - { - "targets": { - "subnet_scan_list": ["10.0.0.0/29", "172.10.20.30"], - "local_network_scan": True, - "blocked_ips": ["10.0.0.3"], - "inaccessible_subnets": ["10.0.0.128/30", "10.0.0.8/29"], - }, - "network_scan": {}, # This is empty since MockIPscanner ignores it - "exploiters": {}, # This is empty since MockExploiter ignores it - }, - 1, - Event(), + targets = ScanTargetConfiguration( + blocked_ips=["10.0.0.3"], + inaccessible_subnets=["10.0.0.128/30", "10.0.0.8/29"], + local_network_scan=True, + subnets=["10.0.0.0/29", "172.10.20.30"], ) + propagation_config = get_propagation_config(default_agent_config, targets) + p.propagate(propagation_config, 1, Event()) + expected_ip_scan_list = [ "10.0.0.0", "10.0.0.1", From 095e49b5436144145730713c25e71e10f2b8b0c5 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Wed, 22 Jun 2022 17:47:27 +0300 Subject: [PATCH 128/196] Agent: Use deserialized config in automated_master.py --- monkey/infection_monkey/master/automated_master.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 573299629..684d60003 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -150,7 +150,7 @@ class AutomatedMaster(IMaster): target=self._run_plugins, name="CredentialCollectorThread", args=( - config["credential_collectors"], + config.credential_collectors, "credential collector", self._collect_credentials, ), @@ -158,7 +158,7 @@ class AutomatedMaster(IMaster): pba_thread = create_daemon_thread( target=self._run_pbas, name="PBAThread", - args=(config["post_breach_actions"].items(), self._run_pba, config["custom_pbas"]), + args=(config.post_breach_actions, self._run_pba, config.custom_pbas), ) credential_collector_thread.start() @@ -181,7 +181,7 @@ class AutomatedMaster(IMaster): payload_thread = create_daemon_thread( target=self._run_plugins, name="PayloadThread", - args=(config["payloads"].items(), "payload", self._run_payload), + args=(config.payloads.items(), "payload", self._run_payload), ) payload_thread.start() payload_thread.join() From 9286e8690008c62054bda04b533f3bd52a791130 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Wed, 22 Jun 2022 17:52:34 +0300 Subject: [PATCH 129/196] Agent: Use deserialized in exploiter.py and propagator.py --- monkey/infection_monkey/master/exploiter.py | 34 ++++++++++------- monkey/infection_monkey/master/propagator.py | 13 ++++--- .../infection_monkey/master/test_exploiter.py | 38 +++++++++++++------ 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/monkey/infection_monkey/master/exploiter.py b/monkey/infection_monkey/master/exploiter.py index 35a212482..46b6799a0 100644 --- a/monkey/infection_monkey/master/exploiter.py +++ b/monkey/infection_monkey/master/exploiter.py @@ -5,9 +5,13 @@ from copy import deepcopy from itertools import chain from queue import Queue from threading import Event -from typing import Callable, Dict, List, Mapping +from typing import Callable, Dict, List from common import OperatingSystems +from common.configuration.agent_sub_configurations import ( + ExploitationConfiguration, + ExploiterConfiguration, +) from infection_monkey.custom_types import PropagationCredentials from infection_monkey.i_puppet import ExploiterResultData, IPuppet from infection_monkey.model import VictimHost @@ -46,7 +50,7 @@ class Exploiter: def exploit_hosts( self, - exploiter_config: Dict, + exploiter_config: ExploitationConfiguration, hosts_to_exploit: Queue, current_depth: int, results_callback: Callback, @@ -56,7 +60,7 @@ class Exploiter: exploiters_to_run = self._process_exploiter_config(exploiter_config) logger.debug( "Agent is configured to run the following exploiters in order: " - f"{', '.join([e['name'] for e in exploiters_to_run])}" + f"{', '.join([e.name for e in exploiters_to_run])}" ) exploit_args = ( @@ -75,24 +79,28 @@ class Exploiter: ) @staticmethod - def _process_exploiter_config(exploiter_config: Mapping) -> List[Mapping]: + def _process_exploiter_config( + exploiter_config: ExploitationConfiguration, + ) -> List[ExploiterConfiguration]: # Run vulnerability exploiters before brute force exploiters to minimize the effect of # account lockout due to invalid credentials - ordered_exploiters = chain( - exploiter_config["vulnerability"], exploiter_config["brute_force"] - ) + ordered_exploiters = chain(exploiter_config.vulnerability, exploiter_config.brute_force) exploiters_to_run = list(deepcopy(ordered_exploiters)) + extended_exploiters = [] for exploiter in exploiters_to_run: # This order allows exploiter-specific options to # override general options for all exploiters. - exploiter["options"] = {**exploiter_config["options"], **exploiter["options"]} + options = {**exploiter_config.options.__dict__, **exploiter.options} + extended_exploiters.append( + ExploiterConfiguration(exploiter.name, options, exploiter.supported_os) + ) - return exploiters_to_run + return extended_exploiters def _exploit_hosts_on_queue( self, - exploiters_to_run: List[Dict], + exploiters_to_run: List[ExploiterConfiguration], hosts_to_exploit: Queue, current_depth: int, results_callback: Callback, @@ -119,7 +127,7 @@ class Exploiter: def _run_all_exploiters( self, - exploiters_to_run: List[Dict], + exploiters_to_run: List[ExploiterConfiguration], victim_host: VictimHost, current_depth: int, results_callback: Callback, @@ -127,7 +135,7 @@ class Exploiter: ): for exploiter in interruptible_iter(exploiters_to_run, stop): - exploiter_name = exploiter["name"] + exploiter_name = exploiter.name victim_os = victim_host.os.get("type") # We want to try all exploiters if the victim's OS is unknown @@ -140,7 +148,7 @@ class Exploiter: continue exploiter_results = self._run_exploiter( - exploiter_name, exploiter["options"], victim_host, current_depth, stop + exploiter_name, exploiter.options, victim_host, current_depth, stop ) results_callback(exploiter_name, victim_host, exploiter_results) diff --git a/monkey/infection_monkey/master/propagator.py b/monkey/infection_monkey/master/propagator.py index a74cf7c86..64edae2ec 100644 --- a/monkey/infection_monkey/master/propagator.py +++ b/monkey/infection_monkey/master/propagator.py @@ -1,10 +1,13 @@ import logging from queue import Queue from threading import Event -from typing import Dict, List +from typing import List -from common.configuration import PropagationConfiguration,\ - NetworkScanConfiguration, ScanTargetConfiguration +from common.configuration import ( + NetworkScanConfiguration, + PropagationConfiguration, + ScanTargetConfiguration, +) from infection_monkey.i_puppet import ( ExploiterResultData, FingerprintData, @@ -139,14 +142,14 @@ class Propagator: def _exploit_hosts( self, - propagation_config: Dict, + propagation_config: PropagationConfiguration, current_depth: int, network_scan_completed: Event, stop: Event, ): logger.info("Exploiting victims") - exploiter_config = propagation_config["exploiters"] + exploiter_config = propagation_config.exploitation self._exploiter.exploit_hosts( exploiter_config, self._hosts_to_exploit, diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py index f5d9c2da4..2af0f3809 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py @@ -8,6 +8,10 @@ import pytest from tests.unit_tests.infection_monkey.master.mock_puppet import MockPuppet from common import OperatingSystems +from common.configuration.agent_sub_configurations import ( + ExploitationConfiguration, + ExploiterConfiguration, +) from infection_monkey.master import Exploiter from infection_monkey.model import VictimHost @@ -35,18 +39,28 @@ def callback(): @pytest.fixture -def exploiter_config(): - return { - "options": {"dropper_path_linux": "/tmp/monkey"}, - "brute_force": [ - {"name": "MSSQLExploiter", "options": {"timeout": 10}}, - {"name": "SSHExploiter", "options": {}}, - {"name": "WmiExploiter", "options": {"timeout": 10}}, - ], - "vulnerability": [ - {"name": "ZerologonExploiter", "options": {}}, - ], - } +def exploiter_config(default_agent_config): + brute_force = [ + ExploiterConfiguration( + name="MSSQLExploiter", options={"timeout": 10} + ), + ExploiterConfiguration( + name="SSHExploiter", options={} + ), + ExploiterConfiguration( + name="WmiExploiter", options={"timeout": 10} + ), + ] + vulnerability = [ + ExploiterConfiguration( + name="ZerologonExploiter", options={} + ) + ] + return ExploitationConfiguration( + options=default_agent_config.propagation.exploitation.options, + brute_force=brute_force, + vulnerability=vulnerability, + ) @pytest.fixture From 86ed174d743e0a1f27912b1800584919280861b2 Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Thu, 23 Jun 2022 15:06:48 +0000 Subject: [PATCH 130/196] Agent: Usa agent config object instead of dict in option_parsing.py --- monkey/infection_monkey/master/option_parsing.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/monkey/infection_monkey/master/option_parsing.py b/monkey/infection_monkey/master/option_parsing.py index c35bf6303..c9262c5c9 100644 --- a/monkey/infection_monkey/master/option_parsing.py +++ b/monkey/infection_monkey/master/option_parsing.py @@ -1,13 +1,12 @@ -from typing import Dict - +from common.configuration import CustomPBAConfiguration from infection_monkey.utils.environment import is_windows_os -def custom_pba_is_enabled(pba_options: Dict) -> bool: +def custom_pba_is_enabled(pba_options: CustomPBAConfiguration) -> bool: if not is_windows_os(): - if pba_options["linux_command"]: + if pba_options.linux_command: return True else: - if pba_options["windows_command"]: + if pba_options.windows_command: return True return False From ab67853192e4f87eb5145f2d4fc797ddb0c9e333 Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Thu, 23 Jun 2022 15:07:33 +0000 Subject: [PATCH 131/196] Agent: Usa agent config object instead of dict automated_master.py --- .../master/automated_master.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 684d60003..90223daad 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,8 +1,9 @@ import logging import threading import time -from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple +from common.configuration import PluginConfiguration from common.utils import Timer from infection_monkey.credential_store import ICredentialsStore from infection_monkey.i_control_channel import IControlChannel, IslandCommunicationError @@ -188,17 +189,17 @@ class AutomatedMaster(IMaster): pba_thread.join() - def _collect_credentials(self, collector: str): - credentials = self._puppet.run_credential_collector(collector, {}) + def _collect_credentials(self, collector: PluginConfiguration): + credentials = self._puppet.run_credential_collector(collector.name, collector.options) if credentials: self._telemetry_messenger.send_telemetry(CredentialsTelem(credentials)) else: logger.debug(f"No credentials were collected by {collector}") - def _run_pba(self, pba: Tuple[str, Dict]): - name = pba[0] - options = pba[1] + def _run_pba(self, pba: PluginConfiguration): + name = pba.name + options = pba.options for pba_data in self._puppet.run_pba(name, options): self._telemetry_messenger.send_telemetry(PostBreachTelem(pba_data)) @@ -210,15 +211,22 @@ class AutomatedMaster(IMaster): self._puppet.run_payload(name, options, self._stop) def _run_pbas( - self, plugins: Iterable[Any], callback: Callable[[Any], None], custom_pba_options: Mapping + self, + plugins: List[PluginConfiguration], + callback: Callable[[Any], None], + custom_pba_options: Dict, ): self._run_plugins(plugins, "post-breach action", callback) if custom_pba_is_enabled(custom_pba_options): - self._run_plugins([("CustomPBA", custom_pba_options)], "post-breach action", callback) + self._run_plugins( + [PluginConfiguration(name="CustomPBA", options=custom_pba_options)], + "post-breach action", + callback, + ) def _run_plugins( - self, plugins: Iterable[Any], plugin_type: str, callback: Callable[[Any], None] + self, plugins: List[PluginConfiguration], plugin_type: str, callback: Callable[[Any], None] ): logger.info(f"Running {plugin_type}s") logger.debug(f"Found {len(plugins)} {plugin_type}(s) to run") From 0f848eb28439f56e34d3952e8f30c16fc8be7aef Mon Sep 17 00:00:00 2001 From: vakaris_zilius Date: Thu, 23 Jun 2022 15:08:20 +0000 Subject: [PATCH 132/196] Agent: Usa agent config object instead of dict should_propagate --- monkey/infection_monkey/utils/propagation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/utils/propagation.py b/monkey/infection_monkey/utils/propagation.py index 004bafdd2..9cfccab51 100644 --- a/monkey/infection_monkey/utils/propagation.py +++ b/monkey/infection_monkey/utils/propagation.py @@ -1,2 +1,5 @@ -def should_propagate(config: dict, current_depth: int) -> bool: - return config["config"]["depth"] > current_depth +from common.configuration import AgentConfiguration + + +def should_propagate(config: AgentConfiguration, current_depth: int) -> bool: + return config.propagation.maximum_depth > current_depth From aff54232e97b52dcf00b3b6aa00df5abf0332870 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 14:44:42 -0400 Subject: [PATCH 133/196] Agent: Remove redundant call to control_channel.get_config() --- monkey/infection_monkey/monkey.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index afcf460b2..cf9283526 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -172,9 +172,8 @@ class InfectionMonkey: ) config = control_channel.get_config() - keep_tunnel_open_time = control_channel.get_config().keep_tunnel_open_time self._monkey_inbound_tunnel = self._control_client.create_control_tunnel( - keep_tunnel_open_time + config.keep_tunnel_open_time ) if self._monkey_inbound_tunnel and should_propagate(config, self._current_depth): self._inbound_tunnel_opened = True From bba7139be69eddfb797a467966aac39692d1e859 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 14:46:03 -0400 Subject: [PATCH 134/196] Agent: Add missing return type hint to _try_communicate_with_island() --- monkey/infection_monkey/master/automated_master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 90223daad..6fe9188d3 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -112,7 +112,7 @@ class AutomatedMaster(IMaster): time.sleep(CHECK_FOR_TERMINATE_INTERVAL_SEC) @staticmethod - def _try_communicate_with_island(fn: Callable[[], Any], max_tries: int): + def _try_communicate_with_island(fn: Callable[[], Any], max_tries: int) -> Any: tries = 0 while tries < max_tries: try: From 6e951ed65d6d6ba026395ec487a7b41b2dda0bc1 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 14:56:24 -0400 Subject: [PATCH 135/196] UT: Remove supported_os from default_config.py "supported_os" was removed from the schema in d079d74b --- .../tests/data_for_tests/monkey_configs/default_config.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/monkey/tests/data_for_tests/monkey_configs/default_config.py b/monkey/tests/data_for_tests/monkey_configs/default_config.py index 84d1f54e2..e232d122d 100644 --- a/monkey/tests/data_for_tests/monkey_configs/default_config.py +++ b/monkey/tests/data_for_tests/monkey_configs/default_config.py @@ -1,4 +1,3 @@ -from common import OperatingSystems from common.configuration import AgentConfigurationSchema flat_config = { @@ -85,39 +84,32 @@ flat_config = { { "name": "MSSQLExploiter", "options": {}, - "supported_os": [OperatingSystems.WINDOWS.name], }, { "name": "PowerShellExploiter", "options": {}, - "supported_os": [OperatingSystems.WINDOWS.name], }, { "name": "SSHExploiter", "options": {}, - "supported_os": [OperatingSystems.LINUX.name], }, { "name": "SmbExploiter", "options": {"smb_download_timeout": 30}, - "supported_os": [OperatingSystems.WINDOWS.name], }, { "name": "WmiExploiter", "options": {"smb_download_timeout": 30}, - "supported_os": [OperatingSystems.WINDOWS.name], }, ], "vulnerability": [ { "name": "HadoopExploiter", "options": {}, - "supported_os": [OperatingSystems.LINUX.name, OperatingSystems.WINDOWS.name], }, { "name": "Log4ShellExploiter", "options": {}, - "supported_os": [OperatingSystems.LINUX.name, OperatingSystems.WINDOWS.name], }, ], }, From 81d3300ec79b5c6423f8a5904a256fd1b5989649 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 15:43:34 -0400 Subject: [PATCH 136/196] Agent: Remove print() that was added by mistake --- monkey/infection_monkey/master/exploiter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monkey/infection_monkey/master/exploiter.py b/monkey/infection_monkey/master/exploiter.py index 46b6799a0..c8bb57299 100644 --- a/monkey/infection_monkey/master/exploiter.py +++ b/monkey/infection_monkey/master/exploiter.py @@ -139,7 +139,6 @@ class Exploiter: victim_os = victim_host.os.get("type") # We want to try all exploiters if the victim's OS is unknown - print(victim_os) if victim_os is not None and victim_os not in SUPPORTED_OS[exploiter_name]: logger.debug( f"Skipping {exploiter_name} because it does not support " From bff92ed7add225d67f67bf2f1fcc74f1a1938c20 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 15:48:25 -0400 Subject: [PATCH 137/196] UT: Fix erroneously abbreviated fixture --- .../infection_monkey/master/test_exploiter.py | 22 +++++------------ .../master/test_ip_scanner.py | 4 ++-- .../master/test_propagator.py | 24 +++++++++---------- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py index 2af0f3809..cc7e497b6 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py @@ -39,25 +39,15 @@ def callback(): @pytest.fixture -def exploiter_config(default_agent_config): +def exploiter_config(default_agent_configuration): brute_force = [ - ExploiterConfiguration( - name="MSSQLExploiter", options={"timeout": 10} - ), - ExploiterConfiguration( - name="SSHExploiter", options={} - ), - ExploiterConfiguration( - name="WmiExploiter", options={"timeout": 10} - ), - ] - vulnerability = [ - ExploiterConfiguration( - name="ZerologonExploiter", options={} - ) + ExploiterConfiguration(name="MSSQLExploiter", options={"timeout": 10}), + ExploiterConfiguration(name="SSHExploiter", options={}), + ExploiterConfiguration(name="WmiExploiter", options={"timeout": 10}), ] + vulnerability = [ExploiterConfiguration(name="ZerologonExploiter", options={})] return ExploitationConfiguration( - options=default_agent_config.propagation.exploitation.options, + options=default_agent_configuration.propagation.exploitation.options, brute_force=brute_force, vulnerability=vulnerability, ) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py index 069f931b9..bf026510f 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_ip_scanner.py @@ -20,7 +20,7 @@ LINUX_OS = "linux" @pytest.fixture -def scan_config(default_agent_config): +def scan_config(default_agent_configuration): tcp_config = TCPScanConfiguration( timeout=3, ports=[ @@ -42,7 +42,7 @@ def scan_config(default_agent_config): tcp_config, icmp_config, fingerprinter_config, - default_agent_config.propagation.network_scan.targets, + default_agent_configuration.propagation.network_scan.targets, ) return scan_config diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py index a2581de60..847d92bcd 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py @@ -140,23 +140,23 @@ class StubExploiter: pass -def get_propagation_config(default_agent_config, scan_target_config: ScanTargetConfiguration): +def get_propagation_config(default_agent_configuration, scan_target_config: ScanTargetConfiguration): network_scan = NetworkScanConfiguration( - default_agent_config.propagation.network_scan.tcp, - default_agent_config.propagation.network_scan.icmp, - default_agent_config.propagation.network_scan.fingerprinters, + default_agent_configuration.propagation.network_scan.tcp, + default_agent_configuration.propagation.network_scan.icmp, + default_agent_configuration.propagation.network_scan.fingerprinters, scan_target_config, ) propagation_config = PropagationConfiguration( - default_agent_config.propagation.maximum_depth, + default_agent_configuration.propagation.maximum_depth, network_scan, - default_agent_config.propagation.exploitation, + default_agent_configuration.propagation.exploitation, ) return propagation_config def test_scan_result_processing( - telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_config + telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_configuration ): p = Propagator( telemetry_messenger_spy, mock_ip_scanner, StubExploiter(), mock_victim_host_factory, [] @@ -167,7 +167,7 @@ def test_scan_result_processing( local_network_scan=False, subnets=["10.0.0.1", "10.0.0.2", "10.0.0.3"], ) - propagation_config = get_propagation_config(default_agent_config, targets) + propagation_config = get_propagation_config(default_agent_configuration, targets) p.propagate(propagation_config, 1, Event()) assert len(telemetry_messenger_spy.telemetries) == 3 @@ -253,7 +253,7 @@ class MockExploiter: def test_exploiter_result_processing( - telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_config + telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_configuration ): p = Propagator( telemetry_messenger_spy, mock_ip_scanner, MockExploiter(), mock_victim_host_factory, [] @@ -265,7 +265,7 @@ def test_exploiter_result_processing( local_network_scan=False, subnets=["10.0.0.1", "10.0.0.2", "10.0.0.3"], ) - propagation_config = get_propagation_config(default_agent_config, targets) + propagation_config = get_propagation_config(default_agent_configuration, targets) p.propagate(propagation_config, 1, Event()) exploit_telems = [t for t in telemetry_messenger_spy.telemetries if isinstance(t, ExploitTelem)] @@ -290,7 +290,7 @@ def test_exploiter_result_processing( def test_scan_target_generation( - telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_config + telemetry_messenger_spy, mock_ip_scanner, mock_victim_host_factory, default_agent_configuration ): local_network_interfaces = [NetworkInterface("10.0.0.9", "/29")] p = Propagator( @@ -306,7 +306,7 @@ def test_scan_target_generation( local_network_scan=True, subnets=["10.0.0.0/29", "172.10.20.30"], ) - propagation_config = get_propagation_config(default_agent_config, targets) + propagation_config = get_propagation_config(default_agent_configuration, targets) p.propagate(propagation_config, 1, Event()) expected_ip_scan_list = [ From 5a95aef94c74eb9175b66313de6da415b59a8dd7 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 15:51:52 -0400 Subject: [PATCH 138/196] Agent: Remove unnecessary parameter --- monkey/infection_monkey/master/exploiter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/monkey/infection_monkey/master/exploiter.py b/monkey/infection_monkey/master/exploiter.py index c8bb57299..60910226d 100644 --- a/monkey/infection_monkey/master/exploiter.py +++ b/monkey/infection_monkey/master/exploiter.py @@ -92,9 +92,7 @@ class Exploiter: # This order allows exploiter-specific options to # override general options for all exploiters. options = {**exploiter_config.options.__dict__, **exploiter.options} - extended_exploiters.append( - ExploiterConfiguration(exploiter.name, options, exploiter.supported_os) - ) + extended_exploiters.append(ExploiterConfiguration(exploiter.name, options)) return extended_exploiters From afeca66d92df422ee581a02ce67b0fc1fa78b355 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 16:27:05 -0400 Subject: [PATCH 139/196] UT: Use AgentConfiguration in test_propagation.py --- .../utils/test_propagation.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py b/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py index 37f7194a6..15c15d0f0 100644 --- a/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py +++ b/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py @@ -1,11 +1,26 @@ +import pytest + +from common.configuration import AgentConfiguration, AgentConfigurationSchema from infection_monkey.utils.propagation import should_propagate -def get_config(max_depth): - return {"config": {"depth": max_depth}} +@pytest.fixture +def get_config(default_agent_configuration): + def _inner(max_depth): + # AgentConfiguration is a frozen dataclass, so we need to deserialize and reserialize to + # modify it. The benefit is that it's impossible to construct an invalid object. The + # downside is the extra steps required to change an object. Maybe we can come up with a + # better all-around solution. It depends how often we need to mutate these objects (probably + # only for tests). + agent_dict = AgentConfigurationSchema().dump(default_agent_configuration) + agent_dict["propagation"]["maximum_depth"] = max_depth + + return AgentConfiguration.from_dict(agent_dict) + + return _inner -def test_should_propagate_current_less_than_max(): +def test_should_propagate_current_less_than_max(get_config): max_depth = 2 current_depth = 1 @@ -14,7 +29,7 @@ def test_should_propagate_current_less_than_max(): assert should_propagate(config, current_depth) is True -def test_should_propagate_current_greater_than_max(): +def test_should_propagate_current_greater_than_max(get_config): max_depth = 2 current_depth = 3 @@ -23,7 +38,7 @@ def test_should_propagate_current_greater_than_max(): assert should_propagate(config, current_depth) is False -def test_should_propagate_current_equal_to_max(): +def test_should_propagate_current_equal_to_max(get_config): max_depth = 2 current_depth = max_depth From ad0f6946bdad1ffbb8038bc299acd14d673da195 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 16:31:28 -0400 Subject: [PATCH 140/196] Agent: Decouple should_propagate() and AgentConfiguration --- .../master/automated_master.py | 2 +- monkey/infection_monkey/monkey.py | 4 +- monkey/infection_monkey/utils/propagation.py | 7 +-- .../utils/test_propagation.py | 45 +++++-------------- 4 files changed, 16 insertions(+), 42 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 6fe9188d3..ced26e58d 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -174,7 +174,7 @@ class AutomatedMaster(IMaster): current_depth = self._current_depth if self._current_depth is not None else 0 logger.info(f"Current depth is {current_depth}") - if should_propagate(self._control_channel.get_config(), self._current_depth): + if should_propagate(config.propagation.maximum_depth, self._current_depth): self._propagator.propagate(config.propagation, current_depth, self._stop) else: logger.info("Skipping propagation: maximum depth reached") diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index cf9283526..23a5a3ace 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -175,7 +175,9 @@ class InfectionMonkey: self._monkey_inbound_tunnel = self._control_client.create_control_tunnel( config.keep_tunnel_open_time ) - if self._monkey_inbound_tunnel and should_propagate(config, self._current_depth): + if self._monkey_inbound_tunnel and should_propagate( + config.propagation.maximum_depth, self._current_depth + ): self._inbound_tunnel_opened = True self._monkey_inbound_tunnel.start() diff --git a/monkey/infection_monkey/utils/propagation.py b/monkey/infection_monkey/utils/propagation.py index 9cfccab51..5036b5745 100644 --- a/monkey/infection_monkey/utils/propagation.py +++ b/monkey/infection_monkey/utils/propagation.py @@ -1,5 +1,2 @@ -from common.configuration import AgentConfiguration - - -def should_propagate(config: AgentConfiguration, current_depth: int) -> bool: - return config.propagation.maximum_depth > current_depth +def should_propagate(maximum_depth: int, current_depth: int) -> bool: + return maximum_depth > current_depth diff --git a/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py b/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py index 15c15d0f0..a4531f92d 100644 --- a/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py +++ b/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py @@ -1,47 +1,22 @@ -import pytest - -from common.configuration import AgentConfiguration, AgentConfigurationSchema from infection_monkey.utils.propagation import should_propagate -@pytest.fixture -def get_config(default_agent_configuration): - def _inner(max_depth): - # AgentConfiguration is a frozen dataclass, so we need to deserialize and reserialize to - # modify it. The benefit is that it's impossible to construct an invalid object. The - # downside is the extra steps required to change an object. Maybe we can come up with a - # better all-around solution. It depends how often we need to mutate these objects (probably - # only for tests). - agent_dict = AgentConfigurationSchema().dump(default_agent_configuration) - agent_dict["propagation"]["maximum_depth"] = max_depth - - return AgentConfiguration.from_dict(agent_dict) - - return _inner - - -def test_should_propagate_current_less_than_max(get_config): - max_depth = 2 +def test_should_propagate_current_less_than_max(): + maximum_depth = 2 current_depth = 1 - config = get_config(max_depth) - - assert should_propagate(config, current_depth) is True + assert should_propagate(maximum_depth, current_depth) is True -def test_should_propagate_current_greater_than_max(get_config): - max_depth = 2 +def test_should_propagate_current_greater_than_max(): + maximum_depth = 2 current_depth = 3 - config = get_config(max_depth) - - assert should_propagate(config, current_depth) is False + assert should_propagate(maximum_depth, current_depth) is False -def test_should_propagate_current_equal_to_max(get_config): - max_depth = 2 - current_depth = max_depth +def test_should_propagate_current_equal_to_max(): + maximum_depth = 2 + current_depth = maximum_depth - config = get_config(max_depth) - - assert should_propagate(config, current_depth) is False + assert should_propagate(maximum_depth, current_depth) is False From 05f640d48777f4fbab3f9a59b105c68154e3a318 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 16:36:01 -0400 Subject: [PATCH 141/196] Agent: Rename should_propagate -> maximum_depth_reached --- monkey/infection_monkey/master/automated_master.py | 4 ++-- monkey/infection_monkey/monkey.py | 4 ++-- monkey/infection_monkey/utils/propagation.py | 2 +- .../infection_monkey/utils/test_propagation.py | 14 +++++++------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index ced26e58d..de55ef904 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -14,7 +14,7 @@ from infection_monkey.network import NetworkInterface from infection_monkey.telemetry.credentials_telem import CredentialsTelem from infection_monkey.telemetry.messengers.i_telemetry_messenger import ITelemetryMessenger from infection_monkey.telemetry.post_breach_telem import PostBreachTelem -from infection_monkey.utils.propagation import should_propagate +from infection_monkey.utils.propagation import maximum_depth_reached from infection_monkey.utils.threading import create_daemon_thread, interruptible_iter from . import Exploiter, IPScanner, Propagator @@ -174,7 +174,7 @@ class AutomatedMaster(IMaster): current_depth = self._current_depth if self._current_depth is not None else 0 logger.info(f"Current depth is {current_depth}") - if should_propagate(config.propagation.maximum_depth, self._current_depth): + if maximum_depth_reached(config.propagation.maximum_depth, self._current_depth): self._propagator.propagate(config.propagation, current_depth, self._stop) else: logger.info("Skipping propagation: maximum depth reached") diff --git a/monkey/infection_monkey/monkey.py b/monkey/infection_monkey/monkey.py index 23a5a3ace..749803c7b 100644 --- a/monkey/infection_monkey/monkey.py +++ b/monkey/infection_monkey/monkey.py @@ -78,7 +78,7 @@ from infection_monkey.utils.monkey_dir import ( remove_monkey_dir, ) from infection_monkey.utils.monkey_log_path import get_agent_log_path -from infection_monkey.utils.propagation import should_propagate +from infection_monkey.utils.propagation import maximum_depth_reached from infection_monkey.utils.signal_handler import register_signal_handlers, reset_signal_handlers logger = logging.getLogger(__name__) @@ -175,7 +175,7 @@ class InfectionMonkey: self._monkey_inbound_tunnel = self._control_client.create_control_tunnel( config.keep_tunnel_open_time ) - if self._monkey_inbound_tunnel and should_propagate( + if self._monkey_inbound_tunnel and maximum_depth_reached( config.propagation.maximum_depth, self._current_depth ): self._inbound_tunnel_opened = True diff --git a/monkey/infection_monkey/utils/propagation.py b/monkey/infection_monkey/utils/propagation.py index 5036b5745..2da2e7bee 100644 --- a/monkey/infection_monkey/utils/propagation.py +++ b/monkey/infection_monkey/utils/propagation.py @@ -1,2 +1,2 @@ -def should_propagate(maximum_depth: int, current_depth: int) -> bool: +def maximum_depth_reached(maximum_depth: int, current_depth: int) -> bool: return maximum_depth > current_depth diff --git a/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py b/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py index a4531f92d..19b2c18b5 100644 --- a/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py +++ b/monkey/tests/unit_tests/infection_monkey/utils/test_propagation.py @@ -1,22 +1,22 @@ -from infection_monkey.utils.propagation import should_propagate +from infection_monkey.utils.propagation import maximum_depth_reached -def test_should_propagate_current_less_than_max(): +def test_maximum_depth_reached__current_less_than_max(): maximum_depth = 2 current_depth = 1 - assert should_propagate(maximum_depth, current_depth) is True + assert maximum_depth_reached(maximum_depth, current_depth) is True -def test_should_propagate_current_greater_than_max(): +def test_maximum_depth_reached__current_greater_than_max(): maximum_depth = 2 current_depth = 3 - assert should_propagate(maximum_depth, current_depth) is False + assert maximum_depth_reached(maximum_depth, current_depth) is False -def test_should_propagate_current_equal_to_max(): +def test_maximum_depth_reached__current_equal_to_max(): maximum_depth = 2 current_depth = maximum_depth - assert should_propagate(maximum_depth, current_depth) is False + assert maximum_depth_reached(maximum_depth, current_depth) is False From 6d156b8fee02e924bf9b4299b3f73e093b5922a6 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Thu, 23 Jun 2022 16:54:27 -0400 Subject: [PATCH 142/196] Island: Return config timeouts in seconds The old config scheme stored timeouts as milliseconds, whereas the new one uses seconds. Seconds are more convenient because most python methods expecting timeouts are expecting floating-point seconds. --- monkey/infection_monkey/master/automated_master.py | 4 ++-- monkey/monkey_island/cc/services/config.py | 4 ++-- .../tests/unit_tests/monkey_island/cc/services/test_config.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index de55ef904..eeb0b48d7 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -23,8 +23,8 @@ from .option_parsing import custom_pba_is_enabled CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC = 5 CHECK_FOR_TERMINATE_INTERVAL_SEC = CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC / 5 SHUTDOWN_TIMEOUT = 5 -NUM_SCAN_THREADS = 16 -NUM_EXPLOIT_THREADS = 6 +NUM_SCAN_THREADS = 1 +NUM_EXPLOIT_THREADS = 1 CHECK_FOR_STOP_AGENT_COUNT = 5 CHECK_FOR_CONFIG_COUNT = 3 diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 69db1c2e1..1a8c7d006 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -460,7 +460,7 @@ class ConfigService: formatted_tcp_scan_config = {} - formatted_tcp_scan_config["timeout"] = config[flat_tcp_timeout_field] + formatted_tcp_scan_config["timeout"] = config[flat_tcp_timeout_field] / 1000 ports = ConfigService._union_tcp_and_http_ports( config[flat_tcp_ports_field], config[flat_http_ports_field] @@ -484,7 +484,7 @@ class ConfigService: flat_ping_timeout_field = "ping_scan_timeout" formatted_icmp_scan_config = {} - formatted_icmp_scan_config["timeout"] = config[flat_ping_timeout_field] + formatted_icmp_scan_config["timeout"] = config[flat_ping_timeout_field] / 1000 config.pop(flat_ping_timeout_field, None) diff --git a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py index 85f3f4823..f170b0865 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py +++ b/monkey/tests/unit_tests/monkey_island/cc/services/test_config.py @@ -99,7 +99,7 @@ def test_format_config_for_agent__propagation(): def test_format_config_for_agent__network_scan(): expected_network_scan_config = { "tcp": { - "timeout": 3000, + "timeout": 3.0, "ports": [ 22, 80, @@ -117,7 +117,7 @@ def test_format_config_for_agent__network_scan(): ], }, "icmp": { - "timeout": 1000, + "timeout": 1.0, }, "targets": { "blocked_ips": ["192.168.1.1", "192.168.1.100"], From 2ff2e5f597ece09d5f75a9fe9b4467bf44ac182b Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 24 Jun 2022 08:33:21 +0200 Subject: [PATCH 143/196] Agent: Fix running of payloads --- monkey/infection_monkey/master/automated_master.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index eeb0b48d7..17012620f 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -182,7 +182,7 @@ class AutomatedMaster(IMaster): payload_thread = create_daemon_thread( target=self._run_plugins, name="PayloadThread", - args=(config.payloads.items(), "payload", self._run_payload), + args=(config.payloads, "payload", self._run_payload), ) payload_thread.start() payload_thread.join() @@ -205,8 +205,8 @@ class AutomatedMaster(IMaster): self._telemetry_messenger.send_telemetry(PostBreachTelem(pba_data)) def _run_payload(self, payload: Tuple[str, Dict]): - name = payload[0] - options = payload[1] + name = payload.name + options = payload.options self._puppet.run_payload(name, options, self._stop) From f9445a2c76dbf58b9ad22d2a23a257713d45070c Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 24 Jun 2022 10:18:51 +0200 Subject: [PATCH 144/196] Agent: Use == to compare OperatingSystems enum --- monkey/infection_monkey/model/host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/model/host.py b/monkey/infection_monkey/model/host.py index 6a1295e58..167bef246 100644 --- a/monkey/infection_monkey/model/host.py +++ b/monkey/infection_monkey/model/host.py @@ -17,7 +17,7 @@ class VictimHost(object): return self.__dict__ def is_windows(self) -> bool: - return OperatingSystems.WINDOWS in self.os["type"] + return OperatingSystems.WINDOWS == self.os["type"] def __hash__(self): return hash(self.ip_addr) From d59dd81f432edf8ba4aa007fc01d3af25194c182 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 24 Jun 2022 10:19:47 +0200 Subject: [PATCH 145/196] Agent: Use OperatingSystems in CachingAgentRepository --- monkey/infection_monkey/exploit/caching_agent_repository.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/exploit/caching_agent_repository.py b/monkey/infection_monkey/exploit/caching_agent_repository.py index 0f86bbd9d..c23aca287 100644 --- a/monkey/infection_monkey/exploit/caching_agent_repository.py +++ b/monkey/infection_monkey/exploit/caching_agent_repository.py @@ -5,6 +5,7 @@ from typing import Mapping import requests +from common import OperatingSystems from common.common_consts.timeouts import MEDIUM_REQUEST_TIMEOUT from . import IAgentRepository @@ -22,13 +23,13 @@ class CachingAgentRepository(IAgentRepository): self._proxies = proxies self._lock = threading.Lock() - def get_agent_binary(self, os: str, architecture: str = None) -> io.BytesIO: + def get_agent_binary(self, os: OperatingSystems, architecture: str = None) -> io.BytesIO: # If multiple calls to get_agent_binary() are made simultaneously before the result of # _download_binary_from_island() is cached, then multiple requests will be sent to the # island. Add a mutex in front of the call to _download_agent_binary_from_island() so # that only one request per OS will be sent to the island. with self._lock: - return io.BytesIO(self._download_binary_from_island(os)) + return io.BytesIO(self._download_binary_from_island(os.value)) @lru_cache(maxsize=None) def _download_binary_from_island(self, os: str) -> bytes: From fb67586a4c27e8616d99a0ad6f4f17dbed6c04d5 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 24 Jun 2022 10:51:58 +0200 Subject: [PATCH 146/196] Agent: Use OperatingSystems.value for urllib.parse.quote --- monkey/infection_monkey/exploit/tools/http_tools.py | 2 +- monkey/infection_monkey/transport/http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/exploit/tools/http_tools.py b/monkey/infection_monkey/exploit/tools/http_tools.py index 92696a5b7..5b73251d9 100644 --- a/monkey/infection_monkey/exploit/tools/http_tools.py +++ b/monkey/infection_monkey/exploit/tools/http_tools.py @@ -57,6 +57,6 @@ class HTTPTools(object): httpd.start() lock.acquire() return ( - "http://%s:%s/%s" % (local_ip, local_port, urllib.parse.quote(host.os["type"])), + "http://%s:%s/%s" % (local_ip, local_port, urllib.parse.quote(host.os["type"].value)), httpd, ) diff --git a/monkey/infection_monkey/transport/http.py b/monkey/infection_monkey/transport/http.py index 63aaa0b36..7bcbcd87d 100644 --- a/monkey/infection_monkey/transport/http.py +++ b/monkey/infection_monkey/transport/http.py @@ -62,7 +62,7 @@ class FileServHTTPRequestHandler(http.server.BaseHTTPRequestHandler): f.close() def send_head(self): - if self.path != "/" + urllib.parse.quote(self.victim_os): + if self.path != "/" + urllib.parse.quote(self.victim_os.value): self.send_error(500, "") return None, 0, 0 try: From b605f16c4f3983cc52931e265cae7ed688154c56 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 24 Jun 2022 11:11:19 +0200 Subject: [PATCH 147/196] Agent: Use == to compare OperatingSystems enum in Log4Shell --- monkey/infection_monkey/exploit/log4shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/exploit/log4shell.py b/monkey/infection_monkey/exploit/log4shell.py index ffbcdd0d6..cab4ed548 100644 --- a/monkey/infection_monkey/exploit/log4shell.py +++ b/monkey/infection_monkey/exploit/log4shell.py @@ -129,7 +129,7 @@ class Log4ShellExploiter(WebRCE): } def _build_java_class(self, exploit_command: str) -> bytes: - if OperatingSystems.LINUX in self.host.os["type"]: + if OperatingSystems.LINUX == self.host.os["type"]: return build_exploit_bytecode(exploit_command, LINUX_EXPLOIT_TEMPLATE_PATH) else: return build_exploit_bytecode(exploit_command, WINDOWS_EXPLOIT_TEMPLATE_PATH) From e1d5d25e9c425521dbcce682629a25f15dc3bf78 Mon Sep 17 00:00:00 2001 From: Ilija Lazoroski Date: Fri, 24 Jun 2022 11:18:39 +0200 Subject: [PATCH 148/196] Agent: Use OperatingSystem.WINDOWS in Powershell --- monkey/infection_monkey/exploit/powershell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monkey/infection_monkey/exploit/powershell.py b/monkey/infection_monkey/exploit/powershell.py index 6ef72963e..c9991b0b4 100644 --- a/monkey/infection_monkey/exploit/powershell.py +++ b/monkey/infection_monkey/exploit/powershell.py @@ -2,6 +2,7 @@ import logging from pathlib import Path, PurePath from typing import List, Optional +from common import OperatingSystems from infection_monkey.exploit.HostExploiter import HostExploiter from infection_monkey.exploit.powershell_utils.auth_options import AuthOptions, get_auth_options from infection_monkey.exploit.powershell_utils.credentials import ( @@ -162,7 +163,7 @@ class PowerShellExploiter(HostExploiter): temp_monkey_binary_filepath.unlink() def _create_local_agent_file(self, binary_path): - agent_binary_bytes = self.agent_repository.get_agent_binary("windows") + agent_binary_bytes = self.agent_repository.get_agent_binary(OperatingSystems.WINDOWS) with open(binary_path, "wb") as f: f.write(agent_binary_bytes.getvalue()) From ffd3464d8a6b4589181563deb7404de76c21130d Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 07:22:07 -0400 Subject: [PATCH 149/196] Agent: Move enum to string conversion to _download_binary_from_island() --- .../infection_monkey/exploit/caching_agent_repository.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/exploit/caching_agent_repository.py b/monkey/infection_monkey/exploit/caching_agent_repository.py index c23aca287..5dd8a603c 100644 --- a/monkey/infection_monkey/exploit/caching_agent_repository.py +++ b/monkey/infection_monkey/exploit/caching_agent_repository.py @@ -29,12 +29,13 @@ class CachingAgentRepository(IAgentRepository): # island. Add a mutex in front of the call to _download_agent_binary_from_island() so # that only one request per OS will be sent to the island. with self._lock: - return io.BytesIO(self._download_binary_from_island(os.value)) + return io.BytesIO(self._download_binary_from_island(os)) @lru_cache(maxsize=None) - def _download_binary_from_island(self, os: str) -> bytes: + def _download_binary_from_island(self, os: OperatingSystems) -> bytes: + os_name = os.name.lower() response = requests.get( # noqa: DUO123 - f"{self._island_url}/api/agent-binaries/{os}", + f"{self._island_url}/api/agent-binaries/{os_name}", verify=False, proxies=self._proxies, timeout=MEDIUM_REQUEST_TIMEOUT, From 858eb2302c252a4473e16304197521780f175304 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 07:23:26 -0400 Subject: [PATCH 150/196] Agent: Rename os -> operating_system in caching_agent_repository The variable name "os" conflicts with the name of Python's `os` library. --- .../exploit/caching_agent_repository.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/monkey/infection_monkey/exploit/caching_agent_repository.py b/monkey/infection_monkey/exploit/caching_agent_repository.py index 5dd8a603c..499e8401c 100644 --- a/monkey/infection_monkey/exploit/caching_agent_repository.py +++ b/monkey/infection_monkey/exploit/caching_agent_repository.py @@ -23,17 +23,20 @@ class CachingAgentRepository(IAgentRepository): self._proxies = proxies self._lock = threading.Lock() - def get_agent_binary(self, os: OperatingSystems, architecture: str = None) -> io.BytesIO: + def get_agent_binary( + self, operating_system: OperatingSystems, architecture: str = None + ) -> io.BytesIO: # If multiple calls to get_agent_binary() are made simultaneously before the result of # _download_binary_from_island() is cached, then multiple requests will be sent to the # island. Add a mutex in front of the call to _download_agent_binary_from_island() so # that only one request per OS will be sent to the island. with self._lock: - return io.BytesIO(self._download_binary_from_island(os)) + return io.BytesIO(self._download_binary_from_island(operating_system)) @lru_cache(maxsize=None) - def _download_binary_from_island(self, os: OperatingSystems) -> bytes: - os_name = os.name.lower() + def _download_binary_from_island(self, operating_system: OperatingSystems) -> bytes: + os_name = operating_system.name.lower() + response = requests.get( # noqa: DUO123 f"{self._island_url}/api/agent-binaries/{os_name}", verify=False, From 2eb1691030324244df6ac56169299d947bdf2588 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 07:34:10 -0400 Subject: [PATCH 151/196] Agent: Use operating_system.value in _download_binary_from_island() --- monkey/infection_monkey/exploit/caching_agent_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/exploit/caching_agent_repository.py b/monkey/infection_monkey/exploit/caching_agent_repository.py index 499e8401c..7d3580258 100644 --- a/monkey/infection_monkey/exploit/caching_agent_repository.py +++ b/monkey/infection_monkey/exploit/caching_agent_repository.py @@ -35,7 +35,7 @@ class CachingAgentRepository(IAgentRepository): @lru_cache(maxsize=None) def _download_binary_from_island(self, operating_system: OperatingSystems) -> bytes: - os_name = operating_system.name.lower() + os_name = operating_system.value response = requests.get( # noqa: DUO123 f"{self._island_url}/api/agent-binaries/{os_name}", From a3db4142bf396e1669a924943af0d06adcf6458b Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 07:36:58 -0400 Subject: [PATCH 152/196] Common: Add a docstring to OperatingSystems --- monkey/common/operating_systems.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monkey/common/operating_systems.py b/monkey/common/operating_systems.py index 67f67da81..2ac2f64b3 100644 --- a/monkey/common/operating_systems.py +++ b/monkey/common/operating_systems.py @@ -2,5 +2,12 @@ from enum import Enum class OperatingSystems(Enum): + """ + An Enum representing all supported operating systems + + This Enum represents all operating systems that Infection Monkey supports. The value of each + member is the member's name in all lower-case characters. + """ + LINUX = "linux" WINDOWS = "windows" From 7b4daaa40f14a10c6dd74063cb533ec293e051a8 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 07:40:24 -0400 Subject: [PATCH 153/196] Agent: Change IAgentRepository to ccept OperatingSystems This was missed in d59dd81f and ffd3464d. --- monkey/infection_monkey/exploit/i_agent_repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/exploit/i_agent_repository.py b/monkey/infection_monkey/exploit/i_agent_repository.py index d825772a0..a00aac55a 100644 --- a/monkey/infection_monkey/exploit/i_agent_repository.py +++ b/monkey/infection_monkey/exploit/i_agent_repository.py @@ -1,6 +1,8 @@ import abc import io +from common import OperatingSystems + # TODO: The Island also has an IAgentRepository with a totally different interface. At the moment, # the Island and Agent have different needs, but at some point we should unify these. @@ -13,10 +15,12 @@ class IAgentRepository(metaclass=abc.ABCMeta): """ @abc.abstractmethod - def get_agent_binary(self, os: str, architecture: str = None) -> io.BytesIO: + def get_agent_binary( + self, operating_system: OperatingSystems, architecture: str = None + ) -> io.BytesIO: """ Retrieve the appropriate agent binary from the repository. - :param str os: The name of the operating system on which the agent binary will run + :param operating_system: The name of the operating system on which the agent binary will run :param str architecture: Reserved :return: A file-like object for the requested agent binary :rtype: io.BytesIO From 02cca3e12ac921129d5a7bf5f15d5e4437264120 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 07:42:01 -0400 Subject: [PATCH 154/196] Agent: Remove unnecessary type hints from IAgentRepository doctring --- monkey/infection_monkey/exploit/i_agent_repository.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monkey/infection_monkey/exploit/i_agent_repository.py b/monkey/infection_monkey/exploit/i_agent_repository.py index a00aac55a..308cf5418 100644 --- a/monkey/infection_monkey/exploit/i_agent_repository.py +++ b/monkey/infection_monkey/exploit/i_agent_repository.py @@ -21,8 +21,7 @@ class IAgentRepository(metaclass=abc.ABCMeta): """ Retrieve the appropriate agent binary from the repository. :param operating_system: The name of the operating system on which the agent binary will run - :param str architecture: Reserved + :param architecture: Reserved :return: A file-like object for the requested agent binary - :rtype: io.BytesIO """ pass From 7bba7113071675155b0a35256ee033a0c50c6543 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 11:08:29 -0400 Subject: [PATCH 155/196] Agent: Revert scan/exploit thread numer change The number of scan and exploit threads was changed accidentally in 6d156b8f. --- monkey/infection_monkey/master/automated_master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 17012620f..e955311b5 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -23,8 +23,8 @@ from .option_parsing import custom_pba_is_enabled CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC = 5 CHECK_FOR_TERMINATE_INTERVAL_SEC = CHECK_ISLAND_FOR_STOP_COMMAND_INTERVAL_SEC / 5 SHUTDOWN_TIMEOUT = 5 -NUM_SCAN_THREADS = 1 -NUM_EXPLOIT_THREADS = 1 +NUM_SCAN_THREADS = 16 +NUM_EXPLOIT_THREADS = 6 CHECK_FOR_STOP_AGENT_COUNT = 5 CHECK_FOR_CONFIG_COUNT = 3 From e3cea20cd578e46d29ee049eddaf4ac28f042aff Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 11:42:24 -0400 Subject: [PATCH 156/196] UT: Move test_agent_configuration.py to configuration/ --- .../common/{ => configuration}/test_agent_configuration.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename monkey/tests/unit_tests/common/{ => configuration}/test_agent_configuration.py (100%) diff --git a/monkey/tests/unit_tests/common/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py similarity index 100% rename from monkey/tests/unit_tests/common/test_agent_configuration.py rename to monkey/tests/unit_tests/common/configuration/test_agent_configuration.py From 5c739716a92f2df2e8e1d6aef9f9e1422a7ea527 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 11:44:00 -0400 Subject: [PATCH 157/196] Common: Rename _dict -> dict_ --- monkey/common/configuration/agent_configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index a636e3d95..36fa24b85 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -25,8 +25,8 @@ class AgentConfiguration: propagation: PropagationConfiguration @staticmethod - def from_dict(_dict: dict): - return AgentConfigurationSchema().load(_dict) + def from_dict(dict_: dict): + return AgentConfigurationSchema().load(dict_) class AgentConfigurationSchema(Schema): From 8605fd40ac35b3eddfc148d76bce36f8ab342a53 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 11:50:49 -0400 Subject: [PATCH 158/196] UT: Add a test for AgentConfiguration.from_dict() --- .../common/configuration/test_agent_configuration.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 7ea80cfc5..13cd106a8 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -1,3 +1,5 @@ +import json + from tests.common.example_agent_configuration import ( AGENT_CONFIGURATION, BLOCKED_IPS, @@ -178,3 +180,12 @@ def test_default_agent_configuration(): config = schema.loads(DEFAULT_AGENT_CONFIGURATION_JSON) assert isinstance(config, AgentConfiguration) + + +def test_from_dict(): + schema = AgentConfigurationSchema() + dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + + config = AgentConfiguration.from_dict(dict_) + + assert schema.dump(config) == dict_ From 1f9a056b0be5e735559babecf6459dfbe12aaa9c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 11:51:33 -0400 Subject: [PATCH 159/196] Agent: Add AgentConfiguration.from_json() --- monkey/common/configuration/agent_configuration.py | 4 ++++ .../common/configuration/test_agent_configuration.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 36fa24b85..8ba9e998b 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -28,6 +28,10 @@ class AgentConfiguration: def from_dict(dict_: dict): return AgentConfigurationSchema().load(dict_) + @staticmethod + def from_json(config_json: dict): + return AgentConfigurationSchema().loads(config_json) + class AgentConfigurationSchema(Schema): keep_tunnel_open_time = fields.Float() diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 13cd106a8..6156a0b69 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -189,3 +189,12 @@ def test_from_dict(): config = AgentConfiguration.from_dict(dict_) assert schema.dump(config) == dict_ + + +def test_from_json(): + schema = AgentConfigurationSchema() + dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + + config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) + + assert schema.dump(config) == dict_ From 28250daffeb5b4b2cf124012260bd856dea11696 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:19:31 -0400 Subject: [PATCH 160/196] Common: Add AgentConfiguration.to_json() --- monkey/common/configuration/agent_configuration.py | 6 ++++++ .../common/configuration/test_agent_configuration.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 8ba9e998b..a1f2eb70c 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from dataclasses import dataclass from typing import List @@ -32,6 +34,10 @@ class AgentConfiguration: def from_json(config_json: dict): return AgentConfigurationSchema().loads(config_json) + @staticmethod + def to_json(config: AgentConfiguration) -> str: + return AgentConfigurationSchema().dumps(config) + class AgentConfigurationSchema(Schema): keep_tunnel_open_time = fields.Float() diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 6156a0b69..134a18a91 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -198,3 +198,11 @@ def test_from_json(): config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) assert schema.dump(config) == dict_ + + +def test_to_json(): + config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) + + assert json.loads(AgentConfiguration.to_json(config)) == json.loads( + DEFAULT_AGENT_CONFIGURATION_JSON + ) From e4eee6a5eb692255c4ba6e543601c226eb725eac Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:23:37 -0400 Subject: [PATCH 161/196] UT: Use from_dict() and from_json() in tests --- .../tests/data_for_tests/monkey_configs/default_config.py | 4 ++-- .../in_memory_agent_configuration_repository.py | 4 ++-- .../common/configuration/test_agent_configuration.py | 6 ++---- .../repository/test_file_agent_configuration_repository.py | 5 ++--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/monkey/tests/data_for_tests/monkey_configs/default_config.py b/monkey/tests/data_for_tests/monkey_configs/default_config.py index e232d122d..6c0771af0 100644 --- a/monkey/tests/data_for_tests/monkey_configs/default_config.py +++ b/monkey/tests/data_for_tests/monkey_configs/default_config.py @@ -1,4 +1,4 @@ -from common.configuration import AgentConfigurationSchema +from common.configuration import AgentConfiguration flat_config = { "keep_tunnel_open_time": 30, @@ -116,4 +116,4 @@ flat_config = { }, } -DEFAULT_CONFIG = AgentConfigurationSchema().load(flat_config) +DEFAULT_CONFIG = AgentConfiguration.from_dict(flat_config) diff --git a/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py b/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py index e737d645c..b18465568 100644 --- a/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py +++ b/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py @@ -1,12 +1,12 @@ from tests.common.example_agent_configuration import AGENT_CONFIGURATION -from common.configuration.agent_configuration import AgentConfigurationSchema +from common.configuration.agent_configuration import AgentConfiguration from monkey_island.cc.repository import IAgentConfigurationRepository class InMemoryAgentConfigurationRepository(IAgentConfigurationRepository): def __init__(self): - self._configuration = AgentConfigurationSchema().load(AGENT_CONFIGURATION) + self._configuration = AgentConfiguration.from_dict(AGENT_CONFIGURATION) def get_configuration(self): return self._configuration diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 134a18a91..23830d619 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -161,7 +161,7 @@ def test_propagation_configuration(): def test_agent_configuration(): schema = AgentConfigurationSchema() - config = schema.load(AGENT_CONFIGURATION) + config = AgentConfiguration.from_dict(AGENT_CONFIGURATION) config_dict = schema.dump(config) assert isinstance(config, AgentConfiguration) @@ -175,9 +175,7 @@ def test_agent_configuration(): def test_default_agent_configuration(): - schema = AgentConfigurationSchema() - - config = schema.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) assert isinstance(config, AgentConfiguration) diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py index 4ab111606..21dc4503b 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -2,7 +2,7 @@ import pytest from tests.common.example_agent_configuration import AGENT_CONFIGURATION from tests.monkey_island import OpenErrorFileRepository, SingleFileRepository -from common.configuration import AgentConfigurationSchema +from common.configuration import AgentConfiguration from monkey_island.cc.repository import FileAgentConfigurationRepository, RetrievalError @@ -12,8 +12,7 @@ def repository(default_agent_configuration): def test_store_agent_config(repository): - schema = AgentConfigurationSchema() - agent_configuration = schema.load(AGENT_CONFIGURATION) + agent_configuration = AgentConfiguration.from_dict(AGENT_CONFIGURATION) repository.store_configuration(agent_configuration) retrieved_agent_configuration = repository.get_configuration() From 6a927266a4c81b89fc90c821590c089077d1a033 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:24:07 -0400 Subject: [PATCH 162/196] Island: Use {from,to}_json() in FileAgentConfigurationRepository --- .../cc/repository/file_agent_configuration_repository.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py index b63ee817c..312e3921e 100644 --- a/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/file_agent_configuration_repository.py @@ -1,6 +1,6 @@ import io -from common.configuration import AgentConfiguration, AgentConfigurationSchema +from common.configuration import AgentConfiguration from monkey_island.cc import repository from monkey_island.cc.repository import ( IAgentConfigurationRepository, @@ -17,21 +17,20 @@ class FileAgentConfigurationRepository(IAgentConfigurationRepository): ): self._default_agent_configuration = default_agent_configuration self._file_repository = file_repository - self._schema = AgentConfigurationSchema() def get_configuration(self) -> AgentConfiguration: try: with self._file_repository.open_file(AGENT_CONFIGURATION_FILE_NAME) as f: configuration_json = f.read().decode() - return self._schema.loads(configuration_json) + return AgentConfiguration.from_json(configuration_json) except repository.FileNotFoundError: return self._default_agent_configuration except Exception as err: raise RetrievalError(f"Error retrieving the agent configuration: {err}") def store_configuration(self, agent_configuration: AgentConfiguration): - configuration_json = self._schema.dumps(agent_configuration) + configuration_json = AgentConfiguration.to_json(agent_configuration) self._file_repository.save_file( AGENT_CONFIGURATION_FILE_NAME, io.BytesIO(configuration_json.encode()) From a1baaae76a90e9ed59146003d5701dfd27b0535a Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:29:54 -0400 Subject: [PATCH 163/196] Common: Use from_json() in build_default_agent_configuration() --- monkey/common/configuration/default_agent_configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/monkey/common/configuration/default_agent_configuration.py b/monkey/common/configuration/default_agent_configuration.py index c83169566..f3295a4b6 100644 --- a/monkey/common/configuration/default_agent_configuration.py +++ b/monkey/common/configuration/default_agent_configuration.py @@ -1,4 +1,4 @@ -from . import AgentConfiguration, AgentConfigurationSchema +from . import AgentConfiguration DEFAULT_AGENT_CONFIGURATION_JSON = """{ "keep_tunnel_open_time": 30, @@ -204,5 +204,4 @@ DEFAULT_AGENT_CONFIGURATION_JSON = """{ def build_default_agent_configuration() -> AgentConfiguration: - schema = AgentConfigurationSchema() - return schema.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + return AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) From 07d1d9c45afcf0d7793893bbb48c4af1cbfdda1c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:33:12 -0400 Subject: [PATCH 164/196] Island: Use {from,to}_json() in resources --- monkey/monkey_island/cc/resources/agent_configuration.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/monkey/monkey_island/cc/resources/agent_configuration.py b/monkey/monkey_island/cc/resources/agent_configuration.py index f0ad73cb8..fd49bd713 100644 --- a/monkey/monkey_island/cc/resources/agent_configuration.py +++ b/monkey/monkey_island/cc/resources/agent_configuration.py @@ -3,7 +3,7 @@ import json import marshmallow from flask import make_response, request -from common.configuration.agent_configuration import AgentConfigurationSchema +from common.configuration.agent_configuration import AgentConfiguration as AgentConfigurationObject from monkey_island.cc.repository import IAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required @@ -14,19 +14,18 @@ class AgentConfiguration(AbstractResource): def __init__(self, agent_configuration_repository: IAgentConfigurationRepository): self._agent_configuration_repository = agent_configuration_repository - self._schema = AgentConfigurationSchema() @jwt_required def get(self): configuration = self._agent_configuration_repository.get_configuration() - configuration_json = self._schema.dumps(configuration) + configuration_json = AgentConfigurationObject.to_json(configuration) return make_response(configuration_json, 200) @jwt_required def post(self): try: - configuration_object = self._schema.loads(request.data) + configuration_object = AgentConfigurationObject.from_json(request.data) self._agent_configuration_repository.store_configuration(configuration_object) return make_response({}, 200) except (marshmallow.exceptions.ValidationError, json.JSONDecodeError) as err: From 4c47eae70bd8b30063e284b01a297b35bd70d618 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:39:47 -0400 Subject: [PATCH 165/196] Common: Encapsulate AgentConfigurationSchema --- monkey/common/configuration/__init__.py | 1 - .../configuration/test_agent_configuration.py | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 107e5a491..1505f0e68 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1,6 +1,5 @@ from .agent_configuration import ( AgentConfiguration, - AgentConfigurationSchema, ) from .agent_sub_configurations import ( CustomPBAConfiguration, diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 23830d619..0de34e4a2 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -25,11 +25,8 @@ from tests.common.example_agent_configuration import ( WINDOWS_FILENAME, ) -from common.configuration import ( - DEFAULT_AGENT_CONFIGURATION_JSON, - AgentConfiguration, - AgentConfigurationSchema, -) +from common.configuration import DEFAULT_AGENT_CONFIGURATION_JSON, AgentConfiguration +from common.configuration.agent_configuration import AgentConfigurationSchema from common.configuration.agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, ExploitationConfigurationSchema, @@ -159,10 +156,8 @@ def test_propagation_configuration(): def test_agent_configuration(): - schema = AgentConfigurationSchema() - config = AgentConfiguration.from_dict(AGENT_CONFIGURATION) - config_dict = schema.dump(config) + config_json = AgentConfiguration.to_json(config) assert isinstance(config, AgentConfiguration) assert config.keep_tunnel_open_time == 30 @@ -171,7 +166,7 @@ def test_agent_configuration(): assert isinstance(config.credential_collectors[0], PluginConfiguration) assert isinstance(config.payloads[0], PluginConfiguration) assert isinstance(config.propagation, PropagationConfiguration) - assert config_dict == AGENT_CONFIGURATION + assert json.loads(config_json) == AGENT_CONFIGURATION def test_default_agent_configuration(): From ea02bec0b486384155c4a10681a27b3fb1248c36 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:42:09 -0400 Subject: [PATCH 166/196] Common: Remove circular dependency in agent_configuration.py --- monkey/common/configuration/agent_configuration.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index a1f2eb70c..8c5b88646 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import List -from marshmallow import Schema, fields, post_load +from marshmallow import Schema, fields from .agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, @@ -28,11 +28,13 @@ class AgentConfiguration: @staticmethod def from_dict(dict_: dict): - return AgentConfigurationSchema().load(dict_) + config_dict = AgentConfigurationSchema().load(dict_) + return AgentConfiguration(**config_dict) @staticmethod def from_json(config_json: dict): - return AgentConfigurationSchema().loads(config_json) + config_dict = AgentConfigurationSchema().loads(config_json) + return AgentConfiguration(**config_dict) @staticmethod def to_json(config: AgentConfiguration) -> str: @@ -46,7 +48,3 @@ class AgentConfigurationSchema(Schema): credential_collectors = fields.List(fields.Nested(PluginConfigurationSchema)) payloads = fields.List(fields.Nested(PluginConfigurationSchema)) propagation = fields.Nested(PropagationConfigurationSchema) - - @post_load - def _make_agent_configuration(self, data, **kwargs): - return AgentConfiguration(**data) From fc9d854c72d7ba64fe721698912e80f02d3e9621 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 12:56:34 -0400 Subject: [PATCH 167/196] Common: Add validation to AgentConfiguration construction --- monkey/common/configuration/agent_configuration.py | 5 +++++ .../common/configuration/test_agent_configuration.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 8c5b88646..71c4bfd49 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -26,6 +26,11 @@ class AgentConfiguration: payloads: List[PluginConfiguration] propagation: PropagationConfiguration + def __post_init__(self): + # This will raise an exception if the object is invalid. Calling this in __post__init() + # makes it impossible to construct an invalid object + AgentConfigurationSchema().dump(self) + @staticmethod def from_dict(dict_: dict): config_dict = AgentConfigurationSchema().load(dict_) diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 0de34e4a2..d4a285364 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -1,5 +1,6 @@ import json +import pytest from tests.common.example_agent_configuration import ( AGENT_CONFIGURATION, BLOCKED_IPS, @@ -169,6 +170,14 @@ def test_agent_configuration(): assert json.loads(config_json) == AGENT_CONFIGURATION +def test_incorrect_type(): + valid_config = AgentConfiguration.from_dict(AGENT_CONFIGURATION) + with pytest.raises(Exception): + valid_config_dict = valid_config.__dict__ + valid_config_dict["keep_tunnel_open_time"] = "not_a_float" + AgentConfiguration(**valid_config_dict) + + def test_default_agent_configuration(): config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) From 94524d124c2d23fb243496adb275bcce46c5a1b4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 13:01:56 -0400 Subject: [PATCH 168/196] Common: Add InvalidConfigurationError --- monkey/common/configuration/__init__.py | 4 +--- monkey/common/configuration/agent_configuration.py | 4 ++++ monkey/common/utils/exceptions.py | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 1505f0e68..06ce30b50 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -1,6 +1,4 @@ -from .agent_configuration import ( - AgentConfiguration, -) +from .agent_configuration import AgentConfiguration, InvalidConfigurationError from .agent_sub_configurations import ( CustomPBAConfiguration, PluginConfiguration, diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 71c4bfd49..fe42e303d 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -17,6 +17,10 @@ from .agent_sub_configurations import ( ) +class InvalidConfigurationError(Exception): + pass + + @dataclass(frozen=True) class AgentConfiguration: keep_tunnel_open_time: float diff --git a/monkey/common/utils/exceptions.py b/monkey/common/utils/exceptions.py index 5935145e7..31cebca32 100644 --- a/monkey/common/utils/exceptions.py +++ b/monkey/common/utils/exceptions.py @@ -38,5 +38,7 @@ class DomainControllerNameFetchError(FailedExploitationError): """Raise on failed attempt to extract domain controller's name""" +# TODO: This has been replaced by common.configuration.InvalidConfigurationError. Use that error +# instead and remove this one. class InvalidConfigurationError(Exception): """Raise when configuration is invalid""" From dbd0d3e0dda0b64b9f10990922fb06556016eab5 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 13:21:39 -0400 Subject: [PATCH 169/196] Common: Encapsulate MarshmallowError --- .../configuration/agent_configuration.py | 25 +++++++++++++++---- .../cc/resources/agent_configuration.py | 4 +-- .../configuration/test_agent_configuration.py | 24 ++++++++++++++++-- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index fe42e303d..87ccce3b0 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import List from marshmallow import Schema, fields +from marshmallow.exceptions import MarshmallowError from .agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, @@ -21,6 +22,11 @@ class InvalidConfigurationError(Exception): pass +INVALID_CONFIGURATION_ERROR_MESSAGE = ( + "Cannot construct an AgentConfiguration object with the supplied, invalid data:" +) + + @dataclass(frozen=True) class AgentConfiguration: keep_tunnel_open_time: float @@ -33,17 +39,26 @@ class AgentConfiguration: def __post_init__(self): # This will raise an exception if the object is invalid. Calling this in __post__init() # makes it impossible to construct an invalid object - AgentConfigurationSchema().dump(self) + try: + AgentConfigurationSchema().dump(self) + except Exception as err: + raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") @staticmethod def from_dict(dict_: dict): - config_dict = AgentConfigurationSchema().load(dict_) - return AgentConfiguration(**config_dict) + try: + config_dict = AgentConfigurationSchema().load(dict_) + return AgentConfiguration(**config_dict) + except MarshmallowError as err: + raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") @staticmethod def from_json(config_json: dict): - config_dict = AgentConfigurationSchema().loads(config_json) - return AgentConfiguration(**config_dict) + try: + config_dict = AgentConfigurationSchema().loads(config_json) + return AgentConfiguration(**config_dict) + except MarshmallowError as err: + raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") @staticmethod def to_json(config: AgentConfiguration) -> str: diff --git a/monkey/monkey_island/cc/resources/agent_configuration.py b/monkey/monkey_island/cc/resources/agent_configuration.py index fd49bd713..1bf311564 100644 --- a/monkey/monkey_island/cc/resources/agent_configuration.py +++ b/monkey/monkey_island/cc/resources/agent_configuration.py @@ -1,9 +1,9 @@ import json -import marshmallow from flask import make_response, request from common.configuration.agent_configuration import AgentConfiguration as AgentConfigurationObject +from common.configuration.agent_configuration import InvalidConfigurationError from monkey_island.cc.repository import IAgentConfigurationRepository from monkey_island.cc.resources.AbstractResource import AbstractResource from monkey_island.cc.resources.request_authentication import jwt_required @@ -28,7 +28,7 @@ class AgentConfiguration(AbstractResource): configuration_object = AgentConfigurationObject.from_json(request.data) self._agent_configuration_repository.store_configuration(configuration_object) return make_response({}, 200) - except (marshmallow.exceptions.ValidationError, json.JSONDecodeError) as err: + except (InvalidConfigurationError, json.JSONDecodeError) as err: return make_response( {"message": f"Invalid configuration supplied: {err}"}, 400, diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index d4a285364..f8c8f4c38 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -26,7 +26,11 @@ from tests.common.example_agent_configuration import ( WINDOWS_FILENAME, ) -from common.configuration import DEFAULT_AGENT_CONFIGURATION_JSON, AgentConfiguration +from common.configuration import ( + DEFAULT_AGENT_CONFIGURATION_JSON, + AgentConfiguration, + InvalidConfigurationError, +) from common.configuration.agent_configuration import AgentConfigurationSchema from common.configuration.agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, @@ -172,7 +176,7 @@ def test_agent_configuration(): def test_incorrect_type(): valid_config = AgentConfiguration.from_dict(AGENT_CONFIGURATION) - with pytest.raises(Exception): + with pytest.raises(InvalidConfigurationError): valid_config_dict = valid_config.__dict__ valid_config_dict["keep_tunnel_open_time"] = "not_a_float" AgentConfiguration(**valid_config_dict) @@ -193,6 +197,14 @@ def test_from_dict(): assert schema.dump(config) == dict_ +def test_from_dict__invalid_data(): + dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + dict_["payloads"] = "payloads" + + with pytest.raises(InvalidConfigurationError): + AgentConfiguration.from_dict(dict_) + + def test_from_json(): schema = AgentConfigurationSchema() dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) @@ -202,6 +214,14 @@ def test_from_json(): assert schema.dump(config) == dict_ +def test_from_json__invalid_data(): + invalid_dict = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + invalid_dict["payloads"] = "payloads" + + with pytest.raises(InvalidConfigurationError): + AgentConfiguration.from_json(json.dumps(invalid_dict)) + + def test_to_json(): config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) From e2f365a1f9c8031b505577d9c15f4508c9397f72 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 13:22:41 -0400 Subject: [PATCH 170/196] Common: Rename dict_ -> config_dict --- monkey/common/configuration/agent_configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 87ccce3b0..e3d039ecc 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -45,9 +45,9 @@ class AgentConfiguration: raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") @staticmethod - def from_dict(dict_: dict): + def from_dict(config_dict: dict): try: - config_dict = AgentConfigurationSchema().load(dict_) + config_dict = AgentConfigurationSchema().load(config_dict) return AgentConfiguration(**config_dict) except MarshmallowError as err: raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") From 334d2a790f5fe9e3475005d1c8213d6b9afc7bec Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 13:24:02 -0400 Subject: [PATCH 171/196] Common: Rename from_dict() -> from_mapping() --- monkey/common/configuration/agent_configuration.py | 6 +++--- monkey/infection_monkey/master/control_channel.py | 2 +- .../tests/data_for_tests/monkey_configs/default_config.py | 2 +- .../in_memory_agent_configuration_repository.py | 2 +- .../common/configuration/test_agent_configuration.py | 8 ++++---- .../test_file_agent_configuration_repository.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index e3d039ecc..72b26d2f0 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import List +from typing import Any, List, Mapping from marshmallow import Schema, fields from marshmallow.exceptions import MarshmallowError @@ -45,9 +45,9 @@ class AgentConfiguration: raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") @staticmethod - def from_dict(config_dict: dict): + def from_mapping(config_mapping: Mapping[str, Any]) -> AgentConfiguration: try: - config_dict = AgentConfigurationSchema().load(config_dict) + config_dict = AgentConfigurationSchema().load(config_mapping) return AgentConfiguration(**config_dict) except MarshmallowError as err: raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") diff --git a/monkey/infection_monkey/master/control_channel.py b/monkey/infection_monkey/master/control_channel.py index 8567a301f..c83942f5d 100644 --- a/monkey/infection_monkey/master/control_channel.py +++ b/monkey/infection_monkey/master/control_channel.py @@ -58,7 +58,7 @@ class ControlChannel(IControlChannel): ) response.raise_for_status() - return AgentConfiguration.from_dict(json.loads(response.text)["config"]) + return AgentConfiguration.from_mapping(json.loads(response.text)["config"]) except ( json.JSONDecodeError, requests.exceptions.ConnectionError, diff --git a/monkey/tests/data_for_tests/monkey_configs/default_config.py b/monkey/tests/data_for_tests/monkey_configs/default_config.py index 6c0771af0..c0eca7c43 100644 --- a/monkey/tests/data_for_tests/monkey_configs/default_config.py +++ b/monkey/tests/data_for_tests/monkey_configs/default_config.py @@ -116,4 +116,4 @@ flat_config = { }, } -DEFAULT_CONFIG = AgentConfiguration.from_dict(flat_config) +DEFAULT_CONFIG = AgentConfiguration.from_mapping(flat_config) diff --git a/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py b/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py index b18465568..e9bcbae62 100644 --- a/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py +++ b/monkey/tests/monkey_island/in_memory_agent_configuration_repository.py @@ -6,7 +6,7 @@ from monkey_island.cc.repository import IAgentConfigurationRepository class InMemoryAgentConfigurationRepository(IAgentConfigurationRepository): def __init__(self): - self._configuration = AgentConfiguration.from_dict(AGENT_CONFIGURATION) + self._configuration = AgentConfiguration.from_mapping(AGENT_CONFIGURATION) def get_configuration(self): return self._configuration diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index f8c8f4c38..4b264c8cb 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -161,7 +161,7 @@ def test_propagation_configuration(): def test_agent_configuration(): - config = AgentConfiguration.from_dict(AGENT_CONFIGURATION) + config = AgentConfiguration.from_mapping(AGENT_CONFIGURATION) config_json = AgentConfiguration.to_json(config) assert isinstance(config, AgentConfiguration) @@ -175,7 +175,7 @@ def test_agent_configuration(): def test_incorrect_type(): - valid_config = AgentConfiguration.from_dict(AGENT_CONFIGURATION) + valid_config = AgentConfiguration.from_mapping(AGENT_CONFIGURATION) with pytest.raises(InvalidConfigurationError): valid_config_dict = valid_config.__dict__ valid_config_dict["keep_tunnel_open_time"] = "not_a_float" @@ -192,7 +192,7 @@ def test_from_dict(): schema = AgentConfigurationSchema() dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) - config = AgentConfiguration.from_dict(dict_) + config = AgentConfiguration.from_mapping(dict_) assert schema.dump(config) == dict_ @@ -202,7 +202,7 @@ def test_from_dict__invalid_data(): dict_["payloads"] = "payloads" with pytest.raises(InvalidConfigurationError): - AgentConfiguration.from_dict(dict_) + AgentConfiguration.from_mapping(dict_) def test_from_json(): diff --git a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py index 21dc4503b..fb7863dc3 100644 --- a/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py +++ b/monkey/tests/unit_tests/monkey_island/cc/repository/test_file_agent_configuration_repository.py @@ -12,7 +12,7 @@ def repository(default_agent_configuration): def test_store_agent_config(repository): - agent_configuration = AgentConfiguration.from_dict(AGENT_CONFIGURATION) + agent_configuration = AgentConfiguration.from_mapping(AGENT_CONFIGURATION) repository.store_configuration(agent_configuration) retrieved_agent_configuration = repository.get_configuration() From 8cb045d63589a98b7b701f697726601c1c2d2be6 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 13:25:09 -0400 Subject: [PATCH 172/196] Common: Fix incorrect type hints on AgentConfiguration.from_json() --- monkey/common/configuration/agent_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index 72b26d2f0..b7dcb90c5 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -53,7 +53,7 @@ class AgentConfiguration: raise InvalidConfigurationError(f"{INVALID_CONFIGURATION_ERROR_MESSAGE}: {err}") @staticmethod - def from_json(config_json: dict): + def from_json(config_json: str) -> AgentConfiguration: try: config_dict = AgentConfigurationSchema().loads(config_json) return AgentConfiguration(**config_dict) From 568eb4ff3bd6de0663fed81a93f2d997caf6d3b4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 13:30:43 -0400 Subject: [PATCH 173/196] Common: Add docstrings to static methods in AgentConfiguration --- .../configuration/agent_configuration.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/monkey/common/configuration/agent_configuration.py b/monkey/common/configuration/agent_configuration.py index b7dcb90c5..097b86382 100644 --- a/monkey/common/configuration/agent_configuration.py +++ b/monkey/common/configuration/agent_configuration.py @@ -46,6 +46,15 @@ class AgentConfiguration: @staticmethod def from_mapping(config_mapping: Mapping[str, Any]) -> AgentConfiguration: + """ + Construct an AgentConfiguration from a Mapping + + :param config_mapping: A Mapping that represents an AgentConfiguration + :return: An AgentConfiguration + :raises: InvalidConfigurationError if the provided Mapping does not represent a valid + AgentConfiguration + """ + try: config_dict = AgentConfigurationSchema().load(config_mapping) return AgentConfiguration(**config_dict) @@ -54,6 +63,14 @@ class AgentConfiguration: @staticmethod def from_json(config_json: str) -> AgentConfiguration: + """ + Construct an AgentConfiguration from a JSON string + + :param config_json: A JSON string that represents an AgentConfiguration + :return: An AgentConfiguration + :raises: InvalidConfigurationError if the provided JSON does not represent a valid + AgentConfiguration + """ try: config_dict = AgentConfigurationSchema().loads(config_json) return AgentConfiguration(**config_dict) @@ -62,6 +79,12 @@ class AgentConfiguration: @staticmethod def to_json(config: AgentConfiguration) -> str: + """ + Serialize an AgentConfiguration to JSON + + :param config: An AgentConfiguration + :return: A JSON string representing the AgentConfiguration + """ return AgentConfigurationSchema().dumps(config) From 84fc78cbf84f115dcb1de9cd85eb465d4268f6ae Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 14:47:15 -0400 Subject: [PATCH 174/196] UT: Remove unused imports from conftest.py --- monkey/tests/unit_tests/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/monkey/tests/unit_tests/conftest.py b/monkey/tests/unit_tests/conftest.py index c54ccce23..3634e52b9 100644 --- a/monkey/tests/unit_tests/conftest.py +++ b/monkey/tests/unit_tests/conftest.py @@ -5,9 +5,6 @@ from typing import Callable, Dict import pytest from _pytest.monkeypatch import MonkeyPatch -from tests.data_for_tests.monkey_configs.default_config import DEFAULT_CONFIG - -from common.configuration import AgentConfiguration MONKEY_BASE_PATH = str(Path(__file__).parent.parent.parent) sys.path.insert(0, MONKEY_BASE_PATH) From b219ca09172ca6833ade893a574384085f587b01 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 14:47:55 -0400 Subject: [PATCH 175/196] UT: Fix line that was too long --- .../unit_tests/infection_monkey/master/test_propagator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py index 847d92bcd..2ebdcd84e 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_propagator.py @@ -140,7 +140,9 @@ class StubExploiter: pass -def get_propagation_config(default_agent_configuration, scan_target_config: ScanTargetConfiguration): +def get_propagation_config( + default_agent_configuration, scan_target_config: ScanTargetConfiguration +): network_scan = NetworkScanConfiguration( default_agent_configuration.propagation.network_scan.tcp, default_agent_configuration.propagation.network_scan.icmp, From 33ec4f7ae963a43c8e977f9bf7729a09a17a24ab Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 15:16:43 -0400 Subject: [PATCH 176/196] Agent: Log configuration when it's received from the Island --- monkey/infection_monkey/master/control_channel.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/control_channel.py b/monkey/infection_monkey/master/control_channel.py index c83942f5d..d68f42bda 100644 --- a/monkey/infection_monkey/master/control_channel.py +++ b/monkey/infection_monkey/master/control_channel.py @@ -1,5 +1,6 @@ import json import logging +from pprint import pformat from typing import Mapping import requests @@ -58,7 +59,10 @@ class ControlChannel(IControlChannel): ) response.raise_for_status() - return AgentConfiguration.from_mapping(json.loads(response.text)["config"]) + config_dict = json.loads(response.text)["config"] + logger.debug(f"Received configuration:\n{pformat(json.loads(response.text))}") + + return AgentConfiguration.from_mapping(config_dict) except ( json.JSONDecodeError, requests.exceptions.ConnectionError, From dc9b91d43096a13c5ae11af8b7d9f6cf02dad02c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 15:32:06 -0400 Subject: [PATCH 177/196] Agent: Use Iterable instead of List in type hint --- monkey/infection_monkey/master/automated_master.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index e955311b5..c347e8627 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,7 +1,7 @@ import logging import threading import time -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple from common.configuration import PluginConfiguration from common.utils import Timer @@ -212,7 +212,7 @@ class AutomatedMaster(IMaster): def _run_pbas( self, - plugins: List[PluginConfiguration], + plugins: Iterable[PluginConfiguration], callback: Callable[[Any], None], custom_pba_options: Dict, ): @@ -226,7 +226,10 @@ class AutomatedMaster(IMaster): ) def _run_plugins( - self, plugins: List[PluginConfiguration], plugin_type: str, callback: Callable[[Any], None] + self, + plugins: Iterable[PluginConfiguration], + plugin_type: str, + callback: Callable[[Any], None], ): logger.info(f"Running {plugin_type}s") logger.debug(f"Found {len(plugins)} {plugin_type}(s) to run") From 8886ebc8b8311408ba73eb8a7a9697399d170ddb Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 15:33:47 -0400 Subject: [PATCH 178/196] Agent: Remove unnecessary local variables --- monkey/infection_monkey/master/automated_master.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index c347e8627..f3624d03d 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -198,10 +198,7 @@ class AutomatedMaster(IMaster): logger.debug(f"No credentials were collected by {collector}") def _run_pba(self, pba: PluginConfiguration): - name = pba.name - options = pba.options - - for pba_data in self._puppet.run_pba(name, options): + for pba_data in self._puppet.run_pba(pba.name, pba.options): self._telemetry_messenger.send_telemetry(PostBreachTelem(pba_data)) def _run_payload(self, payload: Tuple[str, Dict]): From 4f7d8be6ba11bbe204c09ade56dd15b2a4e58dc4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 15:35:41 -0400 Subject: [PATCH 179/196] Agent: Use PluginConfiguration in _run_payload() --- monkey/infection_monkey/master/automated_master.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index f3624d03d..003c9873c 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,7 +1,7 @@ import logging import threading import time -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, List, Optional from common.configuration import PluginConfiguration from common.utils import Timer @@ -201,11 +201,8 @@ class AutomatedMaster(IMaster): for pba_data in self._puppet.run_pba(pba.name, pba.options): self._telemetry_messenger.send_telemetry(PostBreachTelem(pba_data)) - def _run_payload(self, payload: Tuple[str, Dict]): - name = payload.name - options = payload.options - - self._puppet.run_payload(name, options, self._stop) + def _run_payload(self, payload: PluginConfiguration): + self._puppet.run_payload(payload.name, payload.options, self._stop) def _run_pbas( self, From 503a0a833f8dd652587cceceea0d28201b9c7f3c Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 19:19:51 -0400 Subject: [PATCH 180/196] Agent: Use Sequence instead of List for type hints --- monkey/infection_monkey/master/exploiter.py | 8 ++++---- monkey/infection_monkey/master/ip_scanner.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/monkey/infection_monkey/master/exploiter.py b/monkey/infection_monkey/master/exploiter.py index 60910226d..53665da38 100644 --- a/monkey/infection_monkey/master/exploiter.py +++ b/monkey/infection_monkey/master/exploiter.py @@ -5,7 +5,7 @@ from copy import deepcopy from itertools import chain from queue import Queue from threading import Event -from typing import Callable, Dict, List +from typing import Callable, Dict, Sequence from common import OperatingSystems from common.configuration.agent_sub_configurations import ( @@ -81,7 +81,7 @@ class Exploiter: @staticmethod def _process_exploiter_config( exploiter_config: ExploitationConfiguration, - ) -> List[ExploiterConfiguration]: + ) -> Sequence[ExploiterConfiguration]: # Run vulnerability exploiters before brute force exploiters to minimize the effect of # account lockout due to invalid credentials ordered_exploiters = chain(exploiter_config.vulnerability, exploiter_config.brute_force) @@ -98,7 +98,7 @@ class Exploiter: def _exploit_hosts_on_queue( self, - exploiters_to_run: List[ExploiterConfiguration], + exploiters_to_run: Sequence[ExploiterConfiguration], hosts_to_exploit: Queue, current_depth: int, results_callback: Callback, @@ -125,7 +125,7 @@ class Exploiter: def _run_all_exploiters( self, - exploiters_to_run: List[ExploiterConfiguration], + exploiters_to_run: Sequence[ExploiterConfiguration], victim_host: VictimHost, current_depth: int, results_callback: Callback, diff --git a/monkey/infection_monkey/master/ip_scanner.py b/monkey/infection_monkey/master/ip_scanner.py index 14391765f..071abe95a 100644 --- a/monkey/infection_monkey/master/ip_scanner.py +++ b/monkey/infection_monkey/master/ip_scanner.py @@ -3,7 +3,7 @@ import queue import threading from queue import Queue from threading import Event -from typing import Callable, Dict, List +from typing import Callable, Dict, Sequence from common.configuration.agent_sub_configurations import ( NetworkScanConfiguration, @@ -34,7 +34,7 @@ class IPScanner: def scan( self, - addresses_to_scan: List[NetworkAddress], + addresses_to_scan: Sequence[NetworkAddress], options: ScanTargetConfiguration, results_callback: Callback, stop: Event, @@ -99,7 +99,7 @@ class IPScanner: def _run_fingerprinters( self, ip: str, - fingerprinters: List[PluginConfiguration], + fingerprinters: Sequence[PluginConfiguration], ping_scan_data: PingScanData, port_scan_data: Dict[int, PortScanData], stop: Event, From fefd2daf2bb94fd48e2d9701433a4213eb5d3a9f Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Fri, 24 Jun 2022 19:21:48 -0400 Subject: [PATCH 181/196] Agent: Use Mapping instead of Dict --- monkey/infection_monkey/master/automated_master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 003c9873c..9333554b7 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,7 +1,7 @@ import logging import threading import time -from typing import Any, Callable, Dict, Iterable, List, Optional +from typing import Any, Callable, Iterable, List, Mapping, Optional from common.configuration import PluginConfiguration from common.utils import Timer @@ -208,7 +208,7 @@ class AutomatedMaster(IMaster): self, plugins: Iterable[PluginConfiguration], callback: Callable[[Any], None], - custom_pba_options: Dict, + custom_pba_options: Mapping, ): self._run_plugins(plugins, "post-breach action", callback) From 4b7ab058c6b69e941c44bb4881f809cc57f577be Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 27 Jun 2022 09:36:56 +0300 Subject: [PATCH 182/196] Agent: Fix typehints in _run_pbas of automated_master.py Typehint was Mapping, when it was using and calling other methods with CustomPBAConfiguration --- monkey/infection_monkey/master/automated_master.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index 9333554b7..ad73b50f4 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -1,9 +1,9 @@ import logging import threading import time -from typing import Any, Callable, Iterable, List, Mapping, Optional +from typing import Any, Callable, Iterable, List, Optional -from common.configuration import PluginConfiguration +from common.configuration import CustomPBAConfiguration, PluginConfiguration from common.utils import Timer from infection_monkey.credential_store import ICredentialsStore from infection_monkey.i_control_channel import IControlChannel, IslandCommunicationError @@ -208,13 +208,13 @@ class AutomatedMaster(IMaster): self, plugins: Iterable[PluginConfiguration], callback: Callable[[Any], None], - custom_pba_options: Mapping, + custom_pba_options: CustomPBAConfiguration, ): self._run_plugins(plugins, "post-breach action", callback) if custom_pba_is_enabled(custom_pba_options): self._run_plugins( - [PluginConfiguration(name="CustomPBA", options=custom_pba_options)], + [PluginConfiguration(name="CustomPBA", options=custom_pba_options.__dict__)], "post-breach action", callback, ) From 7179f9128c16c535af252ba834aedf5312445b76 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 27 Jun 2022 10:01:15 +0300 Subject: [PATCH 183/196] Agent: Fix typehints in clear_command_history.py --- .../post_breach/actions/clear_command_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monkey/infection_monkey/post_breach/actions/clear_command_history.py b/monkey/infection_monkey/post_breach/actions/clear_command_history.py index 2641051cc..7a5a350f5 100644 --- a/monkey/infection_monkey/post_breach/actions/clear_command_history.py +++ b/monkey/infection_monkey/post_breach/actions/clear_command_history.py @@ -16,7 +16,7 @@ class ClearCommandHistory(PBA): super().__init__(telemetry_messenger, name=POST_BREACH_CLEAR_CMD_HISTORY) def run(self, options: Dict) -> Iterable[PostBreachData]: - results = [pba.run() for pba in self.clear_command_history_pba_list()] + results = [pba.run(options) for pba in self.clear_command_history_pba_list()] if results: # `self.command` is empty here self.pba_data.append(PostBreachData(self.name, self.command, results)) @@ -53,7 +53,7 @@ class ClearCommandHistory(PBA): linux_cmd=linux_cmds, ) - def run(self) -> Tuple[str, bool]: + def run(self, options: Dict) -> Tuple[str, bool]: if self.command: try: output = subprocess.check_output( # noqa: DUO116 From c080f030117280d9a55a0937bcdf617fe0b75a51 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 27 Jun 2022 10:02:45 +0300 Subject: [PATCH 184/196] Agent: Fix _filter_none_values to be a static method --- monkey/monkey_island/cc/services/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monkey/monkey_island/cc/services/config.py b/monkey/monkey_island/cc/services/config.py index 1a8c7d006..6d14505c3 100644 --- a/monkey/monkey_island/cc/services/config.py +++ b/monkey/monkey_island/cc/services/config.py @@ -179,6 +179,7 @@ class ConfigService: should_encrypt=True, ) + @staticmethod def _filter_none_values(data): if isinstance(data, dict): return { From 232d6ba344b6e33225353386accdf05b10ca69c7 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 27 Jun 2022 11:20:27 +0300 Subject: [PATCH 185/196] Agent: Fix string formatting in http_tools.py Move line 60 to f formatting from the old %s style --- monkey/infection_monkey/exploit/tools/http_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/exploit/tools/http_tools.py b/monkey/infection_monkey/exploit/tools/http_tools.py index 5b73251d9..b27f6cf6f 100644 --- a/monkey/infection_monkey/exploit/tools/http_tools.py +++ b/monkey/infection_monkey/exploit/tools/http_tools.py @@ -57,6 +57,6 @@ class HTTPTools(object): httpd.start() lock.acquire() return ( - "http://%s:%s/%s" % (local_ip, local_port, urllib.parse.quote(host.os["type"].value)), + f"http://{local_ip}:{local_port}/{urllib.parse.quote(host.os['type'].value)}", httpd, ) From bf1d360e50e9a9c9a3f0dbc3afa3a4bf4f402945 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 06:55:39 -0400 Subject: [PATCH 186/196] UT: Remove disused DEFAULT_CONFIG --- .../monkey_configs/default_config.py | 119 ------------------ 1 file changed, 119 deletions(-) delete mode 100644 monkey/tests/data_for_tests/monkey_configs/default_config.py diff --git a/monkey/tests/data_for_tests/monkey_configs/default_config.py b/monkey/tests/data_for_tests/monkey_configs/default_config.py deleted file mode 100644 index c0eca7c43..000000000 --- a/monkey/tests/data_for_tests/monkey_configs/default_config.py +++ /dev/null @@ -1,119 +0,0 @@ -from common.configuration import AgentConfiguration - -flat_config = { - "keep_tunnel_open_time": 30, - "post_breach_actions": [ - {"name": "CommunicateAsBackdoorUser", "options": {}}, - {"name": "ModifyShellStartupFiles", "options": {}}, - {"name": "HiddenFiles", "options": {}}, - {"name": "TrapCommand", "options": {}}, - {"name": "ChangeSetuidSetgid", "options": {}}, - {"name": "ScheduleJobs", "options": {}}, - {"name": "Timestomping", "options": {}}, - {"name": "AccountDiscovery", "options": {}}, - {"name": "ProcessListCollection", "options": {}}, - ], - "credential_collectors": [ - {"name": "MimikatzCollector", "options": {}}, - {"name": "SSHCollector", "options": {}}, - ], - "payloads": [ - { - "name": "ransomware", - "options": { - "encryption": { - "enabled": True, - "directories": {"linux_target_dir": "", "windows_target_dir": ""}, - }, - "other_behaviors": {"readme": True}, - }, - } - ], - "custom_pbas": { - "linux_command": "", - "linux_filename": "", - "windows_command": "", - "windows_filename": "", - }, - "propagation": { - "network_scan": { - "tcp": { - "timeout": 3000, - "ports": [ - 22, - 80, - 135, - 443, - 445, - 2222, - 3306, - 3389, - 5985, - 5986, - 7001, - 8008, - 8080, - 8088, - 8983, - 9200, - 9600, - ], - }, - "icmp": {"timeout": 1000}, - "fingerprinters": [ - {"name": "elastic", "options": {}}, - { - "name": "http", - "options": {"http_ports": [80, 443, 7001, 8008, 8080, 8983, 9200, 9600]}, - }, - {"name": "mssql", "options": {}}, - {"name": "smb", "options": {}}, - {"name": "ssh", "options": {}}, - ], - "targets": { - "blocked_ips": [], - "inaccessible_subnets": [], - "local_network_scan": True, - "subnets": [], - }, - }, - "maximum_depth": 2, - "exploitation": { - "options": {"http_ports": [80, 443, 7001, 8008, 8080, 8983, 9200, 9600]}, - "brute_force": [ - { - "name": "MSSQLExploiter", - "options": {}, - }, - { - "name": "PowerShellExploiter", - "options": {}, - }, - { - "name": "SSHExploiter", - "options": {}, - }, - { - "name": "SmbExploiter", - "options": {"smb_download_timeout": 30}, - }, - { - "name": "WmiExploiter", - "options": {"smb_download_timeout": 30}, - }, - ], - "vulnerability": [ - { - "name": "HadoopExploiter", - "options": {}, - }, - { - "name": "Log4ShellExploiter", - "options": {}, - }, - ], - }, - }, -} - -DEFAULT_CONFIG = AgentConfiguration.from_mapping(flat_config) From 90259c1b7a8264bf107199240592d167793ad533 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 07:07:53 -0400 Subject: [PATCH 187/196] UT: Remove dependency on DEFAULT_AGENT_CONFIGURATION_JSON --- .../configuration/test_agent_configuration.py | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index 4b264c8cb..e06a4cf3e 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -1,4 +1,5 @@ import json +from copy import deepcopy import pytest from tests.common.example_agent_configuration import ( @@ -26,11 +27,7 @@ from tests.common.example_agent_configuration import ( WINDOWS_FILENAME, ) -from common.configuration import ( - DEFAULT_AGENT_CONFIGURATION_JSON, - AgentConfiguration, - InvalidConfigurationError, -) +from common.configuration import AgentConfiguration, InvalidConfigurationError from common.configuration.agent_configuration import AgentConfigurationSchema from common.configuration.agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, @@ -182,15 +179,9 @@ def test_incorrect_type(): AgentConfiguration(**valid_config_dict) -def test_default_agent_configuration(): - config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) - - assert isinstance(config, AgentConfiguration) - - def test_from_dict(): schema = AgentConfigurationSchema() - dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + dict_ = deepcopy(AGENT_CONFIGURATION) config = AgentConfiguration.from_mapping(dict_) @@ -198,7 +189,7 @@ def test_from_dict(): def test_from_dict__invalid_data(): - dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + dict_ = deepcopy(AGENT_CONFIGURATION) dict_["payloads"] = "payloads" with pytest.raises(InvalidConfigurationError): @@ -207,15 +198,16 @@ def test_from_dict__invalid_data(): def test_from_json(): schema = AgentConfigurationSchema() - dict_ = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + dict_ = deepcopy(AGENT_CONFIGURATION) - config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) + config = AgentConfiguration.from_json(json.dumps(dict_)) + assert isinstance(config, AgentConfiguration) assert schema.dump(config) == dict_ def test_from_json__invalid_data(): - invalid_dict = json.loads(DEFAULT_AGENT_CONFIGURATION_JSON) + invalid_dict = deepcopy(AGENT_CONFIGURATION) invalid_dict["payloads"] = "payloads" with pytest.raises(InvalidConfigurationError): @@ -223,8 +215,6 @@ def test_from_json__invalid_data(): def test_to_json(): - config = AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) + config = deepcopy(AGENT_CONFIGURATION) - assert json.loads(AgentConfiguration.to_json(config)) == json.loads( - DEFAULT_AGENT_CONFIGURATION_JSON - ) + assert json.loads(AgentConfiguration.to_json(config)) == AGENT_CONFIGURATION From e6d3854f74cae6393018b805618e2533160ccbb4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 08:22:41 -0400 Subject: [PATCH 188/196] Common: Remove DEFAULT_AGENT_CONFIGURATION_JSON It's easier to maintain object than a JSON string for the default configuration. --- monkey/common/configuration/__init__.py | 3 +- .../default_agent_configuration.py | 300 ++++++------------ .../monkey_island/cc/services/initialize.py | 4 +- monkey/tests/unit_tests/conftest.py | 4 +- 4 files changed, 109 insertions(+), 202 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index 06ce30b50..c7fefc11b 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -12,6 +12,5 @@ from .agent_sub_configurations import ( PropagationConfiguration, ) from .default_agent_configuration import ( - DEFAULT_AGENT_CONFIGURATION_JSON, - build_default_agent_configuration, + DEFAULT_AGENT_CONFIGURATION, ) diff --git a/monkey/common/configuration/default_agent_configuration.py b/monkey/common/configuration/default_agent_configuration.py index f3295a4b6..4eb8496a2 100644 --- a/monkey/common/configuration/default_agent_configuration.py +++ b/monkey/common/configuration/default_agent_configuration.py @@ -1,207 +1,115 @@ from . import AgentConfiguration +from .agent_sub_configurations import ( + CustomPBAConfiguration, + ExploitationConfiguration, + ExploitationOptionsConfiguration, + ExploiterConfiguration, + ICMPScanConfiguration, + NetworkScanConfiguration, + PluginConfiguration, + PropagationConfiguration, + ScanTargetConfiguration, + TCPScanConfiguration, +) -DEFAULT_AGENT_CONFIGURATION_JSON = """{ - "keep_tunnel_open_time": 30, - "post_breach_actions": [ - { - "name": "CommunicateAsBackdoorUser", - "options": {} - }, - { - "name": "ModifyShellStartupFiles", - "options": {} - }, - { - "name": "HiddenFiles", - "options": {} - }, - { - "name": "TrapCommand", - "options": {} - }, - { - "name": "ChangeSetuidSetgid", - "options": {} - }, - { - "name": "ScheduleJobs", - "options": {} - }, - { - "name": "Timestomping", - "options": {} - }, - { - "name": "AccountDiscovery", - "options": {} - }, - { - "name": "ProcessListCollection", - "options": {} - } - ], - "credential_collectors": [ - { - "name": "MimikatzCollector", - "options": {} - }, - { - "name": "SSHCollector", - "options": {} - } - ], - "payloads": [ - { - "name": "ransomware", - "options": { - "encryption": { - "enabled": true, - "directories": { - "linux_target_dir": "", - "windows_target_dir": "" - } - }, - "other_behaviors": { - "readme": true - } - } - } - ], - "custom_pbas": { - "linux_command": "", - "linux_filename": "", - "windows_command": "", - "windows_filename": "" - }, - "propagation": { - "maximum_depth": 2, - "network_scan": { - "tcp": { - "timeout": 3000, - "ports": [ - 22, - 80, - 135, - 443, - 445, - 2222, - 3306, - 3389, - 5985, - 5986, - 7001, - 8008, - 8080, - 8088, - 8983, - 9200, - 9600 - ] - }, - "icmp": { - "timeout": 1000 - }, - "fingerprinters": [ - { - "name": "elastic", - "options": {} - }, - { - "name": "http", - "options": { - "http_ports": [ - 80, - 443, - 7001, - 8008, - 8080, - 8983, - 9200, - 9600 - ] - } - }, - { - "name": "mssql", - "options": {} - }, - { - "name": "smb", - "options": {} - }, - { - "name": "ssh", - "options": {} - } - ], - "targets": { - "blocked_ips": [], - "inaccessible_subnets": [], - "local_network_scan": true, - "subnets": [] - } - }, - "exploitation": { - "options": { - "http_ports": [ - 80, - 443, - 7001, - 8008, - 8080, - 8983, - 9200, - 9600 - ] - }, - "brute_force": [ - { - "name": "MSSQLExploiter", - "options": {} +PBAS = [ + "CommunicateAsBackdoorUser", + "ModifyShellStartupFiles", + "HiddenFiles", + "TrapCommand", + "ChangeSetuidSetgid", + "ScheduleJobs", + "Timestomping", + "AccountDiscovery", + "ProcessListCollection", +] - }, - { - "name": "PowerShellExploiter", - "options": {} +CREDENTIAL_COLLECTORS = ["MimikatzCollector", "SSHCollector"] - }, - { - "name": "SSHExploiter", - "options": {} +PBA_CONFIGURATION = [PluginConfiguration(pba, {}) for pba in PBAS] +CREDENTIAL_COLLECTOR_CONFIGURATION = [ + PluginConfiguration(collector, {}) for collector in CREDENTIAL_COLLECTORS +] - }, - { - "name": "SmbExploiter", - "options": { - "smb_download_timeout": 30 - } +RANSOMWARE_OPTIONS = { + "encryption": { + "enabled": True, + "directories": {"linux_target_dir": "", "windows_target_dir": ""}, + }, + "other_behaviors": {"readme": True}, +} - }, - { - "name": "WmiExploiter", - "options": { - "smb_download_timeout": 30 - } +PAYLOAD_CONFIGURATION = [PluginConfiguration("ransomware", RANSOMWARE_OPTIONS)] - } - ], - "vulnerability": [ - { - "name": "HadoopExploiter", - "options": {} +CUSTOM_PBA_CONFIGURATION = CustomPBAConfiguration( + linux_command="", linux_filename="", windows_command="", windows_filename="" +) - }, - { - "name": "Log4ShellExploiter", - "options": {} +TCP_PORTS = [ + 22, + 80, + 135, + 443, + 445, + 2222, + 3306, + 3389, + 5985, + 5986, + 7001, + 8008, + 8080, + 8088, + 8983, + 9200, + 9600, +] - } - ] - } - } - } -""" +TCP_SCAN_CONFIGURATION = TCPScanConfiguration(timeout=3.0, ports=TCP_PORTS) +ICMP_CONFIGURATION = ICMPScanConfiguration(timeout=1.0) +HTTP_PORTS = [80, 443, 7001, 8008, 8080, 8983, 9200, 9600] +FINGERPRINTERS = [ + PluginConfiguration("elastic", {}), + PluginConfiguration("http", {"http_ports": HTTP_PORTS}), + PluginConfiguration("mssql", {}), + PluginConfiguration("smb", {}), + PluginConfiguration("ssh", {}), +] +SCAN_TARGET_CONFIGURATION = ScanTargetConfiguration([], [], True, []) +NETWORK_SCAN_CONFIGURATION = NetworkScanConfiguration( + TCP_SCAN_CONFIGURATION, ICMP_CONFIGURATION, FINGERPRINTERS, SCAN_TARGET_CONFIGURATION +) -def build_default_agent_configuration() -> AgentConfiguration: - return AgentConfiguration.from_json(DEFAULT_AGENT_CONFIGURATION_JSON) +EXPLOITATION_OPTIONS_CONFIGURATION = ExploitationOptionsConfiguration(HTTP_PORTS) +BRUTE_FORCE_EXPLOITERS = [ + ExploiterConfiguration("MSSQLExploiter", {}), + ExploiterConfiguration("PowerShellExploiter", {}), + ExploiterConfiguration("SSHExploiter", {}), + ExploiterConfiguration("SmbExploiter", {"smb_download_timeout": 30}), + ExploiterConfiguration("WmiExploiter", {"smb_download_timeout": 30}), +] + +VULNERABILITY_EXPLOITERS = [ + ExploiterConfiguration("Log4ShellExploiter", {}), + ExploiterConfiguration("HadoopExploiter", {}), +] + +EXPLOITATION_CONFIGURATION = ExploitationConfiguration( + EXPLOITATION_OPTIONS_CONFIGURATION, BRUTE_FORCE_EXPLOITERS, VULNERABILITY_EXPLOITERS +) + +PROPAGATION_CONFIGURATION = PropagationConfiguration( + maximum_depth=2, + network_scan=NETWORK_SCAN_CONFIGURATION, + exploitation=EXPLOITATION_CONFIGURATION, +) + +DEFAULT_AGENT_CONFIGURATION = AgentConfiguration( + keep_tunnel_open_time=30, + custom_pbas=CUSTOM_PBA_CONFIGURATION, + post_breach_actions=PBA_CONFIGURATION, + credential_collectors=CREDENTIAL_COLLECTOR_CONFIGURATION, + payloads=PAYLOAD_CONFIGURATION, + propagation=PROPAGATION_CONFIGURATION, +) diff --git a/monkey/monkey_island/cc/services/initialize.py b/monkey/monkey_island/cc/services/initialize.py index 52343bbf5..922c3654b 100644 --- a/monkey/monkey_island/cc/services/initialize.py +++ b/monkey/monkey_island/cc/services/initialize.py @@ -3,7 +3,7 @@ from pathlib import Path from common import DIContainer from common.aws import AWSInstance -from common.configuration import AgentConfiguration, build_default_agent_configuration +from common.configuration import DEFAULT_AGENT_CONFIGURATION, AgentConfiguration from common.utils.file_utils import get_binary_io_sha256_hash from monkey_island.cc.repository import ( AgentBinaryRepository, @@ -32,7 +32,7 @@ def initialize_services(data_dir: Path) -> DIContainer: container.register_convention(Path, "data_dir", data_dir) container.register_convention( - AgentConfiguration, "default_agent_configuration", build_default_agent_configuration() + AgentConfiguration, "default_agent_configuration", DEFAULT_AGENT_CONFIGURATION ) container.register_instance(AWSInstance, AWSInstance()) diff --git a/monkey/tests/unit_tests/conftest.py b/monkey/tests/unit_tests/conftest.py index 3634e52b9..51528ba00 100644 --- a/monkey/tests/unit_tests/conftest.py +++ b/monkey/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ from _pytest.monkeypatch import MonkeyPatch MONKEY_BASE_PATH = str(Path(__file__).parent.parent.parent) sys.path.insert(0, MONKEY_BASE_PATH) -from common.configuration import AgentConfiguration, build_default_agent_configuration # noqa: E402 +from common.configuration import DEFAULT_AGENT_CONFIGURATION, AgentConfiguration # noqa: E402 @pytest.fixture(scope="session") @@ -60,4 +60,4 @@ def load_monkey_config(data_for_tests_dir) -> Callable[[str], Dict]: @pytest.fixture def default_agent_configuration() -> AgentConfiguration: - return build_default_agent_configuration() + return DEFAULT_AGENT_CONFIGURATION From 4ef17ccc9bc700dcf009e568ad4876b8cc1788d3 Mon Sep 17 00:00:00 2001 From: vakarisz Date: Mon, 27 Jun 2022 14:25:35 +0300 Subject: [PATCH 189/196] Agent: Replace ExploiterConfig with PluginConfig ExploiterConfig was exact match of PluginConfig, so they got merged --- monkey/common/configuration/__init__.py | 1 - .../agent_sub_configuration_schemas.py | 14 ++------------ .../configuration/agent_sub_configurations.py | 10 ++-------- .../configuration/default_agent_configuration.py | 15 +++++++-------- monkey/infection_monkey/master/exploiter.py | 10 +++++----- .../configuration/test_agent_configuration.py | 3 +-- .../infection_monkey/master/test_exploiter.py | 10 +++++----- 7 files changed, 22 insertions(+), 41 deletions(-) diff --git a/monkey/common/configuration/__init__.py b/monkey/common/configuration/__init__.py index c7fefc11b..fc1f3c84d 100644 --- a/monkey/common/configuration/__init__.py +++ b/monkey/common/configuration/__init__.py @@ -7,7 +7,6 @@ from .agent_sub_configurations import ( TCPScanConfiguration, NetworkScanConfiguration, ExploitationOptionsConfiguration, - ExploiterConfiguration, ExploitationConfiguration, PropagationConfiguration, ) diff --git a/monkey/common/configuration/agent_sub_configuration_schemas.py b/monkey/common/configuration/agent_sub_configuration_schemas.py index 4d2ee2d8e..bf4d3b8c7 100644 --- a/monkey/common/configuration/agent_sub_configuration_schemas.py +++ b/monkey/common/configuration/agent_sub_configuration_schemas.py @@ -4,7 +4,6 @@ from .agent_sub_configurations import ( CustomPBAConfiguration, ExploitationConfiguration, ExploitationOptionsConfiguration, - ExploiterConfiguration, ICMPScanConfiguration, NetworkScanConfiguration, PluginConfiguration, @@ -81,19 +80,10 @@ class ExploitationOptionsConfigurationSchema(Schema): return ExploitationOptionsConfiguration(**data) -class ExploiterConfigurationSchema(Schema): - name = fields.Str() - options = fields.Mapping() - - @post_load - def _make_exploiter_configuration(self, data, **kwargs): - return ExploiterConfiguration(**data) - - class ExploitationConfigurationSchema(Schema): options = fields.Nested(ExploitationOptionsConfigurationSchema) - brute_force = fields.List(fields.Nested(ExploiterConfigurationSchema)) - vulnerability = fields.List(fields.Nested(ExploiterConfigurationSchema)) + brute_force = fields.List(fields.Nested(PluginConfigurationSchema)) + vulnerability = fields.List(fields.Nested(PluginConfigurationSchema)) @post_load def _make_exploitation_options_configuration(self, data, **kwargs): diff --git a/monkey/common/configuration/agent_sub_configurations.py b/monkey/common/configuration/agent_sub_configurations.py index c4a0c704c..d93b4d774 100644 --- a/monkey/common/configuration/agent_sub_configurations.py +++ b/monkey/common/configuration/agent_sub_configurations.py @@ -48,17 +48,11 @@ class ExploitationOptionsConfiguration: http_ports: List[int] -@dataclass(frozen=True) -class ExploiterConfiguration: - name: str - options: Dict - - @dataclass(frozen=True) class ExploitationConfiguration: options: ExploitationOptionsConfiguration - brute_force: List[ExploiterConfiguration] - vulnerability: List[ExploiterConfiguration] + brute_force: List[PluginConfiguration] + vulnerability: List[PluginConfiguration] @dataclass(frozen=True) diff --git a/monkey/common/configuration/default_agent_configuration.py b/monkey/common/configuration/default_agent_configuration.py index 4eb8496a2..251676017 100644 --- a/monkey/common/configuration/default_agent_configuration.py +++ b/monkey/common/configuration/default_agent_configuration.py @@ -3,7 +3,6 @@ from .agent_sub_configurations import ( CustomPBAConfiguration, ExploitationConfiguration, ExploitationOptionsConfiguration, - ExploiterConfiguration, ICMPScanConfiguration, NetworkScanConfiguration, PluginConfiguration, @@ -83,16 +82,16 @@ NETWORK_SCAN_CONFIGURATION = NetworkScanConfiguration( EXPLOITATION_OPTIONS_CONFIGURATION = ExploitationOptionsConfiguration(HTTP_PORTS) BRUTE_FORCE_EXPLOITERS = [ - ExploiterConfiguration("MSSQLExploiter", {}), - ExploiterConfiguration("PowerShellExploiter", {}), - ExploiterConfiguration("SSHExploiter", {}), - ExploiterConfiguration("SmbExploiter", {"smb_download_timeout": 30}), - ExploiterConfiguration("WmiExploiter", {"smb_download_timeout": 30}), + PluginConfiguration("MSSQLExploiter", {}), + PluginConfiguration("PowerShellExploiter", {}), + PluginConfiguration("SSHExploiter", {}), + PluginConfiguration("SmbExploiter", {"smb_download_timeout": 30}), + PluginConfiguration("WmiExploiter", {"smb_download_timeout": 30}), ] VULNERABILITY_EXPLOITERS = [ - ExploiterConfiguration("Log4ShellExploiter", {}), - ExploiterConfiguration("HadoopExploiter", {}), + PluginConfiguration("Log4ShellExploiter", {}), + PluginConfiguration("HadoopExploiter", {}), ] EXPLOITATION_CONFIGURATION = ExploitationConfiguration( diff --git a/monkey/infection_monkey/master/exploiter.py b/monkey/infection_monkey/master/exploiter.py index 53665da38..6171576f2 100644 --- a/monkey/infection_monkey/master/exploiter.py +++ b/monkey/infection_monkey/master/exploiter.py @@ -10,7 +10,7 @@ from typing import Callable, Dict, Sequence from common import OperatingSystems from common.configuration.agent_sub_configurations import ( ExploitationConfiguration, - ExploiterConfiguration, + PluginConfiguration, ) from infection_monkey.custom_types import PropagationCredentials from infection_monkey.i_puppet import ExploiterResultData, IPuppet @@ -81,7 +81,7 @@ class Exploiter: @staticmethod def _process_exploiter_config( exploiter_config: ExploitationConfiguration, - ) -> Sequence[ExploiterConfiguration]: + ) -> Sequence[PluginConfiguration]: # Run vulnerability exploiters before brute force exploiters to minimize the effect of # account lockout due to invalid credentials ordered_exploiters = chain(exploiter_config.vulnerability, exploiter_config.brute_force) @@ -92,13 +92,13 @@ class Exploiter: # This order allows exploiter-specific options to # override general options for all exploiters. options = {**exploiter_config.options.__dict__, **exploiter.options} - extended_exploiters.append(ExploiterConfiguration(exploiter.name, options)) + extended_exploiters.append(PluginConfiguration(exploiter.name, options)) return extended_exploiters def _exploit_hosts_on_queue( self, - exploiters_to_run: Sequence[ExploiterConfiguration], + exploiters_to_run: Sequence[PluginConfiguration], hosts_to_exploit: Queue, current_depth: int, results_callback: Callback, @@ -125,7 +125,7 @@ class Exploiter: def _run_all_exploiters( self, - exploiters_to_run: Sequence[ExploiterConfiguration], + exploiters_to_run: Sequence[PluginConfiguration], victim_host: VictimHost, current_depth: int, results_callback: Callback, diff --git a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py index e06a4cf3e..fecb6a6f6 100644 --- a/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py +++ b/monkey/tests/unit_tests/common/configuration/test_agent_configuration.py @@ -33,7 +33,6 @@ from common.configuration.agent_sub_configuration_schemas import ( CustomPBAConfigurationSchema, ExploitationConfigurationSchema, ExploitationOptionsConfigurationSchema, - ExploiterConfigurationSchema, ICMPScanConfigurationSchema, NetworkScanConfigurationSchema, PluginConfigurationSchema, @@ -126,7 +125,7 @@ def test_exploitation_options_configuration_schema(): def test_exploiter_configuration_schema(): name = "bond" options = {"gun": "Walther PPK", "car": "Aston Martin DB5"} - schema = ExploiterConfigurationSchema() + schema = PluginConfigurationSchema() config = schema.load({"name": name, "options": options}) diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py index cc7e497b6..4a44cca95 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_exploiter.py @@ -10,7 +10,7 @@ from tests.unit_tests.infection_monkey.master.mock_puppet import MockPuppet from common import OperatingSystems from common.configuration.agent_sub_configurations import ( ExploitationConfiguration, - ExploiterConfiguration, + PluginConfiguration, ) from infection_monkey.master import Exploiter from infection_monkey.model import VictimHost @@ -41,11 +41,11 @@ def callback(): @pytest.fixture def exploiter_config(default_agent_configuration): brute_force = [ - ExploiterConfiguration(name="MSSQLExploiter", options={"timeout": 10}), - ExploiterConfiguration(name="SSHExploiter", options={}), - ExploiterConfiguration(name="WmiExploiter", options={"timeout": 10}), + PluginConfiguration(name="MSSQLExploiter", options={"timeout": 10}), + PluginConfiguration(name="SSHExploiter", options={}), + PluginConfiguration(name="WmiExploiter", options={"timeout": 10}), ] - vulnerability = [ExploiterConfiguration(name="ZerologonExploiter", options={})] + vulnerability = [PluginConfiguration(name="ZerologonExploiter", options={})] return ExploitationConfiguration( options=default_agent_configuration.propagation.exploitation.options, brute_force=brute_force, From a0de4b8f316283caf649e59e12ef06c2c6aa0f35 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:06:15 -0400 Subject: [PATCH 190/196] Agent: Use derived current_depth in AutomatedMaster --- monkey/infection_monkey/master/automated_master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/infection_monkey/master/automated_master.py b/monkey/infection_monkey/master/automated_master.py index ad73b50f4..214c19004 100644 --- a/monkey/infection_monkey/master/automated_master.py +++ b/monkey/infection_monkey/master/automated_master.py @@ -174,7 +174,7 @@ class AutomatedMaster(IMaster): current_depth = self._current_depth if self._current_depth is not None else 0 logger.info(f"Current depth is {current_depth}") - if maximum_depth_reached(config.propagation.maximum_depth, self._current_depth): + if maximum_depth_reached(config.propagation.maximum_depth, current_depth): self._propagator.propagate(config.propagation, current_depth, self._stop) else: logger.info("Skipping propagation: maximum depth reached") From 851296d5faf5ec1ce7f15e8004490de616f40d14 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:09:07 -0400 Subject: [PATCH 191/196] UT: Remove disused automated_master_config --- .../automated_master_config.json | 107 ------------------ .../unit_tests/infection_monkey/conftest.py | 5 - .../master/test_automated_master.py | 4 +- 3 files changed, 2 insertions(+), 114 deletions(-) delete mode 100644 monkey/tests/data_for_tests/monkey_configs/automated_master_config.json diff --git a/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json b/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json deleted file mode 100644 index c0c035b9d..000000000 --- a/monkey/tests/data_for_tests/monkey_configs/automated_master_config.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "config": { - "propagation": { - "network_scan": { - "tcp": { - "timeout_ms": 3000, - "ports": [ - 22, - 2222, - 445, - 135, - 3389, - 80, - 8080, - 443, - 8008, - 3306, - 7001, - 8088, - 9200 - ] - }, - "icmp": { - "timeout_ms": 1000 - }, - "fingerprinters": [ - "SMBFinger", - "SSHFinger", - "HTTPFinger", - "MySQLFinger", - "MSSQLFinger", - "ElasticFinger" - ] - }, - "targets": { - "blocked_ips": ["192.168.1.1", "192.168.1.100"], - "inaccessible_subnets": ["10.0.0.0/24", "10.0.10.0/24"], - "local_network_scan": true, - "subnet_scan_list": [ - "192.168.1.50", - "192.168.56.0/24", - "10.0.33.0/30", - "10.0.0.1", - "10.0.0.2" - ] - }, - "exploiters": { - "options": {}, - "brute_force": [ - {"name": "MSSQLExploiter", "options": {}}, - {"name": "PowerShellExploiter", "options": {}}, - {"name": "SmbExploiter", "options": {}}, - {"name": "SSHExploiter", "options": {}}, - {"name": "WmiExploiter", "options": {}} - ], - "vulnerability": [ - {"name": "HadoopExploiter", "options": {}}, - {"name": "ShellShockExploiter", "options": {}}, - {"name": "ZerologonExploiter", "options": {}} - ] - } - }, - "PBA_linux_filename": "", - "PBA_windows_filename": "", - "custom_pbas": { - "linux_command": "", - "windows_command": "" - }, - "depth": 2, - "exploit_lm_hash_list": ["DEADBEEF", "FACADE"], - "exploit_ntlm_hash_list": ["BEADED", "ACCEDE", "DECADE"], - "exploit_password_list": ["p1", "p2", "p3"], - "exploit_ssh_keys": "hidden", - "exploit_user_list": ["u1", "u2", "u3"], - "exploiter_classes": [], - "max_depth": 2, - "post_breach_actions": { - "CommunicateAsBackdoorUser": {}, - "ModifyShellStartupFiles": {}, - "HiddenFiles": {}, - "TrapCommand": {}, - "ChangeSetuidSetgid": {}, - "ScheduleJobs": {}, - "Timestomping": {}, - "AccountDiscovery": {}, - "Custom": { - "linux_command": "chmod u+x my_exec && ./my_exec", - "windows_cmd": "powershell test_driver.ps1", - "linux_filename": "my_exec", - "windows_filename": "test_driver.ps1" - } - }, - "payloads": { - "ransomware": { - "encryption": { - "directories": {"linux_target_dir": "", "windows_target_dir": ""}, - "enabled": true - }, - "other_behaviors": {"readme": true} - } - }, - "credential_collectors": [ - "MimikatzCollector", - "SSHCollector" - ] - } -} diff --git a/monkey/tests/unit_tests/infection_monkey/conftest.py b/monkey/tests/unit_tests/infection_monkey/conftest.py index 14a193112..533572f98 100644 --- a/monkey/tests/unit_tests/infection_monkey/conftest.py +++ b/monkey/tests/unit_tests/infection_monkey/conftest.py @@ -15,8 +15,3 @@ class TelemetryMessengerSpy(ITelemetryMessenger): @pytest.fixture def telemetry_messenger_spy(): return TelemetryMessengerSpy() - - -@pytest.fixture -def automated_master_config(load_monkey_config): - return load_monkey_config("automated_master_config.json") diff --git a/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py b/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py index cf0112d59..9029ce480 100644 --- a/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py +++ b/monkey/tests/unit_tests/infection_monkey/master/test_automated_master.py @@ -41,14 +41,14 @@ def test_stop_if_cant_get_config_from_island(monkeypatch): @pytest.fixture -def sleep_and_return_config(automated_master_config): +def sleep_and_return_config(default_agent_configuration): # Ensure that should_agent_stop times out before get_config() returns to prevent the # Propagator's sub-threads from hanging get_config_sleep_time = INTERVAL * (CHECK_FOR_STOP_AGENT_COUNT + 1) def _inner(): time.sleep(get_config_sleep_time) - return automated_master_config + return default_agent_configuration return _inner From 2fbab063cd0b55c65ccc9a582e5465711bbe46af Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:12:37 -0400 Subject: [PATCH 192/196] Common: Remove empty file --- monkey/common/agent_configuration | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 monkey/common/agent_configuration diff --git a/monkey/common/agent_configuration b/monkey/common/agent_configuration deleted file mode 100644 index e69de29bb..000000000 From b5d7b800bbf05fc759e91c84d62b167e5a3d4b24 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:15:26 -0400 Subject: [PATCH 193/196] Agent: Add TODO about GUID --- monkey/infection_monkey/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monkey/infection_monkey/config.py b/monkey/infection_monkey/config.py index bcf95ee72..a4c39ee13 100644 --- a/monkey/infection_monkey/config.py +++ b/monkey/infection_monkey/config.py @@ -1,3 +1,4 @@ import uuid +# TODO: Find a better place for this GUID = str(uuid.getnode()) From 08bac8ef39f55288fbf55d96d9a49b22673dfc21 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:24:32 -0400 Subject: [PATCH 194/196] Island: Added "raises" to store_configuration() docstring --- .../cc/repository/i_agent_configuration_repository.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py index 1e85fadf5..ef63121c2 100644 --- a/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py +++ b/monkey/monkey_island/cc/repository/i_agent_configuration_repository.py @@ -15,7 +15,7 @@ class IAgentConfigurationRepository(ABC): :return: The agent configuration as retrieved from the repository, or the default configuration if the repository is empty - :raises RetrievalError: if the configuration could not be retrieved + :raises RetrievalError: If the configuration could not be retrieved """ pass @@ -25,5 +25,6 @@ class IAgentConfigurationRepository(ABC): Store the agent configuration in the repository :param agent_configuration: The agent configuration to store in the repository + :raises StorageError: If the configuration could not be stored """ pass From b49dfcf79da3a348ddb55e6b44c960c4e73f30db Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:30:33 -0400 Subject: [PATCH 195/196] Island: Change error -> message in AgentConfiguration resource --- monkey/monkey_island/cc/resources/agent_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monkey/monkey_island/cc/resources/agent_configuration.py b/monkey/monkey_island/cc/resources/agent_configuration.py index 1bf311564..4715680a0 100644 --- a/monkey/monkey_island/cc/resources/agent_configuration.py +++ b/monkey/monkey_island/cc/resources/agent_configuration.py @@ -30,6 +30,6 @@ class AgentConfiguration(AbstractResource): return make_response({}, 200) except (InvalidConfigurationError, json.JSONDecodeError) as err: return make_response( - {"message": f"Invalid configuration supplied: {err}"}, + {"error": f"Invalid configuration supplied: {err}"}, 400, ) From 13a7e4ea317bf92200b3f4047596b585f9d63ae4 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Mon, 27 Jun 2022 09:36:52 -0400 Subject: [PATCH 196/196] Project: Remove OperatingSystems enums from vulture allowlist --- vulture_allowlist.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vulture_allowlist.py b/vulture_allowlist.py index dad5117bb..e1d56f689 100644 --- a/vulture_allowlist.py +++ b/vulture_allowlist.py @@ -195,8 +195,6 @@ _make_tcp_scan_configuration # unused method (monkey/common/configuration/agent _make_network_scan_configuration # unused method (monkey/common/configuration/agent_configuration.py:110) _make_propagation_configuration # unused method (monkey/common/configuration/agent_configuration.py:167) _make_agent_configuration # unused method (monkey/common/configuration/agent_configuration.py:192) -LINUX # unused variable (monkey/common/operating_systems.py:5) -WINDOWS # unused variable (monkey/common/operating_systems.py:6) # TODO DELETE AFTER RESOURCE REFACTORING