Merge pull request #507 from guardicore/os_compatibility_env

OS compatibility environment
This commit is contained in:
Shay Nehmad 2020-03-30 17:06:33 +03:00 committed by GitHub
commit bee46c0ec5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 815 additions and 0 deletions

View File

@ -0,0 +1,57 @@
# Monkey maker
## About
Monkey maker is an environment on AWS that
is designed for monkey binary building.
This environment is deployed using terraform scripts
located in this directory.
## Setup
To setup you need to put `accessKeys` file into `./aws_keys` directory.
Contents of `accessKeys` file should be as follows:
```ini
[default]
aws_access_key_id = <...>
aws_secret_access_key = <...>
```
Also review `./terraform/config.tf` file.
Launch the environment by going into `terraform` folder and running
```
terraform init
terraform apply
```
## Usage
To login to windows use Administrator: %HwuzI!Uzsyfa=cB*XaQ6xxHqopfj)h) credentials
You'll find docker files in `/home/ubuntu/docker_envs/linux/...`
To build docker image for 32 bit linux:
```
cd /home/ubuntu/docker_envs/linux/py3-32
sudo docker build -t builder32 .
```
To build docker image for 64 bit linux:
```
cd /home/ubuntu/docker_envs/linux/py3-64
sudo docker build -t builder64 .
```
To build 32 bit monkey binary:
```
cd /home/ubuntu/monkey_folder/monkey
sudo docker run -v "$(pwd):/src" builder32 -c "export SRCDIR=/src/infection_monkey && /entrypoint.sh"
```
To build 64 bit monkey binary:
```
cd /home/ubuntu/monkey_folder/monkey
sudo docker run -v "$(pwd):/src" builder64 -c "export SRCDIR=/src/infection_monkey && /entrypoint.sh"
```

4
envs/monkey_maker/aws_keys/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore

View File

@ -0,0 +1,5 @@
provider "aws" {
version = "~> 2.0"
region = "eu-central-1"
shared_credentials_file = "../aws_keys/accessKeys"
}

View File

@ -0,0 +1,61 @@
resource "aws_vpc" "monkey_maker" {
cidr_block = "10.0.0.0/24"
enable_dns_support = true
tags = {
Name = "monkey_maker_vpc"
}
}
resource "aws_internet_gateway" "monkey_maker_gateway" {
vpc_id = "${aws_vpc.monkey_maker.id}"
tags = {
Name = "monkey_maker_gateway"
}
}
// create routing table which points to the internet gateway
resource "aws_route_table" "monkey_maker_route" {
vpc_id = "${aws_vpc.monkey_maker.id}"
route {
cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.monkey_maker_gateway.id}"
}
tags = {
Name = "monkey_maker_route"
}
}
// associate the routing table with the subnet
resource "aws_route_table_association" "subnet-association" {
subnet_id = "${aws_subnet.main.id}"
route_table_id = "${aws_route_table.monkey_maker_route.id}"
}
resource "aws_subnet" "main" {
vpc_id = "${aws_vpc.monkey_maker.id}"
cidr_block = "10.0.0.0/24"
tags = {
Name = "Main"
}
}
resource "aws_security_group" "monkey_maker_sg" {
name = "monkey_maker_sg"
description = "Allow remote access to the island"
vpc_id = "${aws_vpc.monkey_maker.id}"
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = {
Name = "monkey_maker_sg"
}
}

View File

@ -0,0 +1,25 @@
resource "aws_instance" "island_windows" {
ami = "ami-033b3ef27f8d1881d"
instance_type = "t2.micro"
private_ip = "10.0.0.251"
subnet_id = "${aws_subnet.main.id}"
key_name = "monkey_maker"
tags = {
Name = "monkey_maker_windows"
}
vpc_security_group_ids = ["${aws_security_group.monkey_maker_sg.id}"]
associate_public_ip_address = true
}
resource "aws_instance" "island_linux" {
ami = "ami-0495203541087740a"
instance_type = "t2.micro"
private_ip = "10.0.0.252"
subnet_id = "${aws_subnet.main.id}"
key_name = "monkey_maker"
tags = {
Name = "monkey_maker_linux"
}
vpc_security_group_ids = ["${aws_security_group.monkey_maker_sg.id}"]
associate_public_ip_address = true
}

View File

@ -0,0 +1,79 @@
# OS compatibility
## About
OS compatibility is an environment on AWS that
is designed to test monkey binary compatibility on
different operating systems.
This environment is deployed using terraform scripts
located in this directory.
## Setup
To setup you need to put `accessKeys` file into `./aws_keys` directory.
Contents of `accessKeys` file should be as follows:
```
[default]
aws_access_key_id = <...>
aws_secret_access_key = <...>
```
Also review `./terraform/config.tf` file.
Launch the environment by going into `terraform` folder and running
```angular2html
terraform init
terraform apply
```
## Usage
0. Add your machine's IP to the `os_compat_island` security group ingress rules.
1. Launch os_compat_ISLAND machine and upload your binaries/update island. Reset island environment.
2. Launch/Reboot all other os_compat test machines (Can be filtered with tag "Purpose: os_compat_instance")
3. Wait until machines boot and run monkey
4. Launch `test_compatibility.py` pytest script with island ip parameter
(e.g. `test_compatibility.py --island 111.111.111.111:5000`)
## Machines
Since island machine is built from custom AMI it already has the following credentials:
Administrator: %tPHGz8ZuQsBnEUgdpz!6f&elGnFy?;.
For windows_2008_r2 Administrator:AGE(MP..txL
The following machines does not download monkey automatically, so you'll have to manually check them:
- os_compat_kali_2019
- os_compat_oracle_6
- os_compat_oracle_7
- windows_2003_r2_32
- windows_2003
- windows_2008_r2
A quick reference for usernames on different machines (if in doubt check official docs):
- Ubuntu: ubuntu
- Oracle: clckwrk
- CentOS: centos
- Everything else: ec2-user
To manually verify the machine is compatible use commands to download and execute the monkey.
Also, add your IP to `os_compat_instance` security group.
Example commands:
- Powershell:
```cmd
[System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy
Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction SilentlyContinue
Invoke-WebRequest -Uri 'https://10.0.0.251:5000/api/monkey/download/monkey-windows-64.exe' -OutFile 'C:\windows\temp\monkey-windows-64.exe' -UseBasicParsing
C:\windows\temp\monkey-windows-64.exe m0nk3y -s 10.0.0.251:5000
```
- Bash:
```shell script
wget --no-check-certificate -q https://10.0.0.251:5000/api/monkey/download/monkey-linux-64 -O ./monkey-linux-64 || curl https://10.0.0.251:5000/api/monkey/download/monkey-linux-64 -k -o monkey-linux-64
chmod +x ./monkey-linux-64
./monkey-linux-64 m0nk3y -s 10.0.0.251:5000
```

View File

@ -0,0 +1,4 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore

View File

@ -0,0 +1,11 @@
import pytest
def pytest_addoption(parser):
parser.addoption("--island", action="store", default="",
help="Specify the Monkey Island address (host+port).")
@pytest.fixture(scope='module')
def island(request):
return request.config.getoption("--island")

View File

@ -0,0 +1,5 @@
provider "aws" {
version = "~> 2.0"
region = "eu-central-1"
shared_credentials_file = "../aws_keys/accessKeys"
}

View File

@ -0,0 +1,92 @@
resource "aws_vpc" "os_compat_vpc" {
cidr_block = "10.0.0.0/24"
enable_dns_support = true
tags = {
Name = "os_compat_vpc"
}
}
resource "aws_internet_gateway" "os_compat_gateway" {
vpc_id = "${aws_vpc.os_compat_vpc.id}"
tags = {
Name = "os_compat_gateway"
}
}
// create routing table which points to the internet gateway
resource "aws_route_table" "os_compat_route" {
vpc_id = "${aws_vpc.os_compat_vpc.id}"
route {
cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.os_compat_gateway.id}"
}
tags = {
Name = "os_compat_route"
}
}
// associate the routing table with the subnet
resource "aws_route_table_association" "subnet-association" {
subnet_id = "${aws_subnet.main.id}"
route_table_id = "${aws_route_table.os_compat_route.id}"
}
resource "aws_subnet" "main" {
vpc_id = "${aws_vpc.os_compat_vpc.id}"
cidr_block = "10.0.0.0/24"
tags = {
Name = "Main"
}
}
resource "aws_security_group" "os_compat_island" {
name = "os_compat_island"
description = "Allow remote access to the island"
vpc_id = "${aws_vpc.os_compat_vpc.id}"
ingress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["10.0.0.0/24"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = {
Name = "os_compat_island"
}
}
resource "aws_security_group" "os_compat_instance" {
name = "os_compat_instance"
description = "Allow remote access to the machines"
vpc_id = "${aws_vpc.os_compat_vpc.id}"
ingress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["10.0.0.0/24"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = {
Name = "os_compat_instance"
}
}

View File

@ -0,0 +1,14 @@
resource "aws_instance" "os_test_machine" {
ami = "${var.ami}"
instance_type = "${var.type}"
private_ip = "${var.ip}"
subnet_id = "${data.aws_subnet.main.id}"
key_name = "os_compat"
tags = {
Name = "os_compat_${var.name}"
Purpose = "os_compat_instance"
}
vpc_security_group_ids = ["${data.aws_security_group.os_compat_instance.id}"]
associate_public_ip_address = true
user_data = "${var.user_data}"
}

View File

@ -0,0 +1,25 @@
variable "ami" {type=string}
variable "ip" {type=string}
variable "name" {type=string}
variable "type" {
type=string
default="t2.micro"
}
variable "user_data" {
type=string
default=""
}
variable "env_vars" {
type = object({
subnet_id = string
vpc_security_group_ids = string
})
}
data "aws_subnet" "main" {
id = "${var.env_vars.subnet_id}"
}
data "aws_security_group" "os_compat_instance" {
id = "${var.env_vars.vpc_security_group_ids}"
}

View File

@ -0,0 +1,371 @@
// Instances of machines in os_compat environment
// !!! Don't forget to add machines to test_compatibility.py if you add here !!!
resource "aws_instance" "island" {
ami = "ami-004f0217ce761fc9a"
instance_type = "t2.micro"
private_ip = "10.0.0.251"
subnet_id = "${aws_subnet.main.id}"
key_name = "os_compat"
tags = {
Name = "os_compat_ISLAND"
}
vpc_security_group_ids = ["${aws_security_group.os_compat_island.id}"]
associate_public_ip_address = true
root_block_device {
volume_size = "30"
volume_type = "standard"
delete_on_termination = true
}
}
locals {
env_vars = {
subnet_id = "${aws_subnet.main.id}"
vpc_security_group_ids = "${aws_security_group.os_compat_instance.id}"
}
user_data_linux_64 = <<EOF
Content-Type: multipart/mixed; boundary="//"
MIME-Version: 1.0
--//
Content-Type: text/cloud-config; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cloud-config.txt"
#cloud-config
cloud_final_modules:
- [scripts-user, always]
--//
Content-Type: text/x-shellscript; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="userdata.txt"
#!/bin/bash
rm ./monkey-linux-64
wget --no-check-certificate -q https://10.0.0.251:5000/api/monkey/download/monkey-linux-64 -O ./monkey-linux-64 || curl https://10.0.0.251:5000/api/monkey/download/monkey-linux-64 -k -o monkey-linux-64
chmod +x ./monkey-linux-64
./monkey-linux-64 m0nk3y -s 10.0.0.251:5000
--//
EOF
user_data_linux_32 = <<EOF
Content-Type: multipart/mixed; boundary="//"
MIME-Version: 1.0
--//
Content-Type: text/cloud-config; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="cloud-config.txt"
#cloud-config
cloud_final_modules:
- [scripts-user, always]
--//
Content-Type: text/x-shellscript; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="userdata.txt"
#!/bin/bash
rm ./monkey-linux-32
wget --no-check-certificate -q https://10.0.0.251:5000/api/monkey/download/monkey-linux-32 -O ./monkey-linux-32 || curl https://10.0.0.251:5000/api/monkey/download/monkey-linux-32 -k -o monkey-linux-32
chmod +x ./monkey-linux-32
./monkey-linux-32 m0nk3y -s 10.0.0.251:5000
--//
EOF
user_data_windows_64 = <<EOF
<powershell>
add-type @"
using System.Net;
using System.Security.Cryptography.X509Certificates;
public class TrustAllCertsPolicy : ICertificatePolicy {
public bool CheckValidationResult(
ServicePoint srvPoint, X509Certificate certificate,
WebRequest request, int certificateProblem) {
return true;
}
}
"@
[System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy
Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction SilentlyContinue
Invoke-WebRequest -Uri 'https://10.0.0.251:5000/api/monkey/download/monkey-windows-64.exe' -OutFile 'C:\windows\temp\monkey-windows-64.exe' -UseBasicParsing
C:\windows\temp\monkey-windows-64.exe m0nk3y -s 10.0.0.251:5000
</powershell>
<persist>true</persist>
EOF
user_data_windows_32 = <<EOF
<powershell>
add-type @"
using System.Net;
using System.Security.Cryptography.X509Certificates;
public class TrustAllCertsPolicy : ICertificatePolicy {
public bool CheckValidationResult(
ServicePoint srvPoint, X509Certificate certificate,
WebRequest request, int certificateProblem) {
return true;
}
}
"@
[System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy
Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction SilentlyContinue
Invoke-WebRequest -Uri 'https://10.0.0.251:5000/api/monkey/download/monkey-windows-32.exe' -OutFile 'C:\windows\temp\monkey-windows-32.exe' -UseBasicParsing
C:\windows\temp\monkey-windows-32.exe m0nk3y -s 10.0.0.251:5000
</powershell>
<persist>true</persist>
EOF
}
module "centos_6" {
source = "./instance_template"
name = "centos_6"
ami = "ami-07fa74e425f2abf29"
ip = "10.0.0.36"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "centos_7" {
source = "./instance_template"
name = "centos_7"
ami = "ami-0034b52a39b9fb0e8"
ip = "10.0.0.37"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "centos_8" {
source = "./instance_template"
name = "centos_8"
ami = "ami-0034c84e4e9c557bd"
ip = "10.0.0.38"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "suse_12" {
source = "./instance_template"
name = "suse_12"
ami = "ami-07b12b913a7e36b08"
ip = "10.0.0.42"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "suse_11" {
source = "./instance_template"
name = "suse_11"
ami = "ami-0083986c"
ip = "10.0.0.41"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "kali_2019" {
source = "./instance_template"
name = "kali_2019"
ami = "ami-05d64b1d0f967d4bf"
ip = "10.0.0.99"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
// Requires m3.medium which usually isn't available
//module "rhel_5" {
// source = "./instance_template"
// name = "rhel_5"
// ami = "ami-a48cbfb9"
// type = "m3.medium"
// ip = "10.0.0.85"
// env_vars = "${local.env_vars}"
// user_data = "${local.user_data_linux_64}"
//}
module "rhel_6" {
source = "./instance_template"
name = "rhel_6"
ami = "ami-0af3f0e0918f47bcf"
ip = "10.0.0.86"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "rhel_7" {
source = "./instance_template"
name = "rhel_7"
ami = "ami-0b5edb134b768706c"
ip = "10.0.0.87"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "rhel_8" {
source = "./instance_template"
name = "rhel_8"
ami = "ami-0badcc5b522737046"
ip = "10.0.0.88"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "debian_7" {
source = "./instance_template"
name = "debian_7"
ami = "ami-0badcc5b522737046"
ip = "10.0.0.77"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "debian_8" {
source = "./instance_template"
name = "debian_8"
ami = "ami-0badcc5b522737046"
ip = "10.0.0.78"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "debian_9" {
source = "./instance_template"
name = "debian_9"
ami = "ami-0badcc5b522737046"
ip = "10.0.0.79"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "oracle_6" {
source = "./instance_template"
name = "oracle_6"
ami = "ami-0f9b69f34108a3770"
ip = "10.0.0.66"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "oracle_7" {
source = "./instance_template"
name = "oracle_7"
ami = "ami-001e494dc0f3372bc"
ip = "10.0.0.67"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "ubuntu_12" {
source = "./instance_template"
name = "ubuntu_12"
ami = "ami-003d0b1d"
ip = "10.0.0.22"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
// Requires m3.medium instance which usually isn't available
// module "ubuntu_12_32" {
// source = "./instance_template"
// name = "ubuntu_12_32"
// ami = "ami-06003c1b"
// ip = "10.0.0.23"
// env_vars = "${local.env_vars}"
// user_data = "${local.user_data_linux_32}"
// }
module "ubuntu_14" {
source = "./instance_template"
name = "ubuntu_14"
ami = "ami-067ee10914e74ffee"
ip = "10.0.0.24"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "ubuntu_19" {
source = "./instance_template"
name = "ubuntu_19"
ami = "ami-001b87954b72ea3ac"
ip = "10.0.0.29"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_linux_64}"
}
module "windows_2003_r2_32" {
source = "./instance_template"
name = "windows_2003_r2_32"
ami = "ami-01e4fa6d"
ip = "10.0.0.4"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}
module "windows_2003" {
source = "./instance_template"
name = "windows_2003"
ami = "ami-9e023183"
ip = "10.0.0.5"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}
module "windows_2008_r2" {
source = "./instance_template"
name = "windows_2008_r2"
ami = "ami-05af5509c2c73e36e"
ip = "10.0.0.8"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}
module "windows_2008_32" {
source = "./instance_template"
name = "windows_2008_32"
ami = "ami-3606352b"
ip = "10.0.0.6"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_32}"
}
module "windows_2012" {
source = "./instance_template"
name = "windows_2012"
ami = "ami-0d8c60e4d3ca36ed6"
ip = "10.0.0.12"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}
module "windows_2012_r2" {
source = "./instance_template"
name = "windows_2012_r2"
ami = "ami-08dcceb529e70f875"
ip = "10.0.0.11"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}
module "windows_2016" {
source = "./instance_template"
name = "windows_2016"
ami = "ami-02a6791b44938cfcd"
ip = "10.0.0.116"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}
module "windows_2019" {
source = "./instance_template"
name = "windows_2019"
ami = "ami-09fe2745618d2af42"
ip = "10.0.0.119"
env_vars = "${local.env_vars}"
user_data = "${local.user_data_windows_64}"
}

View File

@ -0,0 +1,62 @@
import pytest
from envs.monkey_zoo.blackbox.island_client.monkey_island_client import MonkeyIslandClient
machine_list = {
"10.0.0.36": "centos_6",
"10.0.0.37": "centos_7",
"10.0.0.38": "centos_8",
"10.0.0.42": "suse_12",
"10.0.0.41": "suse_11",
"10.0.0.99": "kali_2019",
"10.0.0.86": "rhel_6",
"10.0.0.87": "rhel_7",
"10.0.0.88": "rhel_8",
"10.0.0.77": "debian_7",
"10.0.0.78": "debian_8",
"10.0.0.79": "debian_9",
"10.0.0.66": "oracle_6",
"10.0.0.67": "oracle_7",
"10.0.0.22": "ubuntu_12",
"10.0.0.24": "ubuntu_14",
"10.0.0.29": "ubuntu_19",
"10.0.0.4": "windows_2003_r2_32",
"10.0.0.5": "windows_2003",
"10.0.0.8": "windows_2008",
"10.0.0.6": "windows_2008_32",
"10.0.0.12": "windows_2012",
"10.0.0.11": "windows_2012_r2",
"10.0.0.116": "windows_2016",
"10.0.0.119": "windows_2019",
}
@pytest.fixture(scope='class')
def island_client(island):
island_client_object = MonkeyIslandClient(island)
yield island_client_object
@pytest.mark.usefixtures('island_client')
# noinspection PyUnresolvedReferences
class TestOSCompatibility(object):
def test_os_compat(self, island_client):
print()
all_monkeys = island_client.get_all_monkeys_from_db()
ips_that_communicated = []
for monkey in all_monkeys:
for ip in monkey['ip_addresses']:
if ip in machine_list:
ips_that_communicated.append(ip)
break
for ip, os in machine_list.items():
if ip not in ips_that_communicated:
print("{} didn't communicate to island".format(os))
if len(ips_that_communicated) < len(machine_list):
assert False