adds integration tests

Signed-off-by: Mike Brown <brownwm@us.ibm.com>

Makefile

@@ -40,6 +40,12 @@ dbuild: runctestimage
 	docker cp $(RUNC_INSTANCE):$(RUNC_BUILD_PATH) .
 	docker rm $(RUNC_INSTANCE)
 
+integration: runctestimage
+	docker run -e TESTFLAGS -t --privileged --rm -v $(CURDIR):/go/src/$(PROJECT) $(RUNC_TEST_IMAGE) make localintegration
+
+localintegration:
+	bats tests/integration${TESTFLAGS}
+		
 install:
 	install -D -m0755 runc /usr/local/sbin/runc
 

script/test_Dockerfile

@@ -18,6 +18,13 @@ RUN apt-get update && apt-get install -y \
     python-minimal \
     --no-install-recommends
 
+# install bats 
+RUN cd /tmp \
+    && git clone https://github.com/sstephenson/bats.git \
+    && cd bats \
+    && git reset --hard 03608115df2071fff4eaaff1605768c275e5f81f \
+    && ./install.sh /usr/local
+
 # install criu
 ENV CRIU_VERSION 1.7
 RUN mkdir -p /usr/src/criu \
@@ -26,8 +33,10 @@ RUN mkdir -p /usr/src/criu \
     && make install-criu
 
 # setup a playground for us to spawn containers in
-RUN mkdir /busybox && \
-    curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' | tar -xC /busybox
+RUN mkdir /busybox \
+    && mkdir /testdata \
+    && curl -o /testdata/busybox.tar -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' \
+    && tar -C /busybox -xf /testdata/busybox.tar 
 
 COPY script/tmpmount /
 WORKDIR /go/src/github.com/opencontainers/runc

tests/integration/README.md

@@ -0,0 +1,83 @@
+# runc Integration Tests
+
+Integration tests provide end-to-end testing of runc.
+
+Note that integration tests do **not** replace unit tests.
+
+As a rule of thumb, code should be tested thoroughly with unit tests.
+Integration tests on the other hand are meant to test a specific feature end
+to end.
+
+Integration tests are written in *bash* using the
+[bats](https://github.com/sstephenson/bats) framework.
+
+## Running integration tests
+
+The easiest way to run integration tests is with Docker:
+```
+$ make integration
+```
+Alternatively, you can run integration tests directly on your host through make:
+```
+$ sudo make localintegration
+```
+Or you can just run them directly using bats 
+```
+$ sudo bats tests/integration
+```
+To run a single test bucket:
+```
+$ make integration TESTFLAGS="/checkpoint.bats"
+```
+
+
+To run them on your host, you will need to setup a development environment plus
+[bats](https://github.com/sstephenson/bats#installing-bats-from-source) 
+For example:
+```
+$ cd ~/go/src/github.com 
+$ git clone https://github.com/sstephenson/bats.git 
+$ cd bats 
+$ ./install.sh /usr/local
+```
+		
+> **Note**: There are known issues running the integration tests using 
+> **devicemapper** as a storage driver, make sure that your docker daemon 
+> is using **aufs** if you want to successfully run the integration tests.
+
+## Writing integration tests
+
+[helper functions]
+(https://github.com/opencontainers/runc/blob/master/test/integration/helpers.bash)
+are provided in order to facilitate writing tests.
+
+```sh
+#!/usr/bin/env bats
+
+# This will load the helpers.
+load helpers
+
+# setup is called at the beginning of every test.
+function setup() {
+  # see functions teardown_hello and setup_hello in helpers.bash, used to
+  # create a pristine environment for running your tests
+  teardown_hello
+  setup_hello
+}
+
+# teardown is called at the end of every test.
+function teardown() {
+  teardown_hello
+}
+
+@test "this is a simple test" {
+  run "$RUNC" start containerid
+  # "run" automatically populates $status, $output and $lines.
+  # Please refer to bats documentation to find out more.
+  [ "$status" -eq 0 ]
+
+  # check expected output
+  [[ "${output}" == *"Hello"* ]]
+}
+
+```

tests/integration/checkpoint.bats

@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "checkpoint and restore" {
+  if [ ! -e "$CRIU" ] ; then
+    skip
+  fi
+  
+  # criu does not work with external terminals so..
+  # setting terminal and root:readonly: to false
+  sed -i 's;"terminal": true;"terminal": false;' config.json  
+  sed -i 's;"readonly": true;"readonly": false;' config.json 
+  sed -i 's/"sh"/"sh","-c","while :; do date; sleep 1; done"/' config.json
+  
+  (  
+    # start busybox (not detached) 
+    run "$RUNC" start test_busybox
+    [ "$status" -eq 0 ]
+  ) &
+  
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  run "$RUNC" state test_busybox
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"running"* ]]
+  
+  # checkpoint the running container 
+  run "$RUNC" --criu "$CRIU" checkpoint test_busybox
+  # if you are having problems getting criu to work uncomment the following dump:
+  #cat /run/opencontainer/containers/test_busybox/criu.work/dump.log
+  [ "$status" -eq 0 ]
+
+  # after checkpoint busybox is no longer running
+  run "$RUNC" state test_busybox
+  [ "$status" -ne 0 ]
+    
+  # restore from checkpoint
+  (
+    run "$RUNC" --criu "$CRIU" restore test_busybox
+    [ "$status" -eq 0 ]
+  ) &
+  
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  # busybox should be back up and running
+  run "$RUNC" state test_busybox
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"running"* ]]
+}

tests/integration/debug.bats

@@ -0,0 +1,70 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_hello
+  setup_hello
+}
+
+function teardown() {
+  teardown_hello
+}
+
+@test "global --debug" {
+  # start hello-world
+  run "$RUNC" --debug start test_hello
+  echo "${output}"
+  [ "$status" -eq 0 ]
+}
+
+@test "global --debug to --log" {
+  # start hello-world
+  run "$RUNC" --log log.out --debug start test_hello
+  [ "$status" -eq 0 ]
+
+  # check output does not include debug info
+  [[ "${output}" != *"level=debug"* ]]
+  
+  # check log.out was generated 
+  [ -e log.out ]
+  
+  # check expected debug output was sent to log.out
+  run cat log.out
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"level=debug"* ]]
+}
+
+@test "global --debug to --log --log-format 'text'" {
+  # start hello-world
+  run "$RUNC" --log log.out --log-format "text" --debug start test_hello
+  [ "$status" -eq 0 ]
+
+  # check output does not include debug info
+  [[ "${output}" != *"level=debug"* ]]
+
+  # check log.out was generated 
+  [ -e log.out ]
+  
+  # check expected debug output was sent to log.out
+  run cat log.out
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"level=debug"* ]]
+}
+
+@test "global --debug to --log --log-format 'json'" {
+  # start hello-world
+  run "$RUNC" --log log.out --log-format "json" --debug start test_hello
+  [ "$status" -eq 0 ]
+  
+  # check output does not include debug info
+  [[ "${output}" != *"level=debug"* ]]
+
+  # check log.out was generated 
+  [ -e log.out ]
+  
+  # check expected debug output was sent to log.out
+  run cat log.out
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *'"level":"debug"'* ]]
+}

tests/integration/delete.bats

@@ -0,0 +1,33 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "runc delete" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  testcontainer test_busybox running
+  
+  run "$RUNC" kill test_busybox KILL
+  # wait for busybox to be in the destroyed state
+  retry 10 1 eval "'$RUNC' state test_busybox | grep -q 'destroyed'" 
+
+  # delete test_busybox
+  run "$RUNC" delete test_busybox
+  
+  run "$RUNC" state test_busybox
+  [ "$status" -ne 0 ]
+}

tests/integration/events.bats

@@ -0,0 +1,113 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+function startup_events() {
+  ("$RUNC" events test_busybox > events.log)
+}
+
+@test "events --stats" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  # generate stats 
+  run "$RUNC" events --stats test_busybox
+  [ "$status" -eq 0 ]
+  [[ "${lines[0]}" == [\{]"\"type\""[:]"\"stats\""[,]"\"id\""[:]"\"test_busybox\""[,]* ]]
+  [[ "${lines[0]}" == *"CgroupStats"* ]]
+}
+
+@test "events --interval default " {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+  
+  # spawn two sub processes (shells) 
+  # the first sub process is an event logger that sends stats events to events.log 
+  # the second sub process waits for an event that incudes test_busybox then 
+  # kills the test_busybox container which causes the event logger to exit
+  ("$RUNC" events test_busybox > events.log) &
+  ( 
+    retry 10 1 eval "grep -q 'test_busybox' events.log"
+    teardown_running_container test_busybox
+  ) &
+  wait # wait for the above sub shells to finish  
+  
+  [ -e events.log ]
+  
+  run cat events.log
+  [ "$status" -eq 0 ]
+  [[ "${lines[0]}" == [\{]"\"type\""[:]"\"stats\""[,]"\"id\""[:]"\"test_busybox\""[,]* ]]
+  [[ "${lines[0]}" == *"CgroupStats"* ]]
+}
+
+@test "events --interval 1s " {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+  
+  # spawn two sub processes (shells) 
+  # the first sub process is an event logger that sends stats events to events.log once a second 
+  # the second sub process tries 3 times for an event that incudes test_busybox 
+  # pausing 1s between each attempt then kills the test_busybox container which 
+  # causes the event logger to exit
+  ("$RUNC" events --interval 1s test_busybox > events.log) &
+  ( 
+    retry 3 1 eval "grep -q 'test_busybox' events.log"
+    teardown_running_container test_busybox
+  ) &
+  wait # wait for the above sub shells to finish  
+  
+  [ -e events.log ]
+  
+  run eval "grep -q 'test_busybox' events.log"
+  [ "$status" -eq 0 ]
+}
+
+@test "events --interval 100ms " {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+  
+  #prove there is no carry over of events.log from a prior test
+  [ ! -e events.log ]
+    
+  # spawn two sub processes (shells) 
+  # the first sub process is an event logger that sends stats events to events.log once every 100ms 
+  # the second sub process tries 3 times for an event that incudes test_busybox 
+  # pausing 100s between each attempt then kills the test_busybox container which 
+  # causes the event logger to exit
+  ("$RUNC" events --interval 100ms test_busybox > events.log) &
+  ( 
+    retry 3 0.100 eval "grep -q 'test_busybox' events.log"
+    teardown_running_container test_busybox
+  ) &
+  wait # wait for the above sub shells to finish  
+  
+  [ -e events.log ]
+  
+  run eval "grep -q 'test_busybox' events.log"
+  [ "$status" -eq 0 ]
+}

tests/integration/exec.bats

@@ -0,0 +1,45 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "runc exec" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  wait_for_container 15 1 test_busybox
+
+  run "$RUNC" exec test_busybox echo Hello from exec 
+  [ "$status" -eq 0 ]
+  echo text echoed = "'""${output}""'" 
+  [[ "${output}" == *"Hello from exec"* ]]
+}
+
+@test "runc exec --pid-file" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  wait_for_container 15 1 test_busybox
+
+  run "$RUNC" exec --pid-file pid.txt test_busybox echo Hello from exec 
+  [ "$status" -eq 0 ]
+  echo text echoed = "'""${output}""'" 
+  [[ "${output}" == *"Hello from exec"* ]]
+
+  # check pid.txt was generated 
+  [ -e pid.txt ]
+
+  run cat pid.txt
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ [0-9]+ ]]
+}

tests/integration/help.bats

@@ -0,0 +1,82 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "runc -h" {
+  run "$RUNC" -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ NAME:+ ]]
+  [[ ${lines[1]} =~ runc\ '-'\ Open\ Container\ Initiative\ runtime+ ]]
+  
+  run "$RUNC" --help
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ NAME:+ ]]
+  [[ ${lines[1]} =~ runc\ '-'\ Open\ Container\ Initiative\ runtime+ ]]
+}
+
+@test "runc command -h" {
+  run "$RUNC" checkpoint -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ checkpoint+ ]]
+  
+  run "$RUNC" delete -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ delete+ ]]
+  
+  run "$RUNC" events -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ events+ ]]
+
+  run "$RUNC" exec -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ exec+ ]]
+  
+  run "$RUNC" kill -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ kill+ ]]
+  
+  run "$RUNC" list -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ NAME:+ ]]
+  [[ ${lines[1]} =~ runc\ list+ ]]
+  
+  run "$RUNC" list --help
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ NAME:+ ]]
+  [[ ${lines[1]} =~ runc\ list+ ]]
+  
+  run "$RUNC" pause -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ pause+ ]]
+  
+  run "$RUNC" restore -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ restore+ ]]
+  
+  run "$RUNC" resume -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ resume+ ]]
+  
+  run "$RUNC" spec -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ spec+ ]]
+  
+  run "$RUNC" start -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ start+ ]]
+  
+  run "$RUNC" state -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ state+ ]]
+  
+  run "$RUNC" delete -h
+  [ "$status" -eq 0 ]
+  [[ ${lines[1]} =~ runc\ delete+ ]]
+  
+}
+
+@test "runc foo -h" {
+  run "$RUNC" foo -h
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"No help topic for 'foo'"* ]]
+}

tests/integration/helpers.bash

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Root directory of integration tests.
+INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
+RUNC="${INTEGRATION_ROOT}/../../runc"
+GOPATH="${INTEGRATION_ROOT}/../../../.."
+
+# Test data path.
+TESTDATA="${INTEGRATION_ROOT}/testdata"
+
+# Busybox image 
+BUSYBOX_IMAGE="$BATS_TMPDIR/busybox.tar"
+BUSYBOX_BUNDLE="$BATS_TMPDIR/busyboxtest"
+
+# hello-world in tar format 
+HELLO_IMAGE="$TESTDATA/hello-world.tar"
+HELLO_BUNDLE="$BATS_TMPDIR/hello-world"
+
+# CRIU PATH
+CRIU="/usr/local/sbin/criu"
+
+# Retry a command $1 times until it succeeds. Wait $2 seconds between retries.
+function retry() {
+  local attempts=$1
+  shift
+  local delay=$1
+  shift
+  local i
+
+  for ((i=0; i < attempts; i++)); do
+    run "$@"
+    if [[ "$status" -eq 0 ]] ; then
+      return 0
+    fi
+    sleep $delay
+  done
+
+  echo "Command \"$@\" failed $attempts times. Output: $output"
+  false
+}
+
+# retry until the given container has state
+function wait_for_container() {
+  local attempts=$1
+  local delay=$2
+  local cid=$3
+  local i
+
+  for ((i=0; i < attempts; i++)); do
+    run "$RUNC" state $cid
+    if [[ "$status" -eq 0 ]] ; then
+      return 0
+    fi
+    sleep $delay
+  done
+
+  echo "runc state failed to return state $statecheck $attempts times. Output: $output"
+  false
+}
+
+# retry until the given container has state
+function wait_for_container_inroot() {
+  local attempts=$1
+  local delay=$2
+  local cid=$3
+  local i
+
+  for ((i=0; i < attempts; i++)); do
+    run "$RUNC" --root $4 state $cid
+    if [[ "$status" -eq 0 ]] ; then
+      return 0
+    fi
+    sleep $delay
+  done
+
+  echo "runc state failed to return state $statecheck $attempts times. Output: $output"
+  false
+}
+
+function testcontainer() {
+  # test state of container 
+  run "$RUNC" state $1
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"$2"* ]]
+}
+
+function setup_busybox() {
+  run mkdir "$BUSYBOX_BUNDLE" 
+  run mkdir "$BUSYBOX_BUNDLE"/rootfs
+  if [ -e "/testdata/busybox.tar" ]; then
+    BUSYBOX_IMAGE="/testdata/busybox.tar"
+  fi
+  if [ ! -e $BUSYBOX_IMAGE ]; then
+    curl -o $BUSYBOX_IMAGE -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' 
+  fi
+  tar -C "$BUSYBOX_BUNDLE"/rootfs -xf "$BUSYBOX_IMAGE"
+  cd "$BUSYBOX_BUNDLE"
+  run "$RUNC" spec
+}
+
+function setup_hello() {
+  run mkdir "$HELLO_BUNDLE" 
+  run mkdir "$HELLO_BUNDLE"/rootfs
+  tar -C "$HELLO_BUNDLE"/rootfs -xf "$HELLO_IMAGE"
+  cd "$HELLO_BUNDLE"
+  "$RUNC" spec
+  sed -i 's;"sh";"/hello";' config.json
+}
+
+function teardown_running_container() {
+  run "$RUNC" list 
+  if [[ "${output}" == *"$1"* ]]; then
+    run "$RUNC" kill $1 KILL
+    retry 10 1 eval "'$RUNC' state '$1' | grep -q 'destroyed'" 
+    run "$RUNC" delete $1
+  fi
+}
+
+function teardown_running_container_inroot() {
+  run "$RUNC" --root $2 list 
+  if [[ "${output}" == *"$1"* ]]; then
+    run "$RUNC" --root $2 kill $1 KILL
+    retry 10 1 eval "'$RUNC' --root '$2' state '$1' | grep -q 'destroyed'" 
+    run "$RUNC" --root $2 delete $1
+  fi
+}
+
+function teardown_busybox() {
+  cd "$INTEGRATION_ROOT"
+  teardown_running_container test_busybox
+  run rm -f -r "$BUSYBOX_BUNDLE"
+}
+
+function teardown_hello() {
+  cd "$INTEGRATION_ROOT"
+  teardown_running_container test_hello
+  run rm -f -r "$HELLO_BUNDLE"
+}

tests/integration/kill.bats

@@ -0,0 +1,33 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+
+@test "kill detached busybox" {
+  
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  testcontainer test_busybox running
+  
+  run "$RUNC" kill test_busybox KILL
+  [ "$status" -eq 0 ]
+  
+  retry 10 1 eval "'$RUNC' state test_busybox | grep -q 'destroyed'"
+  
+  run "$RUNC" delete test_busybox
+  [ "$status" -eq 0 ]
+}

tests/integration/list.bats

@@ -0,0 +1,53 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_running_container_inroot test_box1 $HELLO_BUNDLE
+  teardown_running_container_inroot test_box2 $HELLO_BUNDLE
+  teardown_running_container_inroot test_box3 $HELLO_BUNDLE
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_running_container_inroot test_box1 $HELLO_BUNDLE
+  teardown_running_container_inroot test_box2 $HELLO_BUNDLE
+  teardown_running_container_inroot test_box3 $HELLO_BUNDLE
+  teardown_busybox
+}
+
+@test "list" {
+  # start a few busyboxes detached
+  run "$RUNC" --root $HELLO_BUNDLE start -d --console /dev/pts/ptmx test_box1
+  [ "$status" -eq 0 ]
+  wait_for_container_inroot 15 1 test_box1 $HELLO_BUNDLE
+  
+  run "$RUNC" --root $HELLO_BUNDLE start -d --console /dev/pts/ptmx test_box2
+  [ "$status" -eq 0 ]
+  wait_for_container_inroot 15 1 test_box2 $HELLO_BUNDLE
+  
+  run "$RUNC" --root $HELLO_BUNDLE start -d --console /dev/pts/ptmx test_box3
+  [ "$status" -eq 0 ]
+  wait_for_container_inroot 15 1 test_box3 $HELLO_BUNDLE
+  
+  run "$RUNC" --root $HELLO_BUNDLE list 
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ ID\ +PID\ +STATUS\ +BUNDLE\ +CREATED+ ]]
+  [[ "${lines[1]}" == *"test_box1"*[0-9]*"running"*$BUSYBOX_BUNDLE*[0-9]* ]]
+  [[ "${lines[2]}" == *"test_box2"*[0-9]*"running"*$BUSYBOX_BUNDLE*[0-9]* ]]
+  [[ "${lines[3]}" == *"test_box3"*[0-9]*"running"*$BUSYBOX_BUNDLE*[0-9]* ]]
+  
+  run "$RUNC" --root $HELLO_BUNDLE list --format table 
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ ID\ +PID\ +STATUS\ +BUNDLE\ +CREATED+ ]]
+  [[ "${lines[1]}" == *"test_box1"*[0-9]*"running"*$BUSYBOX_BUNDLE*[0-9]* ]]
+  [[ "${lines[2]}" == *"test_box2"*[0-9]*"running"*$BUSYBOX_BUNDLE*[0-9]* ]]
+  [[ "${lines[3]}" == *"test_box3"*[0-9]*"running"*$BUSYBOX_BUNDLE*[0-9]* ]]
+  
+  run "$RUNC" --root $HELLO_BUNDLE list --format json
+  [ "$status" -eq 0 ]
+  [[ "${lines[0]}" == [\[][\{]"\"id\""[:]"\"test_box1\""[,]"\"pid\""[:]*[0-9][,]"\"status\""[:]*"\"running\""[,]"\"bundle\""[:]*$BUSYBOX_BUNDLE*[,]"\"created\""[:]*[0-9]*[\}]* ]]
+  [[ "${lines[0]}" == *[,][\{]"\"id\""[:]"\"test_box2\""[,]"\"pid\""[:]*[0-9][,]"\"status\""[:]*"\"running\""[,]"\"bundle\""[:]*$BUSYBOX_BUNDLE*[,]"\"created\""[:]*[0-9]*[\}]* ]]
+  [[ "${lines[0]}" == *[,][\{]"\"id\""[:]"\"test_box3\""[,]"\"pid\""[:]*[0-9][,]"\"status\""[:]*"\"running\""[,]"\"bundle\""[:]*$BUSYBOX_BUNDLE*[,]"\"created\""[:]*[0-9]*[\}][\]] ]]
+}

tests/integration/pause.bats

@@ -0,0 +1,34 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "runc pause and resume" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+  
+  wait_for_container 15 1 test_busybox
+  
+  # pause busybox 
+  run "$RUNC" pause test_busybox 
+  [ "$status" -eq 0 ]
+
+  # test state of busybox is paused
+  testcontainer test_busybox paused
+  
+  # resume busybox 
+  run "$RUNC" resume test_busybox 
+  [ "$status" -eq 0 ]
+  
+  # test state of busybox is back to running
+  testcontainer test_busybox running
+}

tests/integration/root.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_running_container_inroot test_dotbox $HELLO_BUNDLE
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_running_container_inroot test_dotbox $HELLO_BUNDLE
+  teardown_busybox
+}
+
+@test "global --root" {
+  # start busybox detached using $HELLO_BUNDLE for state  
+  run "$RUNC" --root $HELLO_BUNDLE start -d --console /dev/pts/ptmx test_dotbox
+  [ "$status" -eq 0 ]
+
+  # start busybox detached in default root 
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+  
+  # check state of the busyboxes are only in their respective root path
+  wait_for_container 15 1 test_busybox
+  wait_for_container_inroot 15 1 test_dotbox $HELLO_BUNDLE
+
+  run "$RUNC" state test_busybox 
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"running"* ]]
+  
+  run "$RUNC" --root $HELLO_BUNDLE state test_dotbox 
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"running"* ]]
+
+  run "$RUNC" --root $HELLO_BUNDLE state test_busybox 
+  [ "$status" -ne 0 ]
+
+  run "$RUNC" state test_dotbox 
+  [ "$status" -ne 0 ]
+  
+  run "$RUNC" kill test_busybox KILL
+  [ "$status" -eq 0 ]
+  retry 10 1 eval "'$RUNC' state test_busybox | grep -q 'destroyed'" 
+  run "$RUNC" delete test_busybox
+  [ "$status" -eq 0 ]
+  
+  run "$RUNC" --root $HELLO_BUNDLE kill test_dotbox KILL
+  [ "$status" -eq 0 ]
+  retry 10 1 eval "'$RUNC' --root $HELLO_BUNDLE state test_dotbox | grep -q 'destroyed'" 
+  run "$RUNC" --root $HELLO_BUNDLE delete test_dotbox  
+  [ "$status" -eq 0 ]
+}

tests/integration/spec.bats

@@ -0,0 +1,89 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  # initial cleanup in case a prior test exited and did not cleanup
+  cd "$INTEGRATION_ROOT"
+  run rm -f -r "$HELLO_BUNDLE"
+	  
+  # setup hello-world for spec generation testing
+  run mkdir "$HELLO_BUNDLE" 
+  run mkdir "$HELLO_BUNDLE"/rootfs
+  run tar -C "$HELLO_BUNDLE"/rootfs -xf "$HELLO_IMAGE"
+}
+
+function teardown() {
+  cd "$INTEGRATION_ROOT"
+  run rm -f -r "$HELLO_BUNDLE"
+}
+
+@test "spec generation cwd" {
+  cd "$HELLO_BUNDLE"
+  # note this test runs from the bundle not the integration root
+  
+  # test that config.json does not exist after the above partial setup 
+  [ ! -e config.json ]
+  
+  # test generation of spec does not return an error 
+  run "$RUNC" spec
+  [ "$status" -eq 0 ]
+  
+  # test generation of spec created our config.json (spec) 
+  [ -e config.json ]
+
+  # test existence of required args parameter in the generated config.json
+  run bash -c "grep -A2 'args' config.json | grep 'sh'"
+  [[ "${output}" == *"sh"* ]]
+  
+  # change the default args parameter from sh to hello
+  sed -i 's;"sh";"/hello";' config.json
+
+  # ensure the generated spec works by starting hello-world
+  run "$RUNC" start test_hello
+  [ "$status" -eq 0 ]
+}
+
+@test "spec generation --bundle" {
+  # note this test runs from the integration root not the bundle 
+  
+  # test that config.json does not exist after the above partial setup 
+  [ ! -e "$HELLO_BUNDLE"/config.json ]
+  
+  # test generation of spec does not return an error 
+  run "$RUNC" spec --bundle "$HELLO_BUNDLE"
+  [ "$status" -eq 0 ]
+  
+  # test generation of spec created our config.json (spec) 
+  [ -e "$HELLO_BUNDLE"/config.json ]
+  
+  # change the default args parameter from sh to hello
+  sed -i 's;"sh";"/hello";' "$HELLO_BUNDLE"/config.json
+
+  # ensure the generated spec works by starting hello-world
+  run "$RUNC" start --bundle "$HELLO_BUNDLE" test_hello
+  [ "$status" -eq 0 ]
+}
+
+@test "spec validator" {
+  cd "$HELLO_BUNDLE"
+  # note this test runs from the temporary bundle directory not the integration root
+  # note this test is brittle against specs changes that lead runc's spec command
+  # todo get the validate program, gojsonschema, and schema/*s.json from godeps? 
+  
+  run git clone https://github.com/opencontainers/specs.git src/specs
+  [ -e src/specs/schema/schema.json ]
+  
+  run bash -c "GOPATH='$GOPATH' go get github.com/xeipuuv/gojsonschema"
+  [ "$status" -eq 0 ]
+  
+  GOPATH="$GOPATH" go build src/specs/schema/validate.go
+  [ -e ./validate ]
+  
+  run "$RUNC" spec 
+  [ -e config.json ]
+  
+  run ./validate src/specs/schema/schema.json config.json
+  [ "$status" -eq 0 ]
+  [[ "${lines[0]}" == *"The document is valid"* ]]
+}

tests/integration/start_detached.bats

@@ -0,0 +1,41 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "runc start detached" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  testcontainer test_busybox running
+}
+
+@test "runc start detached --pid-file" {
+  # start busybox detached
+  run "$RUNC" start --pid-file pid.txt -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  testcontainer test_busybox running
+
+  # check pid.txt was generated 
+  [ -e pid.txt ]
+
+  run cat pid.txt
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ [0-9]+ ]]
+}

tests/integration/start_hello.bats

@@ -0,0 +1,47 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_hello
+  setup_hello
+}
+
+function teardown() {
+  teardown_hello
+}
+
+@test "runc start" {
+  # start hello-world
+  run "$RUNC" start test_hello  
+  [ "$status" -eq 0 ]
+  
+  # check expected output
+  [[ "${output}" == *"Hello"* ]]
+}
+
+@test "runc start with rootfs set to ." {
+  cp config.json rootfs/.
+  rm config.json
+  cd rootfs
+  sed -i 's;"rootfs";".";' config.json
+
+  # start hello-world
+  run "$RUNC" start test_hello
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"Hello"* ]]
+}
+
+@test "runc start --pid-file" {
+  # start hello-world
+  run "$RUNC" start --pid-file pid.txt test_hello
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"Hello"* ]]
+  
+  # check pid.txt was generated 
+  [ -e pid.txt ]
+
+  run cat pid.txt
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ [0-9]+ ]]
+}

tests/integration/state.bats

@@ -0,0 +1,50 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "state" {
+  run "$RUNC" state test_busybox
+  [ "$status" -ne 0 ]
+  
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  testcontainer test_busybox running
+    
+  # pause busybox 
+  run "$RUNC" pause test_busybox 
+  [ "$status" -eq 0 ]
+
+  # test state of busybox is paused
+  testcontainer test_busybox paused
+  
+  # resume busybox 
+  run "$RUNC" resume test_busybox 
+  [ "$status" -eq 0 ]
+  
+  # test state of busybox is back to running
+  testcontainer test_busybox running
+  
+  run "$RUNC" kill test_busybox KILL
+  # wait for busybox to be in the destroyed state
+  retry 10 1 eval "'$RUNC' state test_busybox | grep -q 'destroyed'" 
+    
+  # delete test_busybox
+  run "$RUNC" delete test_busybox
+  
+  run "$RUNC" state test_busybox
+  [ "$status" -ne 0 ]
+}

tests/integration/version.bats

@@ -0,0 +1,11 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "runc version" {
+  run "$RUNC" -v
+  [ "$status" -eq 0 ]
+  [[ ${lines[0]} =~ runc\ version\ [0-9]+\.[0-9]+\.[0-9]+ ]]
+  [[ ${lines[1]} =~ commit:+ ]] 
+  [[ ${lines[2]} =~ spec:\ [0-9]+\.[0-9]+\.[0-9]+ ]]
+}