Merge pull request #146 from crosbymichael/refactor-execin

Refactor execin send config over pipe

namespaces/execin.go

@@ -3,70 +3,88 @@
 package namespaces
 
 import (
-	"encoding/json"
+	"io"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strconv"
+	"syscall"
 
 	"github.com/docker/libcontainer"
 	"github.com/docker/libcontainer/label"
+	"github.com/docker/libcontainer/syncpipe"
 	"github.com/docker/libcontainer/system"
 )
 
-// ExecIn uses an existing pid and joins the pid's namespaces with the new command.
+// ExecIn reexec's the initPath with the argv 0 rewrite to "nsenter" so that it is able to run the
-func ExecIn(container *libcontainer.Config, state *libcontainer.State, args []string) error {
+// setns code in a single threaded environment joining the existing containers' namespaces.
-	// Enter the namespace and then finish setup
+func ExecIn(container *libcontainer.Config, state *libcontainer.State, userArgs []string, initPath string,
-	args, err := GetNsEnterCommand(strconv.Itoa(state.InitPid), container, "", args)
+	stdin io.Reader, stdout, stderr io.Writer, console string, startCallback func(*exec.Cmd)) (int, error) {
-	if err != nil {
-		return err
-	}
 
-	finalArgs := append([]string{os.Args[0]}, args...)
+	args := []string{"nsenter", "--nspid", strconv.Itoa(state.InitPid)}
-
-	if err := system.Execv(finalArgs[0], finalArgs[0:], os.Environ()); err != nil {
-		return err
-	}
-
-	panic("unreachable")
-}
-
-func getContainerJson(container *libcontainer.Config) (string, error) {
-	// TODO(vmarmol): If this gets too long, send it over a pipe to the child.
-	// Marshall the container into JSON since it won't be available in the namespace.
-	containerJson, err := json.Marshal(container)
-	if err != nil {
-		return "", err
-	}
-	return string(containerJson), nil
-}
-
-func GetNsEnterCommand(initPid string, container *libcontainer.Config, console string, args []string) ([]string, error) {
-	containerJson, err := getContainerJson(container)
-	if err != nil {
-		return nil, err
-	}
-
-	out := []string{
-		"--nspid", initPid,
-		"--containerjson", containerJson,
-	}
 
 	if console != "" {
-		out = append(out, "--console", console)
+		args = append(args, "--console", console)
 	}
-	out = append(out, "nsenter")
-	out = append(out, "--")
-	out = append(out, args...)
 
-	return out, nil
+	cmd := &exec.Cmd{
+		Path: initPath,
+		Args: append(args, append([]string{"--"}, userArgs...)...),
+	}
+
+	if filepath.Base(initPath) == initPath {
+		if lp, err := exec.LookPath(initPath); err == nil {
+			cmd.Path = lp
+		}
+	}
+
+	pipe, err := syncpipe.NewSyncPipe()
+	if err != nil {
+		return -1, err
+	}
+	defer pipe.Close()
+
+	// Note: these are only used in non-tty mode
+	// if there is a tty for the container it will be opened within the namespace and the
+	// fds will be duped to stdin, stdiout, and stderr
+	cmd.Stdin = stdin
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+
+	cmd.ExtraFiles = []*os.File{pipe.Child()}
+
+	if err := cmd.Start(); err != nil {
+		return -1, err
+	}
+	pipe.CloseChild()
+
+	if err := pipe.SendToChild(container); err != nil {
+		cmd.Process.Kill()
+		cmd.Wait()
+		return -1, err
+	}
+
+	if startCallback != nil {
+		startCallback(cmd)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		if _, ok := err.(*exec.ExitError); !ok {
+			return -1, err
+		}
+	}
+
+	return cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil
 }
 
-// Run a command in a container after entering the namespace.
+// Finalize expects that the setns calls have been setup and that is has joined an
-func NsEnter(container *libcontainer.Config, args []string) error {
+// existing namespace
-	// clear the current processes env and replace it with the environment
+func FinalizeSetns(container *libcontainer.Config, args []string) error {
-	// defined on the container
+	// clear the current processes env and replace it with the environment defined on the container
 	if err := LoadContainerEnvironment(container); err != nil {
 		return err
 	}
+
 	if err := FinalizeNamespace(container); err != nil {
 		return err
 	}
@@ -80,5 +98,6 @@ func NsEnter(container *libcontainer.Config, args []string) error {
 	if err := system.Execv(args[0], args[0:], container.Env); err != nil {
 		return err
 	}
+
 	panic("unreachable")
 }

namespaces/init.go

@@ -48,8 +48,8 @@ func Init(container *libcontainer.Config, uncleanRootfs, consolePath string, syn
 	}
 
 	// We always read this as it is a way to sync with the parent as well
-	networkState, err := syncPipe.ReadFromParent()
+	var networkState *network.NetworkState
-	if err != nil {
+	if err := syncPipe.ReadFromParent(&networkState); err != nil {
 		return err
 	}
 

namespaces/nsenter/README.md

@@ -0,0 +1,6 @@
+## nsenter
+
+The `nsenter` package registers a special init constructor that is called before the Go runtime has 
+a chance to boot.  This provides us the ability to `setns` on existing namespaces and avoid the issues
+that the Go runtime has with multiple threads.  This constructor is only called if this package is 
+registered, imported, in your go application and the argv 0 is `nsenter`.

namespaces/nsenter/nsenter.c

@@ -71,7 +71,7 @@ int setns(int fd, int nstype)
 void print_usage()
 {
 	fprintf(stderr,
-		"<binary> nsenter --nspid <pid> --containerjson <container_json> -- cmd1 arg1 arg2...\n");
+		"nsenter --nspid <pid> --console <console> -- cmd1 arg1 arg2...\n");
 }
 
 void nsenter()
@@ -80,53 +80,35 @@ void nsenter()
 	char **argv;
 	get_args(&argc, &argv);
 
-	// Ignore if this is not for us.
+    // check argv 0 to ensure that we are supposed to setns
-	if (argc < 6) {
+    // we use strncmp to test for a value of "nsenter" but also allows alternate implmentations
-		return;
+    // after the setns code path to continue to use the argv 0 to determine actions to be run
-	}
+    // resulting in the ability to specify "nsenter-mknod", "nsenter-exec", etc...
-	int found_nsenter = 0;
+    if (strncmp(argv[0], kNsEnter, strlen(kNsEnter)) != 0) {
-	for (c = 0; c < argc; ++c) {
-		if (strcmp(argv[c], kNsEnter) == 0) {
-			found_nsenter = 1;
-			break;
-		}
-	}
-	if (!found_nsenter) {
         return;
     }
+
 	static const struct option longopts[] = {
 		{"nspid", required_argument, NULL, 'n'},
-		{"containerjson", required_argument, NULL, 'c'},
+		{"console", required_argument, NULL, 't'},
-		{"console", optional_argument, NULL, 't'},
 		{NULL, 0, NULL, 0}
 	};
 
 	pid_t init_pid = -1;
 	char *init_pid_str = NULL;
-	char *container_json = NULL;
 	char *console = NULL;
-	opterr = 0;
+	while ((c = getopt_long_only(argc, argv, "n:c:", longopts, NULL)) != -1) {
-	while ((c =
-		getopt_long_only(argc, argv, "-n:s:c:", longopts,
-				 NULL)) != -1) {
 		switch (c) {
 		case 'n':
 			init_pid_str = optarg;
 			break;
-		case 'c':
-			container_json = optarg;
-			break;
 		case 't':
 			console = optarg;
 			break;
 		}
 	}
 
-	if (strcmp(argv[optind - 2], kNsEnter) != 0) {
+	if (init_pid_str == NULL) {
-		return;
-	}
-
-	if (container_json == NULL || init_pid_str == NULL) {
 		print_usage();
 		exit(1);
 	}
@@ -228,6 +210,7 @@ void nsenter()
 		} else if (WIFSIGNALED(status)) {
 			kill(getpid(), WTERMSIG(status));
 		}
+
 		exit(1);
 	}
 

namespaces/nsenter/nsenter.go

@@ -1,6 +1,6 @@
 // +build linux
 
-package namespaces
+package nsenter
 
 /*
 __attribute__((constructor)) init() {

namespaces/nsenter/nsenter_unsupported.go

@@ -0,0 +1,3 @@
+// +build !linux !cgo
+
+package nsenter

nsinit/cli.go

@@ -7,7 +7,14 @@ import (
 	"github.com/codegangsta/cli"
 )
 
-var logPath = os.Getenv("log")
+var (
+	logPath = os.Getenv("log")
+	argvs   = make(map[string]func())
+)
+
+func init() {
+	argvs["nsenter"] = nsenter
+}
 
 func preload(context *cli.Context) error {
 	if logPath != "" {
@@ -20,21 +27,33 @@ func preload(context *cli.Context) error {
 }
 
 func NsInit() {
+	// we need to check our argv 0 for any registred functions to run instead of the
+	// normal cli code path
+
+	action, exists := argvs[os.Args[0]]
+	if exists {
+		action()
+
+		return
+	}
+
 	app := cli.NewApp()
+
 	app.Name = "nsinit"
 	app.Version = "0.1"
 	app.Author = "libcontainer maintainers"
 	app.Flags = []cli.Flag{
 		cli.StringFlag{Name: "nspid"},
-		cli.StringFlag{Name: "containerjson"},
+		cli.StringFlag{Name: "console"},
-		cli.StringFlag{Name: "console"}}
+	}
+
 	app.Before = preload
+
 	app.Commands = []cli.Command{
 		execCommand,
 		initCommand,
 		statsCommand,
 		configCommand,
-		nsenterCommand,
 		pauseCommand,
 		unpauseCommand,
 	}

nsinit/exec.go

@@ -36,7 +36,7 @@ func execAction(context *cli.Context) {
 	}
 
 	if state != nil {
-		err = namespaces.ExecIn(container, state, []string(context.Args()))
+		exitCode, err = startInExistingContainer(container, state, context)
 	} else {
 		exitCode, err = startContainer(container, dataPath, []string(context.Args()))
 	}
@@ -48,6 +48,63 @@ func execAction(context *cli.Context) {
 	os.Exit(exitCode)
 }
 
+// the process for execing a new process inside an existing container is that we have to exec ourself
+// with the nsenter argument so that the C code can setns an the namespaces that we require.  Then that
+// code path will drop us into the path that we can do the final setup of the namespace and exec the users
+// application.
+func startInExistingContainer(config *libcontainer.Config, state *libcontainer.State, context *cli.Context) (int, error) {
+	var (
+		master  *os.File
+		console string
+		err     error
+
+		sigc = make(chan os.Signal, 10)
+
+		stdin  = os.Stdin
+		stdout = os.Stdout
+		stderr = os.Stderr
+	)
+	signal.Notify(sigc)
+
+	if config.Tty {
+		stdin = nil
+		stdout = nil
+		stderr = nil
+
+		master, console, err = consolepkg.CreateMasterAndConsole()
+		if err != nil {
+			return -1, err
+		}
+
+		go io.Copy(master, os.Stdin)
+		go io.Copy(os.Stdout, master)
+
+		state, err := term.SetRawTerminal(os.Stdin.Fd())
+		if err != nil {
+			return -1, err
+		}
+
+		defer term.RestoreTerminal(os.Stdin.Fd(), state)
+	}
+
+	startCallback := func(cmd *exec.Cmd) {
+		go func() {
+			resizeTty(master)
+
+			for sig := range sigc {
+				switch sig {
+				case syscall.SIGWINCH:
+					resizeTty(master)
+				default:
+					cmd.Process.Signal(sig)
+				}
+			}
+		}()
+	}
+
+	return namespaces.ExecIn(config, state, context.Args(), os.Args[0], stdin, stdout, stderr, console, startCallback)
+}
+
 // startContainer starts the container. Returns the exit status or -1 and an
 // error.
 //

nsinit/nsenter.go

@@ -2,36 +2,41 @@ package nsinit
 
 import (
 	"log"
-	"strconv"
+	"os"
 
-	"github.com/codegangsta/cli"
+	"github.com/docker/libcontainer"
 	"github.com/docker/libcontainer/namespaces"
+	_ "github.com/docker/libcontainer/namespaces/nsenter"
+	"github.com/docker/libcontainer/syncpipe"
 )
 
-var nsenterCommand = cli.Command{
+func findUserArgs() []string {
-	Name:   "nsenter",
+	i := 0
-	Usage:  "init process for entering an existing namespace",
+	for _, a := range os.Args {
-	Action: nsenterAction,
+		i++
+
+		if a == "--" {
+			break
+		}
+	}
+
+	return os.Args[i:]
 }
 
-func nsenterAction(context *cli.Context) {
+// this expects that we already have our namespaces setup by the C initializer
-	args := context.Args()
+// we are expected to finalize the namespace and exec the user's application
-
+func nsenter() {
-	if len(args) == 0 {
+	syncPipe, err := syncpipe.NewSyncPipeFromFd(0, 3)
-		args = []string{"/bin/bash"}
-	}
-
-	container, err := loadContainerFromJson(context.GlobalString("containerjson"))
 	if err != nil {
-		log.Fatalf("unable to load container: %s", err)
+		log.Fatalf("unable to create sync pipe: %s", err)
 	}
 
-	nspid, err := strconv.Atoi(context.GlobalString("nspid"))
+	var config *libcontainer.Config
-	if nspid <= 0 || err != nil {
+	if err := syncPipe.ReadFromParent(&config); err != nil {
-		log.Fatalf("cannot enter into namespaces without valid pid: %q - %s", nspid, err)
+		log.Fatalf("reading container config from parent: %s", err)
 	}
 
-	if err := namespaces.NsEnter(container, args); err != nil {
+	if err := namespaces.FinalizeSetns(config, findUserArgs()); err != nil {
 		log.Fatalf("failed to nsenter: %s", err)
 	}
 }

syncpipe/sync_pipe.go

@@ -6,8 +6,6 @@ import (
 	"io/ioutil"
 	"os"
 	"syscall"
-
-	"github.com/docker/libcontainer/network"
 )
 
 // SyncPipe allows communication to and from the child processes
@@ -39,8 +37,8 @@ func (s *SyncPipe) Parent() *os.File {
 	return s.parent
 }
 
-func (s *SyncPipe) SendToChild(networkState *network.NetworkState) error {
+func (s *SyncPipe) SendToChild(v interface{}) error {
-	data, err := json.Marshal(networkState)
+	data, err := json.Marshal(v)
 	if err != nil {
 		return err
 	}
@@ -63,18 +61,19 @@ func (s *SyncPipe) ReadFromChild() error {
 	return nil
 }
 
-func (s *SyncPipe) ReadFromParent() (*network.NetworkState, error) {
+func (s *SyncPipe) ReadFromParent(v interface{}) error {
 	data, err := ioutil.ReadAll(s.child)
 	if err != nil {
-		return nil, fmt.Errorf("error reading from sync pipe %s", err)
+		return fmt.Errorf("error reading from sync pipe %s", err)
 	}
-	var networkState *network.NetworkState
+
 	if len(data) > 0 {
-		if err := json.Unmarshal(data, &networkState); err != nil {
+		if err := json.Unmarshal(data, v); err != nil {
-			return nil, err
+			return err
 		}
 	}
-	return networkState, nil
+
+	return nil
 }
 
 func (s *SyncPipe) ReportChildError(err error) {

syncpipe/sync_pipe_test.go

@@ -3,10 +3,12 @@ package syncpipe
 import (
 	"fmt"
 	"testing"
-
-	"github.com/docker/libcontainer/network"
 )
 
+type testStruct struct {
+	Name string
+}
+
 func TestSendErrorFromChild(t *testing.T) {
 	pipe, err := NewSyncPipe()
 	if err != nil {
@@ -46,16 +48,16 @@ func TestSendPayloadToChild(t *testing.T) {
 
 	expected := "libcontainer"
 
-	if err := pipe.SendToChild(&network.NetworkState{VethHost: expected}); err != nil {
+	if err := pipe.SendToChild(testStruct{Name: expected}); err != nil {
 		t.Fatal(err)
 	}
 
-	payload, err := pipe.ReadFromParent()
+	var s *testStruct
-	if err != nil {
+	if err := pipe.ReadFromParent(&s); err != nil {
 		t.Fatal(err)
 	}
 
-	if payload.VethHost != expected {
+	if s.Name != expected {
-		t.Fatalf("expected veth host %q but received %q", expected, payload.VethHost)
+		t.Fatalf("expected name %q but received %q", expected, s.Name)
 	}
 }