Add cause to error messages

This is the inital port of the libcontainer.Error to added a cause to
all the existing error messages.  Going forward, when an error can be
wrapped because it is not being checked at the higher levels for
something like `os.IsNotExist` we can add more information to the error
message like cause and stack file/line information.  This will help
higher level tools to know what cause a container start or operation to
fail.

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

libcontainer/container_linux.go

@@ -141,7 +141,7 @@ func (c *linuxContainer) State() (*State, error) {
 func (c *linuxContainer) Processes() ([]int, error) {
 	pids, err := c.cgroupManager.GetAllPids()
 	if err != nil {
-		return nil, newSystemError(err)
+		return nil, newSystemErrorWithCause(err, "getting all container pids from cgroups")
 	}
 	return pids, nil
 }
@@ -152,14 +152,14 @@ func (c *linuxContainer) Stats() (*Stats, error) {
 		stats = &Stats{}
 	)
 	if stats.CgroupStats, err = c.cgroupManager.GetStats(); err != nil {
-		return stats, newSystemError(err)
+		return stats, newSystemErrorWithCause(err, "getting container stats from cgroups")
 	}
 	for _, iface := range c.config.Networks {
 		switch iface.Type {
 		case "veth":
 			istats, err := getNetworkInterfaceStats(iface.HostInterfaceName)
 			if err != nil {
-				return stats, newSystemError(err)
+				return stats, newSystemErrorWithCausef(err, "getting network stats for interface %q", iface.HostInterfaceName)
 			}
 			stats.Interfaces = append(stats.Interfaces, istats)
 		}
@@ -184,14 +184,14 @@ func (c *linuxContainer) Start(process *Process) error {
 	doInit := status == Destroyed
 	parent, err := c.newParentProcess(process, doInit)
 	if err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "creating new parent process")
 	}
 	if err := parent.start(); err != nil {
 		// terminate the process to ensure that it properly is reaped.
 		if err := parent.terminate(); err != nil {
 			logrus.Warn(err)
 		}
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "starting container process")
 	}
 	// generate a timestamp indicating when the container was started
 	c.created = time.Now().UTC()
@@ -211,12 +211,12 @@ func (c *linuxContainer) Start(process *Process) error {
 				Root:       c.config.Rootfs,
 				BundlePath: utils.SearchLabels(c.config.Labels, "bundle"),
 			}
-			for _, hook := range c.config.Hooks.Poststart {
+			for i, hook := range c.config.Hooks.Poststart {
 				if err := hook.Run(s); err != nil {
 					if err := parent.terminate(); err != nil {
 						logrus.Warn(err)
 					}
-					return newSystemError(err)
+					return newSystemErrorWithCausef(err, "running poststart hook %d", i)
 				}
 			}
 		}
@@ -226,7 +226,7 @@ func (c *linuxContainer) Start(process *Process) error {
 
 func (c *linuxContainer) Signal(s os.Signal) error {
 	if err := c.initProcess.signal(s); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "signaling init process")
 	}
 	return nil
 }
@@ -234,11 +234,11 @@ func (c *linuxContainer) Signal(s os.Signal) error {
 func (c *linuxContainer) newParentProcess(p *Process, doInit bool) (parentProcess, error) {
 	parentPipe, childPipe, err := newPipe()
 	if err != nil {
-		return nil, newSystemError(err)
+		return nil, newSystemErrorWithCause(err, "creating new init pipe")
 	}
 	cmd, err := c.commandTemplate(p, childPipe)
 	if err != nil {
-		return nil, newSystemError(err)
+		return nil, newSystemErrorWithCause(err, "creating new command template")
 	}
 	if !doInit {
 		return c.newSetnsProcess(p, cmd, parentPipe, childPipe)
@@ -299,7 +299,7 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe,
 	cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE="+string(initSetns))
 	state, err := c.currentState()
 	if err != nil {
-		return nil, newSystemError(err)
+		return nil, newSystemErrorWithCause(err, "getting container's current state")
 	}
 	// for setns process, we dont have to set cloneflags as the process namespaces
 	// will only be set via setns syscall
@@ -955,9 +955,9 @@ func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Proc
 				Pid:     int(notify.GetPid()),
 				Root:    c.config.Rootfs,
 			}
-			for _, hook := range c.config.Hooks.Prestart {
+			for i, hook := range c.config.Hooks.Prestart {
 				if err := hook.Run(s); err != nil {
-					return newSystemError(err)
+					return newSystemErrorWithCausef(err, "running prestart hook %d", i)
 				}
 			}
 		}
@@ -1046,7 +1046,7 @@ func (c *linuxContainer) isRunning() (bool, error) {
 		if err == syscall.ESRCH {
 			return false, nil
 		}
-		return false, newSystemError(err)
+		return false, newSystemErrorWithCausef(err, "sending signal 0 to pid %d", c.initProcess.pid())
 	}
 	return true, nil
 }
@@ -1057,7 +1057,7 @@ func (c *linuxContainer) isPaused() (bool, error) {
 		if os.IsNotExist(err) {
 			return false, nil
 		}
-		return false, newSystemError(err)
+		return false, newSystemErrorWithCause(err, "checking if container is paused")
 	}
 	return bytes.Equal(bytes.TrimSpace(data), []byte("FROZEN")), nil
 }
@@ -1125,7 +1125,7 @@ func (c *linuxContainer) orderNamespacePaths(namespaces map[configs.NamespaceTyp
 			}
 			// only set to join this namespace if it exists
 			if _, err := os.Lstat(p); err != nil {
-				return nil, newSystemError(err)
+				return nil, newSystemErrorWithCausef(err, "running lstat on namespace path %q", p)
 			}
 			// do not allow namespace path with comma as we use it to separate
 			// the namespace paths

libcontainer/generic_error.go

@@ -1,6 +1,7 @@
 package libcontainer
 
 import (
+	"fmt"
 	"io"
 	"text/template"
 	"time"
@@ -51,6 +52,21 @@ func newGenericError(err error, c ErrorCode) Error {
 }
 
 func newSystemError(err error) Error {
+	return createSystemError(err, "")
+}
+
+func newSystemErrorWithCausef(err error, cause string, v ...interface{}) Error {
+	return createSystemError(err, fmt.Sprintf(cause, v...))
+}
+
+func newSystemErrorWithCause(err error, cause string) Error {
+	return createSystemError(err, cause)
+}
+
+// createSystemError creates the specified error with the correct number of
+// stack frames skipped. This is only to be called by the other functions for
+// formatting the error.
+func createSystemError(err error, cause string) Error {
 	if le, ok := err.(Error); ok {
 		return le
 	}
@@ -58,7 +74,8 @@ func newSystemError(err error) Error {
 		Timestamp: time.Now(),
 		Err:       err,
 		ECode:     SystemError,
-		Stack:     stacktrace.Capture(1),
+		Cause:     cause,
+		Stack:     stacktrace.Capture(2),
 	}
 	if err != nil {
 		gerr.Message = err.Error()
@@ -70,12 +87,17 @@ type genericError struct {
 	Timestamp time.Time
 	ECode     ErrorCode
 	Err       error `json:"-"`
+	Cause     string
 	Message   string
 	Stack     stacktrace.Stacktrace
 }
 
 func (e *genericError) Error() string {
-	return e.Message
+	if e.Cause == "" {
+		return e.Message
+	}
+	frame := e.Stack.Frames[0]
+	return fmt.Sprintf("%s:%d: %s caused %q", frame.File, frame.Line, e.Cause, e.Message)
 }
 
 func (e *genericError) Code() ErrorCode {

libcontainer/process_linux.go

@@ -70,47 +70,47 @@ func (p *setnsProcess) start() (err error) {
 	err = p.cmd.Start()
 	p.childPipe.Close()
 	if err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "starting setns process")
 	}
 	if p.bootstrapData != nil {
 		if _, err := io.Copy(p.parentPipe, p.bootstrapData); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "copying bootstrap data to pipe")
 		}
 	}
 	if err = p.execSetns(); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "executing setns process")
 	}
 	if len(p.cgroupPaths) > 0 {
 		if err := cgroups.EnterPid(p.cgroupPaths, p.pid()); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCausef(err, "adding pid %d to cgroups", p.pid())
 		}
 	}
 	// set oom_score_adj
 	if err := setOomScoreAdj(p.config.Config.OomScoreAdj, p.pid()); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "setting oom score")
 	}
 	// set rlimits, this has to be done here because we lose permissions
 	// to raise the limits once we enter a user-namespace
 	if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "setting rlimits for process")
 	}
 	if err := utils.WriteJSON(p.parentPipe, p.config); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "writing config to pipe")
 	}
 
 	if err := syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "calling shutdown on init pipe")
 	}
 	// wait for the child process to fully complete and receive an error message
 	// if one was encoutered
 	var ierr *genericError
 	if err := json.NewDecoder(p.parentPipe).Decode(&ierr); err != nil && err != io.EOF {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "decoding init error from pipe")
 	}
 	// Must be done after Shutdown so the child will exit and we can wait for it.
 	if ierr != nil {
 		p.wait()
-		return newSystemError(ierr)
+		return ierr
 	}
 	return nil
 }
@@ -123,7 +123,7 @@ func (p *setnsProcess) execSetns() error {
 	status, err := p.cmd.Process.Wait()
 	if err != nil {
 		p.cmd.Wait()
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "waiting on setns process to finish")
 	}
 	if !status.Success() {
 		p.cmd.Wait()
@@ -132,7 +132,7 @@ func (p *setnsProcess) execSetns() error {
 	var pid *pid
 	if err := json.NewDecoder(p.parentPipe).Decode(&pid); err != nil {
 		p.cmd.Wait()
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "reading pid from init pipe")
 	}
 	process, err := os.FindProcess(pid.Pid)
 	if err != nil {
@@ -231,26 +231,26 @@ func (p *initProcess) start() error {
 	p.childPipe.Close()
 	if err != nil {
 		p.process.ops = nil
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "starting init process command")
 	}
 	if _, err := io.Copy(p.parentPipe, p.bootstrapData); err != nil {
 		return err
 	}
 	if err := p.execSetns(); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "running exec setns process for init")
 	}
 	// Save the standard descriptor names before the container process
 	// can potentially move them (e.g., via dup2()).  If we don't do this now,
 	// we won't know at checkpoint time which file descriptor to look up.
 	fds, err := getPipeFds(p.pid())
 	if err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCausef(err, "getting pipe fds for pid %d", p.pid())
 	}
 	p.setExternalDescriptors(fds)
 	// Do this before syncing with child so that no children
 	// can escape the cgroup
 	if err := p.manager.Apply(p.pid()); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "applying cgroup configuration for process")
 	}
 	defer func() {
 		if err != nil {
@@ -259,10 +259,10 @@ func (p *initProcess) start() error {
 		}
 	}()
 	if err := p.createNetworkInterfaces(); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "creating nework interfaces")
 	}
 	if err := p.sendConfig(); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "sending config to init process")
 	}
 	var (
 		procSync   syncT
@@ -278,21 +278,21 @@ loop:
 			if err == io.EOF {
 				break loop
 			}
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "decoding sync type from init pipe")
 		}
 		switch procSync.Type {
 		case procReady:
 			if err := p.manager.Set(p.config.Config); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "setting cgroup config for ready process")
 			}
 			// set oom_score_adj
 			if err := setOomScoreAdj(p.config.Config.OomScoreAdj, p.pid()); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "setting oom score for ready process")
 			}
 			// set rlimits, this has to be done here because we lose permissions
 			// to raise the limits once we enter a user-namespace
 			if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "setting rlimits for ready process")
 			}
 			// call prestart hooks
 			if !p.config.Config.Namespaces.Contains(configs.NEWNS) {
@@ -303,16 +303,16 @@ loop:
 						Pid:     p.pid(),
 						Root:    p.config.Config.Rootfs,
 					}
-					for _, hook := range p.config.Config.Hooks.Prestart {
+					for i, hook := range p.config.Config.Hooks.Prestart {
 						if err := hook.Run(s); err != nil {
-							return newSystemError(err)
+							return newSystemErrorWithCausef(err, "running prestart hook %d", i)
 						}
 					}
 				}
 			}
 			// Sync with child.
 			if err := utils.WriteJSON(p.parentPipe, syncT{procRun}); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "reading syncT run type")
 			}
 			sentRun = true
 		case procHooks:
@@ -324,22 +324,22 @@ loop:
 					Root:       p.config.Config.Rootfs,
 					BundlePath: utils.SearchLabels(p.config.Config.Labels, "bundle"),
 				}
-				for _, hook := range p.config.Config.Hooks.Prestart {
+				for i, hook := range p.config.Config.Hooks.Prestart {
 					if err := hook.Run(s); err != nil {
-						return newSystemError(err)
+						return newSystemErrorWithCausef(err, "running prestart hook %d", i)
 					}
 				}
 			}
 			// Sync with child.
 			if err := utils.WriteJSON(p.parentPipe, syncT{procResume}); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "reading syncT resume type")
 			}
 			sentResume = true
 		case procError:
 			// wait for the child process to fully complete and receive an error message
 			// if one was encoutered
 			if err := dec.Decode(&ierr); err != nil && err != io.EOF {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "decoding proc error from init")
 			}
 			if ierr != nil {
 				break loop
@@ -347,22 +347,22 @@ loop:
 			// Programmer error.
 			panic("No error following JSON procError payload.")
 		default:
-			return newSystemError(fmt.Errorf("invalid JSON synchronisation payload from child"))
+			return newSystemError(fmt.Errorf("invalid JSON payload from child"))
 		}
 	}
 	if !sentRun {
-		return newSystemError(fmt.Errorf("could not synchronise with container process: %v", ierr))
+		return newSystemErrorWithCause(ierr, "container init failed")
 	}
 	if p.config.Config.Namespaces.Contains(configs.NEWNS) && !sentResume {
 		return newSystemError(fmt.Errorf("could not synchronise after executing prestart hooks with container process"))
 	}
 	if err := syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "shutting down init pipe")
 	}
 	// Must be done after Shutdown so the child will exit and we can wait for it.
 	if ierr != nil {
 		p.wait()
-		return newSystemError(ierr)
+		return ierr
 	}
 	return nil
 }

libcontainer/rootfs_linux.go

@@ -39,35 +39,35 @@ func needsSetupDev(config *configs.Config) bool {
 // new mount namespace.
 func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWriter) (err error) {
 	if err := prepareRoot(config); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "preparing rootfs")
 	}
 
 	setupDev := needsSetupDev(config)
 	for _, m := range config.Mounts {
 		for _, precmd := range m.PremountCmds {
 			if err := mountCmd(precmd); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "running premount command")
 			}
 		}
 		if err := mountToRootfs(m, config.Rootfs, config.MountLabel); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCausef(err, "mounting %q to rootfs %q", m.Destination, config.Rootfs)
 		}
 
 		for _, postcmd := range m.PostmountCmds {
 			if err := mountCmd(postcmd); err != nil {
-				return newSystemError(err)
+				return newSystemErrorWithCause(err, "running postmount command")
 			}
 		}
 	}
 	if setupDev {
 		if err := createDevices(config); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "creating device nodes")
 		}
 		if err := setupPtmx(config, console); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "setting up ptmx")
 		}
 		if err := setupDevSymlinks(config.Rootfs); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "setting up /dev symlinks")
 		}
 	}
 	// Signal the parent to run the pre-start hooks.
@@ -78,7 +78,7 @@ func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWrit
 		return err
 	}
 	if err := syscall.Chdir(config.Rootfs); err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCausef(err, "changing dir to %q", config.Rootfs)
 	}
 	if config.NoPivotRoot {
 		err = msMoveRoot(config.Rootfs)
@@ -86,11 +86,11 @@ func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWrit
 		err = pivotRoot(config.Rootfs, config.PivotDir)
 	}
 	if err != nil {
-		return newSystemError(err)
+		return newSystemErrorWithCause(err, "jailing process inside rootfs")
 	}
 	if setupDev {
 		if err := reOpenDevNull(); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "reopening /dev/null inside container")
 		}
 	}
 	// remount dev as ro if specifed
@@ -98,7 +98,7 @@ func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWrit
 		if m.Destination == "/dev" {
 			if m.Flags&syscall.MS_RDONLY != 0 {
 				if err := remountReadonly(m.Destination); err != nil {
-					return newSystemError(err)
+					return newSystemErrorWithCausef(err, "remounting %q as readonly", m.Destination)
 				}
 			}
 			break
@@ -107,7 +107,7 @@ func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWrit
 	// set rootfs ( / ) as readonly
 	if config.Readonlyfs {
 		if err := setReadonly(); err != nil {
-			return newSystemError(err)
+			return newSystemErrorWithCause(err, "setting rootfs as readonly")
 		}
 	}
 	syscall.Umask(0022)
@@ -115,14 +115,12 @@ func setupRootfs(config *configs.Config, console *linuxConsole, pipe io.ReadWrit
 }
 
 func mountCmd(cmd configs.Command) error {
-
 	command := exec.Command(cmd.Path, cmd.Args[:]...)
 	command.Env = cmd.Env
 	command.Dir = cmd.Dir
 	if out, err := command.CombinedOutput(); err != nil {
 		return fmt.Errorf("%#v failed: %s: %v", cmd, string(out), err)
 	}
-
 	return nil
 }