diff --git a/cgroups/cgroups.go b/cgroups/cgroups.go index 64ece568..59845486 100644 --- a/cgroups/cgroups.go +++ b/cgroups/cgroups.go @@ -37,4 +37,5 @@ type Cgroup struct { type ActiveCgroup interface { Cleanup() error + Paths() (map[string]string, error) } diff --git a/cgroups/fs/apply_raw.go b/cgroups/fs/apply_raw.go index 2d1a1523..e20cdbb9 100644 --- a/cgroups/fs/apply_raw.go +++ b/cgroups/fs/apply_raw.go @@ -21,12 +21,16 @@ var ( "perf_event": &PerfEventGroup{}, "freezer": &FreezerGroup{}, } + CgroupProcesses = "cgroup.procs" ) type subsystem interface { - Set(*data) error + // Returns the stats, as 'stats', corresponding to the cgroup under 'path'. + GetStats(path string, stats *cgroups.Stats) error + // Removes the cgroup represented by 'data'. Remove(*data) error - GetStats(string, *cgroups.Stats) error + // Creates and joins the cgroup represented by data. + Set(*data) error } type data struct { @@ -149,6 +153,18 @@ func (raw *data) parent(subsystem string) (string, error) { return filepath.Join(raw.root, subsystem, initPath), nil } +func (raw *data) Paths() (map[string]string, error) { + paths := make(map[string]string) + for sysname := range subsystems { + path, err := raw.path(sysname) + if err != nil { + return nil, err + } + paths[sysname] = path + } + return paths, nil +} + func (raw *data) path(subsystem string) (string, error) { // If the cgroup name/path is absolute do not look relative to the cgroup of the init process. if filepath.IsAbs(raw.cgroup) { @@ -169,7 +185,7 @@ func (raw *data) join(subsystem string) (string, error) { if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { return "", err } - if err := writeFile(path, "cgroup.procs", strconv.Itoa(raw.pid)); err != nil { + if err := writeFile(path, CgroupProcesses, strconv.Itoa(raw.pid)); err != nil { return "", err } return path, nil diff --git a/cgroups/fs/cpuset.go b/cgroups/fs/cpuset.go index 9570125f..88477394 100644 --- a/cgroups/fs/cpuset.go +++ b/cgroups/fs/cpuset.go @@ -20,19 +20,10 @@ func (s *CpusetGroup) Set(d *data) error { if err != nil { return err } - if err := s.ensureParent(dir); err != nil { - return err - } - // because we are not using d.join we need to place the pid into the procs file - // unlike the other subsystems - if err := writeFile(dir, "cgroup.procs", strconv.Itoa(d.pid)); err != nil { - return err - } - if err := writeFile(dir, "cpuset.cpus", d.c.CpusetCpus); err != nil { - return err - } + return s.SetDir(dir, d.c.CpusetCpus, d.pid) } + return nil } @@ -44,6 +35,24 @@ func (s *CpusetGroup) GetStats(path string, stats *cgroups.Stats) error { return nil } +func (s *CpusetGroup) SetDir(dir, value string, pid int) error { + if err := s.ensureParent(dir); err != nil { + return err + } + + // because we are not using d.join we need to place the pid into the procs file + // unlike the other subsystems + if err := writeFile(dir, "cgroup.procs", strconv.Itoa(pid)); err != nil { + return err + } + + if err := writeFile(dir, "cpuset.cpus", value); err != nil { + return err + } + + return nil +} + func (s *CpusetGroup) getSubsystemSettings(parent string) (cpus []byte, mems []byte, err error) { if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil { return diff --git a/cgroups/fs/memory.go b/cgroups/fs/memory.go index c27150d2..ea92934a 100644 --- a/cgroups/fs/memory.go +++ b/cgroups/fs/memory.go @@ -14,7 +14,7 @@ type MemoryGroup struct { func (s *MemoryGroup) Set(d *data) error { dir, err := d.join("memory") - // only return an error for memory if it was not specified + // only return an error for memory if it was specified if err != nil && (d.c.Memory != 0 || d.c.MemoryReservation != 0 || d.c.MemorySwap != 0) { return err } diff --git a/cgroups/systemd/apply_systemd.go b/cgroups/systemd/apply_systemd.go index 01e5bf49..7fcd99bc 100644 --- a/cgroups/systemd/apply_systemd.go +++ b/cgroups/systemd/apply_systemd.go @@ -21,7 +21,7 @@ import ( ) type systemdCgroup struct { - cleanupDirs []string + cgroup *cgroups.Cgroup } type subsystem interface { @@ -84,39 +84,15 @@ func getIfaceForUnit(unitName string) string { return "Unit" } -type cgroupArg struct { - File string - Value string -} - func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { var ( unitName = getUnitName(c) slice = "system.slice" properties []systemd1.Property - cpuArgs []cgroupArg - cpusetArgs []cgroupArg - memoryArgs []cgroupArg - res systemdCgroup + res = &systemdCgroup{} ) - // First set up things not supported by systemd - - // -1 disables memorySwap - if c.MemorySwap >= 0 && (c.Memory != 0 || c.MemorySwap > 0) { - memorySwap := c.MemorySwap - - if memorySwap == 0 { - // By default, MemorySwap is set to twice the size of RAM. - memorySwap = c.Memory * 2 - } - - memoryArgs = append(memoryArgs, cgroupArg{"memory.memsw.limit_in_bytes", strconv.FormatInt(memorySwap, 10)}) - } - - if c.CpusetCpus != "" { - cpusetArgs = append(cpusetArgs, cgroupArg{"cpuset.cpus", c.CpusetCpus}) - } + res.cgroup = c if c.Slice != "" { slice = c.Slice @@ -150,201 +126,84 @@ func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { return nil, err } - // To work around the lack of /dev/pts/* support above we need to manually add these - // so, ask systemd for the cgroup used - props, err := theConn.GetUnitTypeProperties(unitName, getIfaceForUnit(unitName)) - if err != nil { - return nil, err - } - - cgroup := props["ControlGroup"].(string) - if !c.AllowAllDevices { - // Atm we can't use the systemd device support because of two missing things: - // * Support for wildcards to allow mknod on any device - // * Support for wildcards to allow /dev/pts support - // - // The second is available in more recent systemd as "char-pts", but not in e.g. v208 which is - // in wide use. When both these are availalable we will be able to switch, but need to keep the old - // implementation for backwards compat. - // - // Note: we can't use systemd to set up the initial limits, and then change the cgroup - // because systemd will re-write the device settings if it needs to re-apply the cgroup context. - // This happens at least for v208 when any sibling unit is started. - - mountpoint, err := cgroups.FindCgroupMountpoint("devices") - if err != nil { + if err := joinDevices(c, pid); err != nil { return nil, err } - - initPath, err := cgroups.GetInitCgroupDir("devices") - if err != nil { - return nil, err - } - - dir := filepath.Join(mountpoint, initPath, c.Parent, c.Name) - - res.cleanupDirs = append(res.cleanupDirs, dir) - - if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) { - return nil, err - } - - if err := ioutil.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil { - return nil, err - } - - if err := writeFile(dir, "devices.deny", "a"); err != nil { - return nil, err - } - - for _, dev := range c.AllowedDevices { - if err := writeFile(dir, "devices.allow", dev.GetCgroupAllowString()); err != nil { - return nil, err - } - } } - if len(cpuArgs) != 0 { - mountpoint, err := cgroups.FindCgroupMountpoint("cpu") - if err != nil { + // -1 disables memorySwap + if c.MemorySwap >= 0 && (c.Memory != 0 || c.MemorySwap > 0) { + if err := joinMemory(c, pid); err != nil { return nil, err } - path := filepath.Join(mountpoint, cgroup) - - for _, arg := range cpuArgs { - if err := ioutil.WriteFile(filepath.Join(path, arg.File), []byte(arg.Value), 0700); err != nil { - return nil, err - } - } - } - - if len(memoryArgs) != 0 { - mountpoint, err := cgroups.FindCgroupMountpoint("memory") - if err != nil { - return nil, err - } - - path := filepath.Join(mountpoint, cgroup) - - for _, arg := range memoryArgs { - if err := ioutil.WriteFile(filepath.Join(path, arg.File), []byte(arg.Value), 0700); err != nil { - return nil, err - } - } } // we need to manually join the freezer cgroup in systemd because it does not currently support it // via the dbus api - freezerPath, err := joinFreezer(c, pid) - if err != nil { + if err := joinFreezer(c, pid); err != nil { return nil, err } - res.cleanupDirs = append(res.cleanupDirs, freezerPath) - if len(cpusetArgs) != 0 { - // systemd does not atm set up the cpuset controller, so we must manually - // join it. Additionally that is a very finicky controller where each - // level must have a full setup as the default for a new directory is "no cpus", - // so we avoid using any hierarchies here, creating a toplevel directory. - mountpoint, err := cgroups.FindCgroupMountpoint("cpuset") - if err != nil { - return nil, err - } - - initPath, err := cgroups.GetInitCgroupDir("cpuset") - if err != nil { - return nil, err - } - - var ( - foundCpus bool - foundMems bool - - rootPath = filepath.Join(mountpoint, initPath) - path = filepath.Join(mountpoint, initPath, c.Parent+"-"+c.Name) - ) - - res.cleanupDirs = append(res.cleanupDirs, path) - - if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { - return nil, err - } - - for _, arg := range cpusetArgs { - if arg.File == "cpuset.cpus" { - foundCpus = true - } - if arg.File == "cpuset.mems" { - foundMems = true - } - if err := ioutil.WriteFile(filepath.Join(path, arg.File), []byte(arg.Value), 0700); err != nil { - return nil, err - } - } - - // These are required, if not specified inherit from parent - if !foundCpus { - s, err := ioutil.ReadFile(filepath.Join(rootPath, "cpuset.cpus")) - if err != nil { - return nil, err - } - - if err := ioutil.WriteFile(filepath.Join(path, "cpuset.cpus"), s, 0700); err != nil { - return nil, err - } - } - - // These are required, if not specified inherit from parent - if !foundMems { - s, err := ioutil.ReadFile(filepath.Join(rootPath, "cpuset.mems")) - if err != nil { - return nil, err - } - - if err := ioutil.WriteFile(filepath.Join(path, "cpuset.mems"), s, 0700); err != nil { - return nil, err - } - } - - if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil { + if c.CpusetCpus != "" { + if err := joinCpuset(c, pid); err != nil { return nil, err } } - return &res, nil + return res, nil } func writeFile(dir, file, data string) error { return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) } +func (c *systemdCgroup) Paths() (map[string]string, error) { + paths := make(map[string]string) + + for sysname := range subsystems { + subsystemPath, err := getSubsystemPath(c.cgroup, sysname) + if err != nil { + // Don't fail if a cgroup hierarchy was not found, just skip this subsystem + if err == cgroups.ErrNotFound { + continue + } + + return nil, err + } + + paths[sysname] = subsystemPath + } + + return paths, nil +} + func (c *systemdCgroup) Cleanup() error { // systemd cleans up, we don't need to do much + paths, err := c.Paths() + if err != nil { + return err + } - for _, path := range c.cleanupDirs { + for _, path := range paths { os.RemoveAll(path) } return nil } -func joinFreezer(c *cgroups.Cgroup, pid int) (string, error) { +func joinFreezer(c *cgroups.Cgroup, pid int) error { path, err := getSubsystemPath(c, "freezer") if err != nil { - return "", err + return err } if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { - return "", err + return err } - if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil { - return "", err - } - - return path, nil + return ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700) } func getSubsystemPath(c *cgroups.Cgroup, subsystem string) (string, error) { @@ -389,20 +248,12 @@ func Freeze(c *cgroups.Cgroup, state cgroups.FreezerState) error { } func GetPids(c *cgroups.Cgroup) ([]int, error) { - unitName := getUnitName(c) - - mountpoint, err := cgroups.FindCgroupMountpoint("cpu") + path, err := getSubsystemPath(c, "cpu") if err != nil { return nil, err } - props, err := theConn.GetUnitTypeProperties(unitName, getIfaceForUnit(unitName)) - if err != nil { - return nil, err - } - cgroup := props["ControlGroup"].(string) - - return cgroups.ReadProcsFile(filepath.Join(mountpoint, cgroup)) + return cgroups.ReadProcsFile(path) } func getUnitName(c *cgroups.Cgroup) string { @@ -437,3 +288,71 @@ func GetStats(c *cgroups.Cgroup) (*cgroups.Stats, error) { return stats, nil } + +// Atm we can't use the systemd device support because of two missing things: +// * Support for wildcards to allow mknod on any device +// * Support for wildcards to allow /dev/pts support +// +// The second is available in more recent systemd as "char-pts", but not in e.g. v208 which is +// in wide use. When both these are availalable we will be able to switch, but need to keep the old +// implementation for backwards compat. +// +// Note: we can't use systemd to set up the initial limits, and then change the cgroup +// because systemd will re-write the device settings if it needs to re-apply the cgroup context. +// This happens at least for v208 when any sibling unit is started. +func joinDevices(c *cgroups.Cgroup, pid int) error { + path, err := getSubsystemPath(c, "devices") + if err != nil { + return err + } + + if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { + return err + } + + if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil { + return err + } + + if err := writeFile(path, "devices.deny", "a"); err != nil { + return err + } + + for _, dev := range c.AllowedDevices { + if err := writeFile(path, "devices.allow", dev.GetCgroupAllowString()); err != nil { + return err + } + } + + return nil +} + +func joinMemory(c *cgroups.Cgroup, pid int) error { + memorySwap := c.MemorySwap + + if memorySwap == 0 { + // By default, MemorySwap is set to twice the size of RAM. + memorySwap = c.Memory * 2 + } + + path, err := getSubsystemPath(c, "memory") + if err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(path, "memory.memsw.limit_in_bytes"), []byte(strconv.FormatInt(memorySwap, 10)), 0700) +} + +// systemd does not atm set up the cpuset controller, so we must manually +// join it. Additionally that is a very finicky controller where each +// level must have a full setup as the default for a new directory is "no cpus" +func joinCpuset(c *cgroups.Cgroup, pid int) error { + path, err := getSubsystemPath(c, "cpuset") + if err != nil { + return err + } + + s := &fs.CpusetGroup{} + + return s.SetDir(path, c.CpusetCpus, pid) +} diff --git a/cgroups/utils.go b/cgroups/utils.go index ce5c4f33..6688ff71 100644 --- a/cgroups/utils.go +++ b/cgroups/utils.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strconv" @@ -166,3 +167,23 @@ func parseCgroupFile(subsystem string, r io.Reader) (string, error) { } return "", ErrNotFound } + +func pathExists(path string) bool { + if _, err := os.Stat(path); err != nil { + return false + } + return true +} + +func EnterPid(cgroupPaths map[string]string, pid int) error { + for _, path := range cgroupPaths { + if pathExists(path) { + if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), + []byte(strconv.Itoa(pid)), 0700); err != nil { + return err + } + } + } + + return nil +} diff --git a/namespaces/exec.go b/namespaces/exec.go index c9b2037c..382abfbc 100644 --- a/namespaces/exec.go +++ b/namespaces/exec.go @@ -56,14 +56,19 @@ func Exec(container *libcontainer.Config, stdin io.Reader, stdout, stderr io.Wri // Do this before syncing with child so that no children // can escape the cgroup - cleaner, err := SetupCgroups(container, command.Process.Pid) + cgroupRef, err := SetupCgroups(container, command.Process.Pid) if err != nil { command.Process.Kill() command.Wait() return -1, err } - if cleaner != nil { - defer cleaner.Cleanup() + defer cgroupRef.Cleanup() + + cgroupPaths, err := cgroupRef.Paths() + if err != nil { + command.Process.Kill() + command.Wait() + return -1, err } var networkState network.NetworkState @@ -77,6 +82,7 @@ func Exec(container *libcontainer.Config, stdin io.Reader, stdout, stderr io.Wri InitPid: command.Process.Pid, InitStartTime: started, NetworkState: networkState, + CgroupPaths: cgroupPaths, } if err := libcontainer.SaveState(dataPath, state); err != nil { diff --git a/namespaces/execin.go b/namespaces/execin.go index 8958332a..8b81edec 100644 --- a/namespaces/execin.go +++ b/namespaces/execin.go @@ -12,6 +12,7 @@ import ( "syscall" "github.com/docker/libcontainer" + "github.com/docker/libcontainer/cgroups" "github.com/docker/libcontainer/label" "github.com/docker/libcontainer/syncpipe" "github.com/docker/libcontainer/system" @@ -59,6 +60,11 @@ func ExecIn(container *libcontainer.Config, state *libcontainer.State, userArgs } pipe.CloseChild() + // Enter cgroups. + if err := EnterCgroups(state, cmd.Process.Pid); err != nil { + return -1, err + } + if err := pipe.SendToChild(container); err != nil { cmd.Process.Kill() cmd.Wait() @@ -102,3 +108,7 @@ func FinalizeSetns(container *libcontainer.Config, args []string) error { panic("unreachable") } + +func EnterCgroups(state *libcontainer.State, pid int) error { + return cgroups.EnterPid(state.CgroupPaths, pid) +} diff --git a/namespaces/nsenter/nsenter.c b/namespaces/nsenter/nsenter.c index 3bc29e2b..2869dd14 100644 --- a/namespaces/nsenter/nsenter.c +++ b/namespaces/nsenter/nsenter.c @@ -31,8 +31,8 @@ void get_args(int *argc, char ***argv) contents_size += kBufSize; contents = (char *)realloc(contents, contents_size); bytes_read = - read(fd, contents + contents_offset, - contents_size - contents_offset); + read(fd, contents + contents_offset, + contents_size - contents_offset); contents_offset += bytes_read; } while (bytes_read > 0); @@ -80,20 +80,20 @@ void nsenter() char **argv; get_args(&argc, &argv); - // check argv 0 to ensure that we are supposed to setns - // we use strncmp to test for a value of "nsenter" but also allows alternate implmentations - // after the setns code path to continue to use the argv 0 to determine actions to be run - // resulting in the ability to specify "nsenter-mknod", "nsenter-exec", etc... - if (strncmp(argv[0], kNsEnter, strlen(kNsEnter)) != 0) { - return; - } + // check argv 0 to ensure that we are supposed to setns + // we use strncmp to test for a value of "nsenter" but also allows alternate implmentations + // after the setns code path to continue to use the argv 0 to determine actions to be run + // resulting in the ability to specify "nsenter-mknod", "nsenter-exec", etc... + if (strncmp(argv[0], kNsEnter, strlen(kNsEnter)) != 0) { + return; + } static const struct option longopts[] = { {"nspid", required_argument, NULL, 'n'}, {"console", required_argument, NULL, 't'}, {NULL, 0, NULL, 0} }; - + pid_t init_pid = -1; char *init_pid_str = NULL; char *console = NULL; diff --git a/state.go b/state.go index ee5d14d2..208b4c62 100644 --- a/state.go +++ b/state.go @@ -18,6 +18,9 @@ type State struct { // Network runtime state. NetworkState network.NetworkState `json:"network_state,omitempty"` + + // Path to all the cgroups setup for a container. Key is cgroup subsystem name. + CgroupPaths map[string]string `json:"cgroup_paths,omitempty"` } // The running state of the container.