jasder
/
runc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #143 from vishh/enter_cgroup 

Enter cgroups as part of NsEnter
This commit is contained in:
Victor Marmol 2014-08-14 14:04:19 -04:00
parent 6940d0ea85 86ba98bbfd
commit 29363e2d2d
10 changed files with 204 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cgroups/cgroups.go
View File

@ -37,4 +37,5 @@ type Cgroup struct {
type ActiveCgroup interface {
Cleanup() error
Paths() (map[string]string, error)
}

22
cgroups/fs/apply_raw.go
View File

@ -21,12 +21,16 @@ var (
"perf_event": &PerfEventGroup{},
"freezer": &FreezerGroup{},
}
CgroupProcesses = "cgroup.procs"
)
type subsystem interface {
Set(*data) error
// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
GetStats(path string, stats *cgroups.Stats) error
// Removes the cgroup represented by 'data'.
Remove(*data) error
GetStats(string, *cgroups.Stats) error
// Creates and joins the cgroup represented by data.
Set(*data) error
}
type data struct {
@ -149,6 +153,18 @@ func (raw *data) parent(subsystem string) (string, error) {
return filepath.Join(raw.root, subsystem, initPath), nil
}
func (raw *data) Paths() (map[string]string, error) {
paths := make(map[string]string)
for sysname := range subsystems {
path, err := raw.path(sysname)
if err != nil {
return nil, err
}
paths[sysname] = path
}
return paths, nil
}
func (raw *data) path(subsystem string) (string, error) {
// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
if filepath.IsAbs(raw.cgroup) {
@ -169,7 +185,7 @@ func (raw *data) join(subsystem string) (string, error) {
if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
return "", err
}
if err := writeFile(path, "cgroup.procs", strconv.Itoa(raw.pid)); err != nil {
if err := writeFile(path, CgroupProcesses, strconv.Itoa(raw.pid)); err != nil {
return "", err
}
return path, nil

31
cgroups/fs/cpuset.go
View File

@ -20,19 +20,10 @@ func (s *CpusetGroup) Set(d *data) error {
if err != nil {
return err
}
if err := s.ensureParent(dir); err != nil {
return err
}
// because we are not using d.join we need to place the pid into the procs file
// unlike the other subsystems
if err := writeFile(dir, "cgroup.procs", strconv.Itoa(d.pid)); err != nil {
return err
}
if err := writeFile(dir, "cpuset.cpus", d.c.CpusetCpus); err != nil {
return err
}
return s.SetDir(dir, d.c.CpusetCpus, d.pid)
}
return nil
}
@ -44,6 +35,24 @@ func (s *CpusetGroup) GetStats(path string, stats *cgroups.Stats) error {
return nil
}
func (s *CpusetGroup) SetDir(dir, value string, pid int) error {
if err := s.ensureParent(dir); err != nil {
return err
}
// because we are not using d.join we need to place the pid into the procs file
// unlike the other subsystems
if err := writeFile(dir, "cgroup.procs", strconv.Itoa(pid)); err != nil {
return err
}
if err := writeFile(dir, "cpuset.cpus", value); err != nil {
return err
}
return nil
}
func (s *CpusetGroup) getSubsystemSettings(parent string) (cpus []byte, mems []byte, err error) {
if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil {
return

2
cgroups/fs/memory.go
View File

@ -14,7 +14,7 @@ type MemoryGroup struct {
func (s *MemoryGroup) Set(d *data) error {
dir, err := d.join("memory")
// only return an error for memory if it was not specified
// only return an error for memory if it was specified
if err != nil && (d.c.Memory != 0 || d.c.MemoryReservation != 0 || d.c.MemorySwap != 0) {
return err
}

301
cgroups/systemd/apply_systemd.go
View File

@ -21,7 +21,7 @@ import (
)
type systemdCgroup struct {
cleanupDirs []string
cgroup *cgroups.Cgroup
}
type subsystem interface {
@ -84,39 +84,15 @@ func getIfaceForUnit(unitName string) string {
return "Unit"
}
type cgroupArg struct {
File string
Value string
}
func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) {
var (
unitName = getUnitName(c)
slice = "system.slice"
properties []systemd1.Property
cpuArgs []cgroupArg
cpusetArgs []cgroupArg
memoryArgs []cgroupArg
res systemdCgroup
res = &systemdCgroup{}
)
// First set up things not supported by systemd
// -1 disables memorySwap
if c.MemorySwap >= 0 && (c.Memory != 0 || c.MemorySwap > 0) {
memorySwap := c.MemorySwap
if memorySwap == 0 {
// By default, MemorySwap is set to twice the size of RAM.
memorySwap = c.Memory * 2
}
memoryArgs = append(memoryArgs, cgroupArg{"memory.memsw.limit_in_bytes", strconv.FormatInt(memorySwap, 10)})
}
if c.CpusetCpus != "" {
cpusetArgs = append(cpusetArgs, cgroupArg{"cpuset.cpus", c.CpusetCpus})
}
res.cgroup = c
if c.Slice != "" {
slice = c.Slice
@ -150,201 +126,84 @@ func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) {
return nil, err
}
// To work around the lack of /dev/pts/* support above we need to manually add these
// so, ask systemd for the cgroup used
props, err := theConn.GetUnitTypeProperties(unitName, getIfaceForUnit(unitName))
if err != nil {
return nil, err
}
cgroup := props["ControlGroup"].(string)
if !c.AllowAllDevices {
// Atm we can't use the systemd device support because of two missing things:
// * Support for wildcards to allow mknod on any device
// * Support for wildcards to allow /dev/pts support
//
// The second is available in more recent systemd as "char-pts", but not in e.g. v208 which is
// in wide use. When both these are availalable we will be able to switch, but need to keep the old
// implementation for backwards compat.
//
// Note: we can't use systemd to set up the initial limits, and then change the cgroup
// because systemd will re-write the device settings if it needs to re-apply the cgroup context.
// This happens at least for v208 when any sibling unit is started.
mountpoint, err := cgroups.FindCgroupMountpoint("devices")
if err != nil {
if err := joinDevices(c, pid); err != nil {
return nil, err
}
initPath, err := cgroups.GetInitCgroupDir("devices")
if err != nil {
return nil, err
}
dir := filepath.Join(mountpoint, initPath, c.Parent, c.Name)
res.cleanupDirs = append(res.cleanupDirs, dir)
if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) {
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil {
return nil, err
}
if err := writeFile(dir, "devices.deny", "a"); err != nil {
return nil, err
}
for _, dev := range c.AllowedDevices {
if err := writeFile(dir, "devices.allow", dev.GetCgroupAllowString()); err != nil {
return nil, err
}
}
}
if len(cpuArgs) != 0 {
mountpoint, err := cgroups.FindCgroupMountpoint("cpu")
if err != nil {
// -1 disables memorySwap
if c.MemorySwap >= 0 && (c.Memory != 0 || c.MemorySwap > 0) {
if err := joinMemory(c, pid); err != nil {
return nil, err
}
path := filepath.Join(mountpoint, cgroup)
for _, arg := range cpuArgs {
if err := ioutil.WriteFile(filepath.Join(path, arg.File), []byte(arg.Value), 0700); err != nil {
return nil, err
}
}
}
if len(memoryArgs) != 0 {
mountpoint, err := cgroups.FindCgroupMountpoint("memory")
if err != nil {
return nil, err
}
path := filepath.Join(mountpoint, cgroup)
for _, arg := range memoryArgs {
if err := ioutil.WriteFile(filepath.Join(path, arg.File), []byte(arg.Value), 0700); err != nil {
return nil, err
}
}
}
// we need to manually join the freezer cgroup in systemd because it does not currently support it
// via the dbus api
freezerPath, err := joinFreezer(c, pid)
if err != nil {
if err := joinFreezer(c, pid); err != nil {
return nil, err
}
res.cleanupDirs = append(res.cleanupDirs, freezerPath)
if len(cpusetArgs) != 0 {
// systemd does not atm set up the cpuset controller, so we must manually
// join it. Additionally that is a very finicky controller where each
// level must have a full setup as the default for a new directory is "no cpus",
// so we avoid using any hierarchies here, creating a toplevel directory.
mountpoint, err := cgroups.FindCgroupMountpoint("cpuset")
if err != nil {
return nil, err
}
initPath, err := cgroups.GetInitCgroupDir("cpuset")
if err != nil {
return nil, err
}
var (
foundCpus bool
foundMems bool
rootPath = filepath.Join(mountpoint, initPath)
path = filepath.Join(mountpoint, initPath, c.Parent+"-"+c.Name)
)
res.cleanupDirs = append(res.cleanupDirs, path)
if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
return nil, err
}
for _, arg := range cpusetArgs {
if arg.File == "cpuset.cpus" {
foundCpus = true
}
if arg.File == "cpuset.mems" {
foundMems = true
}
if err := ioutil.WriteFile(filepath.Join(path, arg.File), []byte(arg.Value), 0700); err != nil {
return nil, err
}
}
// These are required, if not specified inherit from parent
if !foundCpus {
s, err := ioutil.ReadFile(filepath.Join(rootPath, "cpuset.cpus"))
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(path, "cpuset.cpus"), s, 0700); err != nil {
return nil, err
}
}
// These are required, if not specified inherit from parent
if !foundMems {
s, err := ioutil.ReadFile(filepath.Join(rootPath, "cpuset.mems"))
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(path, "cpuset.mems"), s, 0700); err != nil {
return nil, err
}
}
if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil {
if c.CpusetCpus != "" {
if err := joinCpuset(c, pid); err != nil {
return nil, err
}
}
return &res, nil
return res, nil
}
func writeFile(dir, file, data string) error {
return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
}
func (c *systemdCgroup) Paths() (map[string]string, error) {
paths := make(map[string]string)
for sysname := range subsystems {
subsystemPath, err := getSubsystemPath(c.cgroup, sysname)
if err != nil {
// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
if err == cgroups.ErrNotFound {
continue
}
return nil, err
}
paths[sysname] = subsystemPath
}
return paths, nil
}
func (c *systemdCgroup) Cleanup() error {
// systemd cleans up, we don't need to do much
paths, err := c.Paths()
if err != nil {
return err
}
for _, path := range c.cleanupDirs {
for _, path := range paths {
os.RemoveAll(path)
}
return nil
}
func joinFreezer(c *cgroups.Cgroup, pid int) (string, error) {
func joinFreezer(c *cgroups.Cgroup, pid int) error {
path, err := getSubsystemPath(c, "freezer")
if err != nil {
return "", err
return err
}
if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
return "", err
return err
}
if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil {
return "", err
}
return path, nil
return ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700)
}
func getSubsystemPath(c *cgroups.Cgroup, subsystem string) (string, error) {
@ -389,20 +248,12 @@ func Freeze(c *cgroups.Cgroup, state cgroups.FreezerState) error {
}
func GetPids(c *cgroups.Cgroup) ([]int, error) {
unitName := getUnitName(c)
mountpoint, err := cgroups.FindCgroupMountpoint("cpu")
path, err := getSubsystemPath(c, "cpu")
if err != nil {
return nil, err
}
props, err := theConn.GetUnitTypeProperties(unitName, getIfaceForUnit(unitName))
if err != nil {
return nil, err
}
cgroup := props["ControlGroup"].(string)
return cgroups.ReadProcsFile(filepath.Join(mountpoint, cgroup))
return cgroups.ReadProcsFile(path)
}
func getUnitName(c *cgroups.Cgroup) string {
@ -437,3 +288,71 @@ func GetStats(c *cgroups.Cgroup) (*cgroups.Stats, error) {
return stats, nil
}
// Atm we can't use the systemd device support because of two missing things:
// * Support for wildcards to allow mknod on any device
// * Support for wildcards to allow /dev/pts support
//
// The second is available in more recent systemd as "char-pts", but not in e.g. v208 which is
// in wide use. When both these are availalable we will be able to switch, but need to keep the old
// implementation for backwards compat.
//
// Note: we can't use systemd to set up the initial limits, and then change the cgroup
// because systemd will re-write the device settings if it needs to re-apply the cgroup context.
// This happens at least for v208 when any sibling unit is started.
func joinDevices(c *cgroups.Cgroup, pid int) error {
path, err := getSubsystemPath(c, "devices")
if err != nil {
return err
}
if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
return err
}
if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"), []byte(strconv.Itoa(pid)), 0700); err != nil {
return err
}
if err := writeFile(path, "devices.deny", "a"); err != nil {
return err
}
for _, dev := range c.AllowedDevices {
if err := writeFile(path, "devices.allow", dev.GetCgroupAllowString()); err != nil {
return err
}
}
return nil
}
func joinMemory(c *cgroups.Cgroup, pid int) error {
memorySwap := c.MemorySwap
if memorySwap == 0 {
// By default, MemorySwap is set to twice the size of RAM.
memorySwap = c.Memory * 2
}
path, err := getSubsystemPath(c, "memory")
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(path, "memory.memsw.limit_in_bytes"), []byte(strconv.FormatInt(memorySwap, 10)), 0700)
}
// systemd does not atm set up the cpuset controller, so we must manually
// join it. Additionally that is a very finicky controller where each
// level must have a full setup as the default for a new directory is "no cpus"
func joinCpuset(c *cgroups.Cgroup, pid int) error {
path, err := getSubsystemPath(c, "cpuset")
if err != nil {
return err
}
s := &fs.CpusetGroup{}
return s.SetDir(path, c.CpusetCpus, pid)
}

21
cgroups/utils.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
@ -166,3 +167,23 @@ func parseCgroupFile(subsystem string, r io.Reader) (string, error) {
}
return "", ErrNotFound
}
func pathExists(path string) bool {
if _, err := os.Stat(path); err != nil {
return false
}
return true
}
func EnterPid(cgroupPaths map[string]string, pid int) error {
for _, path := range cgroupPaths {
if pathExists(path) {
if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"),
[]byte(strconv.Itoa(pid)), 0700); err != nil {
return err
}
}
}
return nil
}

12
namespaces/exec.go
View File

@ -56,14 +56,19 @@ func Exec(container *libcontainer.Config, stdin io.Reader, stdout, stderr io.Wri
// Do this before syncing with child so that no children
// can escape the cgroup
cleaner, err := SetupCgroups(container, command.Process.Pid)
cgroupRef, err := SetupCgroups(container, command.Process.Pid)
if err != nil {
command.Process.Kill()
command.Wait()
return -1, err
}
if cleaner != nil {
defer cleaner.Cleanup()
defer cgroupRef.Cleanup()
cgroupPaths, err := cgroupRef.Paths()
if err != nil {
command.Process.Kill()
command.Wait()
return -1, err
}
var networkState network.NetworkState
@ -77,6 +82,7 @@ func Exec(container *libcontainer.Config, stdin io.Reader, stdout, stderr io.Wri
InitPid: command.Process.Pid,
InitStartTime: started,
NetworkState: networkState,
CgroupPaths: cgroupPaths,
}
if err := libcontainer.SaveState(dataPath, state); err != nil {

10
namespaces/execin.go
View File

@ -12,6 +12,7 @@ import (
"syscall"
"github.com/docker/libcontainer"
"github.com/docker/libcontainer/cgroups"
"github.com/docker/libcontainer/label"
"github.com/docker/libcontainer/syncpipe"
"github.com/docker/libcontainer/system"
@ -59,6 +60,11 @@ func ExecIn(container *libcontainer.Config, state *libcontainer.State, userArgs
}
pipe.CloseChild()
// Enter cgroups.
if err := EnterCgroups(state, cmd.Process.Pid); err != nil {
return -1, err
}
if err := pipe.SendToChild(container); err != nil {
cmd.Process.Kill()
cmd.Wait()
@ -102,3 +108,7 @@ func FinalizeSetns(container *libcontainer.Config, args []string) error {
panic("unreachable")
}
func EnterCgroups(state *libcontainer.State, pid int) error {
return cgroups.EnterPid(state.CgroupPaths, pid)
}

20
namespaces/nsenter/nsenter.c
View File

@ -31,8 +31,8 @@ void get_args(int *argc, char ***argv)
contents_size += kBufSize;
contents = (char *)realloc(contents, contents_size);
bytes_read =
read(fd, contents + contents_offset,
contents_size - contents_offset);
read(fd, contents + contents_offset,
contents_size - contents_offset);
contents_offset += bytes_read;
}
while (bytes_read > 0);
@ -80,20 +80,20 @@ void nsenter()
char **argv;
get_args(&argc, &argv);
// check argv 0 to ensure that we are supposed to setns
// we use strncmp to test for a value of "nsenter" but also allows alternate implmentations
// after the setns code path to continue to use the argv 0 to determine actions to be run
// resulting in the ability to specify "nsenter-mknod", "nsenter-exec", etc...
if (strncmp(argv[0], kNsEnter, strlen(kNsEnter)) != 0) {
return;
}
// check argv 0 to ensure that we are supposed to setns
// we use strncmp to test for a value of "nsenter" but also allows alternate implmentations
// after the setns code path to continue to use the argv 0 to determine actions to be run
// resulting in the ability to specify "nsenter-mknod", "nsenter-exec", etc...
if (strncmp(argv[0], kNsEnter, strlen(kNsEnter)) != 0) {
return;
}
static const struct option longopts[] = {
{"nspid", required_argument, NULL, 'n'},
{"console", required_argument, NULL, 't'},
{NULL, 0, NULL, 0}
};
pid_t init_pid = -1;
char *init_pid_str = NULL;
char *console = NULL;

3
state.go
View File

@ -18,6 +18,9 @@ type State struct {
// Network runtime state.
NetworkState network.NetworkState `json:"network_state,omitempty"`
// Path to all the cgroups setup for a container. Key is cgroup subsystem name.
CgroupPaths map[string]string `json:"cgroup_paths,omitempty"`
}
// The running state of the container.