diff --git a/libcontainer/cgroups/fs/memory.go b/libcontainer/cgroups/fs/memory.go index 6b4a9eac..43a9dda8 100644 --- a/libcontainer/cgroups/fs/memory.go +++ b/libcontainer/cgroups/fs/memory.go @@ -5,6 +5,7 @@ package fs import ( "bufio" "fmt" + "math" "os" "path/filepath" "strconv" @@ -12,6 +13,7 @@ import ( "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/system" ) type MemoryGroup struct { @@ -33,7 +35,7 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) { } } // We have to set kernel memory here, as we can't change it once - // processes have been attached. + // processes have been attached to the cgroup. if err := s.SetKernelMemory(path, d.config); err != nil { return err } @@ -55,9 +57,44 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) { } func (s *MemoryGroup) SetKernelMemory(path string, cgroup *configs.Cgroup) error { - // This has to be done separately because it has special constraints (it - // can't be done after there are processes attached to the cgroup). - if cgroup.Resources.KernelMemory > 0 { + // This has to be done separately because it has special + // constraints (it can only be initialized before setting up a + // hierarchy or adding a task to the cgroups. However, if + // sucessfully initialized, it can be updated anytime afterwards) + if cgroup.Resources.KernelMemory != 0 { + kmemInitialized := false + // Is kmem.limit_in_bytes already set? + kmemValue, err := getCgroupParamUint(path, "memory.kmem.limit_in_bytes") + if err != nil { + return err + } + switch system.GetLongBit() { + case 32: + kmemInitialized = uint32(kmemValue) != uint32(math.MaxUint32) + case 64: + kmemInitialized = kmemValue != uint64(math.MaxUint64) + } + + if !kmemInitialized { + // If hierarchy is set, we can't change the limit + usesHierarchy, err := getCgroupParamUint(path, "memory.use_hierarchy") + if err != nil { + return err + } + if usesHierarchy != 0 { + return fmt.Errorf("cannot initialize kmem.limit_in_bytes if use_hierarchy is already set") + } + + // If there's already tasks in the cgroup, we can't change the limit either + tasks, err := getCgroupParamString(path, "tasks") + if err != nil { + return err + } + if tasks != "" { + return fmt.Errorf("cannot initialize kmem.limit_in_bytes after task have joined this cgroup") + } + } + if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil { return err } @@ -113,6 +150,10 @@ func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error { return err } + if err := s.SetKernelMemory(path, cgroup); err != nil { + return err + } + if cgroup.Resources.MemoryReservation != 0 { if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil { return err diff --git a/libcontainer/cgroups/systemd/apply_systemd.go b/libcontainer/cgroups/systemd/apply_systemd.go index 5365bc88..dbf760ad 100644 --- a/libcontainer/cgroups/systemd/apply_systemd.go +++ b/libcontainer/cgroups/systemd/apply_systemd.go @@ -214,11 +214,9 @@ func (m *Manager) Apply(pid int) error { newProp("BlockIOWeight", uint64(c.Resources.BlkioWeight))) } - // We need to set kernel memory before processes join cgroup because - // kmem.limit_in_bytes can only be set when the cgroup is empty. - // And swap memory limit needs to be set after memory limit, only - // memory limit is handled by systemd, so it's kind of ugly here. - if c.Resources.KernelMemory > 0 { + // We have to set kernel memory here, as we can't change it once + // processes have been attached to the cgroup. + if c.Resources.KernelMemory != 0 { if err := setKernelMemory(c); err != nil { return err } @@ -469,11 +467,5 @@ func setKernelMemory(c *configs.Cgroup) error { return err } - if err := os.MkdirAll(path, 0755); err != nil { - return err - } - - // This doesn't get called by manager.Set, so we need to do it here. - s := &fs.MemoryGroup{} - return s.SetKernelMemory(path, c) + return os.MkdirAll(path, 0755) } diff --git a/libcontainer/system/sysconfig.go b/libcontainer/system/sysconfig.go index b3a07cba..3e3d7e01 100644 --- a/libcontainer/system/sysconfig.go +++ b/libcontainer/system/sysconfig.go @@ -10,3 +10,7 @@ import "C" func GetClockTicks() int { return int(C.sysconf(C._SC_CLK_TCK)) } + +func GetLongBit() int { + return int(C.sysconf(C._SC_LONG_BIT)) +} diff --git a/main.go b/main.go index 51b34648..1ceac446 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,7 @@ const ( version = "0.1.1" specConfig = "config.json" usage = `Open Container Initiative runtime - + runc is a command line client for running applications packaged according to the Open Container Initiative (OCI) format and is a compliant implementation of the Open Container Initiative specification. @@ -30,7 +30,7 @@ direct child of the process supervisor. Containers are configured using bundles. A bundle for a container is a directory that includes a specification file named "` + specConfig + `" and a root filesystem. -The root filesystem contains the contents of the container. +The root filesystem contains the contents of the container. To start a new instance of a container: @@ -99,6 +99,7 @@ func main() { specCommand, startCommand, stateCommand, + updateCommand, } app.Before = func(context *cli.Context) error { if context.GlobalBool("debug") { diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats new file mode 100644 index 00000000..a48024d0 --- /dev/null +++ b/tests/integration/cgroups.bats @@ -0,0 +1,77 @@ +#!/usr/bin/env bats + +load helpers + +UPDATE_TEST_RUNC_ROOT="$BATS_TMPDIR/runc-cgroups-integration-test" + +CGROUP_MEMORY="" + +TEST_CGROUP_NAME="runc-cgroups-integration-test" + +function init_cgroup_path() { + base_path=$(grep "rw," /proc/self/mountinfo | grep -i -m 1 'MEMORY$' | cut -d ' ' -f 5) + CGROUP_MEMORY="${base_path}/${TEST_CGROUP_NAME}" +} + +function teardown() { + rm -f $BATS_TMPDIR/runc-update-integration-test.json + teardown_running_container_inroot test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT + teardown_busybox +} + +function setup() { + teardown + setup_busybox + + init_cgroup_path +} + +function check_cgroup_value() { + cgroup=$1 + source=$2 + expected=$3 + + current=$(cat $cgroup/$source) + echo $cgroup/$source + echo "current" $current "!?" "$expected" + [ "$current" -eq "$expected" ] +} + +@test "cgroups-kernel-memory-initialized" { + # Add cgroup path + sed -i 's/\("linux": {\)/\1\n "cgroupsPath": "runc-cgroups-integration-test",/' ${BUSYBOX_BUNDLE}/config.json + + # Set some initial known values + DATA=$(cat <<-EOF + "memory": { + "kernel": 16777216 + }, +EOF + ) + DATA=$(echo ${DATA} | sed 's/\n/\\n/g') + sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json + + # start a detached busybox to work with + "$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_cgroups_kmem + [ "$status" -eq 0 ] + wait_for_container_inroot 15 1 test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT + + # update kernel memory limit + "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_cgroups_kmem --kernel-memory 50331648 + [ "$status" -eq 0 ] + check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 +} + +@test "cgroups-kernel-memory-uninitialized" { + # Add cgroup path + sed -i 's/\("linux": {\)/\1\n "cgroupsPath": "runc-cgroups-integration-test",/' ${BUSYBOX_BUNDLE}/config.json + + # start a detached busybox to work with + run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_cgroups_kmem + [ "$status" -eq 0 ] + wait_for_container_inroot 15 1 test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT + + # update kernel memory limit + run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_cgroups_kmem --kernel-memory 50331648 + [ ! "$status" -eq 0 ] +} diff --git a/tests/integration/update.bats b/tests/integration/update.bats new file mode 100644 index 00000000..9bda96fc --- /dev/null +++ b/tests/integration/update.bats @@ -0,0 +1,205 @@ +#!/usr/bin/env bats + +load helpers + +UPDATE_TEST_RUNC_ROOT="$BATS_TMPDIR/runc-update-integration-test" + +CGROUP_MEMORY="" +CGROUP_CPUSET="" +CGROUP_CPU="" +CGROUP_BLKIO="" + +function init_cgroup_path() { + for g in MEMORY CPUSET CPU BLKIO; do + base_path=$(grep "rw," /proc/self/mountinfo | grep -i -m 1 "$g\$" | cut -d ' ' -f 5) + eval CGROUP_${g}="${base_path}/runc-update-integration-test" + done +} + +function teardown() { + rm -f $BATS_TMPDIR/runc-update-integration-test.json + teardown_running_container_inroot test_update $UPDATE_TEST_RUNC_ROOT + teardown_busybox +} + +function setup() { + teardown + setup_busybox + + # Add cgroup path + sed -i 's/\("linux": {\)/\1\n "cgroupsPath": "runc-update-integration-test",/' ${BUSYBOX_BUNDLE}/config.json + + # Set some initial known values + DATA=$(cat < $BATS_TMPDIR/runc-update-integration-test.json + + run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update -r $BATS_TMPDIR/runc-update-integration-test.json test_update + [ "$status" -eq 0 ] + check_cgroup_value $CGROUP_BLKIO "blkio.weight" 1000 + check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000 + check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000 + check_cgroup_value $CGROUP_CPU "cpu.shares" 100 + check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0 + check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216 + check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432 + check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824 +} diff --git a/update.go b/update.go new file mode 100644 index 00000000..bbfc97c4 --- /dev/null +++ b/update.go @@ -0,0 +1,186 @@ +// +build linux + +package main + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + + "github.com/codegangsta/cli" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func u64Ptr(i uint64) *uint64 { return &i } +func u16Ptr(i uint16) *uint16 { return &i } + +var updateCommand = cli.Command{ + Name: "update", + Usage: "update container resource constraints", + ArgsUsage: ` `, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "resources, r", + Value: "", + Usage: `path to the file containing the resources to update or '-' to read from the standard input. + +The accepted format is as follow (unchanged values can be omitted): + +{ + "memory": { + "limit": 0, + "reservation": 0, + "swap": 0, + "kernel": 0 + }, + "cpu": { + "shares": 0, + "quota": 0, + "period": 0, + "cpus": "", + "mems": "" + }, + "blockIO": { + "blkioWeight": 0 + }, +} + +Note: if data is to be read from a file or the standard output all +other options are ignored. +`, + }, + + cli.IntFlag{ + Name: "blkio-weight", + Usage: "Specifies per cgroup weight, range is from 10 to 1000.", + }, + cli.StringFlag{ + Name: "cpu-period", + Usage: "CPU period to be used for hardcapping (in usecs). 0 to use system default.", + }, + cli.StringFlag{ + Name: "cpu-quota", + Usage: "CPU hardcap limit (in usecs). Allowed cpu time in a given period.", + }, + cli.StringFlag{ + Name: "cpu-share", + Usage: "CPU shares (relative weight vs. other containers)", + }, + cli.StringFlag{ + Name: "cpuset-cpus", + Usage: "CPU(s) to use", + }, + cli.StringFlag{ + Name: "cpuset-mems", + Usage: "Memory node(s) to use", + }, + cli.StringFlag{ + Name: "kernel-memory", + Usage: "Kernel memory limit (in bytes) for tcp buffer", + }, + cli.StringFlag{ + Name: "memory", + Usage: "Memory limit (in bytes)", + }, + cli.StringFlag{ + Name: "memory-reservation", + Usage: "Memory reservation or soft_limit (in bytes)", + }, + cli.StringFlag{ + Name: "memory-swap", + Usage: "Total memory usage (memory + swap); set `-1` to enable unlimited swap", + }, + }, + Action: func(context *cli.Context) { + container, err := getContainer(context) + if err != nil { + fatal(err) + } + + r := specs.Resources{ + Memory: &specs.Memory{ + Limit: u64Ptr(0), + Reservation: u64Ptr(0), + Swap: u64Ptr(0), + Kernel: u64Ptr(0), + }, + CPU: &specs.CPU{ + Shares: u64Ptr(0), + Quota: u64Ptr(0), + Period: u64Ptr(0), + Cpus: sPtr(""), + Mems: sPtr(""), + }, + BlockIO: &specs.BlockIO{ + Weight: u16Ptr(0), + }, + } + + config := container.Config() + + if in := context.String("resources"); in != "" { + var ( + f *os.File + err error + ) + switch in { + case "-": + f = os.Stdin + default: + f, err = os.Open(in) + if err != nil { + fatal(err) + } + } + err = json.NewDecoder(f).Decode(&r) + if err != nil { + fatal(err) + } + } else { + if val := context.Int("blkio-weight"); val != 0 { + r.BlockIO.Weight = u16Ptr(uint16(val)) + } + if val := context.String("cpuset-cpus"); val != "" { + r.CPU.Cpus = &val + } + if val := context.String("cpuset-mems"); val != "" { + r.CPU.Mems = &val + } + + for opt, dest := range map[string]*uint64{ + "cpu-period": r.CPU.Period, + "cpu-quota": r.CPU.Quota, + "cpu-share": r.CPU.Shares, + "kernel-memory": r.Memory.Kernel, + "memory": r.Memory.Limit, + "memory-reservation": r.Memory.Reservation, + "memory-swap": r.Memory.Swap, + } { + if val := context.String(opt); val != "" { + var err error + *dest, err = strconv.ParseUint(val, 10, 64) + if err != nil { + fatal(fmt.Errorf("invalid value for %s: %s", opt, err)) + } + } + } + } + + // Update the value + config.Cgroups.Resources.BlkioWeight = *r.BlockIO.Weight + config.Cgroups.Resources.CpuPeriod = int64(*r.CPU.Period) + config.Cgroups.Resources.CpuQuota = int64(*r.CPU.Quota) + config.Cgroups.Resources.CpuShares = int64(*r.CPU.Shares) + config.Cgroups.Resources.CpusetCpus = *r.CPU.Cpus + config.Cgroups.Resources.CpusetMems = *r.CPU.Mems + config.Cgroups.Resources.KernelMemory = int64(*r.Memory.Kernel) + config.Cgroups.Resources.Memory = int64(*r.Memory.Limit) + config.Cgroups.Resources.MemoryReservation = int64(*r.Memory.Reservation) + config.Cgroups.Resources.MemorySwap = int64(*r.Memory.Swap) + + if err := container.Set(config); err != nil { + fatal(err) + } + }, +}