Merge pull request #790 from mlaventure/runc-update-cgroup-kmem-limit

Runc update cgroup kmem limit
This commit is contained in:
Qiang Huang 2016-05-09 14:01:18 +08:00
commit d49ece5a83
7 changed files with 524 additions and 18 deletions

View File

@ -5,6 +5,7 @@ package fs
import (
"bufio"
"fmt"
"math"
"os"
"path/filepath"
"strconv"
@ -12,6 +13,7 @@ import (
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/configs"
"github.com/opencontainers/runc/libcontainer/system"
)
type MemoryGroup struct {
@ -33,7 +35,7 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
}
}
// We have to set kernel memory here, as we can't change it once
// processes have been attached.
// processes have been attached to the cgroup.
if err := s.SetKernelMemory(path, d.config); err != nil {
return err
}
@ -55,9 +57,44 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
}
func (s *MemoryGroup) SetKernelMemory(path string, cgroup *configs.Cgroup) error {
// This has to be done separately because it has special constraints (it
// can't be done after there are processes attached to the cgroup).
if cgroup.Resources.KernelMemory > 0 {
// This has to be done separately because it has special
// constraints (it can only be initialized before setting up a
// hierarchy or adding a task to the cgroups. However, if
// sucessfully initialized, it can be updated anytime afterwards)
if cgroup.Resources.KernelMemory != 0 {
kmemInitialized := false
// Is kmem.limit_in_bytes already set?
kmemValue, err := getCgroupParamUint(path, "memory.kmem.limit_in_bytes")
if err != nil {
return err
}
switch system.GetLongBit() {
case 32:
kmemInitialized = uint32(kmemValue) != uint32(math.MaxUint32)
case 64:
kmemInitialized = kmemValue != uint64(math.MaxUint64)
}
if !kmemInitialized {
// If hierarchy is set, we can't change the limit
usesHierarchy, err := getCgroupParamUint(path, "memory.use_hierarchy")
if err != nil {
return err
}
if usesHierarchy != 0 {
return fmt.Errorf("cannot initialize kmem.limit_in_bytes if use_hierarchy is already set")
}
// If there's already tasks in the cgroup, we can't change the limit either
tasks, err := getCgroupParamString(path, "tasks")
if err != nil {
return err
}
if tasks != "" {
return fmt.Errorf("cannot initialize kmem.limit_in_bytes after task have joined this cgroup")
}
}
if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil {
return err
}
@ -113,6 +150,10 @@ func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
return err
}
if err := s.SetKernelMemory(path, cgroup); err != nil {
return err
}
if cgroup.Resources.MemoryReservation != 0 {
if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil {
return err

View File

@ -214,11 +214,9 @@ func (m *Manager) Apply(pid int) error {
newProp("BlockIOWeight", uint64(c.Resources.BlkioWeight)))
}
// We need to set kernel memory before processes join cgroup because
// kmem.limit_in_bytes can only be set when the cgroup is empty.
// And swap memory limit needs to be set after memory limit, only
// memory limit is handled by systemd, so it's kind of ugly here.
if c.Resources.KernelMemory > 0 {
// We have to set kernel memory here, as we can't change it once
// processes have been attached to the cgroup.
if c.Resources.KernelMemory != 0 {
if err := setKernelMemory(c); err != nil {
return err
}
@ -469,11 +467,5 @@ func setKernelMemory(c *configs.Cgroup) error {
return err
}
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
// This doesn't get called by manager.Set, so we need to do it here.
s := &fs.MemoryGroup{}
return s.SetKernelMemory(path, c)
return os.MkdirAll(path, 0755)
}

View File

@ -10,3 +10,7 @@ import "C"
func GetClockTicks() int {
return int(C.sysconf(C._SC_CLK_TCK))
}
func GetLongBit() int {
return int(C.sysconf(C._SC_LONG_BIT))
}

View File

@ -99,6 +99,7 @@ func main() {
specCommand,
startCommand,
stateCommand,
updateCommand,
}
app.Before = func(context *cli.Context) error {
if context.GlobalBool("debug") {

View File

@ -0,0 +1,77 @@
#!/usr/bin/env bats
load helpers
UPDATE_TEST_RUNC_ROOT="$BATS_TMPDIR/runc-cgroups-integration-test"
CGROUP_MEMORY=""
TEST_CGROUP_NAME="runc-cgroups-integration-test"
function init_cgroup_path() {
base_path=$(grep "rw," /proc/self/mountinfo | grep -i -m 1 'MEMORY$' | cut -d ' ' -f 5)
CGROUP_MEMORY="${base_path}/${TEST_CGROUP_NAME}"
}
function teardown() {
rm -f $BATS_TMPDIR/runc-update-integration-test.json
teardown_running_container_inroot test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT
teardown_busybox
}
function setup() {
teardown
setup_busybox
init_cgroup_path
}
function check_cgroup_value() {
cgroup=$1
source=$2
expected=$3
current=$(cat $cgroup/$source)
echo $cgroup/$source
echo "current" $current "!?" "$expected"
[ "$current" -eq "$expected" ]
}
@test "cgroups-kernel-memory-initialized" {
# Add cgroup path
sed -i 's/\("linux": {\)/\1\n "cgroupsPath": "runc-cgroups-integration-test",/' ${BUSYBOX_BUNDLE}/config.json
# Set some initial known values
DATA=$(cat <<-EOF
"memory": {
"kernel": 16777216
},
EOF
)
DATA=$(echo ${DATA} | sed 's/\n/\\n/g')
sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json
# start a detached busybox to work with
"$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_cgroups_kmem
[ "$status" -eq 0 ]
wait_for_container_inroot 15 1 test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT
# update kernel memory limit
"$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_cgroups_kmem --kernel-memory 50331648
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648
}
@test "cgroups-kernel-memory-uninitialized" {
# Add cgroup path
sed -i 's/\("linux": {\)/\1\n "cgroupsPath": "runc-cgroups-integration-test",/' ${BUSYBOX_BUNDLE}/config.json
# start a detached busybox to work with
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_cgroups_kmem
[ "$status" -eq 0 ]
wait_for_container_inroot 15 1 test_cgroups_kmem $UPDATE_TEST_RUNC_ROOT
# update kernel memory limit
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_cgroups_kmem --kernel-memory 50331648
[ ! "$status" -eq 0 ]
}

View File

@ -0,0 +1,205 @@
#!/usr/bin/env bats
load helpers
UPDATE_TEST_RUNC_ROOT="$BATS_TMPDIR/runc-update-integration-test"
CGROUP_MEMORY=""
CGROUP_CPUSET=""
CGROUP_CPU=""
CGROUP_BLKIO=""
function init_cgroup_path() {
for g in MEMORY CPUSET CPU BLKIO; do
base_path=$(grep "rw," /proc/self/mountinfo | grep -i -m 1 "$g\$" | cut -d ' ' -f 5)
eval CGROUP_${g}="${base_path}/runc-update-integration-test"
done
}
function teardown() {
rm -f $BATS_TMPDIR/runc-update-integration-test.json
teardown_running_container_inroot test_update $UPDATE_TEST_RUNC_ROOT
teardown_busybox
}
function setup() {
teardown
setup_busybox
# Add cgroup path
sed -i 's/\("linux": {\)/\1\n "cgroupsPath": "runc-update-integration-test",/' ${BUSYBOX_BUNDLE}/config.json
# Set some initial known values
DATA=$(cat <<EOF
"memory": {
"limit": 33554432,
"reservation": 25165824,
"kernel": 16777216
},
"cpu": {
"shares": 100,
"quota": 500000,
"period": 1000000,
"cpus": "0"
},
"blockio": {
"blkioWeight": 1000
},
EOF
)
DATA=$(echo ${DATA} | sed 's/\n/\\n/g')
sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json
}
function check_cgroup_value() {
cgroup=$1
source=$2
expected=$3
current=$(cat $cgroup/$source)
[ "$current" -eq "$expected" ]
}
@test "update" {
# start a few busyboxes detached
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT start -d --console /dev/pts/ptmx test_update
[ "$status" -eq 0 ]
wait_for_container_inroot 15 1 test_update $UPDATE_TEST_RUNC_ROOT
init_cgroup_path
# check that initial values were properly set
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 1000
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000
check_cgroup_value $CGROUP_CPU "cpu.shares" 100
check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0
check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216
check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432
check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824
# update blkio-weight
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --blkio-weight 500
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 500
# update cpu-period
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --cpu-period 900000
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 900000
# update cpu-quota
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --cpu-quota 600000
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 600000
# update cpu-shares
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --cpu-share 200
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_CPU "cpu.shares" 200
# update cpuset if supported (i.e. we're running on a multicore cpu)
cpu_count=$(grep '^processor' /proc/cpuinfo | wc -l)
if [ $cpu_count -ge 1 ]; then
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --cpuset-cpus "1"
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 1
fi
# update memory limit
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --memory 67108864
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 67108864
# update memory soft limit
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --memory-reservation 33554432
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 33554432
# update memory swap (if available)
if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --memory-swap 96468992
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" 96468992
fi
# update kernel memory limit
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --kernel-memory 50331648
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648
# Revert to the test initial value via json on stding
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update -r - test_update <<EOF
{
"memory": {
"limit": 33554432,
"reservation": 25165824,
"kernel": 16777216
},
"cpu": {
"shares": 100,
"quota": 500000,
"period": 1000000,
"cpus": "0"
},
"blockIO": {
"blkioWeight": 1000
}
}
EOF
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 1000
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000
check_cgroup_value $CGROUP_CPU "cpu.shares" 100
check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0
check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216
check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432
check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824
# redo all the changes at once
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update test_update --blkio-weight 500 \
--cpu-period 900000 --cpu-quota 600000 --cpu-share 200 --memory 67108864 \
--memory-reservation 33554432 --kernel-memory 50331648
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 500
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 900000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 600000
check_cgroup_value $CGROUP_CPU "cpu.shares" 200
check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648
check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 67108864
check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 33554432
# reset to initial test value via json file
DATA=$(cat <<"EOF"
{
"memory": {
"limit": 33554432,
"reservation": 25165824,
"kernel": 16777216
},
"cpu": {
"shares": 100,
"quota": 500000,
"period": 1000000,
"cpus": "0"
},
"blockIO": {
"blkioWeight": 1000
}
}
EOF
)
echo $DATA > $BATS_TMPDIR/runc-update-integration-test.json
run "$RUNC" --root $UPDATE_TEST_RUNC_ROOT update -r $BATS_TMPDIR/runc-update-integration-test.json test_update
[ "$status" -eq 0 ]
check_cgroup_value $CGROUP_BLKIO "blkio.weight" 1000
check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000
check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000
check_cgroup_value $CGROUP_CPU "cpu.shares" 100
check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0
check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216
check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432
check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824
}

186
update.go Normal file
View File

@ -0,0 +1,186 @@
// +build linux
package main
import (
"encoding/json"
"fmt"
"os"
"strconv"
"github.com/codegangsta/cli"
"github.com/opencontainers/runtime-spec/specs-go"
)
func u64Ptr(i uint64) *uint64 { return &i }
func u16Ptr(i uint16) *uint16 { return &i }
var updateCommand = cli.Command{
Name: "update",
Usage: "update container resource constraints",
ArgsUsage: `<container-id> <ps options>`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "resources, r",
Value: "",
Usage: `path to the file containing the resources to update or '-' to read from the standard input.
The accepted format is as follow (unchanged values can be omitted):
{
"memory": {
"limit": 0,
"reservation": 0,
"swap": 0,
"kernel": 0
},
"cpu": {
"shares": 0,
"quota": 0,
"period": 0,
"cpus": "",
"mems": ""
},
"blockIO": {
"blkioWeight": 0
},
}
Note: if data is to be read from a file or the standard output all
other options are ignored.
`,
},
cli.IntFlag{
Name: "blkio-weight",
Usage: "Specifies per cgroup weight, range is from 10 to 1000.",
},
cli.StringFlag{
Name: "cpu-period",
Usage: "CPU period to be used for hardcapping (in usecs). 0 to use system default.",
},
cli.StringFlag{
Name: "cpu-quota",
Usage: "CPU hardcap limit (in usecs). Allowed cpu time in a given period.",
},
cli.StringFlag{
Name: "cpu-share",
Usage: "CPU shares (relative weight vs. other containers)",
},
cli.StringFlag{
Name: "cpuset-cpus",
Usage: "CPU(s) to use",
},
cli.StringFlag{
Name: "cpuset-mems",
Usage: "Memory node(s) to use",
},
cli.StringFlag{
Name: "kernel-memory",
Usage: "Kernel memory limit (in bytes) for tcp buffer",
},
cli.StringFlag{
Name: "memory",
Usage: "Memory limit (in bytes)",
},
cli.StringFlag{
Name: "memory-reservation",
Usage: "Memory reservation or soft_limit (in bytes)",
},
cli.StringFlag{
Name: "memory-swap",
Usage: "Total memory usage (memory + swap); set `-1` to enable unlimited swap",
},
},
Action: func(context *cli.Context) {
container, err := getContainer(context)
if err != nil {
fatal(err)
}
r := specs.Resources{
Memory: &specs.Memory{
Limit: u64Ptr(0),
Reservation: u64Ptr(0),
Swap: u64Ptr(0),
Kernel: u64Ptr(0),
},
CPU: &specs.CPU{
Shares: u64Ptr(0),
Quota: u64Ptr(0),
Period: u64Ptr(0),
Cpus: sPtr(""),
Mems: sPtr(""),
},
BlockIO: &specs.BlockIO{
Weight: u16Ptr(0),
},
}
config := container.Config()
if in := context.String("resources"); in != "" {
var (
f *os.File
err error
)
switch in {
case "-":
f = os.Stdin
default:
f, err = os.Open(in)
if err != nil {
fatal(err)
}
}
err = json.NewDecoder(f).Decode(&r)
if err != nil {
fatal(err)
}
} else {
if val := context.Int("blkio-weight"); val != 0 {
r.BlockIO.Weight = u16Ptr(uint16(val))
}
if val := context.String("cpuset-cpus"); val != "" {
r.CPU.Cpus = &val
}
if val := context.String("cpuset-mems"); val != "" {
r.CPU.Mems = &val
}
for opt, dest := range map[string]*uint64{
"cpu-period": r.CPU.Period,
"cpu-quota": r.CPU.Quota,
"cpu-share": r.CPU.Shares,
"kernel-memory": r.Memory.Kernel,
"memory": r.Memory.Limit,
"memory-reservation": r.Memory.Reservation,
"memory-swap": r.Memory.Swap,
} {
if val := context.String(opt); val != "" {
var err error
*dest, err = strconv.ParseUint(val, 10, 64)
if err != nil {
fatal(fmt.Errorf("invalid value for %s: %s", opt, err))
}
}
}
}
// Update the value
config.Cgroups.Resources.BlkioWeight = *r.BlockIO.Weight
config.Cgroups.Resources.CpuPeriod = int64(*r.CPU.Period)
config.Cgroups.Resources.CpuQuota = int64(*r.CPU.Quota)
config.Cgroups.Resources.CpuShares = int64(*r.CPU.Shares)
config.Cgroups.Resources.CpusetCpus = *r.CPU.Cpus
config.Cgroups.Resources.CpusetMems = *r.CPU.Mems
config.Cgroups.Resources.KernelMemory = int64(*r.Memory.Kernel)
config.Cgroups.Resources.Memory = int64(*r.Memory.Limit)
config.Cgroups.Resources.MemoryReservation = int64(*r.Memory.Reservation)
config.Cgroups.Resources.MemorySwap = int64(*r.Memory.Swap)
if err := container.Set(config); err != nil {
fatal(err)
}
},
}