From e9d3ac025d0782b7bfa9673a57a66fdc47b3c32c Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 16 Sep 2015 15:58:55 +0200 Subject: [PATCH] runtime: config: linux: Edit BlockIO struct `WeightDevice`, `ThrottleReadBpsDevice`, `ThrottleWriteBpsDevice`, `ThrottleReadIOpsDevice`, `ThrottleWriteIOpsDevice` are now slices to well defined structs to allow setting multiple devices in their respective blkio file. By using a string to represents those values it wasn't possible to set correct values when multiple devices were passed in the config (either newline separated or comma separated). Signed-off-by: Antonio Murdaca --- runtime-config-linux.md | 167 ++++++++++++++++++++++++++++++---------- runtime_config_linux.go | 40 ++++++++-- 2 files changed, 158 insertions(+), 49 deletions(-) diff --git a/runtime-config-linux.md b/runtime-config-linux.md index f6bf82ee..a1e5da57 100644 --- a/runtime-config-linux.md +++ b/runtime-config-linux.md @@ -50,16 +50,16 @@ within the container. Devices is an array specifying the list of devices to be created in the container. Next parameters can be specified: -* type - type of device: 'c', 'b', 'u' or 'p'. More info in `man mknod` -* path - full path to device inside container -* major, minor - major, minor numbers for device. More info in `man mknod`. +* **type** - type of device: `c`, `b`, `u` or `p`. More info in `man mknod` +* **path** - full path to device inside container +* **major, minor** - major, minor numbers for device. More info in `man mknod`. There is special value: `-1`, which means `*` for `device` cgroup setup. -* permissions - cgroup permissions for device. A composition of 'r' - (read), 'w' (write), and 'm' (mknod). -* fileMode - file mode for device file -* uid - uid of device owner -* gid - gid of device owner +* **permissions** - cgroup permissions for device. A composition of `r` + (read), `w` (write), and `m` (mknod). +* **fileMode** - file mode for device file +* **uid** - uid of device owner +* **gid** - gid of device owner ```json "devices": [ @@ -146,45 +146,128 @@ The cgroups will be created if they don't exist. `cgroupsPath` can be used to either control the cgroups hierarchy for containers or to run a new process in an existing container. -Optionally, cgroups limits can be specified via `resources`. +You can configure a container's cgroups via the `resources` field of the Linux configuration. +Do not specify `resources` unless limits have to be updated. +For example, to run a new process in an existing container without updating limits, `resources` need not be specified. + +#### Disable out-of-memory killer ```json - "resources": { - "disableOOMKiller": false, - "memory": { - "limit": 0, - "reservation": 0, - "swap": 0, - "kernel": 0, - "swappiness": -1 - }, - "cpu": { - "shares": 0, - "quota": 0, - "period": 0, - "realtimeRuntime": 0, - "realtimePeriod": 0, - "cpus": "", - "mems": "" - }, - "blockIO": { - "blkioWeight": 0, - "blkioWeightDevice": "", - "blkioThrottleReadBpsDevice": "", - "blkioThrottleWriteBpsDevice": "", - "blkioThrottleReadIopsDevice": "", - "blkioThrottleWriteIopsDevice": "" - }, - "hugepageLimits": null, - "network": { - "classId": "", - "priorities": null - } + "disableOOMKiller": false +``` + +#### Memory + +```json + "memory": { + "limit": 0, + "reservation": 0, + "swap": 0, + "kernel": 0, + "swappiness": -1 } ``` -Do not specify `resources` unless limits have to be updated. -For example, to run a new process in an existing container without updating limits, `resources` need not be specified. +#### CPU + +```json + "cpu": { + "shares": 0, + "quota": 0, + "period": 0, + "realtimeRuntime": 0, + "realtimePeriod": 0, + "cpus": "", + "mems": "" + } +``` + +#### Block IO Controller + +`blockIO` represents the cgroup subsystem `blkio` which implements the block io controller. +For more information, see the [kernel cgroups documentation about `blkio`](https://www.kernel.org/doc/Documentation/cgroups/blkio-controller.txt). + +The following parameters can be specified to setup the block io controller: + +* **`blkioWeight`** *(uint16, optional)* - specifies per-cgroup weight. This is default weight of the group on all devices until and unless overridden by per-device rules. The range is from 10 to 1000. + +* **`blkioLeafWeight`** *(uint16, optional)* - equivalents of `blkioWeight` for the purpose of deciding how much weight tasks in the given cgroup has while competing with the cgroup's child cgroups. The range is from 10 to 1000. + +* **`blkioWeightDevice`** *(array, optional)* - specifies the list of devices which will be bandwidth rate limited. The following parameters can be specified per-device: + * **`major, minor`** *(int64, required)* - major, minor numbers for device. More info in `man mknod`. + * **`weight`** *(uint16, optional)* - bandwidth rate for the device, range is from 10 to 1000. + * **`leafWeight`** *(uint16, optional)* - bandwidth rate for the device while competing with the cgroup's child cgroups, range is from 10 to 1000, cfq scheduler only. + + You must specify at least one of `weight` or `leafWeight` in a given entry, and can specify both. + +* **`blkioThrottleReadBpsDevice`**, **`blkioThrottleWriteBpsDevice`**, **`blkioThrottleReadIOPSDevice`**, **`blkioThrottleWriteIOPSDevice`** *(array, optional)* - specify the list of devices which will be IO rate limited. The following parameters can be specified per-device: + * **`major, minor`** *(int64, required)* - major, minor numbers for device. More info in `man mknod`. + * **`rate`** *(uint64, required)* - IO rate limit for the device + +###### Example + +```json + "blockIO": { + "blkioWeight": 0, + "blkioLeafWeight": 0, + "blkioWeightDevice": [ + { + "major": 8, + "minor": 0, + "weight": 500, + "leafWeight": 300 + }, + { + "major": 8, + "minor": 16, + "weight": 500 + } + ], + "blkioThrottleReadBpsDevice": [ + { + "major": 8, + "minor": 0, + "rate": 600 + } + ], + "blkioThrottleWriteIOPSDevice": [ + { + "major": 8, + "minor": 16, + "rate": 300 + } + ] + } +``` + +#### Huge page limits + +```json + "hugepageLimits": [ + { + "pageSize": "2MB", + "limit": 9223372036854771712 + } + ] +``` + +#### Network + +```json + "network": { + "classId": "ClassId", + "priorities": [ + { + "name": "eth0", + "priority": 500 + }, + { + "name": "eth1", + "priority": 1000 + } + ] + } +``` ## Sysctl diff --git a/runtime_config_linux.go b/runtime_config_linux.go index 38e49923..04790237 100644 --- a/runtime_config_linux.go +++ b/runtime_config_linux.go @@ -104,20 +104,46 @@ type InterfacePriority struct { Priority int64 `json:"priority"` } +// blockIODevice holds major:minor format supported in blkio cgroup +type blockIODevice struct { + // Major is the device's major number. + Major int64 `json:"major"` + // Minor is the device's minor number. + Minor int64 `json:"minor"` +} + +// WeightDevice struct holds a `major:minor weight` pair for blkioWeightDevice +type WeightDevice struct { + blockIODevice + // Weight is the bandwidth rate for the device, range is from 10 to 1000 + Weight uint16 `json:"weight"` + // LeafWeight is the bandwidth rate for the device while competing with the cgroup's child cgroups, range is from 10 to 1000, cfq scheduler only + LeafWeight uint16 `json:"leafWeight"` +} + +// ThrottleDevice struct holds a `major:minor rate_per_second` pair +type ThrottleDevice struct { + blockIODevice + // Rate is the IO rate limit per cgroup per device + Rate uint64 `json:"rate"` +} + // BlockIO for Linux cgroup 'blkio' resource management type BlockIO struct { // Specifies per cgroup weight, range is from 10 to 1000 - Weight int64 `json:"blkioWeight"` + Weight uint16 `json:"blkioWeight"` + // Specifies tasks' weight in the given cgroup while competing with the cgroup's child cgroups, range is from 10 to 1000, cfq scheduler only + LeafWeight uint16 `json:"blkioLeafWeight"` // Weight per cgroup per device, can override BlkioWeight - WeightDevice string `json:"blkioWeightDevice"` + WeightDevice []*WeightDevice `json:"blkioWeightDevice"` // IO read rate limit per cgroup per device, bytes per second - ThrottleReadBpsDevice string `json:"blkioThrottleReadBpsDevice"` - // IO write rate limit per cgroup per divice, bytes per second - ThrottleWriteBpsDevice string `json:"blkioThrottleWriteBpsDevice"` + ThrottleReadBpsDevice []*ThrottleDevice `json:"blkioThrottleReadBpsDevice"` + // IO write rate limit per cgroup per device, bytes per second + ThrottleWriteBpsDevice []*ThrottleDevice `json:"blkioThrottleWriteBpsDevice"` // IO read rate limit per cgroup per device, IO per second - ThrottleReadIOpsDevice string `json:"blkioThrottleReadIopsDevice"` + ThrottleReadIOPSDevice []*ThrottleDevice `json:"blkioThrottleReadIOPSDevice"` // IO write rate limit per cgroup per device, IO per second - ThrottleWriteIOpsDevice string `json:"blkioThrottleWriteIopsDevice"` + ThrottleWriteIOPSDevice []*ThrottleDevice `json:"blkioThrottleWriteIOPSDevice"` } // Memory for Linux cgroup 'memory' resource management