From 3f6a31b71ed73864023dd8005200b674b64877fb Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Wed, 25 Mar 2020 18:30:44 -0700 Subject: [PATCH] tests/integration: simplify cgroup paths init 1. Consolidate all the cgroup-related initialization code to a single place, init_cgroup_paths(), so we can see which variables are set. 2. Lazily call init_cgroup_paths() from all places that require it. 3. Don't set globals KMEM and RT_PERIOD. 4. Slightly clarlify variables naming: - use OCI_CGROUPS_PATH for cgroupsPath in config.json - use REL_CGROUPS_PATH for relative cgroups path 5. Do not hardcode the list of cgroup subsystems -- get it from /proc/cgroup. 6. Preliminary support for cgroupv2 unified hierarchy (not yet working). Signed-off-by: Kir Kolyshkin --- tests/integration/helpers.bash | 62 ++++++++++++++++++++++------------ tests/integration/update.bats | 15 ++------ 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 161fb84e..0fc78f77 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -36,20 +36,6 @@ ROOT=$(mktemp -d "$BATS_TMPDIR/runc.XXXXXX") # Path to console socket. CONSOLE_SOCKET="$BATS_TMPDIR/console.sock" -# Cgroup paths -CGROUP_MEMORY_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\/ { print $5; exit }') -CGROUP_CPU_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\/ { print $5; exit }') -if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then - CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope" -else - CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup" -fi -CGROUP_MEMORY="${CGROUP_MEMORY_BASE_PATH}${CGROUPS_PATH}" - -# CONFIG_MEMCG_KMEM support -KMEM="${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" -RT_PERIOD="${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" - # Check if we're in rootless mode. ROOTLESS=$(id -u) @@ -119,14 +105,44 @@ function runc_rootless_cgroup() { mv "$bundle/config.json"{.tmp,} } -# Helper function to set cgroupsPath to the value of $CGROUPS_PATH +function init_cgroup_paths() { + # init once + test -n "$CGROUP_UNIFIED" && return + + if [ -n "${RUNC_USE_SYSTEMD}" ] ; then + REL_CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope" + OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test" + else + REL_CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup" + OCI_CGROUPS_PATH=$REL_CGROUPS_PATH + fi + + if stat -f -c %t /sys/fs/cgroup | grep -qFw 63677270; then + CGROUP_UNIFIED=yes + # "pseudo" controllers do not appear in /sys/fs/cgroup/cgroup.controllers. + # - devices (since kernel 4.15) + # - freezer (since kernel 5.2) + # Assume these are always available, as it is hard to detect + CGROUP_SUBSYSTEMS=$(cat /sys/fs/cgroup/cgroup.controllers; echo devices freezer) + CGROUP_BASE_PATH=/sys/fs/cgroup + CGROUP_PATH=${CGROUP_BASE_PATH}${REL_CGROUPS_PATH} + else + CGROUP_UNIFIED=no + CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups) + for g in ${CGROUP_SUBSYSTEMS}; do + base_path=$(gawk '$(NF-2) == "cgroup" && $NF ~ /\<'${g}'\>/ { print $5; exit }' /proc/self/mountinfo) + test -z "$base_path" && continue + eval CGROUP_${g^^}_BASE_PATH="${base_path}" + eval CGROUP_${g^^}="${base_path}${REL_CGROUPS_PATH}" + done + fi +} + +# Helper function to set cgroupsPath to the value of $OCI_CGROUPS_PATH function set_cgroups_path() { bundle="${1:-.}" - cgroups_path="/runc-cgroups-integration-test/test-cgroup" - if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then - cgroups_path="machine.slice:runc-cgroups:integration-test" - fi - sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${cgroups_path}"'",#' "$bundle/config.json" + init_cgroup_paths + sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${OCI_CGROUPS_PATH}"'",#' "$bundle/config.json" } # Helper to check a value in cgroups. @@ -191,12 +207,14 @@ function requires() { fi ;; cgroups_kmem) - if [ ! -e "$KMEM" ]; then + init_cgroup_paths + if [ ! -e "${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" ]; then skip "Test requires ${var}" fi ;; cgroups_rt) - if [ ! -e "$RT_PERIOD" ]; then + init_cgroup_paths + if [ ! -e "${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" ]; then skip "Test requires ${var}" fi ;; diff --git a/tests/integration/update.bats b/tests/integration/update.bats index bdfc1697..d8cf5761 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -49,14 +49,6 @@ EOF runc run -d --console-socket $CONSOLE_SOCKET test_update [ "$status" -eq 0 ] - # get the cgroup paths - for g in MEMORY CPUSET CPU BLKIO PIDS; do - base_path=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<'${g}'\>/ { print $5; exit }') - eval CGROUP_${g}="${base_path}${CGROUPS_PATH}" - done - - CGROUP_SYSTEM_MEMORY=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<'MEMORY'\>/ { print $5; exit }') - # check that initial values were properly set check_cgroup_value "cpu.cfs_period_us" 1000000 check_cgroup_value "cpu.cfs_quota_us" 500000 @@ -111,7 +103,7 @@ EOF runc update test_update --memory-swap -1 [ "$status" -eq 0 ] # Get System memory swap limit - SYSTEM_MEMORY_SW=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.memsw.limit_in_bytes") + SYSTEM_MEMORY_SW=$(cat "${CGROUP_MEMORY_BASE_PATH}/memory.memsw.limit_in_bytes") check_cgroup_value "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY_SW} # update memory swap @@ -125,7 +117,7 @@ EOF [ "$status" -eq 0 ] # Get System memory limit - SYSTEM_MEMORY=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.limit_in_bytes") + SYSTEM_MEMORY=$(cat "${CGROUP_MEMORY_BASE_PATH}/memory.limit_in_bytes") # check memory limited is gone check_cgroup_value "memory.limit_in_bytes" ${SYSTEM_MEMORY} @@ -239,9 +231,6 @@ EOF runc run -d --console-socket $CONSOLE_SOCKET test_update_rt [ "$status" -eq 0 ] - # get the cgroup paths - eval CGROUP_CPU="${CGROUP_CPU_BASE_PATH}${CGROUPS_PATH}" - runc update -r - test_update_rt <