Merge pull request #2335 from kolyshkin/cgroupv2-cpt

Fix cgroupv2 checkpoint/restore

Vagrantfile

@@ -15,9 +15,27 @@ Vagrant.configure("2") do |config|
   config.vm.provision "shell", inline: <<-SHELL
     cat << EOF | dnf -y shell
 update
-install podman make golang-go libseccomp-devel bats jq
+install podman make golang-go libseccomp-devel bats jq \
+ patch protobuf protobuf-c protobuf-c-devel protobuf-compiler \
+ protobuf-devel protobuf-python libnl3-devel libcap-devel libnet-devel \
+ nftables-devel libbsd-devel gnutls-devel
 ts run
 EOF
     dnf clean all
+
+    # TODO: remove this after criu 3.14 is released
+    rpm -e --nodeps criu || true
+    CRIU_VERSION=v3.13
+    mkdir -p /usr/src/criu \
+        && curl -fsSL https://github.com/checkpoint-restore/criu/archive/${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \
+        && cd /usr/src/criu \
+        && echo 1 > .gitid \
+        && curl -sSL https://github.com/checkpoint-restore/criu/commit/4c27b3db4f4325a311d8bfa9a50ea3efb4d6e377.patch | patch -p1 \
+        && curl -sSL https://github.com/checkpoint-restore/criu/commit/aac41164b2cd7f0d2047f207b32844524682e43f.patch | patch -p1 \
+        && curl -sSL https://github.com/checkpoint-restore/criu/commit/6f19249b2565f3f7c0a1f8f65b4ae180e8f7f34b.patch | patch -p1 \
+        && curl -sSL https://github.com/checkpoint-restore/criu/commit/378337a496ca759848180bc5411e4446298c5e4e.patch | patch -p1 \
+        && make install-criu \
+        && cd - \
+        && rm -rf /usr/src/criu
   SHELL
 end

libcontainer/container_linux.go

@@ -1095,18 +1095,19 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error {
 		go waitForCriuLazyServer(statusRead, criuOpts.StatusFd)
 	}
 
-	//no need to dump these information in pre-dump
+	// no need to dump all this in pre-dump
 	if !criuOpts.PreDump {
+		hasCgroupns := c.config.Namespaces.Contains(configs.NEWCGROUP)
 		for _, m := range c.config.Mounts {
 			switch m.Device {
 			case "bind":
 				c.addCriuDumpMount(req, m)
 			case "cgroup":
-				if cgroups.IsCgroup2UnifiedMode() {
-					c.addCriuDumpMount(req, m)
+				if cgroups.IsCgroup2UnifiedMode() || hasCgroupns {
+					// real mount(s)
 					continue
 				}
-				// cgroup v1
+				// a set of "external" bind mounts
 				binds, err := getCgroupMounts(m)
 				if err != nil {
 					return err
@@ -1184,7 +1185,14 @@ func (c *linuxContainer) restoreNetwork(req *criurpc.CriuReq, criuOpts *CriuOpts
 func (c *linuxContainer) makeCriuRestoreMountpoints(m *configs.Mount) error {
 	switch m.Device {
 	case "cgroup":
-		// Do nothing for cgroup, CRIU should handle it
+		// No mount point(s) need to be created:
+		//
+		// * for v1, mount points are saved by CRIU because
+		//   /sys/fs/cgroup is a tmpfs mount
+		//
+		// * for v2, /sys/fs/cgroup is a real mount, but
+		//   the mountpoint appears as soon as /sys is mounted
+		return nil
 	case "bind":
 		// The prepareBindMount() function checks if source
 		// exists. So it cannot be used for other filesystem types.
@@ -1192,7 +1200,7 @@ func (c *linuxContainer) makeCriuRestoreMountpoints(m *configs.Mount) error {
 			return err
 		}
 	default:
-		// for all other file-systems just create the mountpoints
+		// for all other filesystems just create the mountpoints
 		dest, err := securejoin.SecureJoin(c.config.Rootfs, m.Destination)
 		if err != nil {
 			return err
@@ -1366,16 +1374,16 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 		return err
 	}
 
+	hasCgroupns := c.config.Namespaces.Contains(configs.NEWCGROUP)
 	for _, m := range c.config.Mounts {
 		switch m.Device {
 		case "bind":
 			c.addCriuRestoreMount(req, m)
 		case "cgroup":
-			if cgroups.IsCgroup2UnifiedMode() {
-				c.addCriuRestoreMount(req, m)
+			if cgroups.IsCgroup2UnifiedMode() || hasCgroupns {
 				continue
 			}
-			// cgroup v1
+			// cgroup v1 is a set of bind mounts, unless cgroupns is used
 			binds, err := getCgroupMounts(m)
 			if err != nil {
 				return err

libcontainer/integration/checkpoint_test.go

@@ -11,7 +11,6 @@ import (
 	"testing"
 
 	"github.com/opencontainers/runc/libcontainer"
-	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
 
 	"golang.org/x/sys/unix"
@@ -80,17 +79,9 @@ func testCheckpoint(t *testing.T, userns bool) {
 		config.GidMappings = []configs.IDMap{{HostID: 0, ContainerID: 0, Size: 1000}}
 		config.Namespaces = append(config.Namespaces, configs.Namespace{Type: configs.NEWUSER})
 	} else {
-		var cgroupDevice string
-
-		if cgroups.IsCgroup2UnifiedMode() {
-			cgroupDevice = "cgroup2"
-		} else {
-			cgroupDevice = "cgroup"
-		}
-
 		config.Mounts = append(config.Mounts, &configs.Mount{
 			Destination: "/sys/fs/cgroup",
-			Device:      cgroupDevice,
+			Device:      "cgroup",
 			Flags:       defaultMountFlags | unix.MS_RDONLY,
 		})
 	}

libcontainer/rootfs_linux.go

@@ -244,7 +244,7 @@ func mountCgroupV1(m *configs.Mount, rootfs, mountLabel string, enableCgroupns b
 			}
 			cgroupmount := &configs.Mount{
 				Source:      "cgroup",
-				Device:      "cgroup",
+				Device:      "cgroup", // this is actually fstype
 				Destination: subsystemPath,
 				Flags:       flags,
 				Data:        filepath.Base(subsystemPath),
@@ -402,27 +402,9 @@ func mountToRootfs(m *configs.Mount, rootfs, mountLabel string, enableCgroupns b
 		}
 	case "cgroup":
 		if cgroups.IsCgroup2UnifiedMode() {
-			if err := mountCgroupV2(m, rootfs, mountLabel, enableCgroupns); err != nil {
-				return err
-			}
-		} else {
-
-			if err := mountCgroupV1(m, rootfs, mountLabel, enableCgroupns); err != nil {
-				return err
-			}
-		}
-		if m.Flags&unix.MS_RDONLY != 0 {
-			// remount cgroup root as readonly
-			mcgrouproot := &configs.Mount{
-				Source:      m.Destination,
-				Device:      "bind",
-				Destination: m.Destination,
-				Flags:       defaultMountFlags | unix.MS_RDONLY | unix.MS_BIND,
-			}
-			if err := remount(mcgrouproot, rootfs); err != nil {
-				return err
-			}
+			return mountCgroupV2(m, rootfs, mountLabel, enableCgroupns)
 		}
+		return mountCgroupV1(m, rootfs, mountLabel, enableCgroupns)
 	default:
 		// ensure that the destination of the mount is resolved of symlinks at mount time because
 		// any previous mounts can invalidate the next mount's destination.

tests/integration/checkpoint.bats

@@ -3,8 +3,6 @@
 load helpers
 
 function setup() {
-  # All checkpoint tests are currently failing on v2
-  requires cgroups_v1
   # XXX: currently criu require root containers.
   requires criu root
 
@@ -43,7 +41,7 @@ function check_pipes() {
 	[[ "${output}" == *"ponG Ping"* ]]
 }
 
-@test "checkpoint and restore" {
+function simple_cr() {
   runc run -d --console-socket $CONSOLE_SOCKET test_busybox
   [ "$status" -eq 0 ]
 
@@ -69,6 +67,20 @@ function check_pipes() {
   done
 }
 
+@test "checkpoint and restore " {
+	simple_cr
+}
+
+@test "checkpoint and restore (cgroupns)" {
+  # cgroupv2 already enables cgroupns so this case was tested above already
+  requires cgroups_v1
+
+  # enable CGROUPNS
+  sed -i 's|\("namespaces": \[\)|\1\n\t\t\t{"type": "cgroup"},|' config.json
+
+  simple_cr
+}
+
 @test "checkpoint --pre-dump and restore" {
   setup_pipes