Merge pull request #637 from hqhq/hq_fix_kmem_cgroup

Fix kmem limit set

cgroups/fs/memory.go

@@ -18,20 +18,29 @@ type MemoryGroup struct {
 }
 
 func (s *MemoryGroup) Apply(d *data) error {
-	dir, err := d.join("memory")
+	path, err := d.path("memory")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
+	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+	if err := s.Set(path, d.c); err != nil {
+		return err
+	}
+
+	// We need to join memory cgroup after set memory limits, because
+	// kmem.limit_in_bytes can only be set when the cgroup is empty.
+	_, err = d.join("memory")
+	if err != nil {
+		return err
+	}
 	defer func() {
 		if err != nil {
-			os.RemoveAll(dir)
+			os.RemoveAll(path)
 		}
 	}()
 
-	if err := s.Set(dir, d.c); err != nil {
-		return err
-	}
-
 	return nil
 }
 

cgroups/systemd/apply_systemd.go

@@ -188,6 +188,16 @@ func (m *Manager) Apply(pid int) error {
 			newProp("BlockIOWeight", uint64(c.BlkioWeight)))
 	}
 
+	// We need to set kernel memory before processes join cgroup because
+	// kmem.limit_in_bytes can only be set when the cgroup is empty.
+	// And swap memory limit needs to be set after memory limit, only
+	// memory limit is handled by systemd, so it's kind of ugly here.
+	if c.KernelMemory > 0 {
+		if err := setKernelMemory(c); err != nil {
+			return err
+		}
+	}
+
 	if _, err := theConn.StartTransientUnit(unitName, "replace", properties...); err != nil {
 		return err
 	}
@@ -462,6 +472,26 @@ func joinDevices(c *configs.Cgroup, pid int) error {
 	return devices.Set(path, c)
 }
 
+func setKernelMemory(c *configs.Cgroup) error {
+	path, err := getSubsystemPath(c, "memory")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+
+	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	if c.KernelMemory > 0 {
+		err = writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(c.KernelMemory, 10))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func joinMemory(c *configs.Cgroup, pid int) error {
 	path, err := getSubsystemPath(c, "memory")
 	if err != nil && !cgroups.IsNotFound(err) {
@@ -476,12 +506,6 @@ func joinMemory(c *configs.Cgroup, pid int) error {
 		}
 	}
 
-	if c.KernelMemory > 0 {
-		err = writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(c.KernelMemory, 10))
-		if err != nil {
-			return err
-		}
-	}
 	if c.MemorySwappiness >= 0 && c.MemorySwappiness <= 100 {
 		err = writeFile(path, "memory.swappiness", strconv.FormatInt(c.MemorySwappiness, 10))
 		if err != nil {

integration/exec_test.go

@@ -497,7 +497,7 @@ func TestCpuShares(t *testing.T) {
 	testCpuShares(t, false)
 }
 
-func TestSystemdCpuShares(t *testing.T) {
+func TestCpuSharesSystemd(t *testing.T) {
 	if !systemd.UseSystemd() {
 		t.Skip("Systemd is unsupported")
 	}
@@ -524,6 +524,37 @@ func testCpuShares(t *testing.T, systemd bool) {
 	}
 }
 
+func TestRunWithKernelMemory(t *testing.T) {
+	testRunWithKernelMemory(t, false)
+}
+
+func TestRunWithKernelMemorySystemd(t *testing.T) {
+	if !systemd.UseSystemd() {
+		t.Skip("Systemd is unsupported")
+	}
+	testRunWithKernelMemory(t, true)
+}
+
+func testRunWithKernelMemory(t *testing.T, systemd bool) {
+	if testing.Short() {
+		return
+	}
+	rootfs, err := newRootfs()
+	ok(t, err)
+	defer remove(rootfs)
+
+	config := newTemplateConfig(rootfs)
+	if systemd {
+		config.Cgroups.Slice = "system.slice"
+	}
+	config.Cgroups.KernelMemory = 52428800
+
+	_, _, err = runContainer(config, "", "ps")
+	if err != nil {
+		t.Fatalf("runContainer failed with kernel memory limit: %v", err)
+	}
+}
+
 func TestContainerState(t *testing.T) {
 	if testing.Short() {
 		return