From 39279b1762f81e0c2d7ebf7f56bf4db23b212dda Mon Sep 17 00:00:00 2001
From: Qiang Huang <h.huangqiang@huawei.com>
Date: Thu, 18 Jun 2015 15:44:30 +0800
Subject: [PATCH] Fix kmem limit set

Currently we can't start container with kmem limit, because we
set kmem limit after processes joined to cgroup, we'll get device
busy error in this case.

Fix it by moving set before join.

Signed-off-by: Qiang Huang <h.huangqiang@huawei.com>
---
 cgroups/fs/memory.go             | 21 +++++++++++++------
 cgroups/systemd/apply_systemd.go | 36 ++++++++++++++++++++++++++------
 integration/exec_test.go         | 33 ++++++++++++++++++++++++++++-
 3 files changed, 77 insertions(+), 13 deletions(-)

diff --git a/cgroups/fs/memory.go b/cgroups/fs/memory.go
index dccdee69..49bfa58a 100644
--- a/cgroups/fs/memory.go
+++ b/cgroups/fs/memory.go
@@ -18,20 +18,29 @@ type MemoryGroup struct {
 }
 
 func (s *MemoryGroup) Apply(d *data) error {
-	dir, err := d.join("memory")
+	path, err := d.path("memory")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
+	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+	if err := s.Set(path, d.c); err != nil {
+		return err
+	}
+
+	// We need to join memory cgroup after set memory limits, because
+	// kmem.limit_in_bytes can only be set when the cgroup is empty.
+	_, err = d.join("memory")
+	if err != nil {
+		return err
+	}
 	defer func() {
 		if err != nil {
-			os.RemoveAll(dir)
+			os.RemoveAll(path)
 		}
 	}()
 
-	if err := s.Set(dir, d.c); err != nil {
-		return err
-	}
-
 	return nil
 }
 
diff --git a/cgroups/systemd/apply_systemd.go b/cgroups/systemd/apply_systemd.go
index fd7f680b..9aa6f0af 100644
--- a/cgroups/systemd/apply_systemd.go
+++ b/cgroups/systemd/apply_systemd.go
@@ -188,6 +188,16 @@ func (m *Manager) Apply(pid int) error {
 			newProp("BlockIOWeight", uint64(c.BlkioWeight)))
 	}
 
+	// We need to set kernel memory before processes join cgroup because
+	// kmem.limit_in_bytes can only be set when the cgroup is empty.
+	// And swap memory limit needs to be set after memory limit, only
+	// memory limit is handled by systemd, so it's kind of ugly here.
+	if c.KernelMemory > 0 {
+		if err := setKernelMemory(c); err != nil {
+			return err
+		}
+	}
+
 	if _, err := theConn.StartTransientUnit(unitName, "replace", properties...); err != nil {
 		return err
 	}
@@ -462,6 +472,26 @@ func joinDevices(c *configs.Cgroup, pid int) error {
 	return devices.Set(path, c)
 }
 
+func setKernelMemory(c *configs.Cgroup) error {
+	path, err := getSubsystemPath(c, "memory")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+
+	if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	if c.KernelMemory > 0 {
+		err = writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(c.KernelMemory, 10))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func joinMemory(c *configs.Cgroup, pid int) error {
 	path, err := getSubsystemPath(c, "memory")
 	if err != nil && !cgroups.IsNotFound(err) {
@@ -476,12 +506,6 @@ func joinMemory(c *configs.Cgroup, pid int) error {
 		}
 	}
 
-	if c.KernelMemory > 0 {
-		err = writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(c.KernelMemory, 10))
-		if err != nil {
-			return err
-		}
-	}
 	if c.MemorySwappiness >= 0 && c.MemorySwappiness <= 100 {
 		err = writeFile(path, "memory.swappiness", strconv.FormatInt(c.MemorySwappiness, 10))
 		if err != nil {
diff --git a/integration/exec_test.go b/integration/exec_test.go
index c218dbd6..50383bac 100644
--- a/integration/exec_test.go
+++ b/integration/exec_test.go
@@ -497,7 +497,7 @@ func TestCpuShares(t *testing.T) {
 	testCpuShares(t, false)
 }
 
-func TestSystemdCpuShares(t *testing.T) {
+func TestCpuSharesSystemd(t *testing.T) {
 	if !systemd.UseSystemd() {
 		t.Skip("Systemd is unsupported")
 	}
@@ -524,6 +524,37 @@ func testCpuShares(t *testing.T, systemd bool) {
 	}
 }
 
+func TestRunWithKernelMemory(t *testing.T) {
+	testRunWithKernelMemory(t, false)
+}
+
+func TestRunWithKernelMemorySystemd(t *testing.T) {
+	if !systemd.UseSystemd() {
+		t.Skip("Systemd is unsupported")
+	}
+	testRunWithKernelMemory(t, true)
+}
+
+func testRunWithKernelMemory(t *testing.T, systemd bool) {
+	if testing.Short() {
+		return
+	}
+	rootfs, err := newRootfs()
+	ok(t, err)
+	defer remove(rootfs)
+
+	config := newTemplateConfig(rootfs)
+	if systemd {
+		config.Cgroups.Slice = "system.slice"
+	}
+	config.Cgroups.KernelMemory = 52428800
+
+	_, _, err = runContainer(config, "", "ps")
+	if err != nil {
+		t.Fatalf("runContainer failed with kernel memory limit: %v", err)
+	}
+}
+
 func TestContainerState(t *testing.T) {
 	if testing.Short() {
 		return