diff --git a/libcontainer/cgroups/systemd/apply_systemd.go b/libcontainer/cgroups/systemd/apply_systemd.go
index 726ed38a..e8defe3a 100644
--- a/libcontainer/cgroups/systemd/apply_systemd.go
+++ b/libcontainer/cgroups/systemd/apply_systemd.go
@@ -5,6 +5,7 @@ package systemd
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math"
 	"os"
 	"path/filepath"
@@ -590,6 +591,15 @@ func setKernelMemory(c *configs.Cgroup) error {
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return err
 	}
+	// do not try to enable the kernel memory if we already have
+	// tasks in the cgroup.
+	content, err := ioutil.ReadFile(filepath.Join(path, "tasks"))
+	if err != nil {
+		return err
+	}
+	if len(content) > 0 {
+		return nil
+	}
 	return fs.EnableKernelMemoryAccounting(path)
 }