diff --git a/libcontainer/cgroups/fs/memory.go b/libcontainer/cgroups/fs/memory.go
index 23c68f83..e104d168 100644
--- a/libcontainer/cgroups/fs/memory.go
+++ b/libcontainer/cgroups/fs/memory.go
@@ -19,14 +19,19 @@ type MemoryGroup struct {
 
 func (s *MemoryGroup) Apply(d *data) (err error) {
 	path, err := d.path("memory")
-	if err != nil {
-		if cgroups.IsNotFound(err) {
-			return nil
-		}
+	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	if err := os.MkdirAll(path, 0755); err != nil {
-		return err
+	if memoryAssigned(d.c) {
+		if path != "" {
+			if err := os.MkdirAll(path, 0755); err != nil {
+				return err
+			}
+		}
+
+		if err := s.Set(path, d.c); err != nil {
+			return err
+		}
 	}
 
 	defer func() {
@@ -35,13 +40,10 @@ func (s *MemoryGroup) Apply(d *data) (err error) {
 		}
 	}()
 
-	if err := s.Set(path, d.c); err != nil {
-		return err
-	}
-
 	// We need to join memory cgroup after set memory limits, because
 	// kmem.limit_in_bytes can only be set when the cgroup is empty.
-	if _, err = d.join("memory"); err != nil {
+	_, err = d.join("memory")
+	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
 
@@ -132,6 +134,15 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
 	return nil
 }
 
+func memoryAssigned(cgroup *configs.Cgroup) bool {
+	return cgroup.Memory != 0 ||
+		cgroup.MemoryReservation != 0 ||
+		cgroup.MemorySwap > 0 ||
+		cgroup.KernelMemory > 0 ||
+		cgroup.OomKillDisable ||
+		cgroup.MemorySwappiness != -1
+}
+
 func getMemoryData(path, name string) (cgroups.MemoryData, error) {
 	memoryData := cgroups.MemoryData{}