diff --git a/libcontainer/cgroups/systemd/v1.go b/libcontainer/cgroups/systemd/v1.go
index 9e8e2fdd..0803cf0b 100644
--- a/libcontainer/cgroups/systemd/v1.go
+++ b/libcontainer/cgroups/systemd/v1.go
@@ -222,7 +222,14 @@ func (m *legacyManager) Destroy() error {
 		return err
 	}
 	unitName := getUnitName(m.cgroups)
-	if err := stopUnit(dbusConnection, unitName); err != nil {
+
+	err = stopUnit(dbusConnection, unitName)
+	// Both on success and on error, cleanup all the cgroups we are aware of.
+	// Some of them were created directly by Apply() and are not managed by systemd.
+	if err := cgroups.RemovePaths(m.paths); err != nil {
+		return err
+	}
+	if err != nil {
 		return err
 	}
 	m.paths = make(map[string]string)
diff --git a/tests/integration/delete.bats b/tests/integration/delete.bats
index bccd52e7..b356e764 100644
--- a/tests/integration/delete.bats
+++ b/tests/integration/delete.bats
@@ -12,24 +12,24 @@ function teardown() {
 }
 
 @test "runc delete" {
-  # run busybox detached
-  runc run -d --console-socket $CONSOLE_SOCKET test_busybox
+  runc run -d --console-socket $CONSOLE_SOCKET testbusyboxdelete
   [ "$status" -eq 0 ]
 
-  # check state
-  testcontainer test_busybox running
+  testcontainer testbusyboxdelete running
 
-  runc kill test_busybox KILL
+  runc kill testbusyboxdelete KILL
   [ "$status" -eq 0 ]
-  # wait for busybox to be in the destroyed state
-  retry 10 1 eval "__runc state test_busybox | grep -q 'stopped'"
+  retry 10 1 eval "__runc state testbusyboxdelete | grep -q 'stopped'"
 
-  # delete test_busybox
-  runc delete test_busybox
+  runc delete testbusyboxdelete
   [ "$status" -eq 0 ]
 
-  runc state test_busybox
+  runc state testbusyboxdelete
   [ "$status" -ne 0 ]
+
+  run find /sys/fs/cgroup -wholename '*testbusyboxdelete*' -type d
+  [ "$status" -eq 0 ]
+  [ "$output" = "" ] || fail "cgroup not cleaned up correctly: $output"
 }
 
 @test "runc delete --force" {