diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py
index d50098a4383..62c7248de28 100644
--- a/django/db/models/fields/related.py
+++ b/django/db/models/fields/related.py
@@ -580,17 +580,20 @@ def create_many_related_manager(superclass, rel):
             )
         do_not_call_in_templates = True
 
-        def _build_clear_filters(self, qs):
-            filters = Q(**{
-                self.source_field_name: self.related_val,
-                '%s__in' % self.target_field_name: qs
-            })
-
+        def _build_remove_filters(self, removed_vals):
+            filters = Q(**{self.source_field_name: self.related_val})
+            # No need to add a subquery condition if removed_vals is a QuerySet without
+            # filters.
+            removed_vals_filters = (not isinstance(removed_vals, QuerySet) or
+                                    removed_vals._has_filters())
+            if removed_vals_filters:
+                filters &= Q(**{'%s__in' % self.target_field_name: removed_vals})
             if self.symmetrical:
-                filters |= Q(**{
-                    self.target_field_name: self.related_val,
-                    '%s__in' % self.source_field_name: qs
-                })
+                symmetrical_filters = Q(**{self.target_field_name: self.related_val})
+                if removed_vals_filters:
+                    symmetrical_filters &= Q(
+                        **{'%s__in' % self.source_field_name: removed_vals})
+                filters |= symmetrical_filters
             return filters
 
         def get_queryset(self):
@@ -654,7 +657,7 @@ def create_many_related_manager(superclass, rel):
             signals.m2m_changed.send(sender=self.through, action="pre_clear",
                 instance=self.instance, reverse=self.reverse,
                 model=self.model, pk_set=None, using=db)
-            filters = self._build_clear_filters(self.using(db))
+            filters = self._build_remove_filters(super(ManyRelatedManager, self).get_queryset().using(db))
             self.through._default_manager.using(db).filter(filters).delete()
 
             signals.m2m_changed.send(sender=self.through, action="post_clear",
@@ -763,10 +766,13 @@ def create_many_related_manager(superclass, rel):
             signals.m2m_changed.send(sender=self.through, action="pre_remove",
                 instance=self.instance, reverse=self.reverse,
                 model=self.model, pk_set=old_ids, using=db)
-
-            old_vals_qs = self.using(db).filter(**{
-                '%s__in' % self.target_field.related_field.attname: old_ids})
-            filters = self._build_clear_filters(old_vals_qs)
+            target_model_qs = super(ManyRelatedManager, self).get_queryset()
+            if target_model_qs._has_filters():
+                old_vals = target_model_qs.using(db).filter(**{
+                    '%s__in' % self.target_field.related_field.attname: old_ids})
+            else:
+                old_vals = old_ids
+            filters = self._build_remove_filters(old_vals)
             self.through._default_manager.using(db).filter(filters).delete()
 
             signals.m2m_changed.send(sender=self.through, action="post_remove",
diff --git a/django/db/models/query.py b/django/db/models/query.py
index 626eb11b126..59a2b45d026 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -1026,6 +1026,14 @@ class QuerySet(object):
         # If we have a new hint for an existing key, overwrite with the new value.
         self._hints.update(hints)
 
+    def _has_filters(self):
+        """
+        Checks if this QuerySet has any filtering going on. Note that this
+        isn't equivalent for checking if all objects are present in results,
+        for example qs[1:]._has_filters() -> True.
+        """
+        return self.query.has_filters()
+
 
 class InstanceCheckMeta(type):
     def __instancecheck__(self, instance):
diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index 662aede6952..e0593ef0a73 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -430,6 +430,9 @@ class Query(object):
 
         return number
 
+    def has_filters(self):
+        return self.where or self.having
+
     def has_results(self, using):
         q = self.clone()
         if not q.distinct: