diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index 8837881579..af5f684943 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -2086,7 +2086,12 @@ class Query(BaseExpression):
             self.deferred_loading = existing.union(field_names), True
         else:
             # Remove names from the set of any existing "immediate load" names.
-            self.deferred_loading = existing.difference(field_names), False
+            if new_existing := existing.difference(field_names):
+                self.deferred_loading = new_existing, False
+            else:
+                self.clear_deferred_loading()
+                if new_only := set(field_names).difference(existing):
+                    self.deferred_loading = new_only, True
 
     def add_immediate_loading(self, field_names):
         """
diff --git a/tests/defer/tests.py b/tests/defer/tests.py
index 08c67cb5a3..d897c5fa42 100644
--- a/tests/defer/tests.py
+++ b/tests/defer/tests.py
@@ -52,6 +52,8 @@ class DeferTests(AssertionMixin, TestCase):
         self.assert_delayed(qs.defer('name').only('name').only('value')[0], 2)
         self.assert_delayed(qs.defer("name").only("value")[0], 2)
         self.assert_delayed(qs.only("name").defer("value")[0], 2)
+        self.assert_delayed(qs.only('name').defer('name').defer('value')[0], 1)
+        self.assert_delayed(qs.only('name').defer('name', 'value')[0], 1)
 
     def test_defer_only_clear(self):
         qs = Primary.objects.all()