diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 3d133cb6726..7649c39262a 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -892,7 +892,9 @@ class SQLCompiler:
                     from_parent = issubclass(model, opts.model) and model is not opts.model
 
                     def local_setter(obj, from_obj):
-                        f.remote_field.set_cached_value(from_obj, obj)
+                        # Set a reverse fk object when relation is non-empty.
+                        if from_obj:
+                            f.remote_field.set_cached_value(from_obj, obj)
 
                     def remote_setter(obj, from_obj):
                         setattr(from_obj, name, obj)
diff --git a/tests/filtered_relation/tests.py b/tests/filtered_relation/tests.py
index 2596dcbdc22..5e85dff16cd 100644
--- a/tests/filtered_relation/tests.py
+++ b/tests/filtered_relation/tests.py
@@ -50,6 +50,12 @@ class FilteredRelationTests(TestCase):
                 (self.author2, self.book3, self.editor_b, self.author2),
             ], lambda x: (x, x.book_join, x.book_join.editor, x.book_join.author))
 
+    def test_select_related_with_empty_relation(self):
+        qs = Author.objects.annotate(
+            book_join=FilteredRelation('book', condition=Q(pk=-1)),
+        ).select_related('book_join').order_by('pk')
+        self.assertSequenceEqual(qs, [self.author1, self.author2])
+
     def test_select_related_foreign_key(self):
         qs = Book.objects.annotate(
             author_join=FilteredRelation('author'),