diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 3a2e29a76f..a56397a3a1 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -123,8 +123,8 @@ class SQLCompiler:
         for expr, (sql, params, is_ref) in order_by:
             # Skip References to the select clause, as all expressions in the
             # select clause are already part of the group by.
-            if not expr.contains_aggregate and not is_ref:
-                expressions.extend(expr.get_source_expressions())
+            if not is_ref:
+                expressions.extend(expr.get_group_by_cols())
         having_group_by = self.having.get_group_by_cols() if self.having else ()
         for expr in having_group_by:
             expressions.append(expr)
diff --git a/tests/aggregation/tests.py b/tests/aggregation/tests.py
index 06c4cc7ccb..62b47ebd1c 100644
--- a/tests/aggregation/tests.py
+++ b/tests/aggregation/tests.py
@@ -9,6 +9,7 @@ from django.db.models import (
     Max, Min, Sum, Value,
 )
 from django.db.models.expressions import Case, Exists, OuterRef, Subquery, When
+from django.db.models.functions import Coalesce
 from django.test import TestCase
 from django.test.testcases import skipUnlessDBFeature
 from django.test.utils import Approximate, CaptureQueriesContext
@@ -1190,6 +1191,32 @@ class AggregateTestCase(TestCase):
             },
         ])
 
+    def test_aggregation_order_by_not_selected_annotation_values(self):
+        result_asc = [
+            self.b4.pk,
+            self.b3.pk,
+            self.b1.pk,
+            self.b2.pk,
+            self.b5.pk,
+            self.b6.pk,
+        ]
+        result_desc = result_asc[::-1]
+        tests = [
+            ('min_related_age', result_asc),
+            ('-min_related_age', result_desc),
+            (F('min_related_age'), result_asc),
+            (F('min_related_age').asc(), result_asc),
+            (F('min_related_age').desc(), result_desc),
+        ]
+        for ordering, expected_result in tests:
+            with self.subTest(ordering=ordering):
+                books_qs = Book.objects.annotate(
+                    min_age=Min('authors__age'),
+                ).annotate(
+                    min_related_age=Coalesce('min_age', 'contact__age'),
+                ).order_by(ordering).values_list('pk', flat=True)
+                self.assertEqual(list(books_qs), expected_result)
+
     @skipUnlessDBFeature('supports_subqueries_in_group_by')
     def test_group_by_subquery_annotation(self):
         """