diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index 7fdf551554..a44adfc760 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -5,7 +5,8 @@ from itertools import chain
 
 from django.core.exceptions import EmptyResultSet, FieldError
 from django.db.models.constants import LOOKUP_SEP
-from django.db.models.expressions import OrderBy, Random, RawSQL, Ref
+from django.db.models.expressions import OrderBy, Random, RawSQL, Ref, Value
+from django.db.models.functions import Cast
 from django.db.models.query_utils import QueryWrapper, select_related_descend
 from django.db.models.sql.constants import (
     CURSOR, GET_ITERATOR_CHUNK_SIZE, MULTI, NO_RESULTS, ORDER_DIR, SINGLE,
@@ -278,6 +279,9 @@ class SQLCompiler:
         order_by = []
         for field in ordering:
             if hasattr(field, 'resolve_expression'):
+                if isinstance(field, Value):
+                    # output_field must be resolved for constants.
+                    field = Cast(field, field.output_field)
                 if not isinstance(field, OrderBy):
                     field = field.asc()
                 if not self.query.standard_ordering:
@@ -299,10 +303,13 @@ class SQLCompiler:
                     True))
                 continue
             if col in self.query.annotations:
-                # References to an expression which is masked out of the SELECT clause
-                order_by.append((
-                    OrderBy(self.query.annotations[col], descending=descending),
-                    False))
+                # References to an expression which is masked out of the SELECT
+                # clause.
+                expr = self.query.annotations[col]
+                if isinstance(expr, Value):
+                    # output_field must be resolved for constants.
+                    expr = Cast(expr, expr.output_field)
+                order_by.append((OrderBy(expr, descending=descending), False))
                 continue
 
             if '.' in field:
diff --git a/tests/ordering/tests.py b/tests/ordering/tests.py
index e5f56917d4..18c32d6d5f 100644
--- a/tests/ordering/tests.py
+++ b/tests/ordering/tests.py
@@ -405,17 +405,35 @@ class OrderingTests(TestCase):
             attrgetter("headline")
         )
 
-    def test_order_by_annotated_constant_value(self):
+    def test_order_by_constant_value(self):
+        # Order by annotated constant from selected columns.
         qs = Article.objects.annotate(
             constant=Value('1', output_field=CharField()),
         ).order_by('constant', '-headline')
         self.assertSequenceEqual(qs, [self.a4, self.a3, self.a2, self.a1])
+        # Order by annotated constant which is out of selected columns.
+        self.assertSequenceEqual(
+            qs.values_list('headline', flat=True), [
+                'Article 4',
+                'Article 3',
+                'Article 2',
+                'Article 1',
+            ],
+        )
+        # Order by constant.
+        qs = Article.objects.order_by(Value('1', output_field=CharField()), '-headline')
+        self.assertSequenceEqual(qs, [self.a4, self.a3, self.a2, self.a1])
 
     def test_order_by_constant_value_without_output_field(self):
         msg = 'Cannot resolve expression type, unknown output_field'
         qs = Article.objects.annotate(constant=Value('1')).order_by('constant')
-        with self.assertRaisesMessage(FieldError, msg):
-            qs.first()
+        for ordered_qs in (
+            qs,
+            qs.values('headline'),
+            Article.objects.order_by(Value('1')),
+        ):
+            with self.subTest(ordered_qs=ordered_qs), self.assertRaisesMessage(FieldError, msg):
+                ordered_qs.first()
 
     def test_related_ordering_duplicate_table_reference(self):
         """