diff --git a/django/utils/timezone.py b/django/utils/timezone.py
index c0ea04c7d8..b8e40b031c 100644
--- a/django/utils/timezone.py
+++ b/django/utils/timezone.py
@@ -262,9 +262,9 @@ class override(object):
     """
     def __init__(self, timezone):
         self.timezone = timezone
-        self.old_timezone = getattr(_active, 'value', None)
 
     def __enter__(self):
+        self.old_timezone = getattr(_active, 'value', None)
         if self.timezone is None:
             deactivate()
         else:
diff --git a/django/utils/translation/__init__.py b/django/utils/translation/__init__.py
index 70eb01d2d7..e8ef6bd353 100644
--- a/django/utils/translation/__init__.py
+++ b/django/utils/translation/__init__.py
@@ -154,9 +154,9 @@ class override(ContextDecorator):
     def __init__(self, language, deactivate=False):
         self.language = language
         self.deactivate = deactivate
-        self.old_language = get_language()
 
     def __enter__(self):
+        self.old_language = get_language()
         if self.language is not None:
             activate(self.language)
         else:
diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py
index 3ffcb08d8b..64e6eb5eed 100644
--- a/tests/i18n/tests.py
+++ b/tests/i18n/tests.py
@@ -74,7 +74,6 @@ class TranslationTests(TestCase):
             deactivate()
 
     def test_override_decorator(self):
-        activate('de')
 
         @translation.override('pl')
         def func_pl():
@@ -85,6 +84,7 @@ class TranslationTests(TestCase):
             self.assertEqual(get_language(), settings.LANGUAGE_CODE)
 
         try:
+            activate('de')
             func_pl()
             self.assertEqual(get_language(), 'de')
             func_none()
@@ -92,6 +92,24 @@ class TranslationTests(TestCase):
         finally:
             deactivate()
 
+    def test_override_exit(self):
+        """
+        Test that the language restored is the one used when the function was
+        called, not the one used when the decorator was initialized. refs #23381
+        """
+        activate('fr')
+        @translation.override('pl')
+        def func_pl():
+            pass
+        deactivate()
+
+        try:
+            activate('en')
+            func_pl()
+            self.assertEqual(get_language(), 'en')
+        finally:
+            deactivate()
+
     def test_lazy_objects(self):
         """
         Format string interpolation should work with *_lazy objects.