[1.7.x] Fixed #23815 -- Prevented UnicodeDecodeError in CSRF middleware

Thanks codeitloadit for the report, living180 for investigations
and Tim Graham for the review.
Backport of 27dd7e7271 from master.

django/middleware/csrf.py

@@ -148,7 +148,11 @@ class CsrfViewMiddleware(object):
                 # Barth et al. found that the Referer header is missing for
                 # same-domain requests in only about 0.2% of cases or less, so
                 # we can use strict Referer checking.
-                referer = request.META.get('HTTP_REFERER')
+                referer = force_text(
+                    request.META.get('HTTP_REFERER'),
+                    strings_only=True,
+                    errors='replace'
+                )
                 if referer is None:
                     return self._reject(request, REASON_NO_REFERER)
 

docs/releases/1.7.3.txt

@@ -17,3 +17,6 @@ Bugfixes
   affect users who have subclassed
   ``django.contrib.auth.hashers.PBKDF2PasswordHasher`` to change the
   default value.
+
+* Fixed a crash in the CSRF middleware when handling non-ASCII referer header
+  (:ticket:`23815`).

tests/csrf_tests/tests.py

@@ -300,6 +300,11 @@ class CsrfViewMiddlewareTest(TestCase):
         req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
         self.assertNotEqual(None, req2)
         self.assertEqual(403, req2.status_code)
+        # Non-ASCII
+        req.META['HTTP_REFERER'] = b'\xd8B\xf6I\xdf'
+        req2 = CsrfViewMiddleware().process_view(req, post_form_view, (), {})
+        self.assertNotEqual(None, req2)
+        self.assertEqual(403, req2.status_code)
 
     @override_settings(ALLOWED_HOSTS=['www.example.com'])
     def test_https_good_referer(self):