From 4d59730fad1fe882ad957b7ad463398329d2c8ea Mon Sep 17 00:00:00 2001
From: Jacob Kaplan-Moss <jacob@jacobian.org>
Date: Tue, 7 Nov 2006 02:20:08 +0000
Subject: [PATCH] Fixed #2934: greatly improved the accuracy if the
 isExistingURL check.  Also introduced a new setting,
 URL_VALIDATOR_USER_AGENT, which is the User-Agent that the validator will use
 to check for URL existance.  Thanks, Jeremy.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@4035 bcc190cf-cafb-0310-a4f2-bffc1f526a37
---
 django/conf/global_settings.py |  4 ++++
 django/core/validators.py      | 23 ++++++++++++++++-------
 docs/settings.txt              | 10 ++++++++++
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py
index 69c775cdec..a50287f537 100644
--- a/django/conf/global_settings.py
+++ b/django/conf/global_settings.py
@@ -228,6 +228,10 @@ MONTH_DAY_FORMAT = 'F j'
 # Hint: you really don't!
 TRANSACTIONS_MANAGED = False
 
+# The User-Agent string to use when checking for URL validity through the
+# isExistingURL validator.
+URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
+
 ##############
 # MIDDLEWARE #
 ##############
diff --git a/django/core/validators.py b/django/core/validators.py
index 4c3f59143e..816b8af8a3 100644
--- a/django/core/validators.py
+++ b/django/core/validators.py
@@ -8,6 +8,7 @@ validator will *always* be run, regardless of whether its associated
 form field is required.
 """
 
+import urllib2
 from django.conf import settings
 from django.utils.translation import gettext, gettext_lazy, ngettext
 from django.utils.functional import Promise, lazy
@@ -223,18 +224,26 @@ def isWellFormedXmlFragment(field_data, all_data):
     isWellFormedXml('<root>%s</root>' % field_data, all_data)
 
 def isExistingURL(field_data, all_data):
-    import urllib2
     try:
-        u = urllib2.urlopen(field_data)
+        headers = {
+            "Accept" : "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
+            "Accept-Language" : "en-us,en;q=0.5",
+            "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
+            "Connection" : "close",
+            "User-Agent": settings.URL_VALIDATOR_USER_AGENT
+            }
+        req = urllib2.Request(field_data,None, headers)
+        u = urllib2.urlopen(req)
     except ValueError:
-        raise ValidationError, gettext("Invalid URL: %s") % field_data
+        raise ValidationError, _("Invalid URL: %s") % field_data
     except urllib2.HTTPError, e:
         # 401s are valid; they just mean authorization is required.
-        if e.code not in ('401',):
-            raise ValidationError, gettext("The URL %s is a broken link.") % field_data
+        # 301 and 302 are redirects; they just mean look somewhere else.
+        if str(e.code) not in ('401','301','302'):
+            raise ValidationError, _("The URL %s is a broken link.") % field_data
     except: # urllib2.URLError, httplib.InvalidURL, etc.
-        raise ValidationError, gettext("The URL %s is a broken link.") % field_data
-
+        raise ValidationError, _("The URL %s is a broken link.") % field_data
+        
 def isValidUSState(field_data, all_data):
     "Checks that the given string is a valid two-letter U.S. state abbreviation"
     states = ['AA', 'AE', 'AK', 'AL', 'AP', 'AR', 'AS', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'FM', 'GA', 'GU', 'HI', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MH', 'MI', 'MN', 'MO', 'MP', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'PR', 'PW', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VA', 'VI', 'VT', 'WA', 'WI', 'WV', 'WY']
diff --git a/docs/settings.txt b/docs/settings.txt
index 272b20f753..976a174abb 100644
--- a/docs/settings.txt
+++ b/docs/settings.txt
@@ -814,6 +814,16 @@ manual configuration option (see below), Django will *not* touch the ``TZ``
 environment variable, and it'll be up to you to ensure your processes are
 running in the correct environment.
 
+URL_VALIDATOR_USER_AGENT
+------------------------
+
+Default: ``Django/<version> (http://www.djangoproject.com/)``
+
+The string to use as the ``User-Agent`` header when checking to see if URLs
+exist (see the ``verify_exists`` option on URLField_).
+
+.. URLField: ../model_api/#urlfield
+
 USE_ETAGS
 ---------