Merge remote-tracking branch 'django/master' into t3011

django/contrib/gis/__init__.py

@@ -0,0 +1,6 @@
+from django.utils import six
+
+if six.PY3:
+    memoryview = memoryview
+else:
+    memoryview = buffer

django/contrib/gis/db/backends/postgis/adapter.py

@@ -1,6 +1,7 @@
 """
  This object provides quoting for GEOS geometries into PostgreSQL/PostGIS.
 """
+from __future__ import unicode_literals
 
 from psycopg2 import Binary
 from psycopg2.extensions import ISQLQuote
@@ -10,7 +11,7 @@ class PostGISAdapter(object):
         "Initializes on the geometry."
         # Getting the WKB (in string form, to allow easy pickling of
         # the adaptor) and the SRID from the geometry.
-        self.ewkb = str(geom.ewkb)
+        self.ewkb = bytes(geom.ewkb)
         self.srid = geom.srid
         self._adapter = Binary(self.ewkb)
 
@@ -39,7 +40,7 @@ class PostGISAdapter(object):
     def getquoted(self):
         "Returns a properly quoted string for use in PostgreSQL/PostGIS."
         # psycopg will figure out whether to use E'\\000' or '\000'
-        return 'ST_GeomFromEWKB(%s)' % self._adapter.getquoted()
+        return str('ST_GeomFromEWKB(%s)' % self._adapter.getquoted().decode())
 
     def prepare_database_save(self, unused):
         return self

django/contrib/gis/db/models/fields.py

@@ -160,7 +160,7 @@ class GeometryField(Field):
         # from the given string input.
         if isinstance(geom, Geometry):
             pass
-        elif isinstance(geom, six.string_types) or hasattr(geom, '__geo_interface__'):
+        elif isinstance(geom, (bytes, six.string_types)) or hasattr(geom, '__geo_interface__'):
             try:
                 geom = Geometry(geom)
             except GeometryException:

django/contrib/gis/db/models/proxy.py

@@ -5,6 +5,7 @@ corresponding to geographic model fields.
 
 Thanks to Robert Coup for providing this functionality (see #4322).
 """
+from django.contrib.gis import memoryview
 from django.utils import six
 
 class GeometryProxy(object):
@@ -54,7 +55,7 @@ class GeometryProxy(object):
         if isinstance(value, self._klass) and (str(value.geom_type).upper() == gtype or gtype == 'GEOMETRY'):
             # Assigning the SRID to the geometry.
             if value.srid is None: value.srid = self._field.srid
-        elif value is None or isinstance(value, six.string_types + (buffer,)):
+        elif value is None or isinstance(value, six.string_types + (memoryview,)):
             # Set with None, WKT, HEX, or WKB
             pass
         else:

django/contrib/gis/db/models/query.py

@@ -1,6 +1,7 @@
 from django.db import connections
 from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet
 
+from django.contrib.gis import memoryview
 from django.contrib.gis.db.models import aggregates
 from django.contrib.gis.db.models.fields import get_srid_info, PointField, LineStringField
 from django.contrib.gis.db.models.sql import AreaField, DistanceField, GeomField, GeoQuery
@@ -676,7 +677,7 @@ class GeoQuerySet(QuerySet):
                     if not backend.geography:
                         if not isinstance(geo_field, PointField):
                             raise ValueError('Spherical distance calculation only supported on PointFields.')
-                        if not str(Geometry(buffer(params[0].ewkb)).geom_type) == 'Point':
+                        if not str(Geometry(memoryview(params[0].ewkb)).geom_type) == 'Point':
                             raise ValueError('Spherical distance calculation only supported with Point Geometry parameters')
                     # The `function` procedure argument needs to be set differently for
                     # geodetic distance calculations.

django/contrib/gis/db/models/sql/compiler.py

@@ -1,3 +1,8 @@
+try:
+    from itertools import zip_longest
+except ImportError:
+    from itertools import izip_longest as zip_longest
+
 from django.utils.six.moves import zip
 
 from django.db.backends.util import truncate_name, typecast_timestamp
@@ -190,7 +195,7 @@ class GeoSQLCompiler(compiler.SQLCompiler):
         if self.connection.ops.oracle or getattr(self.query, 'geo_values', False):
             # We resolve the rest of the columns if we're on Oracle or if
             # the `geo_values` attribute is defined.
-            for value, field in map(None, row[index_start:], fields):
+            for value, field in zip_longest(row[index_start:], fields):
                 values.append(self.query.convert_values(value, field, self.connection))
         else:
             values.extend(row[index_start:])

django/contrib/gis/gdal/__init__.py

@@ -41,7 +41,7 @@ try:
     from django.contrib.gis.gdal.srs import SpatialReference, CoordTransform
     from django.contrib.gis.gdal.geometries import OGRGeometry
     HAS_GDAL = True
-except:
+except ImportError:
     HAS_GDAL = False
 
 try:

django/contrib/gis/gdal/datasource.py

@@ -45,6 +45,7 @@ from django.contrib.gis.gdal.layer import Layer
 # Getting the ctypes prototypes for the DataSource.
 from django.contrib.gis.gdal.prototypes import ds as capi
 
+from django.utils.encoding import force_bytes
 from django.utils import six
 from django.utils.six.moves import xrange
 
@@ -73,7 +74,7 @@ class DataSource(GDALBase):
             ds_driver = Driver.ptr_type()
             try:
                 # OGROpen will auto-detect the data source type.
-                ds = capi.open_ds(ds_input, self._write, byref(ds_driver))
+                ds = capi.open_ds(force_bytes(ds_input), self._write, byref(ds_driver))
             except OGRException:
                 # Making the error message more clear rather than something
                 # like "Invalid pointer returned from OGROpen".
@@ -102,7 +103,7 @@ class DataSource(GDALBase):
     def __getitem__(self, index):
         "Allows use of the index [] operator to get a layer at the index."
         if isinstance(index, six.string_types):
-            l = capi.get_layer_by_name(self.ptr, index)
+            l = capi.get_layer_by_name(self.ptr, force_bytes(index))
             if not l: raise OGRIndexError('invalid OGR Layer name given: "%s"' % index)
         elif isinstance(index, int):
             if index < 0 or index >= self.layer_count:

django/contrib/gis/gdal/driver.py

@@ -5,6 +5,7 @@ from django.contrib.gis.gdal.error import OGRException
 from django.contrib.gis.gdal.prototypes import ds as capi
 
 from django.utils import six
+from django.utils.encoding import force_bytes
 
 # For more information, see the OGR C API source code:
 #  http://www.gdal.org/ogr/ogr__api_8h.html
@@ -36,7 +37,7 @@ class Driver(GDALBase):
                 name = dr_input
 
             # Attempting to get the OGR driver by the string name.
-            dr = capi.get_driver_by_name(name)
+            dr = capi.get_driver_by_name(force_bytes(name))
         elif isinstance(dr_input, int):
             self._register()
             dr = capi.get_driver(dr_input)

django/contrib/gis/gdal/envelope.py

@@ -52,7 +52,7 @@ class Envelope(object):
         elif len(args) == 4:
             # Individual parameters passed in.
             #  Thanks to ww for the help
-            self._from_sequence(map(float, args))
+            self._from_sequence([float(a) for a in args])
         else:
             raise OGRException('Incorrect number (%d) of arguments.' % len(args))
 

django/contrib/gis/gdal/feature.py

@@ -7,6 +7,7 @@ from django.contrib.gis.gdal.geometries import OGRGeometry, OGRGeomType
 # ctypes function prototypes
 from django.contrib.gis.gdal.prototypes import ds as capi, geom as geom_api
 
+from django.utils.encoding import force_bytes
 from django.utils import six
 from django.utils.six.moves import xrange
 
@@ -107,6 +108,7 @@ class Feature(GDALBase):
 
     def index(self, field_name):
         "Returns the index of the given field name."
-        i = capi.get_field_index(self.ptr, field_name)
+        i = capi.get_field_index(self.ptr, force_bytes(field_name))
-        if i < 0: raise OGRIndexError('invalid OFT field name given: "%s"' % field_name)
+        if i < 0:
+            raise OGRIndexError('invalid OFT field name given: "%s"' % field_name)
         return i

django/contrib/gis/gdal/geometries.py

@@ -43,6 +43,8 @@ import sys
 from binascii import a2b_hex, b2a_hex
 from ctypes import byref, string_at, c_char_p, c_double, c_ubyte, c_void_p
 
+from django.contrib.gis import memoryview
+
 # Getting GDAL prerequisites
 from django.contrib.gis.gdal.base import GDALBase
 from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope
@@ -76,16 +78,11 @@ class OGRGeometry(GDALBase):
 
         # If HEX, unpack input to to a binary buffer.
         if str_instance and hex_regex.match(geom_input):
-            geom_input = buffer(a2b_hex(geom_input.upper()))
+            geom_input = memoryview(a2b_hex(geom_input.upper().encode()))
             str_instance = False
 
         # Constructing the geometry,
         if str_instance:
-            # Checking if unicode
-            if isinstance(geom_input, six.text_type):
-                # Encoding to ASCII, WKT or HEX doesn't need any more.
-                geom_input = geom_input.encode('ascii')
-
             wkt_m = wkt_regex.match(geom_input)
             json_m = json_regex.match(geom_input)
             if wkt_m:
@@ -96,19 +93,19 @@ class OGRGeometry(GDALBase):
                     # OGR_G_CreateFromWkt doesn't work with LINEARRING WKT.
                     #  See http://trac.osgeo.org/gdal/ticket/1992.
                     g = capi.create_geom(OGRGeomType(wkt_m.group('type')).num)
-                    capi.import_wkt(g, byref(c_char_p(wkt_m.group('wkt'))))
+                    capi.import_wkt(g, byref(c_char_p(wkt_m.group('wkt').encode())))
                 else:
-                    g = capi.from_wkt(byref(c_char_p(wkt_m.group('wkt'))), None, byref(c_void_p()))
+                    g = capi.from_wkt(byref(c_char_p(wkt_m.group('wkt').encode())), None, byref(c_void_p()))
             elif json_m:
-                g = capi.from_json(geom_input)
+                g = capi.from_json(geom_input.encode())
             else:
                 # Seeing if the input is a valid short-hand string
                 # (e.g., 'Point', 'POLYGON').
                 ogr_t = OGRGeomType(geom_input)
                 g = capi.create_geom(OGRGeomType(geom_input).num)
-        elif isinstance(geom_input, buffer):
+        elif isinstance(geom_input, memoryview):
             # WKB was passed in
-            g = capi.from_wkb(str(geom_input), None, byref(c_void_p()), len(geom_input))
+            g = capi.from_wkb(bytes(geom_input), None, byref(c_void_p()), len(geom_input))
         elif isinstance(geom_input, OGRGeomType):
             # OGRGeomType was passed in, an empty geometry will be created.
             g = capi.create_geom(geom_input.num)
@@ -141,7 +138,7 @@ class OGRGeometry(GDALBase):
             srs = srs.wkt
         else:
             srs = None
-        return str(self.wkb), srs
+        return bytes(self.wkb), srs
 
     def __setstate__(self, state):
         wkb, srs = state
@@ -354,7 +351,7 @@ class OGRGeometry(GDALBase):
         buf = (c_ubyte * sz)()
         wkb = capi.to_wkb(self.ptr, byteorder, byref(buf))
         # Returning a buffer of the string at the pointer.
-        return buffer(string_at(buf, sz))
+        return memoryview(string_at(buf, sz))
 
     @property
     def wkt(self):

django/contrib/gis/gdal/layer.py

@@ -14,6 +14,7 @@ from django.contrib.gis.gdal.srs import SpatialReference
 # GDAL ctypes function prototypes.
 from django.contrib.gis.gdal.prototypes import ds as capi, geom as geom_api, srs as srs_api
 
+from django.utils.encoding import force_bytes
 from django.utils import six
 from django.utils.six.moves import xrange
 
@@ -38,7 +39,7 @@ class Layer(GDALBase):
         self._ds = ds
         self._ldefn = capi.get_layer_defn(self._ptr)
         # Does the Layer support random reading?
-        self._random_read = self.test_capability('RandomRead')
+        self._random_read = self.test_capability(b'RandomRead')
 
     def __getitem__(self, index):
         "Gets the Feature at the specified index."
@@ -212,4 +213,4 @@ class Layer(GDALBase):
           'FastFeatureCount', 'FastGetExtent', 'CreateField', 'Transactions',
           'DeleteFeature', and 'FastSetNextByIndex'.
         """
-        return bool(capi.test_capability(self.ptr, capability))
+        return bool(capi.test_capability(self.ptr, force_bytes(capability)))

django/contrib/gis/gdal/libgdal.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import re
 from ctypes import c_char_p, CDLL
@@ -65,7 +67,7 @@ _version_info.restype = c_char_p
 
 def gdal_version():
     "Returns only the GDAL version number information."
-    return _version_info('RELEASE_NAME')
+    return _version_info(b'RELEASE_NAME')
 
 def gdal_full_version():
     "Returns the full GDAL version information."
@@ -86,7 +88,7 @@ def gdal_release_date(date=False):
 
 version_regex = re.compile(r'^(?P<major>\d+)\.(?P<minor>\d+)(\.(?P<subminor>\d+))?')
 def gdal_version_info():
-    ver = gdal_version()
+    ver = gdal_version().decode()
     m = version_regex.match(ver)
     if not m: raise OGRException('Could not parse GDAL version string "%s"' % ver)
     return dict([(key, m.group(key)) for key in ('major', 'minor', 'subminor')])

django/contrib/gis/gdal/prototypes/errcheck.py

@@ -30,9 +30,10 @@ def check_const_string(result, func, cargs, offset=None):
     if offset:
         check_err(result)
         ptr = ptr_byref(cargs, offset)
-        return ptr.value
+        return ptr.value.decode()
     else:
-        return result
+        if result is not None:
+            return result.decode()
 
 def check_string(result, func, cargs, offset=-1, str_result=False):
     """
@@ -47,13 +48,13 @@ def check_string(result, func, cargs, offset=-1, str_result=False):
         # For routines that return a string.
         ptr = result
         if not ptr: s = None
-        else: s = string_at(result)
+        else: s = string_at(result).decode()
     else:
         # Error-code return specified.
         check_err(result)
         ptr = ptr_byref(cargs, offset)
         # Getting the string value
-        s = ptr.value
+        s = ptr.value.decode()
     # Correctly freeing the allocated memory beind GDAL pointer
     # w/the VSIFree routine.
     if ptr: lgdal.VSIFree(ptr)
@@ -125,4 +126,4 @@ def check_str_arg(result, func, cargs):
     """
     dbl = result
     ptr = cargs[-1]._obj
-    return dbl, ptr.value
+    return dbl, ptr.value.decode()

django/contrib/gis/gdal/srs.py

@@ -34,6 +34,8 @@ from django.contrib.gis.gdal.error import SRSException
 from django.contrib.gis.gdal.prototypes import srs as capi
 
 from django.utils import six
+from django.utils.encoding import force_bytes, force_text
+
 
 #### Spatial Reference class. ####
 class SpatialReference(GDALBase):
@@ -51,7 +53,6 @@ class SpatialReference(GDALBase):
         EPSG code, a PROJ.4 string, and/or a projection "well known" shorthand
         string (one of 'WGS84', 'WGS72', 'NAD27', 'NAD83').
         """
-        buf = c_char_p('')
         srs_type = 'user'
 
         if isinstance(srs_input, six.string_types):
@@ -79,6 +80,7 @@ class SpatialReference(GDALBase):
             srs = srs_input
         else:
             # Creating a new SRS pointer, using the string buffer.
+            buf = c_char_p(b'')
             srs = capi.new_srs(buf)
 
         # If the pointer is NULL, throw an exception.
@@ -137,15 +139,16 @@ class SpatialReference(GDALBase):
         """
         if not isinstance(target, six.string_types) or not isinstance(index, int):
             raise TypeError
-        return capi.get_attr_value(self.ptr, target, index)
+        value = capi.get_attr_value(self.ptr, force_bytes(target), index)
+        return force_text(value, 'ascii', strings_only=True)
 
     def auth_name(self, target):
         "Returns the authority name for the given string target node."
-        return capi.get_auth_name(self.ptr, target)
+        return capi.get_auth_name(self.ptr, force_bytes(target))
 
     def auth_code(self, target):
         "Returns the authority code for the given string target node."
-        return capi.get_auth_code(self.ptr, target)
+        return capi.get_auth_code(self.ptr, force_bytes(target))
 
     def clone(self):
         "Returns a clone of this SpatialReference object."
@@ -219,12 +222,14 @@ class SpatialReference(GDALBase):
         and will automatically determines whether to return the linear
         or angular units.
         """
+        units, name = None, None
         if self.projected or self.local:
-            return capi.linear_units(self.ptr, byref(c_char_p()))
+            units, name = capi.linear_units(self.ptr, byref(c_char_p()))
         elif self.geographic:
-            return capi.angular_units(self.ptr, byref(c_char_p()))
+            units, name = capi.angular_units(self.ptr, byref(c_char_p()))
-        else:
+        if name is not None:
-            return (None, None)
+            name.decode()
+        return (units, name)
 
     #### Spheroid/Ellipsoid Properties ####
     @property
@@ -283,7 +288,7 @@ class SpatialReference(GDALBase):
 
     def import_user_input(self, user_input):
         "Imports the Spatial Reference from the given user input string."
-        capi.from_user_input(self.ptr, user_input)
+        capi.from_user_input(self.ptr, force_bytes(user_input))
 
     def import_wkt(self, wkt):
         "Imports the Spatial Reference from OGC WKT (string)"

django/contrib/gis/gdal/tests/test_ds.py

@@ -9,7 +9,7 @@ ds_list = (TestDS('test_point', nfeat=5, nfld=3, geom='POINT', gtype=1, driver='
                   fields={'dbl' : OFTReal, 'int' : OFTInteger, 'str' : OFTString,},
                   extent=(-1.35011,0.166623,-0.524093,0.824508), # Got extent from QGIS
                   srs_wkt='GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]',
-                  field_values={'dbl' : [float(i) for i in range(1, 6)], 'int' : range(1, 6), 'str' : [str(i) for i in range(1, 6)]},
+                  field_values={'dbl' : [float(i) for i in range(1, 6)], 'int' : list(range(1, 6)), 'str' : [str(i) for i in range(1, 6)]},
                   fids=range(5)),
            TestDS('test_vrt', ext='vrt', nfeat=3, nfld=3, geom='POINT', gtype='Point25D', driver='VRT',
                   fields={'POINT_X' : OFTString, 'POINT_Y' : OFTString, 'NUM' : OFTString}, # VRT uses CSV, which all types are OFTString.
@@ -200,7 +200,7 @@ class DataSourceTest(unittest.TestCase):
 
         # Setting the spatial filter with a tuple/list with the extent of
         # a buffer centering around Pueblo.
-        self.assertRaises(ValueError, lyr._set_spatial_filter, range(5))
+        self.assertRaises(ValueError, lyr._set_spatial_filter, list(range(5)))
         filter_extent = (-105.609252, 37.255001, -103.609252, 39.255001)
         lyr.spatial_filter = (-105.609252, 37.255001, -103.609252, 39.255001)
         self.assertEqual(OGRGeometry.from_bbox(filter_extent), lyr.spatial_filter)

django/contrib/gis/gdal/tests/test_geom.py

@@ -92,7 +92,7 @@ class OGRGeomTest(unittest.TestCase, TestDataMixin):
         "Testing HEX input/output."
         for g in self.geometries.hex_wkt:
             geom1 = OGRGeometry(g.wkt)
-            self.assertEqual(g.hex, geom1.hex)
+            self.assertEqual(g.hex.encode(), geom1.hex)
             # Constructing w/HEX
             geom2 = OGRGeometry(g.hex)
             self.assertEqual(geom1, geom2)
@@ -102,7 +102,7 @@ class OGRGeomTest(unittest.TestCase, TestDataMixin):
         for g in self.geometries.hex_wkt:
             geom1 = OGRGeometry(g.wkt)
             wkb = geom1.wkb
-            self.assertEqual(b2a_hex(wkb).upper(), g.hex)
+            self.assertEqual(b2a_hex(wkb).upper(), g.hex.encode())
             # Constructing w/WKB.
             geom2 = OGRGeometry(wkb)
             self.assertEqual(geom1, geom2)

django/contrib/gis/geometry/test_data.py

@@ -101,6 +101,6 @@ class TestDataMixin(object):
         if GEOMETRIES is None:
             # Load up the test geometry data from fixture into global.
             gzf = gzip.GzipFile(os.path.join(TEST_DATA, 'geometries.json.gz'))
-            geometries = json.loads(gzf.read())
+            geometries = json.loads(gzf.read().decode())
             GEOMETRIES = TestGeomSet(**strconvert(geometries))
         return GEOMETRIES

django/contrib/gis/geos/factory.py

@@ -1,7 +1,9 @@
+from django.contrib.gis import memoryview
 from django.contrib.gis.geos.geometry import GEOSGeometry, wkt_regex, hex_regex
 
 from django.utils import six
 
+
 def fromfile(file_h):
     """
     Given a string file name, returns a GEOSGeometry. The file may contain WKB,
@@ -14,11 +16,19 @@ def fromfile(file_h):
     else:
         buf = file_h.read()
 
-    # If we get WKB need to wrap in buffer(), so run through regexes.
+    # If we get WKB need to wrap in memoryview(), so run through regexes.
-    if wkt_regex.match(buf) or hex_regex.match(buf):
+    if isinstance(buf, bytes):
-        return GEOSGeometry(buf)
+        try:
+            decoded = buf.decode()
+            if wkt_regex.match(decoded) or hex_regex.match(decoded):
+                return GEOSGeometry(decoded)
+        except UnicodeDecodeError:
+            pass
     else:
-        return GEOSGeometry(buffer(buf))
+        return GEOSGeometry(buf)
+
+    return GEOSGeometry(memoryview(buf))
+
 
 def fromstr(string, **kwargs):
     "Given a string value, returns a GEOSGeometry object."

django/contrib/gis/geos/geometry.py

@@ -2,9 +2,12 @@
  This module contains the 'base' GEOSGeometry object -- all GEOS Geometries
  inherit from this object.
 """
+from __future__ import unicode_literals
+
 # Python, ctypes and types dependencies.
 from ctypes import addressof, byref, c_double
 
+from django.contrib.gis import memoryview
 # super-class for mutable list behavior
 from django.contrib.gis.geos.mutable_list import ListMixin
 
@@ -28,6 +31,8 @@ from django.contrib.gis.geos.prototypes.io import wkt_r, wkt_w, wkb_r, wkb_w, ew
 from django.contrib.gis.geometry.regex import hex_regex, wkt_regex, json_regex
 
 from django.utils import six
+from django.utils.encoding import force_bytes, force_text
+
 
 class GEOSGeometry(GEOSBase, ListMixin):
     "A class that, generally, encapsulates a GEOS geometry."
@@ -54,19 +59,17 @@ class GEOSGeometry(GEOSBase, ListMixin):
         The `srid` keyword is used to specify the Source Reference Identifier
         (SRID) number for this Geometry.  If not set, the SRID will be None.
         """
+        if isinstance(geo_input, bytes):
+            geo_input = force_text(geo_input)
         if isinstance(geo_input, six.string_types):
-            if isinstance(geo_input, six.text_type):
-                # Encoding to ASCII, WKT or HEXEWKB doesn't need any more.
-                geo_input = geo_input.encode('ascii')
-
             wkt_m = wkt_regex.match(geo_input)
             if wkt_m:
                 # Handling WKT input.
                 if wkt_m.group('srid'): srid = int(wkt_m.group('srid'))
-                g = wkt_r().read(wkt_m.group('wkt'))
+                g = wkt_r().read(force_bytes(wkt_m.group('wkt')))
             elif hex_regex.match(geo_input):
                 # Handling HEXEWKB input.
-                g = wkb_r().read(geo_input)
+                g = wkb_r().read(force_bytes(geo_input))
             elif gdal.HAS_GDAL and json_regex.match(geo_input):
                 # Handling GeoJSON input.
                 g = wkb_r().read(gdal.OGRGeometry(geo_input).wkb)
@@ -75,7 +78,7 @@ class GEOSGeometry(GEOSBase, ListMixin):
         elif isinstance(geo_input, GEOM_PTR):
             # When the input is a pointer to a geomtry (GEOM_PTR).
             g = geo_input
-        elif isinstance(geo_input, buffer):
+        elif isinstance(geo_input, memoryview):
             # When the input is a buffer (WKB).
             g = wkb_r().read(geo_input)
         elif isinstance(geo_input, GEOSGeometry):
@@ -139,12 +142,12 @@ class GEOSGeometry(GEOSBase, ListMixin):
     def __getstate__(self):
         # The pickled state is simply a tuple of the WKB (in string form)
         # and the SRID.
-        return str(self.wkb), self.srid
+        return bytes(self.wkb), self.srid
 
     def __setstate__(self, state):
         # Instantiating from the tuple state that was pickled.
         wkb, srid = state
-        ptr = wkb_r().read(buffer(wkb))
+        ptr = wkb_r().read(memoryview(wkb))
         if not ptr: raise GEOSException('Invalid Geometry loaded from pickled state.')
         self.ptr = ptr
         self._post_init(srid)
@@ -216,7 +219,7 @@ class GEOSGeometry(GEOSBase, ListMixin):
     @property
     def geom_type(self):
         "Returns a string representing the Geometry type, e.g. 'Polygon'"
-        return capi.geos_type(self.ptr)
+        return capi.geos_type(self.ptr).decode()
 
     @property
     def geom_typeid(self):
@@ -283,7 +286,7 @@ class GEOSGeometry(GEOSBase, ListMixin):
         """
         if not GEOS_PREPARE:
             raise GEOSException('Upgrade GEOS to 3.1 to get validity reason.')
-        return capi.geos_isvalidreason(self.ptr)
+        return capi.geos_isvalidreason(self.ptr).decode()
 
     #### Binary predicates. ####
     def contains(self, other):
@@ -337,7 +340,7 @@ class GEOSGeometry(GEOSBase, ListMixin):
         """
         if not isinstance(pattern, six.string_types) or len(pattern) > 9:
             raise GEOSException('invalid intersection matrix pattern')
-        return capi.geos_relatepattern(self.ptr, other.ptr, pattern)
+        return capi.geos_relatepattern(self.ptr, other.ptr, force_bytes(pattern))
 
     def touches(self, other):
         """
@@ -379,7 +382,7 @@ class GEOSGeometry(GEOSBase, ListMixin):
     @property
     def wkt(self):
         "Returns the WKT (Well-Known Text) representation of this Geometry."
-        return wkt_w().write(self)
+        return wkt_w().write(self).decode()
 
     @property
     def hex(self):
@@ -589,7 +592,7 @@ class GEOSGeometry(GEOSBase, ListMixin):
 
     def relate(self, other):
         "Returns the DE-9IM intersection matrix for this Geometry and the other."
-        return capi.geos_relate(self.ptr, other.ptr)
+        return capi.geos_relate(self.ptr, other.ptr).decode()
 
     def simplify(self, tolerance=0.0, preserve_topology=False):
         """

django/contrib/gis/geos/libgeos.py

@@ -57,6 +57,7 @@ lgeos = CDLL(lib_path)
 #  typedef void (*GEOSMessageHandler)(const char *fmt, ...);
 NOTICEFUNC = CFUNCTYPE(None, c_char_p, c_char_p)
 def notice_h(fmt, lst, output_h=sys.stdout):
+    fmt, lst = fmt.decode(), lst.decode()
     try:
         warn_msg = fmt % lst
     except:
@@ -66,6 +67,7 @@ notice_h = NOTICEFUNC(notice_h)
 
 ERRORFUNC = CFUNCTYPE(None, c_char_p, c_char_p)
 def error_h(fmt, lst, output_h=sys.stderr):
+    fmt, lst = fmt.decode(), lst.decode()
     try:
         err_msg = fmt % lst
     except:

django/contrib/gis/geos/prototypes/io.py

@@ -1,5 +1,6 @@
 import threading
 from ctypes import byref, c_char_p, c_int, c_char, c_size_t, Structure, POINTER
+from django.contrib.gis import memoryview
 from django.contrib.gis.geos.base import GEOSBase
 from django.contrib.gis.geos.libgeos import GEOM_PTR
 from django.contrib.gis.geos.prototypes.errcheck import check_geom, check_string, check_sized_string
@@ -7,6 +8,7 @@ from django.contrib.gis.geos.prototypes.geom import c_uchar_p, geos_char_p
 from django.contrib.gis.geos.prototypes.threadsafe import GEOSFunc
 
 from django.utils import six
+from django.utils.encoding import force_bytes
 
 ### The WKB/WKT Reader/Writer structures and pointers ###
 class WKTReader_st(Structure): pass
@@ -120,8 +122,9 @@ class _WKTReader(IOBase):
     ptr_type = WKT_READ_PTR
 
     def read(self, wkt):
-        if not isinstance(wkt, six.string_types): raise TypeError
+        if not isinstance(wkt, (bytes, six.string_types)):
-        return wkt_reader_read(self.ptr, wkt)
+            raise TypeError
+        return wkt_reader_read(self.ptr, force_bytes(wkt))
 
 class _WKBReader(IOBase):
     _constructor = wkb_reader_create
@@ -130,10 +133,10 @@ class _WKBReader(IOBase):
 
     def read(self, wkb):
         "Returns a _pointer_ to C GEOS Geometry object from the given WKB."
-        if isinstance(wkb, buffer):
+        if isinstance(wkb, memoryview):
-            wkb_s = str(wkb)
+            wkb_s = bytes(wkb)
             return wkb_reader_read(self.ptr, wkb_s, len(wkb_s))
-        elif isinstance(wkb, six.string_types):
+        elif isinstance(wkb, (bytes, six.string_types)):
             return wkb_reader_read_hex(self.ptr, wkb, len(wkb))
         else:
             raise TypeError
@@ -155,7 +158,7 @@ class WKBWriter(IOBase):
 
     def write(self, geom):
         "Returns the WKB representation of the given geometry."
-        return buffer(wkb_writer_write(self.ptr, geom.ptr, byref(c_size_t())))
+        return memoryview(wkb_writer_write(self.ptr, geom.ptr, byref(c_size_t())))
 
     def write_hex(self, geom):
         "Returns the HEXEWKB representation of the given geometry."
@@ -188,8 +191,8 @@ class WKBWriter(IOBase):
         return bool(ord(wkb_writer_get_include_srid(self.ptr)))
 
     def _set_include_srid(self, include):
-        if bool(include): flag = chr(1)
+        if bool(include): flag = b'\x01'
-        else: flag = chr(0)
+        else: flag = b'\x00'
         wkb_writer_set_include_srid(self.ptr, flag)
 
     srid = property(_get_include_srid, _set_include_srid)

django/contrib/gis/geos/tests/test_geos.py

@@ -1,7 +1,12 @@
+from __future__ import unicode_literals
+
 import ctypes
 import json
 import random
+from binascii import a2b_hex, b2a_hex
+from io import BytesIO
 
+from django.contrib.gis import memoryview
 from django.contrib.gis.geos import (GEOSException, GEOSIndexError, GEOSGeometry,
     GeometryCollection, Point, MultiPoint, Polygon, MultiPolygon, LinearRing,
     LineString, MultiLineString, fromfile, fromstr, geos_version_info)
@@ -9,6 +14,7 @@ from django.contrib.gis.geos.base import gdal, numpy, GEOSBase
 from django.contrib.gis.geos.libgeos import GEOS_PREPARE
 from django.contrib.gis.geometry.test_data import TestDataMixin
 
+from django.utils.encoding import force_bytes
 from django.utils import six
 from django.utils.six.moves import xrange
 from django.utils import unittest
@@ -64,7 +70,7 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
         # result in a TypeError when trying to assign it to the `ptr` property.
         # Thus, memmory addresses (integers) and pointers of the incorrect type
         # (in `bad_ptrs`) will not be allowed.
-        bad_ptrs = (5, ctypes.c_char_p('foobar'))
+        bad_ptrs = (5, ctypes.c_char_p(b'foobar'))
         for bad_ptr in bad_ptrs:
             # Equivalent to `fg.ptr = bad_ptr`
             self.assertRaises(TypeError, fg1._set_ptr, bad_ptr)
@@ -80,18 +86,16 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
         "Testing HEX output."
         for g in self.geometries.hex_wkt:
             geom = fromstr(g.wkt)
-            self.assertEqual(g.hex, geom.hex)
+            self.assertEqual(g.hex, geom.hex.decode())
 
     def test_hexewkb(self):
         "Testing (HEX)EWKB output."
-        from binascii import a2b_hex
-
         # For testing HEX(EWKB).
-        ogc_hex = '01010000000000000000000000000000000000F03F'
+        ogc_hex = b'01010000000000000000000000000000000000F03F'
         # `SELECT ST_AsHEXEWKB(ST_GeomFromText('POINT(0 1)', 4326));`
-        hexewkb_2d = '0101000020E61000000000000000000000000000000000F03F'
+        hexewkb_2d = b'0101000020E61000000000000000000000000000000000F03F'
         # `SELECT ST_AsHEXEWKB(ST_GeomFromEWKT('SRID=4326;POINT(0 1 2)'));`
-        hexewkb_3d = '01010000A0E61000000000000000000000000000000000F03F0000000000000040'
+        hexewkb_3d = b'01010000A0E61000000000000000000000000000000000F03F0000000000000040'
 
         pnt_2d = Point(0, 1, srid=4326)
         pnt_3d = Point(0, 1, 2, srid=4326)
@@ -118,9 +122,9 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
                 self.fail('Should have raised GEOSException.')
 
         # Same for EWKB.
-        self.assertEqual(buffer(a2b_hex(hexewkb_2d)), pnt_2d.ewkb)
+        self.assertEqual(memoryview(a2b_hex(hexewkb_2d)), pnt_2d.ewkb)
         if GEOS_PREPARE:
-            self.assertEqual(buffer(a2b_hex(hexewkb_3d)), pnt_3d.ewkb)
+            self.assertEqual(memoryview(a2b_hex(hexewkb_3d)), pnt_3d.ewkb)
         else:
             try:
                 ewkb = pnt_3d.ewkb
@@ -150,7 +154,7 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
                 pass
 
         # Bad WKB
-        self.assertRaises(GEOSException, GEOSGeometry, buffer('0'))
+        self.assertRaises(GEOSException, GEOSGeometry, memoryview(b'0'))
 
         print("\nEND - expecting GEOS_ERROR; safe to ignore.\n")
 
@@ -164,11 +168,10 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
 
     def test_wkb(self):
         "Testing WKB output."
-        from binascii import b2a_hex
         for g in self.geometries.hex_wkt:
             geom = fromstr(g.wkt)
             wkb = geom.wkb
-            self.assertEqual(b2a_hex(wkb).upper(), g.hex)
+            self.assertEqual(b2a_hex(wkb).decode().upper(), g.hex)
 
     def test_create_hex(self):
         "Testing creation from HEX."
@@ -180,9 +183,8 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
 
     def test_create_wkb(self):
         "Testing creation from WKB."
-        from binascii import a2b_hex
         for g in self.geometries.hex_wkt:
-            wkb = buffer(a2b_hex(g.hex))
+            wkb = memoryview(a2b_hex(g.hex.encode()))
             geom_h = GEOSGeometry(wkb)
             # we need to do this so decimal places get normalised
             geom_t = fromstr(g.wkt)
@@ -212,13 +214,12 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
 
     def test_fromfile(self):
         "Testing the fromfile() factory."
-        from io import BytesIO
         ref_pnt = GEOSGeometry('POINT(5 23)')
 
         wkt_f = BytesIO()
-        wkt_f.write(ref_pnt.wkt)
+        wkt_f.write(force_bytes(ref_pnt.wkt))
         wkb_f = BytesIO()
-        wkb_f.write(str(ref_pnt.wkb))
+        wkb_f.write(bytes(ref_pnt.wkb))
 
         # Other tests use `fromfile()` on string filenames so those
         # aren't tested here.
@@ -439,8 +440,8 @@ class GEOSTest(unittest.TestCase, TestDataMixin):
                 self.assertEqual(r.geom_typeid, 2)
 
             # Testing polygon construction.
-            self.assertRaises(TypeError, Polygon.__init__, 0, [1, 2, 3])
+            self.assertRaises(TypeError, Polygon, 0, [1, 2, 3])
-            self.assertRaises(TypeError, Polygon.__init__, 'foo')
+            self.assertRaises(TypeError, Polygon, 'foo')
 
             # Polygon(shell, (hole1, ... holeN))
             rings = tuple(r for r in poly)

django/contrib/gis/geos/tests/test_io.py

@@ -1,8 +1,13 @@
+from __future__ import unicode_literals
+
 import binascii
 import unittest
+
+from django.contrib.gis import memoryview
 from django.contrib.gis.geos import GEOSGeometry, WKTReader, WKTWriter, WKBReader, WKBWriter, geos_version_info
 from django.utils import six
 
+
 class GEOSIOTest(unittest.TestCase):
 
     def test01_wktreader(self):
@@ -12,15 +17,15 @@ class GEOSIOTest(unittest.TestCase):
 
         # read() should return a GEOSGeometry
         ref = GEOSGeometry(wkt)
-        g1 = wkt_r.read(wkt)
+        g1 = wkt_r.read(wkt.encode())
-        g2 = wkt_r.read(six.text_type(wkt))
+        g2 = wkt_r.read(wkt)
 
         for geom in (g1, g2):
             self.assertEqual(ref, geom)
 
         # Should only accept six.string_types objects.
         self.assertRaises(TypeError, wkt_r.read, 1)
-        self.assertRaises(TypeError, wkt_r.read, buffer('foo'))
+        self.assertRaises(TypeError, wkt_r.read, memoryview(b'foo'))
 
     def test02_wktwriter(self):
         # Creating a WKTWriter instance, testing its ptr property.
@@ -29,14 +34,14 @@ class GEOSIOTest(unittest.TestCase):
 
         ref = GEOSGeometry('POINT (5 23)')
         ref_wkt = 'POINT (5.0000000000000000 23.0000000000000000)'
-        self.assertEqual(ref_wkt, wkt_w.write(ref))
+        self.assertEqual(ref_wkt, wkt_w.write(ref).decode())
 
     def test03_wkbreader(self):
         # Creating a WKBReader instance
         wkb_r = WKBReader()
 
-        hex = '000000000140140000000000004037000000000000'
+        hex = b'000000000140140000000000004037000000000000'
-        wkb = buffer(binascii.a2b_hex(hex))
+        wkb = memoryview(binascii.a2b_hex(hex))
         ref = GEOSGeometry(hex)
 
         # read() should return a GEOSGeometry on either a hex string or
@@ -56,10 +61,10 @@ class GEOSIOTest(unittest.TestCase):
         # Representations of 'POINT (5 23)' in hex -- one normal and
         # the other with the byte order changed.
         g = GEOSGeometry('POINT (5 23)')
-        hex1 = '010100000000000000000014400000000000003740'
+        hex1 = b'010100000000000000000014400000000000003740'
-        wkb1 = buffer(binascii.a2b_hex(hex1))
+        wkb1 = memoryview(binascii.a2b_hex(hex1))
-        hex2 = '000000000140140000000000004037000000000000'
+        hex2 = b'000000000140140000000000004037000000000000'
-        wkb2 = buffer(binascii.a2b_hex(hex2))
+        wkb2 = memoryview(binascii.a2b_hex(hex2))
 
         self.assertEqual(hex1, wkb_w.write_hex(g))
         self.assertEqual(wkb1, wkb_w.write(g))
@@ -81,10 +86,10 @@ class GEOSIOTest(unittest.TestCase):
         g = GEOSGeometry('POINT (5 23 17)')
         g.srid = 4326
 
-        hex3d = '0101000080000000000000144000000000000037400000000000003140'
+        hex3d = b'0101000080000000000000144000000000000037400000000000003140'
-        wkb3d = buffer(binascii.a2b_hex(hex3d))
+        wkb3d = memoryview(binascii.a2b_hex(hex3d))
-        hex3d_srid = '01010000A0E6100000000000000000144000000000000037400000000000003140'
+        hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140'
-        wkb3d_srid = buffer(binascii.a2b_hex(hex3d_srid))
+        wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid))
 
         # Ensuring bad output dimensions are not accepted
         for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None):
@@ -100,7 +105,7 @@ class GEOSIOTest(unittest.TestCase):
             self.assertEqual(hex3d, wkb_w.write_hex(g))
             self.assertEqual(wkb3d, wkb_w.write(g))
 
-            # Telling the WKBWriter to inlcude the srid in the representation.
+            # Telling the WKBWriter to include the srid in the representation.
             wkb_w.srid = True
             self.assertEqual(hex3d_srid, wkb_w.write_hex(g))
             self.assertEqual(wkb3d_srid, wkb_w.write(g))

django/contrib/gis/tests/geoapp/test_regress.py

@@ -1,4 +1,5 @@
-from __future__ import absolute_import
+# -*- encoding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
 
 from datetime import datetime
 
@@ -26,7 +27,7 @@ class GeoRegressionTests(TestCase):
 
     def test_kmz(self):
         "Testing `render_to_kmz` with non-ASCII data. See #11624."
-        name = '\xc3\x85land Islands'.decode('iso-8859-1')
+        name = "Åland Islands"
         places = [{'name' : name,
                   'description' : name,
                   'kml' : '<Point><coordinates>5.0,23.0</coordinates></Point>'

django/forms/fields.py

@@ -341,7 +341,7 @@ class BaseTemporalField(Field):
             for format in self.input_formats:
                 try:
                     return self.strptime(value, format)
-                except ValueError:
+                except (ValueError, TypeError):
                     continue
         raise ValidationError(self.error_messages['invalid'])
 

docs/intro/tutorial03.txt

@@ -555,7 +555,7 @@ with the :ttag:`url` template tag:
 
     If ``{% url 'polls.views.detail' poll.id %}`` (with quotes) doesn't work,
     but ``{% url polls.views.detail poll.id %}`` (without quotes) does, that
-    means you're using a version of Django ≤ 1.4. In this case, add the
+    means you're using a version of Django < 1.5. In this case, add the
     following declaration at the top of your template:
 
     .. code-block:: html+django

tests/regressiontests/forms/tests/fields.py

@@ -356,6 +356,11 @@ class FieldsTests(SimpleTestCase):
         self.assertEqual(datetime.date(2006, 10, 25), f.clean(' 25 October 2006 '))
         self.assertRaisesMessage(ValidationError, "'Enter a valid date.'", f.clean, '   ')
 
+    def test_datefield_5(self):
+        # Test null bytes (#18982)
+        f = DateField()
+        self.assertRaisesMessage(ValidationError, "'Enter a valid date.'", f.clean, 'a\x00b')
+
     # TimeField ###################################################################
 
     def test_timefield_1(self):