diff --git a/django/contrib/gis/__init__.py b/django/contrib/gis/__init__.py index e69de29bb2..c996fdfdc2 100644 --- a/django/contrib/gis/__init__.py +++ b/django/contrib/gis/__init__.py @@ -0,0 +1,6 @@ +from django.utils import six + +if six.PY3: + memoryview = memoryview +else: + memoryview = buffer diff --git a/django/contrib/gis/db/backends/postgis/adapter.py b/django/contrib/gis/db/backends/postgis/adapter.py index 863ee78acd..8bb514d760 100644 --- a/django/contrib/gis/db/backends/postgis/adapter.py +++ b/django/contrib/gis/db/backends/postgis/adapter.py @@ -1,6 +1,7 @@ """ This object provides quoting for GEOS geometries into PostgreSQL/PostGIS. """ +from __future__ import unicode_literals from psycopg2 import Binary from psycopg2.extensions import ISQLQuote @@ -10,7 +11,7 @@ class PostGISAdapter(object): "Initializes on the geometry." # Getting the WKB (in string form, to allow easy pickling of # the adaptor) and the SRID from the geometry. - self.ewkb = str(geom.ewkb) + self.ewkb = bytes(geom.ewkb) self.srid = geom.srid self._adapter = Binary(self.ewkb) @@ -39,7 +40,7 @@ class PostGISAdapter(object): def getquoted(self): "Returns a properly quoted string for use in PostgreSQL/PostGIS." # psycopg will figure out whether to use E'\\000' or '\000' - return 'ST_GeomFromEWKB(%s)' % self._adapter.getquoted() + return str('ST_GeomFromEWKB(%s)' % self._adapter.getquoted().decode()) def prepare_database_save(self, unused): return self diff --git a/django/contrib/gis/db/models/fields.py b/django/contrib/gis/db/models/fields.py index 17630d0899..c8b8901d59 100644 --- a/django/contrib/gis/db/models/fields.py +++ b/django/contrib/gis/db/models/fields.py @@ -160,7 +160,7 @@ class GeometryField(Field): # from the given string input. if isinstance(geom, Geometry): pass - elif isinstance(geom, six.string_types) or hasattr(geom, '__geo_interface__'): + elif isinstance(geom, (bytes, six.string_types)) or hasattr(geom, '__geo_interface__'): try: geom = Geometry(geom) except GeometryException: diff --git a/django/contrib/gis/db/models/proxy.py b/django/contrib/gis/db/models/proxy.py index 413610fc5c..1fdc5036ba 100644 --- a/django/contrib/gis/db/models/proxy.py +++ b/django/contrib/gis/db/models/proxy.py @@ -5,6 +5,7 @@ corresponding to geographic model fields. Thanks to Robert Coup for providing this functionality (see #4322). """ +from django.contrib.gis import memoryview from django.utils import six class GeometryProxy(object): @@ -54,7 +55,7 @@ class GeometryProxy(object): if isinstance(value, self._klass) and (str(value.geom_type).upper() == gtype or gtype == 'GEOMETRY'): # Assigning the SRID to the geometry. if value.srid is None: value.srid = self._field.srid - elif value is None or isinstance(value, six.string_types + (buffer,)): + elif value is None or isinstance(value, six.string_types + (memoryview,)): # Set with None, WKT, HEX, or WKB pass else: diff --git a/django/contrib/gis/db/models/query.py b/django/contrib/gis/db/models/query.py index d87e151aea..b994252002 100644 --- a/django/contrib/gis/db/models/query.py +++ b/django/contrib/gis/db/models/query.py @@ -1,6 +1,7 @@ from django.db import connections from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet +from django.contrib.gis import memoryview from django.contrib.gis.db.models import aggregates from django.contrib.gis.db.models.fields import get_srid_info, PointField, LineStringField from django.contrib.gis.db.models.sql import AreaField, DistanceField, GeomField, GeoQuery @@ -676,7 +677,7 @@ class GeoQuerySet(QuerySet): if not backend.geography: if not isinstance(geo_field, PointField): raise ValueError('Spherical distance calculation only supported on PointFields.') - if not str(Geometry(buffer(params[0].ewkb)).geom_type) == 'Point': + if not str(Geometry(memoryview(params[0].ewkb)).geom_type) == 'Point': raise ValueError('Spherical distance calculation only supported with Point Geometry parameters') # The `function` procedure argument needs to be set differently for # geodetic distance calculations. diff --git a/django/contrib/gis/db/models/sql/compiler.py b/django/contrib/gis/db/models/sql/compiler.py index 5c8d2647f7..233ca5a03e 100644 --- a/django/contrib/gis/db/models/sql/compiler.py +++ b/django/contrib/gis/db/models/sql/compiler.py @@ -1,3 +1,8 @@ +try: + from itertools import zip_longest +except ImportError: + from itertools import izip_longest as zip_longest + from django.utils.six.moves import zip from django.db.backends.util import truncate_name, typecast_timestamp @@ -190,7 +195,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): if self.connection.ops.oracle or getattr(self.query, 'geo_values', False): # We resolve the rest of the columns if we're on Oracle or if # the `geo_values` attribute is defined. - for value, field in map(None, row[index_start:], fields): + for value, field in zip_longest(row[index_start:], fields): values.append(self.query.convert_values(value, field, self.connection)) else: values.extend(row[index_start:]) diff --git a/django/contrib/gis/gdal/__init__.py b/django/contrib/gis/gdal/__init__.py index adff96b47a..f477f05982 100644 --- a/django/contrib/gis/gdal/__init__.py +++ b/django/contrib/gis/gdal/__init__.py @@ -41,7 +41,7 @@ try: from django.contrib.gis.gdal.srs import SpatialReference, CoordTransform from django.contrib.gis.gdal.geometries import OGRGeometry HAS_GDAL = True -except: +except ImportError: HAS_GDAL = False try: diff --git a/django/contrib/gis/gdal/datasource.py b/django/contrib/gis/gdal/datasource.py index 4ceddc6c72..fa3d86afba 100644 --- a/django/contrib/gis/gdal/datasource.py +++ b/django/contrib/gis/gdal/datasource.py @@ -45,6 +45,7 @@ from django.contrib.gis.gdal.layer import Layer # Getting the ctypes prototypes for the DataSource. from django.contrib.gis.gdal.prototypes import ds as capi +from django.utils.encoding import force_bytes from django.utils import six from django.utils.six.moves import xrange @@ -73,7 +74,7 @@ class DataSource(GDALBase): ds_driver = Driver.ptr_type() try: # OGROpen will auto-detect the data source type. - ds = capi.open_ds(ds_input, self._write, byref(ds_driver)) + ds = capi.open_ds(force_bytes(ds_input), self._write, byref(ds_driver)) except OGRException: # Making the error message more clear rather than something # like "Invalid pointer returned from OGROpen". @@ -102,7 +103,7 @@ class DataSource(GDALBase): def __getitem__(self, index): "Allows use of the index [] operator to get a layer at the index." if isinstance(index, six.string_types): - l = capi.get_layer_by_name(self.ptr, index) + l = capi.get_layer_by_name(self.ptr, force_bytes(index)) if not l: raise OGRIndexError('invalid OGR Layer name given: "%s"' % index) elif isinstance(index, int): if index < 0 or index >= self.layer_count: diff --git a/django/contrib/gis/gdal/driver.py b/django/contrib/gis/gdal/driver.py index de4dc61c63..55a5d77d66 100644 --- a/django/contrib/gis/gdal/driver.py +++ b/django/contrib/gis/gdal/driver.py @@ -5,6 +5,7 @@ from django.contrib.gis.gdal.error import OGRException from django.contrib.gis.gdal.prototypes import ds as capi from django.utils import six +from django.utils.encoding import force_bytes # For more information, see the OGR C API source code: # http://www.gdal.org/ogr/ogr__api_8h.html @@ -36,7 +37,7 @@ class Driver(GDALBase): name = dr_input # Attempting to get the OGR driver by the string name. - dr = capi.get_driver_by_name(name) + dr = capi.get_driver_by_name(force_bytes(name)) elif isinstance(dr_input, int): self._register() dr = capi.get_driver(dr_input) diff --git a/django/contrib/gis/gdal/envelope.py b/django/contrib/gis/gdal/envelope.py index ab0940e476..f145526af0 100644 --- a/django/contrib/gis/gdal/envelope.py +++ b/django/contrib/gis/gdal/envelope.py @@ -52,7 +52,7 @@ class Envelope(object): elif len(args) == 4: # Individual parameters passed in. # Thanks to ww for the help - self._from_sequence(map(float, args)) + self._from_sequence([float(a) for a in args]) else: raise OGRException('Incorrect number (%d) of arguments.' % len(args)) diff --git a/django/contrib/gis/gdal/feature.py b/django/contrib/gis/gdal/feature.py index 292004873d..b8737cd1a0 100644 --- a/django/contrib/gis/gdal/feature.py +++ b/django/contrib/gis/gdal/feature.py @@ -7,6 +7,7 @@ from django.contrib.gis.gdal.geometries import OGRGeometry, OGRGeomType # ctypes function prototypes from django.contrib.gis.gdal.prototypes import ds as capi, geom as geom_api +from django.utils.encoding import force_bytes from django.utils import six from django.utils.six.moves import xrange @@ -107,6 +108,7 @@ class Feature(GDALBase): def index(self, field_name): "Returns the index of the given field name." - i = capi.get_field_index(self.ptr, field_name) - if i < 0: raise OGRIndexError('invalid OFT field name given: "%s"' % field_name) + i = capi.get_field_index(self.ptr, force_bytes(field_name)) + if i < 0: + raise OGRIndexError('invalid OFT field name given: "%s"' % field_name) return i diff --git a/django/contrib/gis/gdal/geometries.py b/django/contrib/gis/gdal/geometries.py index 373ece777d..eb67059245 100644 --- a/django/contrib/gis/gdal/geometries.py +++ b/django/contrib/gis/gdal/geometries.py @@ -43,6 +43,8 @@ import sys from binascii import a2b_hex, b2a_hex from ctypes import byref, string_at, c_char_p, c_double, c_ubyte, c_void_p +from django.contrib.gis import memoryview + # Getting GDAL prerequisites from django.contrib.gis.gdal.base import GDALBase from django.contrib.gis.gdal.envelope import Envelope, OGREnvelope @@ -76,16 +78,11 @@ class OGRGeometry(GDALBase): # If HEX, unpack input to to a binary buffer. if str_instance and hex_regex.match(geom_input): - geom_input = buffer(a2b_hex(geom_input.upper())) + geom_input = memoryview(a2b_hex(geom_input.upper().encode())) str_instance = False # Constructing the geometry, if str_instance: - # Checking if unicode - if isinstance(geom_input, six.text_type): - # Encoding to ASCII, WKT or HEX doesn't need any more. - geom_input = geom_input.encode('ascii') - wkt_m = wkt_regex.match(geom_input) json_m = json_regex.match(geom_input) if wkt_m: @@ -96,19 +93,19 @@ class OGRGeometry(GDALBase): # OGR_G_CreateFromWkt doesn't work with LINEARRING WKT. # See http://trac.osgeo.org/gdal/ticket/1992. g = capi.create_geom(OGRGeomType(wkt_m.group('type')).num) - capi.import_wkt(g, byref(c_char_p(wkt_m.group('wkt')))) + capi.import_wkt(g, byref(c_char_p(wkt_m.group('wkt').encode()))) else: - g = capi.from_wkt(byref(c_char_p(wkt_m.group('wkt'))), None, byref(c_void_p())) + g = capi.from_wkt(byref(c_char_p(wkt_m.group('wkt').encode())), None, byref(c_void_p())) elif json_m: - g = capi.from_json(geom_input) + g = capi.from_json(geom_input.encode()) else: # Seeing if the input is a valid short-hand string # (e.g., 'Point', 'POLYGON'). ogr_t = OGRGeomType(geom_input) g = capi.create_geom(OGRGeomType(geom_input).num) - elif isinstance(geom_input, buffer): + elif isinstance(geom_input, memoryview): # WKB was passed in - g = capi.from_wkb(str(geom_input), None, byref(c_void_p()), len(geom_input)) + g = capi.from_wkb(bytes(geom_input), None, byref(c_void_p()), len(geom_input)) elif isinstance(geom_input, OGRGeomType): # OGRGeomType was passed in, an empty geometry will be created. g = capi.create_geom(geom_input.num) @@ -141,7 +138,7 @@ class OGRGeometry(GDALBase): srs = srs.wkt else: srs = None - return str(self.wkb), srs + return bytes(self.wkb), srs def __setstate__(self, state): wkb, srs = state @@ -354,7 +351,7 @@ class OGRGeometry(GDALBase): buf = (c_ubyte * sz)() wkb = capi.to_wkb(self.ptr, byteorder, byref(buf)) # Returning a buffer of the string at the pointer. - return buffer(string_at(buf, sz)) + return memoryview(string_at(buf, sz)) @property def wkt(self): diff --git a/django/contrib/gis/gdal/layer.py b/django/contrib/gis/gdal/layer.py index 2357fbb88a..d7bf6969ca 100644 --- a/django/contrib/gis/gdal/layer.py +++ b/django/contrib/gis/gdal/layer.py @@ -14,6 +14,7 @@ from django.contrib.gis.gdal.srs import SpatialReference # GDAL ctypes function prototypes. from django.contrib.gis.gdal.prototypes import ds as capi, geom as geom_api, srs as srs_api +from django.utils.encoding import force_bytes from django.utils import six from django.utils.six.moves import xrange @@ -38,7 +39,7 @@ class Layer(GDALBase): self._ds = ds self._ldefn = capi.get_layer_defn(self._ptr) # Does the Layer support random reading? - self._random_read = self.test_capability('RandomRead') + self._random_read = self.test_capability(b'RandomRead') def __getitem__(self, index): "Gets the Feature at the specified index." @@ -212,4 +213,4 @@ class Layer(GDALBase): 'FastFeatureCount', 'FastGetExtent', 'CreateField', 'Transactions', 'DeleteFeature', and 'FastSetNextByIndex'. """ - return bool(capi.test_capability(self.ptr, capability)) + return bool(capi.test_capability(self.ptr, force_bytes(capability))) diff --git a/django/contrib/gis/gdal/libgdal.py b/django/contrib/gis/gdal/libgdal.py index 27a5b8172e..0d2889d704 100644 --- a/django/contrib/gis/gdal/libgdal.py +++ b/django/contrib/gis/gdal/libgdal.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import os import re from ctypes import c_char_p, CDLL @@ -65,7 +67,7 @@ _version_info.restype = c_char_p def gdal_version(): "Returns only the GDAL version number information." - return _version_info('RELEASE_NAME') + return _version_info(b'RELEASE_NAME') def gdal_full_version(): "Returns the full GDAL version information." @@ -86,7 +88,7 @@ def gdal_release_date(date=False): version_regex = re.compile(r'^(?P\d+)\.(?P\d+)(\.(?P\d+))?') def gdal_version_info(): - ver = gdal_version() + ver = gdal_version().decode() m = version_regex.match(ver) if not m: raise OGRException('Could not parse GDAL version string "%s"' % ver) return dict([(key, m.group(key)) for key in ('major', 'minor', 'subminor')]) diff --git a/django/contrib/gis/gdal/prototypes/errcheck.py b/django/contrib/gis/gdal/prototypes/errcheck.py index d8ff1c7dcf..9103022896 100644 --- a/django/contrib/gis/gdal/prototypes/errcheck.py +++ b/django/contrib/gis/gdal/prototypes/errcheck.py @@ -30,9 +30,10 @@ def check_const_string(result, func, cargs, offset=None): if offset: check_err(result) ptr = ptr_byref(cargs, offset) - return ptr.value + return ptr.value.decode() else: - return result + if result is not None: + return result.decode() def check_string(result, func, cargs, offset=-1, str_result=False): """ @@ -47,13 +48,13 @@ def check_string(result, func, cargs, offset=-1, str_result=False): # For routines that return a string. ptr = result if not ptr: s = None - else: s = string_at(result) + else: s = string_at(result).decode() else: # Error-code return specified. check_err(result) ptr = ptr_byref(cargs, offset) # Getting the string value - s = ptr.value + s = ptr.value.decode() # Correctly freeing the allocated memory beind GDAL pointer # w/the VSIFree routine. if ptr: lgdal.VSIFree(ptr) @@ -125,4 +126,4 @@ def check_str_arg(result, func, cargs): """ dbl = result ptr = cargs[-1]._obj - return dbl, ptr.value + return dbl, ptr.value.decode() diff --git a/django/contrib/gis/gdal/srs.py b/django/contrib/gis/gdal/srs.py index cdeaaca690..1a110b0114 100644 --- a/django/contrib/gis/gdal/srs.py +++ b/django/contrib/gis/gdal/srs.py @@ -34,6 +34,8 @@ from django.contrib.gis.gdal.error import SRSException from django.contrib.gis.gdal.prototypes import srs as capi from django.utils import six +from django.utils.encoding import force_bytes, force_text + #### Spatial Reference class. #### class SpatialReference(GDALBase): @@ -51,7 +53,6 @@ class SpatialReference(GDALBase): EPSG code, a PROJ.4 string, and/or a projection "well known" shorthand string (one of 'WGS84', 'WGS72', 'NAD27', 'NAD83'). """ - buf = c_char_p('') srs_type = 'user' if isinstance(srs_input, six.string_types): @@ -79,6 +80,7 @@ class SpatialReference(GDALBase): srs = srs_input else: # Creating a new SRS pointer, using the string buffer. + buf = c_char_p(b'') srs = capi.new_srs(buf) # If the pointer is NULL, throw an exception. @@ -137,15 +139,16 @@ class SpatialReference(GDALBase): """ if not isinstance(target, six.string_types) or not isinstance(index, int): raise TypeError - return capi.get_attr_value(self.ptr, target, index) + value = capi.get_attr_value(self.ptr, force_bytes(target), index) + return force_text(value, 'ascii', strings_only=True) def auth_name(self, target): "Returns the authority name for the given string target node." - return capi.get_auth_name(self.ptr, target) + return capi.get_auth_name(self.ptr, force_bytes(target)) def auth_code(self, target): "Returns the authority code for the given string target node." - return capi.get_auth_code(self.ptr, target) + return capi.get_auth_code(self.ptr, force_bytes(target)) def clone(self): "Returns a clone of this SpatialReference object." @@ -219,12 +222,14 @@ class SpatialReference(GDALBase): and will automatically determines whether to return the linear or angular units. """ + units, name = None, None if self.projected or self.local: - return capi.linear_units(self.ptr, byref(c_char_p())) + units, name = capi.linear_units(self.ptr, byref(c_char_p())) elif self.geographic: - return capi.angular_units(self.ptr, byref(c_char_p())) - else: - return (None, None) + units, name = capi.angular_units(self.ptr, byref(c_char_p())) + if name is not None: + name.decode() + return (units, name) #### Spheroid/Ellipsoid Properties #### @property @@ -283,7 +288,7 @@ class SpatialReference(GDALBase): def import_user_input(self, user_input): "Imports the Spatial Reference from the given user input string." - capi.from_user_input(self.ptr, user_input) + capi.from_user_input(self.ptr, force_bytes(user_input)) def import_wkt(self, wkt): "Imports the Spatial Reference from OGC WKT (string)" diff --git a/django/contrib/gis/gdal/tests/test_ds.py b/django/contrib/gis/gdal/tests/test_ds.py index 22394a2888..094f65b468 100644 --- a/django/contrib/gis/gdal/tests/test_ds.py +++ b/django/contrib/gis/gdal/tests/test_ds.py @@ -9,7 +9,7 @@ ds_list = (TestDS('test_point', nfeat=5, nfld=3, geom='POINT', gtype=1, driver=' fields={'dbl' : OFTReal, 'int' : OFTInteger, 'str' : OFTString,}, extent=(-1.35011,0.166623,-0.524093,0.824508), # Got extent from QGIS srs_wkt='GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]', - field_values={'dbl' : [float(i) for i in range(1, 6)], 'int' : range(1, 6), 'str' : [str(i) for i in range(1, 6)]}, + field_values={'dbl' : [float(i) for i in range(1, 6)], 'int' : list(range(1, 6)), 'str' : [str(i) for i in range(1, 6)]}, fids=range(5)), TestDS('test_vrt', ext='vrt', nfeat=3, nfld=3, geom='POINT', gtype='Point25D', driver='VRT', fields={'POINT_X' : OFTString, 'POINT_Y' : OFTString, 'NUM' : OFTString}, # VRT uses CSV, which all types are OFTString. @@ -200,7 +200,7 @@ class DataSourceTest(unittest.TestCase): # Setting the spatial filter with a tuple/list with the extent of # a buffer centering around Pueblo. - self.assertRaises(ValueError, lyr._set_spatial_filter, range(5)) + self.assertRaises(ValueError, lyr._set_spatial_filter, list(range(5))) filter_extent = (-105.609252, 37.255001, -103.609252, 39.255001) lyr.spatial_filter = (-105.609252, 37.255001, -103.609252, 39.255001) self.assertEqual(OGRGeometry.from_bbox(filter_extent), lyr.spatial_filter) diff --git a/django/contrib/gis/gdal/tests/test_geom.py b/django/contrib/gis/gdal/tests/test_geom.py index dda22036e3..b22bb62109 100644 --- a/django/contrib/gis/gdal/tests/test_geom.py +++ b/django/contrib/gis/gdal/tests/test_geom.py @@ -92,7 +92,7 @@ class OGRGeomTest(unittest.TestCase, TestDataMixin): "Testing HEX input/output." for g in self.geometries.hex_wkt: geom1 = OGRGeometry(g.wkt) - self.assertEqual(g.hex, geom1.hex) + self.assertEqual(g.hex.encode(), geom1.hex) # Constructing w/HEX geom2 = OGRGeometry(g.hex) self.assertEqual(geom1, geom2) @@ -102,7 +102,7 @@ class OGRGeomTest(unittest.TestCase, TestDataMixin): for g in self.geometries.hex_wkt: geom1 = OGRGeometry(g.wkt) wkb = geom1.wkb - self.assertEqual(b2a_hex(wkb).upper(), g.hex) + self.assertEqual(b2a_hex(wkb).upper(), g.hex.encode()) # Constructing w/WKB. geom2 = OGRGeometry(wkb) self.assertEqual(geom1, geom2) diff --git a/django/contrib/gis/geometry/test_data.py b/django/contrib/gis/geometry/test_data.py index 505f0e4f4b..833e62f224 100644 --- a/django/contrib/gis/geometry/test_data.py +++ b/django/contrib/gis/geometry/test_data.py @@ -101,6 +101,6 @@ class TestDataMixin(object): if GEOMETRIES is None: # Load up the test geometry data from fixture into global. gzf = gzip.GzipFile(os.path.join(TEST_DATA, 'geometries.json.gz')) - geometries = json.loads(gzf.read()) + geometries = json.loads(gzf.read().decode()) GEOMETRIES = TestGeomSet(**strconvert(geometries)) return GEOMETRIES diff --git a/django/contrib/gis/geos/factory.py b/django/contrib/gis/geos/factory.py index fbd7d5a3e9..2e5fa4f331 100644 --- a/django/contrib/gis/geos/factory.py +++ b/django/contrib/gis/geos/factory.py @@ -1,7 +1,9 @@ +from django.contrib.gis import memoryview from django.contrib.gis.geos.geometry import GEOSGeometry, wkt_regex, hex_regex from django.utils import six + def fromfile(file_h): """ Given a string file name, returns a GEOSGeometry. The file may contain WKB, @@ -14,11 +16,19 @@ def fromfile(file_h): else: buf = file_h.read() - # If we get WKB need to wrap in buffer(), so run through regexes. - if wkt_regex.match(buf) or hex_regex.match(buf): - return GEOSGeometry(buf) + # If we get WKB need to wrap in memoryview(), so run through regexes. + if isinstance(buf, bytes): + try: + decoded = buf.decode() + if wkt_regex.match(decoded) or hex_regex.match(decoded): + return GEOSGeometry(decoded) + except UnicodeDecodeError: + pass else: - return GEOSGeometry(buffer(buf)) + return GEOSGeometry(buf) + + return GEOSGeometry(memoryview(buf)) + def fromstr(string, **kwargs): "Given a string value, returns a GEOSGeometry object." diff --git a/django/contrib/gis/geos/geometry.py b/django/contrib/gis/geos/geometry.py index 4e5409de1d..6dbb6b2cb3 100644 --- a/django/contrib/gis/geos/geometry.py +++ b/django/contrib/gis/geos/geometry.py @@ -2,9 +2,12 @@ This module contains the 'base' GEOSGeometry object -- all GEOS Geometries inherit from this object. """ +from __future__ import unicode_literals + # Python, ctypes and types dependencies. from ctypes import addressof, byref, c_double +from django.contrib.gis import memoryview # super-class for mutable list behavior from django.contrib.gis.geos.mutable_list import ListMixin @@ -28,6 +31,8 @@ from django.contrib.gis.geos.prototypes.io import wkt_r, wkt_w, wkb_r, wkb_w, ew from django.contrib.gis.geometry.regex import hex_regex, wkt_regex, json_regex from django.utils import six +from django.utils.encoding import force_bytes, force_text + class GEOSGeometry(GEOSBase, ListMixin): "A class that, generally, encapsulates a GEOS geometry." @@ -54,19 +59,17 @@ class GEOSGeometry(GEOSBase, ListMixin): The `srid` keyword is used to specify the Source Reference Identifier (SRID) number for this Geometry. If not set, the SRID will be None. """ + if isinstance(geo_input, bytes): + geo_input = force_text(geo_input) if isinstance(geo_input, six.string_types): - if isinstance(geo_input, six.text_type): - # Encoding to ASCII, WKT or HEXEWKB doesn't need any more. - geo_input = geo_input.encode('ascii') - wkt_m = wkt_regex.match(geo_input) if wkt_m: # Handling WKT input. if wkt_m.group('srid'): srid = int(wkt_m.group('srid')) - g = wkt_r().read(wkt_m.group('wkt')) + g = wkt_r().read(force_bytes(wkt_m.group('wkt'))) elif hex_regex.match(geo_input): # Handling HEXEWKB input. - g = wkb_r().read(geo_input) + g = wkb_r().read(force_bytes(geo_input)) elif gdal.HAS_GDAL and json_regex.match(geo_input): # Handling GeoJSON input. g = wkb_r().read(gdal.OGRGeometry(geo_input).wkb) @@ -75,7 +78,7 @@ class GEOSGeometry(GEOSBase, ListMixin): elif isinstance(geo_input, GEOM_PTR): # When the input is a pointer to a geomtry (GEOM_PTR). g = geo_input - elif isinstance(geo_input, buffer): + elif isinstance(geo_input, memoryview): # When the input is a buffer (WKB). g = wkb_r().read(geo_input) elif isinstance(geo_input, GEOSGeometry): @@ -139,12 +142,12 @@ class GEOSGeometry(GEOSBase, ListMixin): def __getstate__(self): # The pickled state is simply a tuple of the WKB (in string form) # and the SRID. - return str(self.wkb), self.srid + return bytes(self.wkb), self.srid def __setstate__(self, state): # Instantiating from the tuple state that was pickled. wkb, srid = state - ptr = wkb_r().read(buffer(wkb)) + ptr = wkb_r().read(memoryview(wkb)) if not ptr: raise GEOSException('Invalid Geometry loaded from pickled state.') self.ptr = ptr self._post_init(srid) @@ -216,7 +219,7 @@ class GEOSGeometry(GEOSBase, ListMixin): @property def geom_type(self): "Returns a string representing the Geometry type, e.g. 'Polygon'" - return capi.geos_type(self.ptr) + return capi.geos_type(self.ptr).decode() @property def geom_typeid(self): @@ -283,7 +286,7 @@ class GEOSGeometry(GEOSBase, ListMixin): """ if not GEOS_PREPARE: raise GEOSException('Upgrade GEOS to 3.1 to get validity reason.') - return capi.geos_isvalidreason(self.ptr) + return capi.geos_isvalidreason(self.ptr).decode() #### Binary predicates. #### def contains(self, other): @@ -337,7 +340,7 @@ class GEOSGeometry(GEOSBase, ListMixin): """ if not isinstance(pattern, six.string_types) or len(pattern) > 9: raise GEOSException('invalid intersection matrix pattern') - return capi.geos_relatepattern(self.ptr, other.ptr, pattern) + return capi.geos_relatepattern(self.ptr, other.ptr, force_bytes(pattern)) def touches(self, other): """ @@ -379,7 +382,7 @@ class GEOSGeometry(GEOSBase, ListMixin): @property def wkt(self): "Returns the WKT (Well-Known Text) representation of this Geometry." - return wkt_w().write(self) + return wkt_w().write(self).decode() @property def hex(self): @@ -589,7 +592,7 @@ class GEOSGeometry(GEOSBase, ListMixin): def relate(self, other): "Returns the DE-9IM intersection matrix for this Geometry and the other." - return capi.geos_relate(self.ptr, other.ptr) + return capi.geos_relate(self.ptr, other.ptr).decode() def simplify(self, tolerance=0.0, preserve_topology=False): """ diff --git a/django/contrib/gis/geos/libgeos.py b/django/contrib/gis/geos/libgeos.py index aed6cf366c..b31a7955a2 100644 --- a/django/contrib/gis/geos/libgeos.py +++ b/django/contrib/gis/geos/libgeos.py @@ -57,6 +57,7 @@ lgeos = CDLL(lib_path) # typedef void (*GEOSMessageHandler)(const char *fmt, ...); NOTICEFUNC = CFUNCTYPE(None, c_char_p, c_char_p) def notice_h(fmt, lst, output_h=sys.stdout): + fmt, lst = fmt.decode(), lst.decode() try: warn_msg = fmt % lst except: @@ -66,6 +67,7 @@ notice_h = NOTICEFUNC(notice_h) ERRORFUNC = CFUNCTYPE(None, c_char_p, c_char_p) def error_h(fmt, lst, output_h=sys.stderr): + fmt, lst = fmt.decode(), lst.decode() try: err_msg = fmt % lst except: diff --git a/django/contrib/gis/geos/prototypes/io.py b/django/contrib/gis/geos/prototypes/io.py index 053b9948a2..1eeab60a4b 100644 --- a/django/contrib/gis/geos/prototypes/io.py +++ b/django/contrib/gis/geos/prototypes/io.py @@ -1,5 +1,6 @@ import threading from ctypes import byref, c_char_p, c_int, c_char, c_size_t, Structure, POINTER +from django.contrib.gis import memoryview from django.contrib.gis.geos.base import GEOSBase from django.contrib.gis.geos.libgeos import GEOM_PTR from django.contrib.gis.geos.prototypes.errcheck import check_geom, check_string, check_sized_string @@ -7,6 +8,7 @@ from django.contrib.gis.geos.prototypes.geom import c_uchar_p, geos_char_p from django.contrib.gis.geos.prototypes.threadsafe import GEOSFunc from django.utils import six +from django.utils.encoding import force_bytes ### The WKB/WKT Reader/Writer structures and pointers ### class WKTReader_st(Structure): pass @@ -120,8 +122,9 @@ class _WKTReader(IOBase): ptr_type = WKT_READ_PTR def read(self, wkt): - if not isinstance(wkt, six.string_types): raise TypeError - return wkt_reader_read(self.ptr, wkt) + if not isinstance(wkt, (bytes, six.string_types)): + raise TypeError + return wkt_reader_read(self.ptr, force_bytes(wkt)) class _WKBReader(IOBase): _constructor = wkb_reader_create @@ -130,10 +133,10 @@ class _WKBReader(IOBase): def read(self, wkb): "Returns a _pointer_ to C GEOS Geometry object from the given WKB." - if isinstance(wkb, buffer): - wkb_s = str(wkb) + if isinstance(wkb, memoryview): + wkb_s = bytes(wkb) return wkb_reader_read(self.ptr, wkb_s, len(wkb_s)) - elif isinstance(wkb, six.string_types): + elif isinstance(wkb, (bytes, six.string_types)): return wkb_reader_read_hex(self.ptr, wkb, len(wkb)) else: raise TypeError @@ -155,7 +158,7 @@ class WKBWriter(IOBase): def write(self, geom): "Returns the WKB representation of the given geometry." - return buffer(wkb_writer_write(self.ptr, geom.ptr, byref(c_size_t()))) + return memoryview(wkb_writer_write(self.ptr, geom.ptr, byref(c_size_t()))) def write_hex(self, geom): "Returns the HEXEWKB representation of the given geometry." @@ -188,8 +191,8 @@ class WKBWriter(IOBase): return bool(ord(wkb_writer_get_include_srid(self.ptr))) def _set_include_srid(self, include): - if bool(include): flag = chr(1) - else: flag = chr(0) + if bool(include): flag = b'\x01' + else: flag = b'\x00' wkb_writer_set_include_srid(self.ptr, flag) srid = property(_get_include_srid, _set_include_srid) diff --git a/django/contrib/gis/geos/tests/test_geos.py b/django/contrib/gis/geos/tests/test_geos.py index 7300ab9c63..c8d3e43a0e 100644 --- a/django/contrib/gis/geos/tests/test_geos.py +++ b/django/contrib/gis/geos/tests/test_geos.py @@ -1,7 +1,12 @@ +from __future__ import unicode_literals + import ctypes import json import random +from binascii import a2b_hex, b2a_hex +from io import BytesIO +from django.contrib.gis import memoryview from django.contrib.gis.geos import (GEOSException, GEOSIndexError, GEOSGeometry, GeometryCollection, Point, MultiPoint, Polygon, MultiPolygon, LinearRing, LineString, MultiLineString, fromfile, fromstr, geos_version_info) @@ -9,6 +14,7 @@ from django.contrib.gis.geos.base import gdal, numpy, GEOSBase from django.contrib.gis.geos.libgeos import GEOS_PREPARE from django.contrib.gis.geometry.test_data import TestDataMixin +from django.utils.encoding import force_bytes from django.utils import six from django.utils.six.moves import xrange from django.utils import unittest @@ -64,7 +70,7 @@ class GEOSTest(unittest.TestCase, TestDataMixin): # result in a TypeError when trying to assign it to the `ptr` property. # Thus, memmory addresses (integers) and pointers of the incorrect type # (in `bad_ptrs`) will not be allowed. - bad_ptrs = (5, ctypes.c_char_p('foobar')) + bad_ptrs = (5, ctypes.c_char_p(b'foobar')) for bad_ptr in bad_ptrs: # Equivalent to `fg.ptr = bad_ptr` self.assertRaises(TypeError, fg1._set_ptr, bad_ptr) @@ -80,18 +86,16 @@ class GEOSTest(unittest.TestCase, TestDataMixin): "Testing HEX output." for g in self.geometries.hex_wkt: geom = fromstr(g.wkt) - self.assertEqual(g.hex, geom.hex) + self.assertEqual(g.hex, geom.hex.decode()) def test_hexewkb(self): "Testing (HEX)EWKB output." - from binascii import a2b_hex - # For testing HEX(EWKB). - ogc_hex = '01010000000000000000000000000000000000F03F' + ogc_hex = b'01010000000000000000000000000000000000F03F' # `SELECT ST_AsHEXEWKB(ST_GeomFromText('POINT(0 1)', 4326));` - hexewkb_2d = '0101000020E61000000000000000000000000000000000F03F' + hexewkb_2d = b'0101000020E61000000000000000000000000000000000F03F' # `SELECT ST_AsHEXEWKB(ST_GeomFromEWKT('SRID=4326;POINT(0 1 2)'));` - hexewkb_3d = '01010000A0E61000000000000000000000000000000000F03F0000000000000040' + hexewkb_3d = b'01010000A0E61000000000000000000000000000000000F03F0000000000000040' pnt_2d = Point(0, 1, srid=4326) pnt_3d = Point(0, 1, 2, srid=4326) @@ -118,9 +122,9 @@ class GEOSTest(unittest.TestCase, TestDataMixin): self.fail('Should have raised GEOSException.') # Same for EWKB. - self.assertEqual(buffer(a2b_hex(hexewkb_2d)), pnt_2d.ewkb) + self.assertEqual(memoryview(a2b_hex(hexewkb_2d)), pnt_2d.ewkb) if GEOS_PREPARE: - self.assertEqual(buffer(a2b_hex(hexewkb_3d)), pnt_3d.ewkb) + self.assertEqual(memoryview(a2b_hex(hexewkb_3d)), pnt_3d.ewkb) else: try: ewkb = pnt_3d.ewkb @@ -150,7 +154,7 @@ class GEOSTest(unittest.TestCase, TestDataMixin): pass # Bad WKB - self.assertRaises(GEOSException, GEOSGeometry, buffer('0')) + self.assertRaises(GEOSException, GEOSGeometry, memoryview(b'0')) print("\nEND - expecting GEOS_ERROR; safe to ignore.\n") @@ -164,11 +168,10 @@ class GEOSTest(unittest.TestCase, TestDataMixin): def test_wkb(self): "Testing WKB output." - from binascii import b2a_hex for g in self.geometries.hex_wkt: geom = fromstr(g.wkt) wkb = geom.wkb - self.assertEqual(b2a_hex(wkb).upper(), g.hex) + self.assertEqual(b2a_hex(wkb).decode().upper(), g.hex) def test_create_hex(self): "Testing creation from HEX." @@ -180,9 +183,8 @@ class GEOSTest(unittest.TestCase, TestDataMixin): def test_create_wkb(self): "Testing creation from WKB." - from binascii import a2b_hex for g in self.geometries.hex_wkt: - wkb = buffer(a2b_hex(g.hex)) + wkb = memoryview(a2b_hex(g.hex.encode())) geom_h = GEOSGeometry(wkb) # we need to do this so decimal places get normalised geom_t = fromstr(g.wkt) @@ -212,13 +214,12 @@ class GEOSTest(unittest.TestCase, TestDataMixin): def test_fromfile(self): "Testing the fromfile() factory." - from io import BytesIO ref_pnt = GEOSGeometry('POINT(5 23)') wkt_f = BytesIO() - wkt_f.write(ref_pnt.wkt) + wkt_f.write(force_bytes(ref_pnt.wkt)) wkb_f = BytesIO() - wkb_f.write(str(ref_pnt.wkb)) + wkb_f.write(bytes(ref_pnt.wkb)) # Other tests use `fromfile()` on string filenames so those # aren't tested here. @@ -439,8 +440,8 @@ class GEOSTest(unittest.TestCase, TestDataMixin): self.assertEqual(r.geom_typeid, 2) # Testing polygon construction. - self.assertRaises(TypeError, Polygon.__init__, 0, [1, 2, 3]) - self.assertRaises(TypeError, Polygon.__init__, 'foo') + self.assertRaises(TypeError, Polygon, 0, [1, 2, 3]) + self.assertRaises(TypeError, Polygon, 'foo') # Polygon(shell, (hole1, ... holeN)) rings = tuple(r for r in poly) diff --git a/django/contrib/gis/geos/tests/test_io.py b/django/contrib/gis/geos/tests/test_io.py index ebf178a807..45a9a220b1 100644 --- a/django/contrib/gis/geos/tests/test_io.py +++ b/django/contrib/gis/geos/tests/test_io.py @@ -1,8 +1,13 @@ +from __future__ import unicode_literals + import binascii import unittest + +from django.contrib.gis import memoryview from django.contrib.gis.geos import GEOSGeometry, WKTReader, WKTWriter, WKBReader, WKBWriter, geos_version_info from django.utils import six + class GEOSIOTest(unittest.TestCase): def test01_wktreader(self): @@ -12,15 +17,15 @@ class GEOSIOTest(unittest.TestCase): # read() should return a GEOSGeometry ref = GEOSGeometry(wkt) - g1 = wkt_r.read(wkt) - g2 = wkt_r.read(six.text_type(wkt)) + g1 = wkt_r.read(wkt.encode()) + g2 = wkt_r.read(wkt) for geom in (g1, g2): self.assertEqual(ref, geom) # Should only accept six.string_types objects. self.assertRaises(TypeError, wkt_r.read, 1) - self.assertRaises(TypeError, wkt_r.read, buffer('foo')) + self.assertRaises(TypeError, wkt_r.read, memoryview(b'foo')) def test02_wktwriter(self): # Creating a WKTWriter instance, testing its ptr property. @@ -29,14 +34,14 @@ class GEOSIOTest(unittest.TestCase): ref = GEOSGeometry('POINT (5 23)') ref_wkt = 'POINT (5.0000000000000000 23.0000000000000000)' - self.assertEqual(ref_wkt, wkt_w.write(ref)) + self.assertEqual(ref_wkt, wkt_w.write(ref).decode()) def test03_wkbreader(self): # Creating a WKBReader instance wkb_r = WKBReader() - hex = '000000000140140000000000004037000000000000' - wkb = buffer(binascii.a2b_hex(hex)) + hex = b'000000000140140000000000004037000000000000' + wkb = memoryview(binascii.a2b_hex(hex)) ref = GEOSGeometry(hex) # read() should return a GEOSGeometry on either a hex string or @@ -56,10 +61,10 @@ class GEOSIOTest(unittest.TestCase): # Representations of 'POINT (5 23)' in hex -- one normal and # the other with the byte order changed. g = GEOSGeometry('POINT (5 23)') - hex1 = '010100000000000000000014400000000000003740' - wkb1 = buffer(binascii.a2b_hex(hex1)) - hex2 = '000000000140140000000000004037000000000000' - wkb2 = buffer(binascii.a2b_hex(hex2)) + hex1 = b'010100000000000000000014400000000000003740' + wkb1 = memoryview(binascii.a2b_hex(hex1)) + hex2 = b'000000000140140000000000004037000000000000' + wkb2 = memoryview(binascii.a2b_hex(hex2)) self.assertEqual(hex1, wkb_w.write_hex(g)) self.assertEqual(wkb1, wkb_w.write(g)) @@ -81,10 +86,10 @@ class GEOSIOTest(unittest.TestCase): g = GEOSGeometry('POINT (5 23 17)') g.srid = 4326 - hex3d = '0101000080000000000000144000000000000037400000000000003140' - wkb3d = buffer(binascii.a2b_hex(hex3d)) - hex3d_srid = '01010000A0E6100000000000000000144000000000000037400000000000003140' - wkb3d_srid = buffer(binascii.a2b_hex(hex3d_srid)) + hex3d = b'0101000080000000000000144000000000000037400000000000003140' + wkb3d = memoryview(binascii.a2b_hex(hex3d)) + hex3d_srid = b'01010000A0E6100000000000000000144000000000000037400000000000003140' + wkb3d_srid = memoryview(binascii.a2b_hex(hex3d_srid)) # Ensuring bad output dimensions are not accepted for bad_outdim in (-1, 0, 1, 4, 423, 'foo', None): @@ -100,7 +105,7 @@ class GEOSIOTest(unittest.TestCase): self.assertEqual(hex3d, wkb_w.write_hex(g)) self.assertEqual(wkb3d, wkb_w.write(g)) - # Telling the WKBWriter to inlcude the srid in the representation. + # Telling the WKBWriter to include the srid in the representation. wkb_w.srid = True self.assertEqual(hex3d_srid, wkb_w.write_hex(g)) self.assertEqual(wkb3d_srid, wkb_w.write(g)) diff --git a/django/contrib/gis/tests/geoapp/test_regress.py b/django/contrib/gis/tests/geoapp/test_regress.py index fffd7d3cab..0e9c5c44a3 100644 --- a/django/contrib/gis/tests/geoapp/test_regress.py +++ b/django/contrib/gis/tests/geoapp/test_regress.py @@ -1,4 +1,5 @@ -from __future__ import absolute_import +# -*- encoding: utf-8 -*- +from __future__ import absolute_import, unicode_literals from datetime import datetime @@ -26,7 +27,7 @@ class GeoRegressionTests(TestCase): def test_kmz(self): "Testing `render_to_kmz` with non-ASCII data. See #11624." - name = '\xc3\x85land Islands'.decode('iso-8859-1') + name = "Åland Islands" places = [{'name' : name, 'description' : name, 'kml' : '5.0,23.0' diff --git a/django/forms/fields.py b/django/forms/fields.py index 124e4f669a..0075325288 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -341,7 +341,7 @@ class BaseTemporalField(Field): for format in self.input_formats: try: return self.strptime(value, format) - except ValueError: + except (ValueError, TypeError): continue raise ValidationError(self.error_messages['invalid']) diff --git a/docs/intro/tutorial03.txt b/docs/intro/tutorial03.txt index 03d4bf68b3..d6f95008de 100644 --- a/docs/intro/tutorial03.txt +++ b/docs/intro/tutorial03.txt @@ -555,7 +555,7 @@ with the :ttag:`url` template tag: If ``{% url 'polls.views.detail' poll.id %}`` (with quotes) doesn't work, but ``{% url polls.views.detail poll.id %}`` (without quotes) does, that - means you're using a version of Django ≤ 1.4. In this case, add the + means you're using a version of Django < 1.5. In this case, add the following declaration at the top of your template: .. code-block:: html+django diff --git a/tests/regressiontests/forms/tests/fields.py b/tests/regressiontests/forms/tests/fields.py index 197ce1abd9..989acbc496 100644 --- a/tests/regressiontests/forms/tests/fields.py +++ b/tests/regressiontests/forms/tests/fields.py @@ -356,6 +356,11 @@ class FieldsTests(SimpleTestCase): self.assertEqual(datetime.date(2006, 10, 25), f.clean(' 25 October 2006 ')) self.assertRaisesMessage(ValidationError, "'Enter a valid date.'", f.clean, ' ') + def test_datefield_5(self): + # Test null bytes (#18982) + f = DateField() + self.assertRaisesMessage(ValidationError, "'Enter a valid date.'", f.clean, 'a\x00b') + # TimeField ################################################################### def test_timefield_1(self):