From 8eca2df3a4db70ed7d721312d789a92274aee397 Mon Sep 17 00:00:00 2001 From: Justin Bronn Date: Mon, 30 Mar 2009 22:15:41 +0000 Subject: [PATCH] Fixed #9686 -- SpatiaLite is now a supported spatial database backend. Thanks to Matthew Hancher for initial patch and hard work in implementing this feature. git-svn-id: http://code.djangoproject.com/svn/django/trunk@10222 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- .../gis/db/backend/spatialite/__init__.py | 59 +++++++ .../gis/db/backend/spatialite/adaptor.py | 8 + .../gis/db/backend/spatialite/creation.py | 61 +++++++ .../gis/db/backend/spatialite/field.py | 81 +++++++++ .../gis/db/backend/spatialite/models.py | 53 ++++++ .../gis/db/backend/spatialite/query.py | 159 ++++++++++++++++++ .../contrib/gis/tests/test_spatialrefsys.py | 18 +- 7 files changed, 433 insertions(+), 6 deletions(-) create mode 100644 django/contrib/gis/db/backend/spatialite/__init__.py create mode 100644 django/contrib/gis/db/backend/spatialite/adaptor.py create mode 100644 django/contrib/gis/db/backend/spatialite/creation.py create mode 100644 django/contrib/gis/db/backend/spatialite/field.py create mode 100644 django/contrib/gis/db/backend/spatialite/models.py create mode 100644 django/contrib/gis/db/backend/spatialite/query.py diff --git a/django/contrib/gis/db/backend/spatialite/__init__.py b/django/contrib/gis/db/backend/spatialite/__init__.py new file mode 100644 index 0000000000..7010025e39 --- /dev/null +++ b/django/contrib/gis/db/backend/spatialite/__init__.py @@ -0,0 +1,59 @@ +__all__ = ['create_test_spatial_db', 'get_geo_where_clause', 'SpatialBackend'] + +from ctypes.util import find_library +from django.conf import settings +from django.db.backends.signals import connection_created + +from django.contrib.gis.db.backend.base import BaseSpatialBackend +from django.contrib.gis.db.backend.spatialite.adaptor import SpatiaLiteAdaptor +from django.contrib.gis.db.backend.spatialite.creation import create_test_spatial_db +from django.contrib.gis.db.backend.spatialite.field import SpatiaLiteField +from django.contrib.gis.db.backend.spatialite.models import GeometryColumns, SpatialRefSys +from django.contrib.gis.db.backend.spatialite.query import * + +# Here we are figuring out the path to the SpatiLite library (`libspatialite`). +# If it's not in the system PATH, it may be set manually in the settings via +# the `SPATIALITE_LIBRARY_PATH` setting. +spatialite_lib = getattr(settings, 'SPATIALITE_LIBRARY_PATH', find_library('spatialite')) +if spatialite_lib: + def initialize_spatialite(sender=None, **kwargs): + """ + This function initializes the pysqlite2 connection to enable the + loading of extensions, and to load up the SpatiaLite library + extension. + """ + from django.db import connection + connection.connection.enable_load_extension(True) + connection.cursor().execute("SELECT load_extension(%s)", (spatialite_lib,)) + connection_created.connect(initialize_spatialite) +else: + # No SpatiaLite library found. + raise Exception('Unable to locate SpatiaLite, needed to use GeoDjango with sqlite3.') + +SpatialBackend = BaseSpatialBackend(name='spatialite', spatialite=True, + area=AREA, + centroid=CENTROID, + contained=CONTAINED, + difference=DIFFERENCE, + distance=DISTANCE, + distance_functions=DISTANCE_FUNCTIONS, + envelope=ENVELOPE, + from_text=GEOM_FROM_TEXT, + gis_terms=SPATIALITE_TERMS, + intersection=INTERSECTION, + length=LENGTH, + num_geom=NUM_GEOM, + num_points=NUM_POINTS, + point_on_surface=POINT_ON_SURFACE, + scale=SCALE, + select=GEOM_SELECT, + sym_difference=SYM_DIFFERENCE, + transform=TRANSFORM, + translate=TRANSLATE, + union=UNION, + unionagg=UNIONAGG, + Adaptor=SpatiaLiteAdaptor, + Field=SpatiaLiteField, + GeometryColumns=GeometryColumns, + SpatialRefSys=SpatialRefSys, + ) diff --git a/django/contrib/gis/db/backend/spatialite/adaptor.py b/django/contrib/gis/db/backend/spatialite/adaptor.py new file mode 100644 index 0000000000..a8683c24de --- /dev/null +++ b/django/contrib/gis/db/backend/spatialite/adaptor.py @@ -0,0 +1,8 @@ +from django.db.backends.sqlite3.base import Database +from django.contrib.gis.db.backend.adaptor import WKTAdaptor + +class SpatiaLiteAdaptor(WKTAdaptor): + "SQLite adaptor for geometry objects." + def __conform__(self, protocol): + if protocol is Database.PrepareProtocol: + return str(self) diff --git a/django/contrib/gis/db/backend/spatialite/creation.py b/django/contrib/gis/db/backend/spatialite/creation.py new file mode 100644 index 0000000000..bb5507fb40 --- /dev/null +++ b/django/contrib/gis/db/backend/spatialite/creation.py @@ -0,0 +1,61 @@ +import os +from django.conf import settings +from django.core.management import call_command +from django.db import connection + +def spatialite_init_file(): + # SPATIALITE_SQL may be placed in settings to tell + # GeoDjango to use a specific user-supplied file. + return getattr(settings, 'SPATIALITE_SQL', 'init_spatialite-2.2.sql') + +def create_test_spatial_db(verbosity=1, autoclobber=False, interactive=False): + "Creates a spatial database based on the settings." + + # Making sure we're using PostgreSQL and psycopg2 + if settings.DATABASE_ENGINE != 'sqlite3': + raise Exception('SpatiaLite database creation only supported on sqlite3 platform.') + + # Getting the test database name using the the SQLite backend's + # `_create_test_db`. Unless `TEST_DATABASE_NAME` is defined, + # it returns ":memory:". + db_name = connection.creation._create_test_db(verbosity, autoclobber) + + # Closing out the current connection to the database set in + # originally in the settings. This makes it so `initialize_spatialite` + # function will be run on the connection for the _test_ database instead. + connection.close() + + # Point to the new database + settings.DATABASE_NAME = db_name + connection.settings_dict["DATABASE_NAME"] = db_name + can_rollback = connection.creation._rollback_works() + settings.DATABASE_SUPPORTS_TRANSACTIONS = can_rollback + connection.settings_dict["DATABASE_SUPPORTS_TRANSACTIONS"] = can_rollback + + # Finally, loading up the SpatiaLite SQL file. + load_spatialite_sql(db_name, verbosity=verbosity) + + if verbosity >= 1: + print 'Creation of spatial database %s successful.' % db_name + + # Syncing the database + call_command('syncdb', verbosity=verbosity, interactive=interactive) + +def load_spatialite_sql(db_name, verbosity=1): + """ + This routine loads up the SpatiaLite SQL file. + """ + # Getting the location of the SpatiaLite SQL file, and confirming + # it exists. + spatialite_sql = spatialite_init_file() + if not os.path.isfile(spatialite_sql): + raise Exception('Could not find the SpatiaLite initialization SQL file: %s' % spatialite_sql) + + # Opening up the SpatiaLite SQL initialization file and executing + # as a script. + sql_fh = open(spatialite_sql, 'r') + try: + cur = connection.cursor() + cur.executescript(sql_fh.read()) + finally: + sql_fh.close() diff --git a/django/contrib/gis/db/backend/spatialite/field.py b/django/contrib/gis/db/backend/spatialite/field.py new file mode 100644 index 0000000000..971f2f6bfa --- /dev/null +++ b/django/contrib/gis/db/backend/spatialite/field.py @@ -0,0 +1,81 @@ +from django.db.models.fields import Field # Django base Field class + +# Quotename & geographic quotename, respectively +from django.db import connection +qn = connection.ops.quote_name +from django.contrib.gis.db.backend.util import gqn +from django.contrib.gis.db.backend.spatialite.query import GEOM_FROM_TEXT, TRANSFORM + +class SpatiaLiteField(Field): + """ + The backend-specific geographic field for SpatiaLite. + """ + + def _add_geom(self, style, db_table): + """ + Constructs the addition of the geometry to the table using the + AddGeometryColumn(...) OpenGIS stored procedure. + + Takes the style object (provides syntax highlighting) and the + database table as parameters. + """ + sql = (style.SQL_KEYWORD('SELECT ') + + style.SQL_TABLE('AddGeometryColumn') + '(' + + style.SQL_TABLE(gqn(db_table)) + ', ' + + style.SQL_FIELD(gqn(self.column)) + ', ' + + style.SQL_FIELD(str(self.srid)) + ', ' + + style.SQL_COLTYPE(gqn(self.geom_type)) + ', ' + + style.SQL_KEYWORD(str(self.dim)) + ');') + + return sql + + def _geom_index(self, style, db_table): + "Creates a spatial index for this geometry field." + sql = (style.SQL_KEYWORD('SELECT ') + + style.SQL_TABLE('CreateSpatialIndex') + '(' + + style.SQL_TABLE(gqn(db_table)) + ', ' + + style.SQL_FIELD(gqn(self.column)) + ');') + return sql + + def post_create_sql(self, style, db_table): + """ + Returns SQL that will be executed after the model has been + created. Geometry columns must be added after creation with the + OpenGIS AddGeometryColumn() function. + """ + # Getting the AddGeometryColumn() SQL necessary to create a OpenGIS + # geometry field. + post_sql = self._add_geom(style, db_table) + + # If the user wants to index this data, then get the indexing SQL as well. + if self.spatial_index: + return (post_sql, self._geom_index(style, db_table)) + else: + return (post_sql,) + + def _post_delete_sql(self, style, db_table): + "Drops the geometry column." + sql = (style.SQL_KEYWORD('SELECT ') + + style.SQL_KEYWORD('DropGeometryColumn') + '(' + + style.SQL_TABLE(gqn(db_table)) + ', ' + + style.SQL_FIELD(gqn(self.column)) + ');') + return sql + + def db_type(self): + """ + SpatiaLite geometry columns are added by stored procedures; + should be None. + """ + return None + + def get_placeholder(self, value): + """ + Provides a proper substitution value for Geometries that are not in the + SRID of the field. Specifically, this routine will substitute in the + Transform() and GeomFromText() function call(s). + """ + if value is None or value.srid == self.srid: + return '%s(%%s,%s)' % (GEOM_FROM_TEXT, self.srid) + else: + # Adding Transform() to the SQL placeholder. + return '%s(%s(%%s,%s), %s)' % (TRANSFORM, GEOM_FROM_TEXT, value.srid, self.srid) diff --git a/django/contrib/gis/db/backend/spatialite/models.py b/django/contrib/gis/db/backend/spatialite/models.py new file mode 100644 index 0000000000..330a90c8be --- /dev/null +++ b/django/contrib/gis/db/backend/spatialite/models.py @@ -0,0 +1,53 @@ +""" + The GeometryColumns and SpatialRefSys models for the SpatiaLite backend. +""" +from django.db import models + +class GeometryColumns(models.Model): + """ + The 'geometry_columns' table from SpatiaLite. + """ + f_table_name = models.CharField(max_length=256) + f_geometry_column = models.CharField(max_length=256) + type = models.CharField(max_length=30) + coord_dimension = models.IntegerField() + srid = models.IntegerField(primary_key=True) + spatial_index_enabled = models.IntegerField() + + class Meta: + db_table = 'geometry_columns' + + @classmethod + def table_name_col(cls): + """ + Returns the name of the metadata column used to store the + the feature table name. + """ + return 'f_table_name' + + @classmethod + def geom_col_name(cls): + """ + Returns the name of the metadata column used to store the + the feature geometry column. + """ + return 'f_geometry_column' + + def __unicode__(self): + return "%s.%s - %dD %s field (SRID: %d)" % \ + (self.f_table_name, self.f_geometry_column, + self.coord_dimension, self.type, self.srid) + +class SpatialRefSys(models.Model): + """ + The 'spatial_ref_sys' table from SpatiaLite. + """ + srid = models.IntegerField(primary_key=True) + auth_name = models.CharField(max_length=256) + auth_srid = models.IntegerField() + ref_sys_name = models.CharField(max_length=256) + proj4text = models.CharField(max_length=2048) + + class Meta: + abstract = True + db_table = 'spatial_ref_sys' diff --git a/django/contrib/gis/db/backend/spatialite/query.py b/django/contrib/gis/db/backend/spatialite/query.py new file mode 100644 index 0000000000..dea6420b70 --- /dev/null +++ b/django/contrib/gis/db/backend/spatialite/query.py @@ -0,0 +1,159 @@ +""" + This module contains the spatial lookup types, and the get_geo_where_clause() + routine for SpatiaLite. +""" +import re +from decimal import Decimal +from django.db import connection +from django.contrib.gis.measure import Distance +from django.contrib.gis.db.backend.util import SpatialOperation, SpatialFunction +qn = connection.ops.quote_name + +GEOM_SELECT = 'AsText(%s)' + +# Dummy func, in case we need it later: +def get_func(str): + return str + +# Functions used by the GeoManager & GeoQuerySet +AREA = get_func('Area') +CENTROID = get_func('Centroid') +CONTAINED = get_func('MbrWithin') +DIFFERENCE = get_func('Difference') +DISTANCE = get_func('Distance') +ENVELOPE = get_func('Envelope') +GEOM_FROM_TEXT = get_func('GeomFromText') +GEOM_FROM_WKB = get_func('GeomFromWKB') +INTERSECTION = get_func('Intersection') +LENGTH = get_func('GLength') # OpenGis defines Length, but this conflicts with an SQLite reserved keyword +NUM_GEOM = get_func('NumGeometries') +NUM_POINTS = get_func('NumPoints') +POINT_ON_SURFACE = get_func('PointOnSurface') +SCALE = get_func('ScaleCoords') +SYM_DIFFERENCE = get_func('SymDifference') +TRANSFORM = get_func('Transform') +TRANSLATE = get_func('ShiftCoords') +UNION = 'GUnion'# OpenGis defines Union, but this conflicts with an SQLite reserved keyword +UNIONAGG = 'GUnion' + +#### Classes used in constructing SpatiaLite spatial SQL #### +class SpatiaLiteOperator(SpatialOperation): + "For SpatiaLite operators (e.g. `&&`, `~`)." + def __init__(self, operator): + super(SpatiaLiteOperator, self).__init__(operator=operator, beg_subst='%s %s %%s') + +class SpatiaLiteFunction(SpatialFunction): + "For SpatiaLite function calls." + def __init__(self, function, **kwargs): + super(SpatiaLiteFunction, self).__init__(get_func(function), **kwargs) + +class SpatiaLiteFunctionParam(SpatiaLiteFunction): + "For SpatiaLite functions that take another parameter." + def __init__(self, func): + super(SpatiaLiteFunctionParam, self).__init__(func, end_subst=', %%s)') + +class SpatiaLiteDistance(SpatiaLiteFunction): + "For SpatiaLite distance operations." + dist_func = 'Distance' + def __init__(self, operator): + super(SpatiaLiteDistance, self).__init__(self.dist_func, end_subst=') %s %s', + operator=operator, result='%%s') + +class SpatiaLiteRelate(SpatiaLiteFunctionParam): + "For SpatiaLite Relate(, ) calls." + pattern_regex = re.compile(r'^[012TF\*]{9}$') + def __init__(self, pattern): + if not self.pattern_regex.match(pattern): + raise ValueError('Invalid intersection matrix pattern "%s".' % pattern) + super(SpatiaLiteRelate, self).__init__('Relate') + + +SPATIALITE_GEOMETRY_FUNCTIONS = { + 'equals' : SpatiaLiteFunction('Equals'), + 'disjoint' : SpatiaLiteFunction('Disjoint'), + 'touches' : SpatiaLiteFunction('Touches'), + 'crosses' : SpatiaLiteFunction('Crosses'), + 'within' : SpatiaLiteFunction('Within'), + 'overlaps' : SpatiaLiteFunction('Overlaps'), + 'contains' : SpatiaLiteFunction('Contains'), + 'intersects' : SpatiaLiteFunction('Intersects'), + 'relate' : (SpatiaLiteRelate, basestring), + # Retruns true if B's bounding box completely contains A's bounding box. + 'contained' : SpatiaLiteFunction('MbrWithin'), + # Returns true if A's bounding box completely contains B's bounding box. + 'bbcontains' : SpatiaLiteFunction('MbrContains'), + # Returns true if A's bounding box overlaps B's bounding box. + 'bboverlaps' : SpatiaLiteFunction('MbrOverlaps'), + # These are implemented here as synonyms for Equals + 'same_as' : SpatiaLiteFunction('Equals'), + 'exact' : SpatiaLiteFunction('Equals'), + } + +# Valid distance types and substitutions +dtypes = (Decimal, Distance, float, int, long) +def get_dist_ops(operator): + "Returns operations for regular distances; spherical distances are not currently supported." + return (SpatiaLiteDistance(operator),) +DISTANCE_FUNCTIONS = { + 'distance_gt' : (get_dist_ops('>'), dtypes), + 'distance_gte' : (get_dist_ops('>='), dtypes), + 'distance_lt' : (get_dist_ops('<'), dtypes), + 'distance_lte' : (get_dist_ops('<='), dtypes), + } + +# Distance functions are a part of SpatiaLite geometry functions. +SPATIALITE_GEOMETRY_FUNCTIONS.update(DISTANCE_FUNCTIONS) + +# Any other lookup types that do not require a mapping. +MISC_TERMS = ['isnull'] + +# These are the SpatiaLite-customized QUERY_TERMS -- a list of the lookup types +# allowed for geographic queries. +SPATIALITE_TERMS = SPATIALITE_GEOMETRY_FUNCTIONS.keys() # Getting the Geometry Functions +SPATIALITE_TERMS += MISC_TERMS # Adding any other miscellaneous terms (e.g., 'isnull') +SPATIALITE_TERMS = dict((term, None) for term in SPATIALITE_TERMS) # Making a dictionary for fast lookups + +#### The `get_geo_where_clause` function for SpatiaLite. #### +def get_geo_where_clause(table_alias, name, lookup_type, geo_annot): + "Returns the SQL WHERE clause for use in SpatiaLite SQL construction." + # Getting the quoted field as `geo_col`. + geo_col = '%s.%s' % (qn(table_alias), qn(name)) + if lookup_type in SPATIALITE_GEOMETRY_FUNCTIONS: + # See if a SpatiaLite geometry function matches the lookup type. + tmp = SPATIALITE_GEOMETRY_FUNCTIONS[lookup_type] + + # Lookup types that are tuples take tuple arguments, e.g., 'relate' and + # distance lookups. + if isinstance(tmp, tuple): + # First element of tuple is the SpatiaLiteOperation instance, and the + # second element is either the type or a tuple of acceptable types + # that may passed in as further parameters for the lookup type. + op, arg_type = tmp + + # Ensuring that a tuple _value_ was passed in from the user + if not isinstance(geo_annot.value, (tuple, list)): + raise TypeError('Tuple required for `%s` lookup type.' % lookup_type) + + # Number of valid tuple parameters depends on the lookup type. + if len(geo_annot.value) != 2: + raise ValueError('Incorrect number of parameters given for `%s` lookup type.' % lookup_type) + + # Ensuring the argument type matches what we expect. + if not isinstance(geo_annot.value[1], arg_type): + raise TypeError('Argument type should be %s, got %s instead.' % (arg_type, type(geo_annot.value[1]))) + + # For lookup type `relate`, the op instance is not yet created (has + # to be instantiated here to check the pattern parameter). + if lookup_type == 'relate': + op = op(geo_annot.value[1]) + elif lookup_type in DISTANCE_FUNCTIONS: + op = op[0] + else: + op = tmp + # Calling the `as_sql` function on the operation instance. + return op.as_sql(geo_col) + elif lookup_type == 'isnull': + # Handling 'isnull' lookup type + return "%s IS %sNULL" % (geo_col, (not geo_annot.value and 'NOT ' or '')) + + raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type)) diff --git a/django/contrib/gis/tests/test_spatialrefsys.py b/django/contrib/gis/tests/test_spatialrefsys.py index 0b133a11f2..4b1cc093eb 100644 --- a/django/contrib/gis/tests/test_spatialrefsys.py +++ b/django/contrib/gis/tests/test_spatialrefsys.py @@ -1,5 +1,5 @@ import unittest -from django.contrib.gis.tests.utils import mysql, no_mysql, oracle, postgis +from django.contrib.gis.tests.utils import mysql, no_mysql, oracle, postgis, spatialite if not mysql: from django.contrib.gis.models import SpatialRefSys @@ -9,7 +9,7 @@ test_srs = ({'srid' : 4326, 'srtext' : 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]', 'proj4' : '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs ', 'spheroid' : 'WGS 84', 'name' : 'WGS 84', - 'geographic' : True, 'projected' : False, + 'geographic' : True, 'projected' : False, 'spatialite' : True, 'ellipsoid' : (6378137.0, 6356752.3, 298.257223563), # From proj's "cs2cs -le" and Wikipedia (semi-minor only) 'eprec' : (1, 1, 9), }, @@ -19,7 +19,7 @@ test_srs = ({'srid' : 4326, 'srtext' : 'PROJCS["NAD83 / Texas South Central",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",30.28333333333333],PARAMETER["standard_parallel_2",28.38333333333333],PARAMETER["latitude_of_origin",27.83333333333333],PARAMETER["central_meridian",-99],PARAMETER["false_easting",600000],PARAMETER["false_northing",4000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AUTHORITY["EPSG","32140"]]', 'proj4' : '+proj=lcc +lat_1=30.28333333333333 +lat_2=28.38333333333333 +lat_0=27.83333333333333 +lon_0=-99 +x_0=600000 +y_0=4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs ', 'spheroid' : 'GRS 1980', 'name' : 'NAD83 / Texas South Central', - 'geographic' : False, 'projected' : True, + 'geographic' : False, 'projected' : True, 'spatialite' : False, 'ellipsoid' : (6378137.0, 6356752.31414, 298.257222101), # From proj's "cs2cs -le" and Wikipedia (semi-minor only) 'eprec' : (1, 5, 10), }, @@ -56,13 +56,19 @@ class SpatialRefSysTest(unittest.TestCase): self.assertEqual(True, sr.spheroid.startswith(sd['spheroid'])) self.assertEqual(sd['geographic'], sr.geographic) self.assertEqual(sd['projected'], sr.projected) - self.assertEqual(True, sr.name.startswith(sd['name'])) + + if not (spatialite and not sd['spatialite']): + # Can't get 'NAD83 / Texas South Central' from PROJ.4 string + # on SpatiaLite + self.assertEqual(True, sr.name.startswith(sd['name'])) # Testing the SpatialReference object directly. - if postgis: + if postgis or spatialite: srs = sr.srs self.assertEqual(sd['proj4'], srs.proj4) - self.assertEqual(sd['srtext'], srs.wkt) + # No `srtext` field in the `spatial_ref_sys` table in SpatiaLite + if not spatialite: + self.assertEqual(sd['srtext'], srs.wkt) @no_mysql def test03_ellipsoid(self):