Fixed #9686 -- SpatiaLite is now a supported spatial database backend. Thanks to Matthew Hancher for initial patch and hard work in implementing this feature.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@10222 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/gis/db/backend/spatialite/__init__.py

@@ -0,0 +1,59 @@
+__all__ = ['create_test_spatial_db', 'get_geo_where_clause', 'SpatialBackend']
+
+from ctypes.util import find_library
+from django.conf import settings
+from django.db.backends.signals import connection_created
+
+from django.contrib.gis.db.backend.base import BaseSpatialBackend
+from django.contrib.gis.db.backend.spatialite.adaptor import SpatiaLiteAdaptor
+from django.contrib.gis.db.backend.spatialite.creation import create_test_spatial_db
+from django.contrib.gis.db.backend.spatialite.field import SpatiaLiteField
+from django.contrib.gis.db.backend.spatialite.models import GeometryColumns, SpatialRefSys
+from django.contrib.gis.db.backend.spatialite.query import *
+
+# Here we are figuring out the path to the SpatiLite library (`libspatialite`).
+# If it's not in the system PATH, it may be set manually in the settings via
+# the `SPATIALITE_LIBRARY_PATH` setting.
+spatialite_lib = getattr(settings, 'SPATIALITE_LIBRARY_PATH', find_library('spatialite'))
+if spatialite_lib:
+    def initialize_spatialite(sender=None, **kwargs):
+        """
+        This function initializes the pysqlite2 connection to enable the
+        loading of extensions, and to load up the SpatiaLite library
+        extension.
+        """
+        from django.db import connection
+        connection.connection.enable_load_extension(True)
+        connection.cursor().execute("SELECT load_extension(%s)", (spatialite_lib,))
+    connection_created.connect(initialize_spatialite)
+else:
+    # No SpatiaLite library found.
+    raise Exception('Unable to locate SpatiaLite, needed to use GeoDjango with sqlite3.')
+
+SpatialBackend = BaseSpatialBackend(name='spatialite', spatialite=True,
+                                    area=AREA,
+                                    centroid=CENTROID,
+                                    contained=CONTAINED,
+                                    difference=DIFFERENCE,
+                                    distance=DISTANCE,
+                                    distance_functions=DISTANCE_FUNCTIONS,
+                                    envelope=ENVELOPE,
+                                    from_text=GEOM_FROM_TEXT,
+                                    gis_terms=SPATIALITE_TERMS,
+                                    intersection=INTERSECTION,
+                                    length=LENGTH,
+                                    num_geom=NUM_GEOM,
+                                    num_points=NUM_POINTS,
+                                    point_on_surface=POINT_ON_SURFACE,
+                                    scale=SCALE,
+                                    select=GEOM_SELECT,
+                                    sym_difference=SYM_DIFFERENCE,
+                                    transform=TRANSFORM,
+                                    translate=TRANSLATE,
+                                    union=UNION,
+                                    unionagg=UNIONAGG,
+                                    Adaptor=SpatiaLiteAdaptor,
+                                    Field=SpatiaLiteField,
+                                    GeometryColumns=GeometryColumns,
+                                    SpatialRefSys=SpatialRefSys,
+                                    )

django/contrib/gis/db/backend/spatialite/adaptor.py

@@ -0,0 +1,8 @@
+from django.db.backends.sqlite3.base import Database
+from django.contrib.gis.db.backend.adaptor import WKTAdaptor
+
+class SpatiaLiteAdaptor(WKTAdaptor):
+    "SQLite adaptor for geometry objects."
+    def __conform__(self, protocol):
+        if protocol is Database.PrepareProtocol:
+            return str(self)

django/contrib/gis/db/backend/spatialite/creation.py

@@ -0,0 +1,61 @@
+import os
+from django.conf import settings
+from django.core.management import call_command
+from django.db import connection
+
+def spatialite_init_file():
+    # SPATIALITE_SQL may be placed in settings to tell
+    # GeoDjango to use a specific user-supplied file.
+    return getattr(settings, 'SPATIALITE_SQL', 'init_spatialite-2.2.sql')
+
+def create_test_spatial_db(verbosity=1, autoclobber=False, interactive=False):
+    "Creates a spatial database based on the settings."
+
+    # Making sure we're using PostgreSQL and psycopg2
+    if settings.DATABASE_ENGINE != 'sqlite3':
+        raise Exception('SpatiaLite database creation only supported on sqlite3 platform.')
+
+    # Getting the test database name using the the SQLite backend's
+    # `_create_test_db`.  Unless `TEST_DATABASE_NAME` is defined,
+    # it returns ":memory:".
+    db_name = connection.creation._create_test_db(verbosity, autoclobber)
+
+    # Closing out the current connection to the database set in
+    # originally in the settings.  This makes it so `initialize_spatialite`
+    # function will be run on the connection for the _test_ database instead.
+    connection.close()
+
+    # Point to the new database
+    settings.DATABASE_NAME = db_name
+    connection.settings_dict["DATABASE_NAME"] = db_name
+    can_rollback = connection.creation._rollback_works()
+    settings.DATABASE_SUPPORTS_TRANSACTIONS = can_rollback
+    connection.settings_dict["DATABASE_SUPPORTS_TRANSACTIONS"] = can_rollback
+
+    # Finally, loading up the SpatiaLite SQL file.
+    load_spatialite_sql(db_name, verbosity=verbosity)
+
+    if verbosity >= 1:
+        print 'Creation of spatial database %s successful.' % db_name
+
+    # Syncing the database
+    call_command('syncdb', verbosity=verbosity, interactive=interactive)
+
+def load_spatialite_sql(db_name, verbosity=1):
+    """
+    This routine loads up the SpatiaLite SQL file.
+    """
+    # Getting the location of the SpatiaLite SQL file, and confirming
+    # it exists.
+    spatialite_sql = spatialite_init_file()
+    if not os.path.isfile(spatialite_sql):
+        raise Exception('Could not find the SpatiaLite initialization SQL file: %s' % spatialite_sql)
+
+    # Opening up the SpatiaLite SQL initialization file and executing
+    # as a script.
+    sql_fh = open(spatialite_sql, 'r')
+    try:
+        cur = connection.cursor()
+        cur.executescript(sql_fh.read())
+    finally:
+        sql_fh.close()

django/contrib/gis/db/backend/spatialite/field.py

@@ -0,0 +1,81 @@
+from django.db.models.fields import Field # Django base Field class
+
+# Quotename & geographic quotename, respectively
+from django.db import connection
+qn = connection.ops.quote_name
+from django.contrib.gis.db.backend.util import gqn
+from django.contrib.gis.db.backend.spatialite.query import GEOM_FROM_TEXT, TRANSFORM
+
+class SpatiaLiteField(Field):
+    """
+    The backend-specific geographic field for SpatiaLite.
+    """
+
+    def _add_geom(self, style, db_table):
+        """
+        Constructs the addition of the geometry to the table using the
+        AddGeometryColumn(...) OpenGIS stored procedure.
+
+        Takes the style object (provides syntax highlighting) and the
+        database table as parameters.
+        """
+        sql = (style.SQL_KEYWORD('SELECT ') +
+               style.SQL_TABLE('AddGeometryColumn') + '(' +
+               style.SQL_TABLE(gqn(db_table)) + ', ' +
+               style.SQL_FIELD(gqn(self.column)) + ', ' +
+               style.SQL_FIELD(str(self.srid)) + ', ' +
+               style.SQL_COLTYPE(gqn(self.geom_type)) + ', ' +
+               style.SQL_KEYWORD(str(self.dim)) + ');')
+
+        return sql
+
+    def _geom_index(self, style, db_table):
+        "Creates a spatial index for this geometry field."
+        sql = (style.SQL_KEYWORD('SELECT ') +
+              style.SQL_TABLE('CreateSpatialIndex') + '(' +
+              style.SQL_TABLE(gqn(db_table)) + ', ' +
+              style.SQL_FIELD(gqn(self.column)) + ');')
+        return sql
+
+    def post_create_sql(self, style, db_table):
+        """
+        Returns SQL that will be executed after the model has been
+        created. Geometry columns must be added after creation with the
+        OpenGIS AddGeometryColumn() function.
+        """
+        # Getting the AddGeometryColumn() SQL necessary to create a OpenGIS
+        # geometry field.
+        post_sql = self._add_geom(style, db_table)
+
+        # If the user wants to index this data, then get the indexing SQL as well.
+        if self.spatial_index:
+            return (post_sql, self._geom_index(style, db_table))
+        else:
+            return (post_sql,)
+
+    def _post_delete_sql(self, style, db_table):
+        "Drops the geometry column."
+        sql = (style.SQL_KEYWORD('SELECT ') +
+               style.SQL_KEYWORD('DropGeometryColumn') + '(' +
+               style.SQL_TABLE(gqn(db_table)) + ', ' +
+               style.SQL_FIELD(gqn(self.column)) +  ');')
+        return sql
+
+    def db_type(self):
+        """
+        SpatiaLite geometry columns are added by stored procedures;
+        should be None.
+        """
+        return None
+
+    def get_placeholder(self, value):
+        """
+        Provides a proper substitution value for Geometries that are not in the
+        SRID of the field.  Specifically, this routine will substitute in the
+        Transform() and GeomFromText() function call(s).
+        """
+        if value is None or value.srid == self.srid:
+            return '%s(%%s,%s)' % (GEOM_FROM_TEXT, self.srid)
+        else:
+            # Adding Transform() to the SQL placeholder.
+            return '%s(%s(%%s,%s), %s)' % (TRANSFORM, GEOM_FROM_TEXT, value.srid, self.srid)

django/contrib/gis/db/backend/spatialite/models.py

@@ -0,0 +1,53 @@
+"""
+ The GeometryColumns and SpatialRefSys models for the SpatiaLite backend.
+"""
+from django.db import models
+
+class GeometryColumns(models.Model):
+    """
+    The 'geometry_columns' table from SpatiaLite.
+    """
+    f_table_name = models.CharField(max_length=256)
+    f_geometry_column = models.CharField(max_length=256)
+    type = models.CharField(max_length=30)
+    coord_dimension = models.IntegerField()
+    srid = models.IntegerField(primary_key=True)
+    spatial_index_enabled = models.IntegerField()
+
+    class Meta:
+        db_table = 'geometry_columns'
+
+    @classmethod
+    def table_name_col(cls):
+        """
+        Returns the name of the metadata column used to store the
+        the feature table name.
+        """
+        return 'f_table_name'
+
+    @classmethod
+    def geom_col_name(cls):
+        """
+        Returns the name of the metadata column used to store the
+        the feature geometry column.
+        """
+        return 'f_geometry_column'
+
+    def __unicode__(self):
+        return "%s.%s - %dD %s field (SRID: %d)" % \
+               (self.f_table_name, self.f_geometry_column,
+                self.coord_dimension, self.type, self.srid)
+
+class SpatialRefSys(models.Model):
+    """
+    The 'spatial_ref_sys' table from SpatiaLite.
+    """
+    srid = models.IntegerField(primary_key=True)
+    auth_name = models.CharField(max_length=256)
+    auth_srid = models.IntegerField()
+    ref_sys_name = models.CharField(max_length=256)
+    proj4text = models.CharField(max_length=2048)
+
+    class Meta:
+        abstract = True
+        db_table = 'spatial_ref_sys'

django/contrib/gis/db/backend/spatialite/query.py

@@ -0,0 +1,159 @@
+"""
+ This module contains the spatial lookup types, and the get_geo_where_clause()
+ routine for SpatiaLite.
+"""
+import re
+from decimal import Decimal
+from django.db import connection
+from django.contrib.gis.measure import Distance
+from django.contrib.gis.db.backend.util import SpatialOperation, SpatialFunction
+qn = connection.ops.quote_name
+
+GEOM_SELECT = 'AsText(%s)'
+
+# Dummy func, in case we need it later:
+def get_func(str):
+    return str
+
+# Functions used by the GeoManager & GeoQuerySet
+AREA = get_func('Area')
+CENTROID = get_func('Centroid')
+CONTAINED = get_func('MbrWithin')
+DIFFERENCE = get_func('Difference')
+DISTANCE = get_func('Distance')
+ENVELOPE = get_func('Envelope')
+GEOM_FROM_TEXT = get_func('GeomFromText')
+GEOM_FROM_WKB = get_func('GeomFromWKB')
+INTERSECTION = get_func('Intersection')
+LENGTH = get_func('GLength') # OpenGis defines Length, but this conflicts with an SQLite reserved keyword
+NUM_GEOM = get_func('NumGeometries')
+NUM_POINTS = get_func('NumPoints')
+POINT_ON_SURFACE = get_func('PointOnSurface')
+SCALE = get_func('ScaleCoords')
+SYM_DIFFERENCE = get_func('SymDifference')
+TRANSFORM = get_func('Transform')
+TRANSLATE = get_func('ShiftCoords')
+UNION = 'GUnion'# OpenGis defines Union, but this conflicts with an SQLite reserved keyword
+UNIONAGG = 'GUnion'
+
+#### Classes used in constructing SpatiaLite spatial SQL ####
+class SpatiaLiteOperator(SpatialOperation):
+    "For SpatiaLite operators (e.g. `&&`, `~`)."
+    def __init__(self, operator):
+        super(SpatiaLiteOperator, self).__init__(operator=operator, beg_subst='%s %s %%s')
+
+class SpatiaLiteFunction(SpatialFunction):
+    "For SpatiaLite function calls."
+    def __init__(self, function, **kwargs):
+        super(SpatiaLiteFunction, self).__init__(get_func(function), **kwargs)
+
+class SpatiaLiteFunctionParam(SpatiaLiteFunction):
+    "For SpatiaLite functions that take another parameter."
+    def __init__(self, func):
+        super(SpatiaLiteFunctionParam, self).__init__(func, end_subst=', %%s)')
+
+class SpatiaLiteDistance(SpatiaLiteFunction):
+    "For SpatiaLite distance operations."
+    dist_func = 'Distance'
+    def __init__(self, operator):
+        super(SpatiaLiteDistance, self).__init__(self.dist_func, end_subst=') %s %s', 
+                                              operator=operator, result='%%s')
+                                                    
+class SpatiaLiteRelate(SpatiaLiteFunctionParam):
+    "For SpatiaLite Relate(<geom>, <pattern>) calls."
+    pattern_regex = re.compile(r'^[012TF\*]{9}$')
+    def __init__(self, pattern):
+        if not self.pattern_regex.match(pattern):
+            raise ValueError('Invalid intersection matrix pattern "%s".' % pattern)
+        super(SpatiaLiteRelate, self).__init__('Relate')
+
+
+SPATIALITE_GEOMETRY_FUNCTIONS = {
+    'equals' : SpatiaLiteFunction('Equals'),
+    'disjoint' : SpatiaLiteFunction('Disjoint'),
+    'touches' : SpatiaLiteFunction('Touches'),
+    'crosses' : SpatiaLiteFunction('Crosses'),
+    'within' : SpatiaLiteFunction('Within'),
+    'overlaps' : SpatiaLiteFunction('Overlaps'),
+    'contains' : SpatiaLiteFunction('Contains'),
+    'intersects' : SpatiaLiteFunction('Intersects'),
+    'relate' : (SpatiaLiteRelate, basestring),
+    # Retruns true if B's bounding box completely contains A's bounding box.
+    'contained' : SpatiaLiteFunction('MbrWithin'),
+    # Returns true if A's bounding box completely contains B's bounding box.
+    'bbcontains' : SpatiaLiteFunction('MbrContains'),
+    # Returns true if A's bounding box overlaps B's bounding box.
+    'bboverlaps' : SpatiaLiteFunction('MbrOverlaps'),
+    # These are implemented here as synonyms for Equals
+    'same_as' : SpatiaLiteFunction('Equals'),
+    'exact' : SpatiaLiteFunction('Equals'),
+    }
+
+# Valid distance types and substitutions
+dtypes = (Decimal, Distance, float, int, long)
+def get_dist_ops(operator):
+    "Returns operations for regular distances; spherical distances are not currently supported."
+    return (SpatiaLiteDistance(operator),)
+DISTANCE_FUNCTIONS = {
+    'distance_gt' : (get_dist_ops('>'), dtypes),
+    'distance_gte' : (get_dist_ops('>='), dtypes),
+    'distance_lt' : (get_dist_ops('<'), dtypes),
+    'distance_lte' : (get_dist_ops('<='), dtypes),
+    }
+
+# Distance functions are a part of SpatiaLite geometry functions.
+SPATIALITE_GEOMETRY_FUNCTIONS.update(DISTANCE_FUNCTIONS)
+
+# Any other lookup types that do not require a mapping.
+MISC_TERMS = ['isnull']
+
+# These are the SpatiaLite-customized QUERY_TERMS -- a list of the lookup types
+# allowed for geographic queries.
+SPATIALITE_TERMS = SPATIALITE_GEOMETRY_FUNCTIONS.keys() # Getting the Geometry Functions
+SPATIALITE_TERMS += MISC_TERMS # Adding any other miscellaneous terms (e.g., 'isnull')
+SPATIALITE_TERMS = dict((term, None) for term in SPATIALITE_TERMS) # Making a dictionary for fast lookups
+
+#### The `get_geo_where_clause` function for SpatiaLite. ####
+def get_geo_where_clause(table_alias, name, lookup_type, geo_annot):
+    "Returns the SQL WHERE clause for use in SpatiaLite SQL construction."
+    # Getting the quoted field as `geo_col`.
+    geo_col = '%s.%s' % (qn(table_alias), qn(name))
+    if lookup_type in SPATIALITE_GEOMETRY_FUNCTIONS:
+        # See if a SpatiaLite geometry function matches the lookup type.
+        tmp = SPATIALITE_GEOMETRY_FUNCTIONS[lookup_type]
+
+        # Lookup types that are tuples take tuple arguments, e.g., 'relate' and 
+        # distance lookups.
+        if isinstance(tmp, tuple):
+            # First element of tuple is the SpatiaLiteOperation instance, and the
+            # second element is either the type or a tuple of acceptable types
+            # that may passed in as further parameters for the lookup type.
+            op, arg_type = tmp
+
+            # Ensuring that a tuple _value_ was passed in from the user
+            if not isinstance(geo_annot.value, (tuple, list)): 
+                raise TypeError('Tuple required for `%s` lookup type.' % lookup_type)
+           
+            # Number of valid tuple parameters depends on the lookup type.
+            if len(geo_annot.value) != 2:
+                raise ValueError('Incorrect number of parameters given for `%s` lookup type.' % lookup_type)
+            
+            # Ensuring the argument type matches what we expect.
+            if not isinstance(geo_annot.value[1], arg_type):
+                raise TypeError('Argument type should be %s, got %s instead.' % (arg_type, type(geo_annot.value[1])))
+
+            # For lookup type `relate`, the op instance is not yet created (has
+            # to be instantiated here to check the pattern parameter).
+            if lookup_type == 'relate': 
+                op = op(geo_annot.value[1])
+            elif lookup_type in DISTANCE_FUNCTIONS:
+                op = op[0]
+        else:
+            op = tmp
+        # Calling the `as_sql` function on the operation instance.
+        return op.as_sql(geo_col)
+    elif lookup_type == 'isnull':
+        # Handling 'isnull' lookup type
+        return "%s IS %sNULL" % (geo_col, (not geo_annot.value and 'NOT ' or ''))
+
+    raise TypeError("Got invalid lookup_type: %s" % repr(lookup_type))

django/contrib/gis/tests/test_spatialrefsys.py

@@ -1,5 +1,5 @@
 import unittest
-from django.contrib.gis.tests.utils import mysql, no_mysql, oracle, postgis
+from django.contrib.gis.tests.utils import mysql, no_mysql, oracle, postgis, spatialite
 if not mysql:
     from django.contrib.gis.models import SpatialRefSys
 
@@ -9,7 +9,7 @@ test_srs = ({'srid' : 4326,
              'srtext' : 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
              'proj4' : '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs ',
              'spheroid' : 'WGS 84', 'name' : 'WGS 84', 
-             'geographic' : True, 'projected' : False,
+             'geographic' : True, 'projected' : False, 'spatialite' : True,
              'ellipsoid' : (6378137.0, 6356752.3, 298.257223563), # From proj's "cs2cs -le" and Wikipedia (semi-minor only)
              'eprec' : (1, 1, 9),
              },
@@ -19,7 +19,7 @@ test_srs = ({'srid' : 4326,
              'srtext' : 'PROJCS["NAD83 / Texas South Central",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",30.28333333333333],PARAMETER["standard_parallel_2",28.38333333333333],PARAMETER["latitude_of_origin",27.83333333333333],PARAMETER["central_meridian",-99],PARAMETER["false_easting",600000],PARAMETER["false_northing",4000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AUTHORITY["EPSG","32140"]]',
              'proj4' : '+proj=lcc +lat_1=30.28333333333333 +lat_2=28.38333333333333 +lat_0=27.83333333333333 +lon_0=-99 +x_0=600000 +y_0=4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs ',
              'spheroid' : 'GRS 1980', 'name' : 'NAD83 / Texas South Central',
-             'geographic' : False, 'projected' : True,
+             'geographic' : False, 'projected' : True, 'spatialite' : False,
              'ellipsoid' : (6378137.0, 6356752.31414, 298.257222101), # From proj's "cs2cs -le" and Wikipedia (semi-minor only)
              'eprec' : (1, 5, 10),
              },
@@ -56,12 +56,18 @@ class SpatialRefSysTest(unittest.TestCase):
             self.assertEqual(True, sr.spheroid.startswith(sd['spheroid']))
             self.assertEqual(sd['geographic'], sr.geographic)
             self.assertEqual(sd['projected'], sr.projected)
+
+            if not (spatialite and not sd['spatialite']):
+                # Can't get 'NAD83 / Texas South Central' from PROJ.4 string
+                # on SpatiaLite
                 self.assertEqual(True, sr.name.startswith(sd['name']))
 
             # Testing the SpatialReference object directly.
-            if postgis:
+            if postgis or spatialite:
                 srs = sr.srs
                 self.assertEqual(sd['proj4'], srs.proj4)
+                # No `srtext` field in the `spatial_ref_sys` table in SpatiaLite
+                if not spatialite:
                     self.assertEqual(sd['srtext'], srs.wkt)
 
     @no_mysql