Fixed KML sitemaps, and added support for generating KMZ and GeoRSS sitemaps; sitemaps now support Google's Geo Sitemap format.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@8502 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/gis/shortcuts.py

@@ -1,11 +1,30 @@
+import cStringIO, zipfile
 from django.http import HttpResponse
 from django.template import loader
 
+def compress_kml(kml):
+    "Returns compressed KMZ from the given KML string."
+    kmz = cStringIO.StringIO()
+    zf = zipfile.ZipFile(kmz, 'a', zipfile.ZIP_DEFLATED, False)
+    zf.writestr('doc.kml', kml)
+    zf.close()
+    kmz.seek(0)
+    return kmz.read()
+
 def render_to_kml(*args, **kwargs):
-    "Renders the response using the MIME type for KML."
+    "Renders the response as KML (using the correct MIME type)."
     return HttpResponse(loader.render_to_string(*args, **kwargs),
                         mimetype='application/vnd.google-earth.kml+xml kml')
 
+def render_to_kmz(*args, **kwargs):
+    """
+    Compresses the KML content and returns as KMZ (using the correct 
+    MIME type).
+    """
+    return HttpResponse(compress_kml(loader.render_to_string(*args, **kwargs)),
+                        mimetype='application/vnd.google-earth.kmz')
+
+
 def render_to_text(*args, **kwargs):
     "Renders the response using the MIME type for plain text."
     return HttpResponse(loader.render_to_string(*args, **kwargs),

django/contrib/gis/sitemaps.py

@@ -1,55 +0,0 @@
-from django.core import urlresolvers
-from django.contrib.sitemaps import Sitemap
-from django.contrib.gis.db.models.fields import GeometryField
-from django.contrib.gis.shortcuts import render_to_kml
-from django.db.models import get_model, get_models
-from django.http import HttpResponse
-
-class KMLSitemap(Sitemap):
-    """
-    A minimal hook to produce KML sitemaps.
-    """
-    def __init__(self, locations=None):
-        if locations is None:
-            self.locations = _build_kml_sources()
-        else:
-            self.locations = locations
-
-    def items(self):
-        return self.locations
-
-    def location(self, obj):
-        return urlresolvers.reverse('django.contrib.gis.sitemaps.kml',
-                                    kwargs={'label':obj[0],
-                                            'field_name':obj[1]})
-
-def _build_kml_sources():
-    "Make a mapping of all available KML sources."
-    ret = []
-    for klass in get_models():
-        for field in klass._meta.fields:
-            if isinstance(field, GeometryField):
-                label = "%s.%s" % (klass._meta.app_label,
-                                   klass._meta.module_name)
-                
-                ret.append((label, field.name))
-    return ret
-
-
-class KMLNotFound(Exception):
-    pass
-
-def kml(request, label, field_name):
-    placemarks = []
-    klass = get_model(*label.split('.'))
-    if not klass:
-        raise KMLNotFound("You must supply a valid app.model label.  Got %s" % label)
-
-    #FIXME: GMaps apparently has a limit on size of displayed kml files
-    #  check if paginating w/ external refs (i.e. linked list) helps.
-    placemarks.extend(list(klass._default_manager.kml(field_name)[:100]))
-
-    #FIXME: other KML features?
-    return render_to_kml('gis/kml/placemarks.kml', {'places' : placemarks})
-
-    

django/contrib/gis/sitemaps/__init__.py

@@ -0,0 +1,4 @@
+# Geo-enabled Sitemap classes.
+from django.contrib.gis.sitemaps.georss import GeoRSSSitemap
+from django.contrib.gis.sitemaps.kml import KMLSitemap, KMZSitemap
+

django/contrib/gis/sitemaps/georss.py

@@ -0,0 +1,53 @@
+from django.core import urlresolvers
+from django.contrib.sitemaps import Sitemap
+
+class GeoRSSSitemap(Sitemap):
+    """
+    A minimal hook to produce sitemaps for GeoRSS feeds.
+    """
+    def __init__(self, feed_dict, slug_dict=None):
+        """
+        This sitemap object initializes on a feed dictionary (as would be passed
+        to `django.contrib.syndication.views.feed`) and a slug dictionary.  
+        If the slug dictionary is not defined, then it's assumed the keys provide
+        the URL parameter to the feed.  However, if you have a complex feed (e.g.,
+        you override `get_object`, then you'll need to provide a slug dictionary.
+        The slug dictionary should have the same keys as the feed dictionary, but 
+        each value in the slug dictionary should be a sequence of slugs that may 
+        be used for valid feeds.  For example, let's say we have a feed that 
+        returns objects for a specific ZIP code in our feed dictionary:
+
+            feed_dict = {'zipcode' : ZipFeed}
+
+        Then we would use a slug dictionary with a list of the zip code slugs
+        corresponding to feeds you want listed in the sitemap:
+
+            slug_dict = {'zipcode' : ['77002', '77054']}
+        """
+        # Setting up.
+        self.feed_dict = feed_dict
+        self.locations = []
+        if slug_dict is None: slug_dict = {}
+        # Getting the feed locations.
+        for section in feed_dict.keys():
+            if slug_dict.get(section, False):
+                for slug in slug_dict[section]:
+                    self.locations.append(('%s/%s' % (section, slug)))
+            else:
+                self.locations.append(section)
+ 
+    def get_urls(self, page=1):
+        """
+        This method is overrridden so the appropriate `geo_format` attribute
+        is placed on each URL element.
+        """
+        urls = Sitemap.get_urls(self)
+        for url in urls: url['geo_format'] = 'georss'
+        return urls
+
+    def items(self):
+        return self.locations
+
+    def location(self, obj):
+        return urlresolvers.reverse('django.contrib.syndication.views.feed',
+                                    args=(obj,), kwargs={'feed_dict' : self.feed_dict})

django/contrib/gis/sitemaps/kml.py

@@ -0,0 +1,63 @@
+from django.core import urlresolvers
+from django.contrib.sitemaps import Sitemap
+from django.contrib.gis.db.models.fields import GeometryField
+from django.db import models
+
+class KMLSitemap(Sitemap):
+    """
+    A minimal hook to produce KML sitemaps.
+    """
+    geo_format = 'kml'
+
+    def __init__(self, locations=None):
+        # If no locations specified, then we try to build for
+        # every model in installed applications.
+        self.locations = self._build_kml_sources(locations)
+        
+    def _build_kml_sources(self, sources):
+        """
+        Goes through the given sources and returns a 3-tuple of
+        the application label, module name, and field name of every
+        GeometryField encountered in the sources.
+
+        If no sources are provided, then all models.
+        """
+        kml_sources = []
+        if sources is None:
+            sources = models.get_models()
+        for source in sources:
+            if isinstance(source, models.base.ModelBase):
+                for field in source._meta.fields:
+                    if isinstance(field, GeometryField):
+                        kml_sources.append((source._meta.app_label,
+                                            source._meta.module_name,
+                                            field.name))
+            elif isinstance(source, (list, tuple)):
+                if len(source) != 3: 
+                    raise ValueError('Must specify a 3-tuple of (app_label, module_name, field_name).')
+                kml_sources.append(source)
+            else:
+                raise TypeError('KML Sources must be a model or a 3-tuple.')
+        return kml_sources
+
+    def get_urls(self, page=1):
+        """
+        This method is overrridden so the appropriate `geo_format` attribute
+        is placed on each URL element.
+        """
+        urls = Sitemap.get_urls(self, page=page)
+        for url in urls: url['geo_format'] = self.geo_format
+        return urls
+
+    def items(self):
+        return self.locations
+
+    def location(self, obj):
+        return urlresolvers.reverse('django.contrib.gis.sitemaps.views.%s' % self.geo_format,
+                                    kwargs={'label' : obj[0], 
+                                            'model' : obj[1],
+                                            'field_name': obj[2],
+                                            }
+                                    )
+class KMZSitemap(KMLSitemap):
+    geo_format = 'kmz'

django/contrib/gis/sitemaps/views.py

@@ -0,0 +1,102 @@
+from django.http import HttpResponse, Http404
+from django.template import loader
+from django.contrib.gis.db.backend import SpatialBackend
+from django.contrib.sites.models import Site
+from django.core import urlresolvers
+from django.core.paginator import EmptyPage, PageNotAnInteger
+from django.db.models import get_model
+from django.utils.encoding import smart_str
+
+from django.contrib.gis.shortcuts import render_to_kml, render_to_kmz
+
+class KMLNotFound(Exception):
+    pass
+
+def index(request, sitemaps):
+    """
+    This view generates a sitemap index that uses the proper view
+    for resolving geographic section sitemap URLs.
+    """
+    current_site = Site.objects.get_current()
+    sites = []
+    protocol = request.is_secure() and 'https' or 'http'
+    for section, site in sitemaps.items():
+        if callable(site):
+            pages = site().paginator.num_pages
+        else:
+            pages = site.paginator.num_pages
+        sitemap_url = urlresolvers.reverse('django.contrib.gis.sitemaps.views.sitemap', kwargs={'section': section})
+        sites.append('%s://%s%s' % (protocol, current_site.domain, sitemap_url))
+      
+        if pages > 1:
+            for page in range(2, pages+1):
+                sites.append('%s://%s%s?p=%s' % (protocol, current_site.domain, sitemap_url, page))
+    xml = loader.render_to_string('sitemap_index.xml', {'sitemaps': sites})
+    return HttpResponse(xml, mimetype='application/xml')
+
+def sitemap(request, sitemaps, section=None):
+    """
+    This view generates a sitemap with additional geographic
+    elements defined by Google.
+    """
+    maps, urls = [], []
+    if section is not None:
+        if section not in sitemaps:
+            raise Http404("No sitemap available for section: %r" % section)
+        maps.append(sitemaps[section])
+    else:
+        maps = sitemaps.values()
+
+    page = request.GET.get("p", 1)
+    for site in maps:
+        try:
+            if callable(site):
+                urls.extend(site().get_urls(page))
+            else:
+                urls.extend(site.get_urls(page))
+        except EmptyPage:
+            raise Http404("Page %s empty" % page)
+        except PageNotAnInteger:
+            raise Http404("No page '%s'" % page)
+    xml = smart_str(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls}))
+    return HttpResponse(xml, mimetype='application/xml')
+
+def kml(request, label, model, field_name=None, compress=False):
+    """
+    This view generates KML for the given app label, model, and field name.
+
+    The model's default manager must be GeoManager, and the field name
+    must be that of a geographic field.
+    """
+    placemarks = []
+    klass = get_model(label, model)
+    if not klass:
+        raise KMLNotFound("You must supply a valid app.model label.  Got %s.%s" % (label, model))
+
+    if SpatialBackend.postgis:
+        # PostGIS will take care of transformation.
+        placemarks = klass._default_manager.kml(field_name=field_name)
+    else:
+        # There's no KML method on Oracle or MySQL, so we use the `kml`
+        # attribute of the lazy geometry instead.
+        placemarks = []
+        if SpatialBackend.oracle:
+            qs = klass._default_manager.transform(4326, field_name=field_name)
+        else:
+            qs = klass._default_manager.all()
+        for mod in qs:
+            setattr(mod, 'kml', getattr(mod, field_name).kml)
+            placemarks.append(mod)
+
+    # Getting the render function and rendering to the correct.
+    if compress:
+        render = render_to_kmz
+    else:
+        render = render_to_kml
+    return render('gis/kml/placemarks.kml', {'places' : placemarks})
+
+def kmz(request, label, model, field_name=None):
+    """
+    This view returns KMZ for the given app label, model, and field name.
+    """
+    return kml(request, label, model, field_name, True)

django/contrib/gis/templates/gis/sitemaps/geo_sitemap.xml

@@ -0,0 +1,17 @@
+{% autoescape off %}<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:geo="http://www.google.com/geo/schemas/sitemap/1.0">
+{% spaceless %}
+{% for url in urlset %}
+  <url>
+    <loc>{{ url.location|escape }}</loc>
+    {% if url.lastmod %}<lastmod>{{ url.lastmod|date:"Y-m-d" }}</lastmod>{% endif %}
+    {% if url.changefreq %}<changefreq>{{ url.changefreq }}</changefreq>{% endif %}
+    {% if url.priority %}<priority>{{ url.priority }}</priority>{% endif %}
+    {% if url.geo_format %}<geo:geo>
+      <geo:format>{{ url.geo_format }}</geo:format>
+    </geo:geo>{% endif %}
+   </url>
+{% endfor %}
+{% endspaceless %}
+</urlset>
+{% endautoescape %}

django/contrib/gis/tests/__init__.py

@@ -74,6 +74,7 @@ def run_gis_tests(test_labels, **kwargs):
     new_installed =  ['django.contrib.contenttypes',
                       'django.contrib.auth',
                       'django.contrib.sites',
+                      'django.contrib.sitemaps',
                       'django.contrib.flatpages',
                       'django.contrib.gis',
                       'django.contrib.redirects',

django/contrib/gis/tests/geoapp/feeds.py

@@ -50,3 +50,14 @@ class TestW3CGeo3(TestGeoRSS1):
     def item_geometry(self, item):
         from django.contrib.gis.geos import Polygon
         return Polygon(((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)))
+
+# The feed dictionary to use for URLs.
+feed_dict = {
+    'rss1' : TestGeoRSS1,
+    'rss2' : TestGeoRSS2,
+    'atom1' : TestGeoAtom1,
+    'atom2' : TestGeoAtom2,
+    'w3cgeo1' : TestW3CGeo1,
+    'w3cgeo2' : TestW3CGeo2,
+    'w3cgeo3' : TestW3CGeo3,
+}

django/contrib/gis/tests/geoapp/sitemaps.py

@@ -0,0 +1,8 @@
+from django.contrib.gis.sitemaps import GeoRSSSitemap, KMLSitemap, KMZSitemap
+from models import City, Country
+from feeds import feed_dict
+
+sitemaps = {'kml' : KMLSitemap([City, Country]),
+            'kmz' : KMZSitemap([City, Country]),
+            'georss' : GeoRSSSitemap(feed_dict),
+            }

django/contrib/gis/tests/geoapp/test_sitemaps.py

@@ -0,0 +1,83 @@
+import unittest, zipfile, cStringIO
+from xml.dom import minidom
+
+from django.test import Client
+from models import City, Country
+
+class GeoSitemapTest(unittest.TestCase):
+    client = Client()
+
+    def assertChildNodes(self, elem, expected):
+        "Taken from regressiontests/syndication/tests.py."
+        actual = set([n.nodeName for n in elem.childNodes])
+        expected = set(expected)
+        self.assertEqual(actual, expected)
+
+    def test_geositemap_index(self):
+        "Tests geographic sitemap index."
+        # Getting the geo index.
+        doc = minidom.parseString(self.client.get('/geoapp/sitemap.xml').content)
+        index = doc.firstChild
+        self.assertEqual(index.getAttribute(u'xmlns'), u'http://www.sitemaps.org/schemas/sitemap/0.9')
+        self.assertEqual(3, len(index.getElementsByTagName('sitemap')))
+
+    def test_geositemap_kml(self):
+        "Tests KML/KMZ geographic sitemaps."
+        for kml_type in ('kml', 'kmz'):
+            doc = minidom.parseString(self.client.get('/geoapp/sitemaps/%s.xml' % kml_type).content)
+
+            # Ensuring the right sitemaps namespaces are present.
+            urlset = doc.firstChild
+            self.assertEqual(urlset.getAttribute(u'xmlns'), u'http://www.sitemaps.org/schemas/sitemap/0.9')
+            self.assertEqual(urlset.getAttribute(u'xmlns:geo'), u'http://www.google.com/geo/schemas/sitemap/1.0')
+        
+            urls = urlset.getElementsByTagName('url')
+            self.assertEqual(2, len(urls)) # Should only be 2 sitemaps.
+            for url in urls:
+                self.assertChildNodes(url, ['loc', 'geo:geo'])
+                # Making sure the 'geo:format' element was properly set.
+                geo_elem = url.getElementsByTagName('geo:geo')[0]
+                geo_format = geo_elem.getElementsByTagName('geo:format')[0]
+                self.assertEqual(kml_type, geo_format.childNodes[0].data)
+
+                # Getting the relative URL since we don't have a real site.
+                kml_url = url.getElementsByTagName('loc')[0].childNodes[0].data.split('http://example.com')[1]
+                
+                if kml_type == 'kml':
+                    kml_doc = minidom.parseString(self.client.get(kml_url).content)
+                elif kml_type == 'kmz':
+                    # Have to decompress KMZ before parsing.
+                    buf = cStringIO.StringIO(self.client.get(kml_url).content)
+                    zf = zipfile.ZipFile(buf)
+                    self.assertEqual(1, len(zf.filelist))
+                    self.assertEqual('doc.kml', zf.filelist[0].filename)
+                    kml_doc = minidom.parseString(zf.read('doc.kml'))
+                
+                # Ensuring the correct number of placemarks are in the KML doc.
+                if 'city' in kml_url:
+                    model = City
+                elif 'country' in kml_url:
+                    model = Country
+                self.assertEqual(model.objects.count(), len(kml_doc.getElementsByTagName('Placemark')))
+
+    def test_geositemap_georss(self):
+        "Tests GeoRSS geographic sitemaps."
+        from feeds import feed_dict
+
+        doc = minidom.parseString(self.client.get('/geoapp/sitemaps/georss.xml').content)
+   
+        # Ensuring the right sitemaps namespaces are present.
+        urlset = doc.firstChild
+        self.assertEqual(urlset.getAttribute(u'xmlns'), u'http://www.sitemaps.org/schemas/sitemap/0.9')
+        self.assertEqual(urlset.getAttribute(u'xmlns:geo'), u'http://www.google.com/geo/schemas/sitemap/1.0')
+
+        # Making sure the correct number of feed URLs were included.
+        urls = urlset.getElementsByTagName('url')
+        self.assertEqual(len(feed_dict), len(urls))
+
+        for url in urls:
+            self.assertChildNodes(url, ['loc', 'geo:geo'])
+            # Making sure the 'geo:format' element was properly set to 'georss'.
+            geo_elem = url.getElementsByTagName('geo:geo')[0]
+            geo_format = geo_elem.getElementsByTagName('geo:format')[0]
+            self.assertEqual('georss', geo_format.childNodes[0].data)

django/contrib/gis/tests/geoapp/tests.py

@@ -559,8 +559,10 @@ class GeoModelTest(unittest.TestCase):
             self.assertEqual(c.mpoly.union(geom), c.union)
 
 from test_feeds import GeoFeedTest
+from test_sitemaps import GeoSitemapTest
 def suite():
     s = unittest.TestSuite()
     s.addTest(unittest.makeSuite(GeoModelTest))
     s.addTest(unittest.makeSuite(GeoFeedTest))
+    s.addTest(unittest.makeSuite(GeoSitemapTest))
     return s

django/contrib/gis/tests/geoapp/tests_mysql.py

@@ -174,8 +174,10 @@ class GeoModelTest(unittest.TestCase):
         self.assertRaises(ImproperlyConfigured, Country.objects.all().gml, field_name='mpoly')
 
 from test_feeds import GeoFeedTest
+from test_sitemaps import GeoSitemapTest
 def suite():
     s = unittest.TestSuite()
     s.addTest(unittest.makeSuite(GeoModelTest))
     s.addTest(unittest.makeSuite(GeoFeedTest))
+    s.addTest(unittest.makeSuite(GeoSitemapTest))
     return s

django/contrib/gis/tests/geoapp/urls.py

@@ -1,16 +1,14 @@
 from django.conf.urls.defaults import *
-from feeds import TestGeoRSS1, TestGeoRSS2, TestGeoAtom1, TestGeoAtom2, TestW3CGeo1, TestW3CGeo2, TestW3CGeo3
+from feeds import feed_dict
-
-feed_dict = {
-    'rss1' : TestGeoRSS1,
-    'rss2' : TestGeoRSS2,
-    'atom1' : TestGeoAtom1,
-    'atom2' : TestGeoAtom2,
-    'w3cgeo1' : TestW3CGeo1,
-    'w3cgeo2' : TestW3CGeo2,
-    'w3cgeo3' : TestW3CGeo3,
-}
 
 urlpatterns = patterns('',
-    (r'^feeds/(?P<url>.*)/$', 'django.contrib.syndication.views.feed', {'feed_dict': feed_dict})
+    (r'^feeds/(?P<url>.*)/$', 'django.contrib.syndication.views.feed', {'feed_dict': feed_dict}),
+)
+
+from sitemaps import sitemaps
+urlpatterns += patterns('django.contrib.gis.sitemaps.views',
+    (r'^sitemap.xml$', 'index', {'sitemaps' : sitemaps}),
+    (r'^sitemaps/(?P<section>\w+)\.xml$', 'sitemap', {'sitemaps' : sitemaps}),
+    (r'^sitemaps/kml/(?P<label>\w+)/(?P<model>\w+)/(?P<field_name>\w+)\.kml$', 'kml'),
+    (r'^sitemaps/kml/(?P<label>\w+)/(?P<model>\w+)/(?P<field_name>\w+)\.kmz$', 'kmz'),                
 )