From 712bb0dde7d935d4a7703635c821ea63f60997c7 Mon Sep 17 00:00:00 2001 From: Justin Bronn Date: Sat, 23 Aug 2008 19:22:23 +0000 Subject: [PATCH] Fixed KML sitemaps, and added support for generating KMZ and GeoRSS sitemaps; sitemaps now support Google's Geo Sitemap format. git-svn-id: http://code.djangoproject.com/svn/django/trunk@8502 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/contrib/gis/shortcuts.py | 21 +++- django/contrib/gis/sitemaps.py | 55 ---------- django/contrib/gis/sitemaps/__init__.py | 4 + django/contrib/gis/sitemaps/georss.py | 53 +++++++++ django/contrib/gis/sitemaps/kml.py | 63 +++++++++++ django/contrib/gis/sitemaps/views.py | 102 ++++++++++++++++++ .../templates/gis/sitemaps/geo_sitemap.xml | 17 +++ django/contrib/gis/tests/__init__.py | 1 + django/contrib/gis/tests/geoapp/feeds.py | 11 ++ django/contrib/gis/tests/geoapp/sitemaps.py | 8 ++ .../contrib/gis/tests/geoapp/test_sitemaps.py | 83 ++++++++++++++ django/contrib/gis/tests/geoapp/tests.py | 2 + .../contrib/gis/tests/geoapp/tests_mysql.py | 2 + django/contrib/gis/tests/geoapp/urls.py | 22 ++-- 14 files changed, 376 insertions(+), 68 deletions(-) delete mode 100644 django/contrib/gis/sitemaps.py create mode 100644 django/contrib/gis/sitemaps/__init__.py create mode 100644 django/contrib/gis/sitemaps/georss.py create mode 100644 django/contrib/gis/sitemaps/kml.py create mode 100644 django/contrib/gis/sitemaps/views.py create mode 100644 django/contrib/gis/templates/gis/sitemaps/geo_sitemap.xml create mode 100644 django/contrib/gis/tests/geoapp/sitemaps.py create mode 100644 django/contrib/gis/tests/geoapp/test_sitemaps.py diff --git a/django/contrib/gis/shortcuts.py b/django/contrib/gis/shortcuts.py index 8eeaed1aaa..5310e58eef 100644 --- a/django/contrib/gis/shortcuts.py +++ b/django/contrib/gis/shortcuts.py @@ -1,11 +1,30 @@ +import cStringIO, zipfile from django.http import HttpResponse from django.template import loader +def compress_kml(kml): + "Returns compressed KMZ from the given KML string." + kmz = cStringIO.StringIO() + zf = zipfile.ZipFile(kmz, 'a', zipfile.ZIP_DEFLATED, False) + zf.writestr('doc.kml', kml) + zf.close() + kmz.seek(0) + return kmz.read() + def render_to_kml(*args, **kwargs): - "Renders the response using the MIME type for KML." + "Renders the response as KML (using the correct MIME type)." return HttpResponse(loader.render_to_string(*args, **kwargs), mimetype='application/vnd.google-earth.kml+xml kml') +def render_to_kmz(*args, **kwargs): + """ + Compresses the KML content and returns as KMZ (using the correct + MIME type). + """ + return HttpResponse(compress_kml(loader.render_to_string(*args, **kwargs)), + mimetype='application/vnd.google-earth.kmz') + + def render_to_text(*args, **kwargs): "Renders the response using the MIME type for plain text." return HttpResponse(loader.render_to_string(*args, **kwargs), diff --git a/django/contrib/gis/sitemaps.py b/django/contrib/gis/sitemaps.py deleted file mode 100644 index d5d11b9c0f..0000000000 --- a/django/contrib/gis/sitemaps.py +++ /dev/null @@ -1,55 +0,0 @@ -from django.core import urlresolvers -from django.contrib.sitemaps import Sitemap -from django.contrib.gis.db.models.fields import GeometryField -from django.contrib.gis.shortcuts import render_to_kml -from django.db.models import get_model, get_models -from django.http import HttpResponse - -class KMLSitemap(Sitemap): - """ - A minimal hook to produce KML sitemaps. - """ - def __init__(self, locations=None): - if locations is None: - self.locations = _build_kml_sources() - else: - self.locations = locations - - def items(self): - return self.locations - - def location(self, obj): - return urlresolvers.reverse('django.contrib.gis.sitemaps.kml', - kwargs={'label':obj[0], - 'field_name':obj[1]}) - -def _build_kml_sources(): - "Make a mapping of all available KML sources." - ret = [] - for klass in get_models(): - for field in klass._meta.fields: - if isinstance(field, GeometryField): - label = "%s.%s" % (klass._meta.app_label, - klass._meta.module_name) - - ret.append((label, field.name)) - return ret - - -class KMLNotFound(Exception): - pass - -def kml(request, label, field_name): - placemarks = [] - klass = get_model(*label.split('.')) - if not klass: - raise KMLNotFound("You must supply a valid app.model label. Got %s" % label) - - #FIXME: GMaps apparently has a limit on size of displayed kml files - # check if paginating w/ external refs (i.e. linked list) helps. - placemarks.extend(list(klass._default_manager.kml(field_name)[:100])) - - #FIXME: other KML features? - return render_to_kml('gis/kml/placemarks.kml', {'places' : placemarks}) - - diff --git a/django/contrib/gis/sitemaps/__init__.py b/django/contrib/gis/sitemaps/__init__.py new file mode 100644 index 0000000000..9b6287f899 --- /dev/null +++ b/django/contrib/gis/sitemaps/__init__.py @@ -0,0 +1,4 @@ +# Geo-enabled Sitemap classes. +from django.contrib.gis.sitemaps.georss import GeoRSSSitemap +from django.contrib.gis.sitemaps.kml import KMLSitemap, KMZSitemap + diff --git a/django/contrib/gis/sitemaps/georss.py b/django/contrib/gis/sitemaps/georss.py new file mode 100644 index 0000000000..45c97fb8a2 --- /dev/null +++ b/django/contrib/gis/sitemaps/georss.py @@ -0,0 +1,53 @@ +from django.core import urlresolvers +from django.contrib.sitemaps import Sitemap + +class GeoRSSSitemap(Sitemap): + """ + A minimal hook to produce sitemaps for GeoRSS feeds. + """ + def __init__(self, feed_dict, slug_dict=None): + """ + This sitemap object initializes on a feed dictionary (as would be passed + to `django.contrib.syndication.views.feed`) and a slug dictionary. + If the slug dictionary is not defined, then it's assumed the keys provide + the URL parameter to the feed. However, if you have a complex feed (e.g., + you override `get_object`, then you'll need to provide a slug dictionary. + The slug dictionary should have the same keys as the feed dictionary, but + each value in the slug dictionary should be a sequence of slugs that may + be used for valid feeds. For example, let's say we have a feed that + returns objects for a specific ZIP code in our feed dictionary: + + feed_dict = {'zipcode' : ZipFeed} + + Then we would use a slug dictionary with a list of the zip code slugs + corresponding to feeds you want listed in the sitemap: + + slug_dict = {'zipcode' : ['77002', '77054']} + """ + # Setting up. + self.feed_dict = feed_dict + self.locations = [] + if slug_dict is None: slug_dict = {} + # Getting the feed locations. + for section in feed_dict.keys(): + if slug_dict.get(section, False): + for slug in slug_dict[section]: + self.locations.append(('%s/%s' % (section, slug))) + else: + self.locations.append(section) + + def get_urls(self, page=1): + """ + This method is overrridden so the appropriate `geo_format` attribute + is placed on each URL element. + """ + urls = Sitemap.get_urls(self) + for url in urls: url['geo_format'] = 'georss' + return urls + + def items(self): + return self.locations + + def location(self, obj): + return urlresolvers.reverse('django.contrib.syndication.views.feed', + args=(obj,), kwargs={'feed_dict' : self.feed_dict}) diff --git a/django/contrib/gis/sitemaps/kml.py b/django/contrib/gis/sitemaps/kml.py new file mode 100644 index 0000000000..d85744f0f9 --- /dev/null +++ b/django/contrib/gis/sitemaps/kml.py @@ -0,0 +1,63 @@ +from django.core import urlresolvers +from django.contrib.sitemaps import Sitemap +from django.contrib.gis.db.models.fields import GeometryField +from django.db import models + +class KMLSitemap(Sitemap): + """ + A minimal hook to produce KML sitemaps. + """ + geo_format = 'kml' + + def __init__(self, locations=None): + # If no locations specified, then we try to build for + # every model in installed applications. + self.locations = self._build_kml_sources(locations) + + def _build_kml_sources(self, sources): + """ + Goes through the given sources and returns a 3-tuple of + the application label, module name, and field name of every + GeometryField encountered in the sources. + + If no sources are provided, then all models. + """ + kml_sources = [] + if sources is None: + sources = models.get_models() + for source in sources: + if isinstance(source, models.base.ModelBase): + for field in source._meta.fields: + if isinstance(field, GeometryField): + kml_sources.append((source._meta.app_label, + source._meta.module_name, + field.name)) + elif isinstance(source, (list, tuple)): + if len(source) != 3: + raise ValueError('Must specify a 3-tuple of (app_label, module_name, field_name).') + kml_sources.append(source) + else: + raise TypeError('KML Sources must be a model or a 3-tuple.') + return kml_sources + + def get_urls(self, page=1): + """ + This method is overrridden so the appropriate `geo_format` attribute + is placed on each URL element. + """ + urls = Sitemap.get_urls(self, page=page) + for url in urls: url['geo_format'] = self.geo_format + return urls + + def items(self): + return self.locations + + def location(self, obj): + return urlresolvers.reverse('django.contrib.gis.sitemaps.views.%s' % self.geo_format, + kwargs={'label' : obj[0], + 'model' : obj[1], + 'field_name': obj[2], + } + ) +class KMZSitemap(KMLSitemap): + geo_format = 'kmz' diff --git a/django/contrib/gis/sitemaps/views.py b/django/contrib/gis/sitemaps/views.py new file mode 100644 index 0000000000..d20f7c6d20 --- /dev/null +++ b/django/contrib/gis/sitemaps/views.py @@ -0,0 +1,102 @@ +from django.http import HttpResponse, Http404 +from django.template import loader +from django.contrib.gis.db.backend import SpatialBackend +from django.contrib.sites.models import Site +from django.core import urlresolvers +from django.core.paginator import EmptyPage, PageNotAnInteger +from django.db.models import get_model +from django.utils.encoding import smart_str + +from django.contrib.gis.shortcuts import render_to_kml, render_to_kmz + +class KMLNotFound(Exception): + pass + +def index(request, sitemaps): + """ + This view generates a sitemap index that uses the proper view + for resolving geographic section sitemap URLs. + """ + current_site = Site.objects.get_current() + sites = [] + protocol = request.is_secure() and 'https' or 'http' + for section, site in sitemaps.items(): + if callable(site): + pages = site().paginator.num_pages + else: + pages = site.paginator.num_pages + sitemap_url = urlresolvers.reverse('django.contrib.gis.sitemaps.views.sitemap', kwargs={'section': section}) + sites.append('%s://%s%s' % (protocol, current_site.domain, sitemap_url)) + + if pages > 1: + for page in range(2, pages+1): + sites.append('%s://%s%s?p=%s' % (protocol, current_site.domain, sitemap_url, page)) + xml = loader.render_to_string('sitemap_index.xml', {'sitemaps': sites}) + return HttpResponse(xml, mimetype='application/xml') + +def sitemap(request, sitemaps, section=None): + """ + This view generates a sitemap with additional geographic + elements defined by Google. + """ + maps, urls = [], [] + if section is not None: + if section not in sitemaps: + raise Http404("No sitemap available for section: %r" % section) + maps.append(sitemaps[section]) + else: + maps = sitemaps.values() + + page = request.GET.get("p", 1) + for site in maps: + try: + if callable(site): + urls.extend(site().get_urls(page)) + else: + urls.extend(site.get_urls(page)) + except EmptyPage: + raise Http404("Page %s empty" % page) + except PageNotAnInteger: + raise Http404("No page '%s'" % page) + xml = smart_str(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls})) + return HttpResponse(xml, mimetype='application/xml') + +def kml(request, label, model, field_name=None, compress=False): + """ + This view generates KML for the given app label, model, and field name. + + The model's default manager must be GeoManager, and the field name + must be that of a geographic field. + """ + placemarks = [] + klass = get_model(label, model) + if not klass: + raise KMLNotFound("You must supply a valid app.model label. Got %s.%s" % (label, model)) + + if SpatialBackend.postgis: + # PostGIS will take care of transformation. + placemarks = klass._default_manager.kml(field_name=field_name) + else: + # There's no KML method on Oracle or MySQL, so we use the `kml` + # attribute of the lazy geometry instead. + placemarks = [] + if SpatialBackend.oracle: + qs = klass._default_manager.transform(4326, field_name=field_name) + else: + qs = klass._default_manager.all() + for mod in qs: + setattr(mod, 'kml', getattr(mod, field_name).kml) + placemarks.append(mod) + + # Getting the render function and rendering to the correct. + if compress: + render = render_to_kmz + else: + render = render_to_kml + return render('gis/kml/placemarks.kml', {'places' : placemarks}) + +def kmz(request, label, model, field_name=None): + """ + This view returns KMZ for the given app label, model, and field name. + """ + return kml(request, label, model, field_name, True) diff --git a/django/contrib/gis/templates/gis/sitemaps/geo_sitemap.xml b/django/contrib/gis/templates/gis/sitemaps/geo_sitemap.xml new file mode 100644 index 0000000000..dbf858e40d --- /dev/null +++ b/django/contrib/gis/templates/gis/sitemaps/geo_sitemap.xml @@ -0,0 +1,17 @@ +{% autoescape off %} + +{% spaceless %} +{% for url in urlset %} + + {{ url.location|escape }} + {% if url.lastmod %}{{ url.lastmod|date:"Y-m-d" }}{% endif %} + {% if url.changefreq %}{{ url.changefreq }}{% endif %} + {% if url.priority %}{{ url.priority }}{% endif %} + {% if url.geo_format %} + {{ url.geo_format }} + {% endif %} + +{% endfor %} +{% endspaceless %} + +{% endautoescape %} diff --git a/django/contrib/gis/tests/__init__.py b/django/contrib/gis/tests/__init__.py index 1bf888a08d..8aa3820fc0 100644 --- a/django/contrib/gis/tests/__init__.py +++ b/django/contrib/gis/tests/__init__.py @@ -74,6 +74,7 @@ def run_gis_tests(test_labels, **kwargs): new_installed = ['django.contrib.contenttypes', 'django.contrib.auth', 'django.contrib.sites', + 'django.contrib.sitemaps', 'django.contrib.flatpages', 'django.contrib.gis', 'django.contrib.redirects', diff --git a/django/contrib/gis/tests/geoapp/feeds.py b/django/contrib/gis/tests/geoapp/feeds.py index 5ab35dc5cd..942b1405f9 100644 --- a/django/contrib/gis/tests/geoapp/feeds.py +++ b/django/contrib/gis/tests/geoapp/feeds.py @@ -50,3 +50,14 @@ class TestW3CGeo3(TestGeoRSS1): def item_geometry(self, item): from django.contrib.gis.geos import Polygon return Polygon(((0, 0), (0, 1), (1, 1), (1, 0), (0, 0))) + +# The feed dictionary to use for URLs. +feed_dict = { + 'rss1' : TestGeoRSS1, + 'rss2' : TestGeoRSS2, + 'atom1' : TestGeoAtom1, + 'atom2' : TestGeoAtom2, + 'w3cgeo1' : TestW3CGeo1, + 'w3cgeo2' : TestW3CGeo2, + 'w3cgeo3' : TestW3CGeo3, +} diff --git a/django/contrib/gis/tests/geoapp/sitemaps.py b/django/contrib/gis/tests/geoapp/sitemaps.py new file mode 100644 index 0000000000..ca785f2458 --- /dev/null +++ b/django/contrib/gis/tests/geoapp/sitemaps.py @@ -0,0 +1,8 @@ +from django.contrib.gis.sitemaps import GeoRSSSitemap, KMLSitemap, KMZSitemap +from models import City, Country +from feeds import feed_dict + +sitemaps = {'kml' : KMLSitemap([City, Country]), + 'kmz' : KMZSitemap([City, Country]), + 'georss' : GeoRSSSitemap(feed_dict), + } diff --git a/django/contrib/gis/tests/geoapp/test_sitemaps.py b/django/contrib/gis/tests/geoapp/test_sitemaps.py new file mode 100644 index 0000000000..a83e044b86 --- /dev/null +++ b/django/contrib/gis/tests/geoapp/test_sitemaps.py @@ -0,0 +1,83 @@ +import unittest, zipfile, cStringIO +from xml.dom import minidom + +from django.test import Client +from models import City, Country + +class GeoSitemapTest(unittest.TestCase): + client = Client() + + def assertChildNodes(self, elem, expected): + "Taken from regressiontests/syndication/tests.py." + actual = set([n.nodeName for n in elem.childNodes]) + expected = set(expected) + self.assertEqual(actual, expected) + + def test_geositemap_index(self): + "Tests geographic sitemap index." + # Getting the geo index. + doc = minidom.parseString(self.client.get('/geoapp/sitemap.xml').content) + index = doc.firstChild + self.assertEqual(index.getAttribute(u'xmlns'), u'http://www.sitemaps.org/schemas/sitemap/0.9') + self.assertEqual(3, len(index.getElementsByTagName('sitemap'))) + + def test_geositemap_kml(self): + "Tests KML/KMZ geographic sitemaps." + for kml_type in ('kml', 'kmz'): + doc = minidom.parseString(self.client.get('/geoapp/sitemaps/%s.xml' % kml_type).content) + + # Ensuring the right sitemaps namespaces are present. + urlset = doc.firstChild + self.assertEqual(urlset.getAttribute(u'xmlns'), u'http://www.sitemaps.org/schemas/sitemap/0.9') + self.assertEqual(urlset.getAttribute(u'xmlns:geo'), u'http://www.google.com/geo/schemas/sitemap/1.0') + + urls = urlset.getElementsByTagName('url') + self.assertEqual(2, len(urls)) # Should only be 2 sitemaps. + for url in urls: + self.assertChildNodes(url, ['loc', 'geo:geo']) + # Making sure the 'geo:format' element was properly set. + geo_elem = url.getElementsByTagName('geo:geo')[0] + geo_format = geo_elem.getElementsByTagName('geo:format')[0] + self.assertEqual(kml_type, geo_format.childNodes[0].data) + + # Getting the relative URL since we don't have a real site. + kml_url = url.getElementsByTagName('loc')[0].childNodes[0].data.split('http://example.com')[1] + + if kml_type == 'kml': + kml_doc = minidom.parseString(self.client.get(kml_url).content) + elif kml_type == 'kmz': + # Have to decompress KMZ before parsing. + buf = cStringIO.StringIO(self.client.get(kml_url).content) + zf = zipfile.ZipFile(buf) + self.assertEqual(1, len(zf.filelist)) + self.assertEqual('doc.kml', zf.filelist[0].filename) + kml_doc = minidom.parseString(zf.read('doc.kml')) + + # Ensuring the correct number of placemarks are in the KML doc. + if 'city' in kml_url: + model = City + elif 'country' in kml_url: + model = Country + self.assertEqual(model.objects.count(), len(kml_doc.getElementsByTagName('Placemark'))) + + def test_geositemap_georss(self): + "Tests GeoRSS geographic sitemaps." + from feeds import feed_dict + + doc = minidom.parseString(self.client.get('/geoapp/sitemaps/georss.xml').content) + + # Ensuring the right sitemaps namespaces are present. + urlset = doc.firstChild + self.assertEqual(urlset.getAttribute(u'xmlns'), u'http://www.sitemaps.org/schemas/sitemap/0.9') + self.assertEqual(urlset.getAttribute(u'xmlns:geo'), u'http://www.google.com/geo/schemas/sitemap/1.0') + + # Making sure the correct number of feed URLs were included. + urls = urlset.getElementsByTagName('url') + self.assertEqual(len(feed_dict), len(urls)) + + for url in urls: + self.assertChildNodes(url, ['loc', 'geo:geo']) + # Making sure the 'geo:format' element was properly set to 'georss'. + geo_elem = url.getElementsByTagName('geo:geo')[0] + geo_format = geo_elem.getElementsByTagName('geo:format')[0] + self.assertEqual('georss', geo_format.childNodes[0].data) diff --git a/django/contrib/gis/tests/geoapp/tests.py b/django/contrib/gis/tests/geoapp/tests.py index b49bfe2fd7..ff6eac9cc7 100644 --- a/django/contrib/gis/tests/geoapp/tests.py +++ b/django/contrib/gis/tests/geoapp/tests.py @@ -559,8 +559,10 @@ class GeoModelTest(unittest.TestCase): self.assertEqual(c.mpoly.union(geom), c.union) from test_feeds import GeoFeedTest +from test_sitemaps import GeoSitemapTest def suite(): s = unittest.TestSuite() s.addTest(unittest.makeSuite(GeoModelTest)) s.addTest(unittest.makeSuite(GeoFeedTest)) + s.addTest(unittest.makeSuite(GeoSitemapTest)) return s diff --git a/django/contrib/gis/tests/geoapp/tests_mysql.py b/django/contrib/gis/tests/geoapp/tests_mysql.py index dce71c6445..040130cc1a 100644 --- a/django/contrib/gis/tests/geoapp/tests_mysql.py +++ b/django/contrib/gis/tests/geoapp/tests_mysql.py @@ -174,8 +174,10 @@ class GeoModelTest(unittest.TestCase): self.assertRaises(ImproperlyConfigured, Country.objects.all().gml, field_name='mpoly') from test_feeds import GeoFeedTest +from test_sitemaps import GeoSitemapTest def suite(): s = unittest.TestSuite() s.addTest(unittest.makeSuite(GeoModelTest)) s.addTest(unittest.makeSuite(GeoFeedTest)) + s.addTest(unittest.makeSuite(GeoSitemapTest)) return s diff --git a/django/contrib/gis/tests/geoapp/urls.py b/django/contrib/gis/tests/geoapp/urls.py index b3fd92972b..edaf2802ef 100644 --- a/django/contrib/gis/tests/geoapp/urls.py +++ b/django/contrib/gis/tests/geoapp/urls.py @@ -1,16 +1,14 @@ from django.conf.urls.defaults import * -from feeds import TestGeoRSS1, TestGeoRSS2, TestGeoAtom1, TestGeoAtom2, TestW3CGeo1, TestW3CGeo2, TestW3CGeo3 - -feed_dict = { - 'rss1' : TestGeoRSS1, - 'rss2' : TestGeoRSS2, - 'atom1' : TestGeoAtom1, - 'atom2' : TestGeoAtom2, - 'w3cgeo1' : TestW3CGeo1, - 'w3cgeo2' : TestW3CGeo2, - 'w3cgeo3' : TestW3CGeo3, -} +from feeds import feed_dict urlpatterns = patterns('', - (r'^feeds/(?P.*)/$', 'django.contrib.syndication.views.feed', {'feed_dict': feed_dict}) + (r'^feeds/(?P.*)/$', 'django.contrib.syndication.views.feed', {'feed_dict': feed_dict}), +) + +from sitemaps import sitemaps +urlpatterns += patterns('django.contrib.gis.sitemaps.views', + (r'^sitemap.xml$', 'index', {'sitemaps' : sitemaps}), + (r'^sitemaps/(?P
\w+)\.xml$', 'sitemap', {'sitemaps' : sitemaps}), + (r'^sitemaps/kml/(?P