diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 8c1db8945d..c012949414 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -128,6 +128,7 @@ SERVER_EMAIL = 'root@localhost' SEND_BROKEN_LINK_EMAILS = False # Database connection info. +# Legacy format DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. DATABASE_NAME = '' # Or path to database file if using sqlite3. DATABASE_USER = '' # Not used with sqlite3. @@ -136,9 +137,13 @@ DATABASE_HOST = '' # Set to empty string for localhost. Not used wit DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. DATABASE_OPTIONS = {} # Set to empty dictionary for default. +# New format DATABASES = { } +# Classes used to implement db routing behaviour +DATABASE_ROUTERS = [] + # The email backend to use. For possible shortcuts see django.core.mail. # The default is to use the SMTP backend. # Third-party backends can be specified by providing a Python path diff --git a/django/contrib/auth/models.py b/django/contrib/auth/models.py index 3328fe4e96..5e76935603 100644 --- a/django/contrib/auth/models.py +++ b/django/contrib/auth/models.py @@ -3,7 +3,7 @@ import urllib from django.contrib import auth from django.core.exceptions import ImproperlyConfigured -from django.db import models, DEFAULT_DB_ALIAS +from django.db import models from django.db.models.manager import EmptyManager from django.contrib.contenttypes.models import ContentType from django.utils.encoding import smart_str diff --git a/django/contrib/contenttypes/generic.py b/django/contrib/contenttypes/generic.py index 2a9f4d3ac6..1b2d8b7d96 100644 --- a/django/contrib/contenttypes/generic.py +++ b/django/contrib/contenttypes/generic.py @@ -5,7 +5,7 @@ Classes allowing "generic" relations through ContentType and object-id fields. from django.core.exceptions import ObjectDoesNotExist from django.db import connection from django.db.models import signals -from django.db import models, DEFAULT_DB_ALIAS +from django.db import models from django.db.models.fields.related import RelatedField, Field, ManyToManyRel from django.db.models.loading import get_model from django.forms import ModelForm @@ -255,7 +255,7 @@ def create_generic_related_manager(superclass): raise TypeError("'%s' instance expected" % self.model._meta.object_name) setattr(obj, self.content_type_field_name, self.content_type) setattr(obj, self.object_id_field_name, self.pk_val) - obj.save(using=self.instance._state.db) + obj.save() add.alters_data = True def remove(self, *objs): diff --git a/django/contrib/contenttypes/models.py b/django/contrib/contenttypes/models.py index ed64d1ea0c..af3abf249f 100644 --- a/django/contrib/contenttypes/models.py +++ b/django/contrib/contenttypes/models.py @@ -1,4 +1,4 @@ -from django.db import models, DEFAULT_DB_ALIAS +from django.db import models from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import smart_unicode diff --git a/django/contrib/gis/db/models/sql/query.py b/django/contrib/gis/db/models/sql/query.py index c7341db757..eeda456595 100644 --- a/django/contrib/gis/db/models/sql/query.py +++ b/django/contrib/gis/db/models/sql/query.py @@ -1,4 +1,4 @@ -from django.db import connections, DEFAULT_DB_ALIAS +from django.db import connections from django.db.models.query import sql from django.contrib.gis.db.models.fields import GeometryField diff --git a/django/db/__init__.py b/django/db/__init__.py index ea353e4412..69996e6ad5 100644 --- a/django/db/__init__.py +++ b/django/db/__init__.py @@ -1,13 +1,12 @@ from django.conf import settings from django.core import signals from django.core.exceptions import ImproperlyConfigured -from django.db.utils import ConnectionHandler, load_backend +from django.db.utils import ConnectionHandler, ConnectionRouter, load_backend, DEFAULT_DB_ALIAS from django.utils.functional import curry -__all__ = ('backend', 'connection', 'connections', 'DatabaseError', +__all__ = ('backend', 'connection', 'connections', 'router', 'DatabaseError', 'IntegrityError', 'DEFAULT_DB_ALIAS') -DEFAULT_DB_ALIAS = 'default' # For backwards compatibility - Port any old database settings over to # the new values. @@ -61,6 +60,7 @@ for alias, database in settings.DATABASES.items(): connections = ConnectionHandler(settings.DATABASES) +router = ConnectionRouter(settings.DATABASE_ROUTERS) # `connection`, `DatabaseError` and `IntegrityError` are convenient aliases # for backend bits. diff --git a/django/db/models/base.py b/django/db/models/base.py index 822116054c..dc4b8f0e7b 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -10,7 +10,7 @@ from django.db.models.fields.related import OneToOneRel, ManyToOneRel, OneToOneF from django.db.models.query import delete_objects, Q from django.db.models.query_utils import CollectedObjects, DeferredAttribute from django.db.models.options import Options -from django.db import connections, transaction, DatabaseError, DEFAULT_DB_ALIAS +from django.db import connections, router, transaction, DatabaseError, DEFAULT_DB_ALIAS from django.db.models import signals from django.db.models.loading import register_models, get_model from django.utils.translation import ugettext_lazy as _ @@ -439,7 +439,7 @@ class Model(object): need for overrides of save() to pass around internal-only parameters ('raw', 'cls', and 'origin'). """ - using = using or self._state.db or DEFAULT_DB_ALIAS + using = using or router.db_for_write(self.__class__, instance=self) connection = connections[using] assert not (force_insert and force_update) if cls is None: @@ -592,7 +592,7 @@ class Model(object): parent_obj._collect_sub_objects(seen_objs) def delete(self, using=None): - using = using or self._state.db or DEFAULT_DB_ALIAS + using = using or router.db_for_write(self.__class__, instance=self) connection = connections[using] assert self._get_pk_val() is not None, "%s object can't be deleted because its %s attribute is set to None." % (self._meta.object_name, self._meta.pk.attname) @@ -719,7 +719,7 @@ class Model(object): # no value, skip the lookup continue if f.primary_key and not getattr(self, '_adding', False): - # no need to check for unique primary key when editting + # no need to check for unique primary key when editing continue lookup_kwargs[str(field_name)] = lookup_value diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 36dde5b7f4..4020d5e268 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -1,4 +1,5 @@ -from django.db import connection, transaction, DEFAULT_DB_ALIAS +from django.conf import settings +from django.db import connection, router, transaction from django.db.backends import util from django.db.models import signals, get_model from django.db.models.fields import (AutoField, Field, IntegerField, @@ -197,7 +198,8 @@ class SingleRelatedObjectDescriptor(object): return getattr(instance, self.cache_name) except AttributeError: params = {'%s__pk' % self.related.field.name: instance._get_pk_val()} - rel_obj = self.related.model._base_manager.using(instance._state.db).get(**params) + db = router.db_for_read(instance.__class__, instance=instance) + rel_obj = self.related.model._base_manager.using(db).get(**params) setattr(instance, self.cache_name, rel_obj) return rel_obj @@ -218,6 +220,15 @@ class SingleRelatedObjectDescriptor(object): raise ValueError('Cannot assign "%r": "%s.%s" must be a "%s" instance.' % (value, instance._meta.object_name, self.related.get_accessor_name(), self.related.opts.object_name)) + elif value is not None: + if instance._state.db is None: + instance._state.db = router.db_for_write(instance.__class__, instance=value) + elif value._state.db is None: + value._state.db = router.db_for_write(value.__class__, instance=instance) + elif value._state.db is not None and instance._state.db is not None: + if not router.allow_relation(value, instance): + raise ValueError('Cannot assign "%r": instance is on database "%s", value is is on database "%s"' % + (value, instance._state.db, value._state.db)) # Set the value of the related field to the value of the related object's related field setattr(value, self.related.field.attname, getattr(instance, self.related.field.rel.get_related_field().attname)) @@ -260,11 +271,11 @@ class ReverseSingleRelatedObjectDescriptor(object): # If the related manager indicates that it should be used for # related fields, respect that. rel_mgr = self.field.rel.to._default_manager - using = instance._state.db or DEFAULT_DB_ALIAS + db = router.db_for_read(self.field.rel.to, instance=instance) if getattr(rel_mgr, 'use_for_related_fields', False): - rel_obj = rel_mgr.using(using).get(**params) + rel_obj = rel_mgr.using(db).get(**params) else: - rel_obj = QuerySet(self.field.rel.to).using(using).get(**params) + rel_obj = QuerySet(self.field.rel.to).using(db).get(**params) setattr(instance, cache_name, rel_obj) return rel_obj @@ -281,14 +292,15 @@ class ReverseSingleRelatedObjectDescriptor(object): raise ValueError('Cannot assign "%r": "%s.%s" must be a "%s" instance.' % (value, instance._meta.object_name, self.field.name, self.field.rel.to._meta.object_name)) - elif value is not None and value._state.db != instance._state.db: + elif value is not None: if instance._state.db is None: - instance._state.db = value._state.db - else:#elif value._state.db is None: - value._state.db = instance._state.db -# elif value._state.db is not None and instance._state.db is not None: -# raise ValueError('Cannot assign "%r": instance is on database "%s", value is is on database "%s"' % -# (value, instance._state.db, value._state.db)) + instance._state.db = router.db_for_write(instance.__class__, instance=value) + elif value._state.db is None: + value._state.db = router.db_for_write(value.__class__, instance=instance) + elif value._state.db is not None and instance._state.db is not None: + if not router.allow_relation(value, instance): + raise ValueError('Cannot assign "%r": instance is on database "%s", value is is on database "%s"' % + (value, instance._state.db, value._state.db)) # If we're setting the value of a OneToOneField to None, we need to clear # out the cache on any old related object. Otherwise, deleting the @@ -370,15 +382,15 @@ class ForeignRelatedObjectsDescriptor(object): class RelatedManager(superclass): def get_query_set(self): - using = instance._state.db or DEFAULT_DB_ALIAS - return superclass.get_query_set(self).using(using).filter(**(self.core_filters)) + db = router.db_for_read(rel_model, instance=instance) + return superclass.get_query_set(self).using(db).filter(**(self.core_filters)) def add(self, *objs): for obj in objs: if not isinstance(obj, self.model): raise TypeError("'%s' instance expected" % self.model._meta.object_name) setattr(obj, rel_field.name, instance) - obj.save(using=instance._state.db) + obj.save() add.alters_data = True def create(self, **kwargs): @@ -390,8 +402,8 @@ class ForeignRelatedObjectsDescriptor(object): # Update kwargs with the related object that this # ForeignRelatedObjectsDescriptor knows about. kwargs.update({rel_field.name: instance}) - using = instance._state.db or DEFAULT_DB_ALIAS - return super(RelatedManager, self).using(using).get_or_create(**kwargs) + db = router.db_for_write(rel_model, instance=instance) + return super(RelatedManager, self).using(db).get_or_create(**kwargs) get_or_create.alters_data = True # remove() and clear() are only provided if the ForeignKey can have a value of null. @@ -402,7 +414,7 @@ class ForeignRelatedObjectsDescriptor(object): # Is obj actually part of this descriptor set? if getattr(obj, rel_field.attname) == val: setattr(obj, rel_field.name, None) - obj.save(using=instance._state.db) + obj.save() else: raise rel_field.rel.to.DoesNotExist("%r is not related to %r." % (obj, instance)) remove.alters_data = True @@ -410,7 +422,7 @@ class ForeignRelatedObjectsDescriptor(object): def clear(self): for obj in self.all(): setattr(obj, rel_field.name, None) - obj.save(using=instance._state.db) + obj.save() clear.alters_data = True manager = RelatedManager() @@ -443,7 +455,8 @@ def create_many_related_manager(superclass, rel=False): raise ValueError("%r instance needs to have a primary key value before a many-to-many relationship can be used." % instance.__class__.__name__) def get_query_set(self): - return superclass.get_query_set(self).using(self.instance._state.db)._next_is_sticky().filter(**(self.core_filters)) + db = router.db_for_read(self.instance.__class__, instance=self.instance) + return superclass.get_query_set(self).using(db)._next_is_sticky().filter(**(self.core_filters)) # If the ManyToMany relation has an intermediary model, # the add and remove methods do not exist. @@ -478,14 +491,16 @@ def create_many_related_manager(superclass, rel=False): if not rel.through._meta.auto_created: opts = through._meta raise AttributeError("Cannot use create() on a ManyToManyField which specifies an intermediary model. Use %s.%s's Manager instead." % (opts.app_label, opts.object_name)) - new_obj = super(ManyRelatedManager, self).using(self.instance._state.db).create(**kwargs) + db = router.db_for_write(self.instance.__class__, instance=self.instance) + new_obj = super(ManyRelatedManager, self).using(db).create(**kwargs) self.add(new_obj) return new_obj create.alters_data = True def get_or_create(self, **kwargs): + db = router.db_for_write(self.instance.__class__, instance=self.instance) obj, created = \ - super(ManyRelatedManager, self).using(self.instance._state.db).get_or_create(**kwargs) + super(ManyRelatedManager, self).using(db).get_or_create(**kwargs) # We only need to add() if created because if we got an object back # from get() then the relationship already exists. if created: @@ -505,15 +520,16 @@ def create_many_related_manager(superclass, rel=False): new_ids = set() for obj in objs: if isinstance(obj, self.model): -# if obj._state.db != self.instance._state.db: -# raise ValueError('Cannot add "%r": instance is on database "%s", value is is on database "%s"' % -# (obj, self.instance._state.db, obj._state.db)) + if not router.allow_relation(obj, self.instance): + raise ValueError('Cannot add "%r": instance is on database "%s", value is is on database "%s"' % + (obj, self.instance._state.db, obj._state.db)) new_ids.add(obj.pk) elif isinstance(obj, Model): raise TypeError("'%s' instance expected" % self.model._meta.object_name) else: new_ids.add(obj) - vals = self.through._default_manager.using(self.instance._state.db).values_list(target_field_name, flat=True) + db = router.db_for_write(self.through.__class__, instance=self.instance) + vals = self.through._default_manager.using(db).values_list(target_field_name, flat=True) vals = vals.filter(**{ source_field_name: self._pk_val, '%s__in' % target_field_name: new_ids, @@ -521,7 +537,7 @@ def create_many_related_manager(superclass, rel=False): new_ids = new_ids - set(vals) # Add the ones that aren't there already for obj_id in new_ids: - self.through._default_manager.using(self.instance._state.db).create(**{ + self.through._default_manager.using(db).create(**{ '%s_id' % source_field_name: self._pk_val, '%s_id' % target_field_name: obj_id, }) @@ -547,7 +563,8 @@ def create_many_related_manager(superclass, rel=False): else: old_ids.add(obj) # Remove the specified objects from the join table - self.through._default_manager.using(self.instance._state.db).filter(**{ + db = router.db_for_write(self.through.__class__, instance=self.instance) + self.through._default_manager.using(db).filter(**{ source_field_name: self._pk_val, '%s__in' % target_field_name: old_ids }).delete() @@ -566,7 +583,8 @@ def create_many_related_manager(superclass, rel=False): signals.m2m_changed.send(sender=rel.through, action="clear", instance=self.instance, reverse=self.reverse, model=self.model, pk_set=None) - self.through._default_manager.using(self.instance._state.db).filter(**{ + db = router.db_for_write(self.through.__class__, instance=self.instance) + self.through._default_manager.using(db).filter(**{ source_field_name: self._pk_val }).delete() diff --git a/django/db/models/manager.py b/django/db/models/manager.py index 453caa8ebe..6a62c254ef 100644 --- a/django/db/models/manager.py +++ b/django/db/models/manager.py @@ -1,10 +1,11 @@ from django.utils import copycompat as copy - -from django.db import DEFAULT_DB_ALIAS +from django.conf import settings +from django.db import router from django.db.models.query import QuerySet, EmptyQuerySet, insert_query, RawQuerySet from django.db.models import signals from django.db.models.fields import FieldDoesNotExist + def ensure_default_manager(sender, **kwargs): """ Ensures that a Model subclass contains a default manager and sets the @@ -87,30 +88,27 @@ class Manager(object): mgr._inherited = True return mgr - def db_manager(self, alias): + def db_manager(self, using): obj = copy.copy(self) - obj._db = alias + obj._db = using return obj @property def db(self): - return self._db or DEFAULT_DB_ALIAS + return self._db or router.db_for_read(self.model) ####################### # PROXIES TO QUERYSET # ####################### def get_empty_query_set(self): - return EmptyQuerySet(self.model) + return EmptyQuerySet(self.model, using=self._db) def get_query_set(self): """Returns a new QuerySet object. Subclasses can override this method to easily customize the behavior of the Manager. """ - qs = QuerySet(self.model) - if self._db is not None: - qs = qs.using(self._db) - return qs + return QuerySet(self.model, using=self._db) def none(self): return self.get_empty_query_set() @@ -200,7 +198,7 @@ class Manager(object): return self.get_query_set()._update(values, **kwargs) def raw(self, raw_query, params=None, *args, **kwargs): - return RawQuerySet(raw_query=raw_query, model=self.model, params=params, using=self.db, *args, **kwargs) + return RawQuerySet(raw_query=raw_query, model=self.model, params=params, using=self._db, *args, **kwargs) class ManagerDescriptor(object): # This class ensures managers aren't accessible via model instances. diff --git a/django/db/models/query.py b/django/db/models/query.py index c8e630ad9d..3b290a6457 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -4,7 +4,7 @@ The main QuerySet implementation. This provides the public API for the ORM. from copy import deepcopy -from django.db import connections, transaction, IntegrityError, DEFAULT_DB_ALIAS +from django.db import connections, router, transaction, IntegrityError from django.db.models.aggregates import Aggregate from django.db.models.fields import DateField from django.db.models.query_utils import Q, select_related_descend, CollectedObjects, CyclicDependency, deferred_class_factory, InvalidQuery @@ -34,6 +34,7 @@ class QuerySet(object): self._result_cache = None self._iter = None self._sticky_filter = False + self._for_write = False ######################## # PYTHON MAGIC METHODS # @@ -345,6 +346,7 @@ class QuerySet(object): and returning the created object. """ obj = self.model(**kwargs) + self._for_write = True obj.save(force_insert=True, using=self.db) return obj @@ -358,6 +360,7 @@ class QuerySet(object): 'get_or_create() must be passed at least one keyword argument' defaults = kwargs.pop('defaults', {}) try: + self._for_write = True return self.get(**kwargs), False except self.model.DoesNotExist: try: @@ -413,6 +416,11 @@ class QuerySet(object): del_query = self._clone() + # The delete is actually 2 queries - one to find related objects, + # and one to delete. Make sure that the discovery of related + # objects is performed on the same database as the deletion. + del_query._for_write = True + # Disable non-supported fields. del_query.query.select_related = False del_query.query.clear_ordering() @@ -442,6 +450,7 @@ class QuerySet(object): """ assert self.query.can_filter(), \ "Cannot update a query once a slice has been taken." + self._for_write = True query = self.query.clone(sql.UpdateQuery) query.add_update_values(kwargs) if not transaction.is_managed(using=self.db): @@ -714,7 +723,9 @@ class QuerySet(object): @property def db(self): "Return the database that will be used if this query is executed now" - return self._db or DEFAULT_DB_ALIAS + if self._for_write: + return self._db or router.db_for_write(self.model) + return self._db or router.db_for_read(self.model) ################### # PRIVATE METHODS # @@ -726,8 +737,8 @@ class QuerySet(object): query = self.query.clone() if self._sticky_filter: query.filter_is_sticky = True - c = klass(model=self.model, query=query) - c._db = self._db + c = klass(model=self.model, query=query, using=self._db) + c._for_write = self._for_write c.__dict__.update(kwargs) if setup and hasattr(c, '_setup_query'): c._setup_query() @@ -988,8 +999,8 @@ class DateQuerySet(QuerySet): class EmptyQuerySet(QuerySet): - def __init__(self, model=None, query=None): - super(EmptyQuerySet, self).__init__(model, query) + def __init__(self, model=None, query=None, using=None): + super(EmptyQuerySet, self).__init__(model, query, using) self._result_cache = [] def __and__(self, other): @@ -1254,7 +1265,7 @@ class RawQuerySet(object): @property def db(self): "Return the database that will be used if this query is executed now" - return self._db or DEFAULT_DB_ALIAS + return self._db or router.db_for_read(self.model) def using(self, alias): """ diff --git a/django/db/utils.py b/django/db/utils.py index 831c63a3dd..50ab35e05e 100644 --- a/django/db/utils.py +++ b/django/db/utils.py @@ -5,6 +5,8 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module +DEFAULT_DB_ALIAS = 'default' + def load_backend(backend_name): try: module = import_module('.base', 'django.db.backends.%s' % backend_name) @@ -55,6 +57,7 @@ class ConnectionHandler(object): conn = self.databases[alias] except KeyError: raise ConnectionDoesNotExist("The connection %s doesn't exist" % alias) + conn.setdefault('ENGINE', 'django.db.backends.dummy') if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']: conn['ENGINE'] = 'django.db.backends.dummy' @@ -82,3 +85,38 @@ class ConnectionHandler(object): def all(self): return [self[alias] for alias in self] + +class ConnectionRouter(object): + def __init__(self, routers): + self.routers = [] + for r in routers: + if isinstance(r, basestring): + module_name, klass_name = r.rsplit('.', 1) + module = import_module(module_name) + router = getattr(module, klass_name)() + else: + router = r + self.routers.append(router) + + def _router_func(action): + def _route_db(self, model, **hints): + chosen_db = None + for router in self.routers: + chosen_db = getattr(router, action)(model, **hints) + if chosen_db: + return chosen_db + try: + return hints['instance']._state.db or DEFAULT_DB_ALIAS + except KeyError: + return DEFAULT_DB_ALIAS + return _route_db + + db_for_read = _router_func('db_for_read') + db_for_write = _router_func('db_for_write') + + def allow_relation(self, obj1, obj2, **hints): + for router in self.routers: + allow = router.allow_relation(obj1, obj2, **hints) + if allow is not None: + return allow + return obj1._state.db == obj2._state.db diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index ca63aaf478..42f93f2d83 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -372,6 +372,22 @@ test database will use the name ``'test_' + DATABASE_NAME``. See :ref:`topics-testing`. + +.. setting:: DATABASE_ROUTERS + +DATABASE_ROUTERS +---------------- + +.. versionadded: 1.2 + +Default: ``[]`` (Empty list) + +The list of routers that will be used to determine which database +to use when performing a database queries. + +See the documentation on :ref:`automatic database routing in multi +database configurations `. + .. setting:: DATE_FORMAT DATE_FORMAT diff --git a/docs/topics/db/multi-db.txt b/docs/topics/db/multi-db.txt index 52863d82e5..77f0b7a4d5 100644 --- a/docs/topics/db/multi-db.txt +++ b/docs/topics/db/multi-db.txt @@ -6,10 +6,10 @@ Multiple databases .. versionadded:: 1.2 -This topic guide describes Django's support for interacting with multiple -databases. Most of the rest of Django's documentation assumes you are -interacting with a single database. If you want to interact with multiple -databases, you'll need to take some additional steps. +This topic guide describes Django's support for interacting with +multiple databases. Most of the rest of Django's documentation assumes +you are interacting with a single database. If you want to interact +with multiple databases, you'll need to take some additional steps. Defining your databases ======================= @@ -22,9 +22,11 @@ a dictionary of settings for that specific connection. The settings in the inner dictionaries are described fully in the :setting:`DATABASES` documentation. -Regardless of how many databases you have, you *must* have a database -named ``'default'``. Any additional databases can have whatever alias -you choose. +Databases can have any alias you choose. However, the alias +``default`` has special significance. Django uses the database with +the alias of ``default`` when no other database has been selected. If +you don't have a ``default`` database, you need to be careful to +always specify the database that you want to use. The following is an example ``settings.py`` snippet defining two databases -- a default PostgreSQL database and a MySQL database called @@ -65,10 +67,10 @@ all databases in our example, you would need to call:: If you don't want every application to be synchronized onto a particular database. you can specify the :djadminopt:`--exclude` -argument to :djadmin:`syncdb`. The :djadminopt:`--exclude` option -lets you prevent a specific application or applications from -being synchronized. For example, if you don't want the ``sales`` -application to be in the ``users`` database, you could run:: +argument to :djadmin:`syncdb`. The :djadminopt:`--exclude` option lets +you prevent a specific application or applications from being +synchronized. For example, if you don't want the ``sales`` application +to be in the ``users`` database, you could run:: $ ./manage.py syncdb --database=users --exclude=sales @@ -86,46 +88,235 @@ operate in the same way as :djadmin:`syncdb` -- they only ever operate on one database at a time, using :djadminopt:`--database` to control the database used. -Selecting a database for a ``QuerySet`` -======================================= +.. _topics-db-multi-db-routing: -You can select the database for a ``QuerySet`` at any point in the ``QuerySet`` -"chain." Just call ``using()`` on the ``QuerySet`` to get another ``QuerySet`` -that uses the specified database. +Automatic database routing +========================== -``using()`` takes a single argument: the alias of the database on which you -want to run the query. For example: +The easiest way to use multiple databases is to set up a database +routing scheme. The default routing scheme ensures that objects remain +'sticky' to their original database (i.e., an object retrieved from +the ``foo`` database will be saved on the same database). However, you +can implement more interesting behaviors by defining a different +routing scheme. -.. code-block:: python +Database routers +---------------- - # This will run on the 'default' database. +A database Router is a class that provides three methods: + +.. method:: db_for_read(model, **hints) + + Suggest the database that should be used for read operations for + objects of type ``model``. + + If a database operation is able to provide any additional + information that might assist in selecting a database, it will be + provided in the ``hints`` dictionary. Details on valid hints are + provided :ref:`below `. + + Returns None if there is no suggestion. + +.. method:: db_for_write(model, **hints) + + Suggest the database that should be used for writes of objects of + type Model. + + If a database operation is able to provide any additional + information that might assist in selecting a database, it will be + provided in the ``hints`` dictionary. Details on valid hints are + provided :ref:`below `. + + Returns None if there is no suggestion. + +.. method:: allow_relation(obj1, obj2, **hints) + + Return True if a relation between obj1 and obj2 should be + allowed, False if the relation should be prevented, or None if + the router has no opinion. This is purely a validation operation, + used by foreign key and many to many operations to determine if a + relation should be allowed between two objects. + +.. _topics-db-multi-db-hints: + +Hints +~~~~~ + +The hints received by the database router can be used to decide which +database should receive a given request. + +At present, the only hint that will be provided is ``instance``, an +object instance that is related to the read or write operation that is +underway. This might be the instance that is being saved, or it might +be an instance that is being added in a many-to-many relation. In some +cases, no instance hint will be provided at all. The router check for +the existence of an instance hint, and determine if hat hint should be +used to alter routing behavior. + +Using routers +------------- + +Database routers are installed using the :setting:`DATABASE_ROUTERS` +setting. This setting defines a list of class names, each specifying a +router that should be used by the master router +(``django.db.router``). + +The master router is used by Django's database operations to allocate +database usage. Whenever a query needs to know which database to use, +it calls the master router, providing a model and a hint (if +available). Django then tries each router in turn until a database +suggestion can be found. If no suggestion can be found, it tries the +current ``_state.db`` of the hint instance. If a hint instance wasn't +provided, or the instance doesn't currently have database state, the +master router will allocate the ``default`` database. + +An example +---------- + +.. admonition:: Example purposes only! + + This example is intended as a demonstration of how the router + infrastructure can be used to alter database usage. It + intentionally ignores some complex issues in order to + demonstrate how routers are used. + + The approach of splitting ``contrib.auth`` onto a different + database won't actually work on Postgres, Oracle, or MySQL with + InnoDB tables. ForeignKeys to a remote database won't work due as + they introduce referential integrity problems. If you're using + SQLite or MySQL with MyISAM tables, there is no referential + integrity checking, so you will be able to define cross-database + foreign keys. + + The master/slave configuration described is also flawed -- it + doesn't provide any solution for handling replication lag (i.e., + query inconsistencies introduced because of the time taken for a + write to propagate to the slaves). It also doesn't consider the + interaction of transactions with the database utiliztion strategy. + +So - what does this mean in practice? Say you want ``contrib.auth`` to +exist on the 'credentials' database, and you want all other models in a +master/slave relationship between the databses 'master', 'slave1' and +'slave2'. To implement this, you would need 2 routers:: + + class AuthRouter(object): + """A router to control all database operations on models in + the contrib.auth application""" + + def db_for_read(self, model, **hints): + "Point all operations on auth models to 'credentials'" + if model._meta.app_label == 'auth': + return 'credentials' + return None + + def db_for_write(self, model, **hints): + "Point all operations on auth models to 'credentials'" + if model._meta.app_label == 'auth': + return 'credentials' + return None + + def allow_relation(self, obj1, obj2, **hints): + "Allow any relation if a model in Auth is involved" + if obj1._meta.app_label == 'auth' or obj2._meta.app_label == 'auth': + return True + return None + + + class MasterSlaveRouter(object): + """A router that sets up a simple master/slave configuration""" + + def db_for_read(self, model, **hints): + "Point all read operations to a random slave" + return random.choice(['slave1','slave2']) + + def db_for_write(self, model, **hints): + "Point all write operations to the master" + return 'master' + + def allow_relation(self, obj1, obj2, **hints): + "Allow any relation between two objects in the db pool" + db_list = ('master','slave1','slave2') + if obj1 in db_list and obj2 in db_list: + return True + return None + +Then, in your settings file, add the following (substituting ``path.to.`` with +the actual python path to the module where you define the routers):: + + DATABASE_ROUTERS = ['path.to.AuthRouter', 'path.to.MasterSlaveRouter'] + +With this setup installed, lets run some Django code:: + + >>> # This retrieval will be performed on the 'credentials' database + >>> fred = User.objects.get(username='fred') + >>> fred.first_name = 'Frederick' + + >>> # This save will also be directed to 'credentials' + >>> fred.save() + + >>> # These retrieval will be randomly allocated to a slave database + >>> dna = Person.objects.get(name='Douglas Adams') + + >>> # A new object has no database allocation when created + >>> mh = Book(title='Mostly Harmless') + + >>> # This assignment will consult the router, and set mh onto + >>> # the same database as the author object + >>> mh.author = dna + + >>> # This save will force the 'mh' instance onto the master database... + >>> mh.save() + + >>> # ... but if we re-retrieve the object, it will come back on a slave + >>> mh = Book.objects.get(title='Mostly Harmless') + +Manually selecting a database +============================= + +Django also provides an API that allows you to maintain complete control +over database usage in your code. A manually specified database allocation +will take priority over a database allocated by a router. + +Manually selecting a database for a ``QuerySet`` +------------------------------------------------ + +You can select the database for a ``QuerySet`` at any point in the +``QuerySet`` "chain." Just call ``using()`` on the ``QuerySet`` to get +another ``QuerySet`` that uses the specified database. + +``using()`` takes a single argument: the alias of the database on +which you want to run the query. For example:: + + >>> # This will run on the 'default' database. >>> Author.objects.all() - - # So will this. + + >>> # So will this. >>> Author.objects.using('default').all() - - # This will run on the 'other' database. + + >>> # This will run on the 'other' database. >>> Author.objects.using('other').all() Selecting a database for ``save()`` -=================================== +----------------------------------- -Use the ``using`` keyword to ``Model.save()`` to specify to which database the -data should be saved. +Use the ``using`` keyword to ``Model.save()`` to specify to which +database the data should be saved. -For example, to save an object to the ``legacy_users`` database, you'd use this:: +For example, to save an object to the ``legacy_users`` database, you'd +use this:: >>> my_object.save(using='legacy_users') -If you don't specify ``using``, the ``save()`` method will always save into the -default database. +If you don't specify ``using``, the ``save()`` method will save into +the default database allocated by the routers. Moving an object from one database to another ---------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you've saved an instance to one database, it might be tempting to use -``save(using=...)`` as a way to migrate the instance to a new database. However, -if you don't take appropriate steps, this could have some unexpected consequences. +If you've saved an instance to one database, it might be tempting to +use ``save(using=...)`` as a way to migrate the instance to a new +database. However, if you don't take appropriate steps, this could +have some unexpected consequences. Consider the following example:: @@ -149,16 +340,17 @@ However, if the primary key of ``p`` is already in use on the will be overridden when ``p`` is saved. You can avoid this in two ways. First, you can clear the primary key -of the instance. If an object has no primary key, Django will treat it as -a new object, avoiding any loss of data on the ``second`` database:: +of the instance. If an object has no primary key, Django will treat it +as a new object, avoiding any loss of data on the ``second`` +database:: >>> p = Person(name='Fred') >>> p.save(using='first') >>> p.pk = None # Clear the primary key. >>> p.save(using='second') # Write a completely new object. -The second option is to use the ``force_insert`` option to ``save()`` to ensure -that Django does a SQL ``INSERT``:: +The second option is to use the ``force_insert`` option to ``save()`` +to ensure that Django does a SQL ``INSERT``:: >>> p = Person(name='Fred') >>> p.save(using='first') @@ -170,51 +362,53 @@ when you try to save onto the ``second`` database, an error will be raised. Selecting a database to delete from -=================================== +----------------------------------- -By default, a call to delete an existing object will be executed on the -same database that was used to retrieve the object in the first place:: +By default, a call to delete an existing object will be executed on +the same database that was used to retrieve the object in the first +place:: >>> u = User.objects.using('legacy_users').get(username='fred') >>> u.delete() # will delete from the `legacy_users` database To specify the database from which a model will be deleted, pass a -``using`` keyword argument to the ``Model.delete()`` method. This argument -works just like the ``using`` keyword argument to ``save()``. +``using`` keyword argument to the ``Model.delete()`` method. This +argument works just like the ``using`` keyword argument to ``save()``. -For example, if you're migrating a user from the ``legacy_users`` database -to the ``new_users`` database, you might use these commands:: +For example, if you're migrating a user from the ``legacy_users`` +database to the ``new_users`` database, you might use these commands:: >>> user_obj.save(using='new_users') >>> user_obj.delete(using='legacy_users') Using managers with multiple databases -====================================== +-------------------------------------- -Use the ``db_manager()`` method on managers to give managers access to a -non-default database. +Use the ``db_manager()`` method on managers to give managers access to +a non-default database. -For example, say you have a custom manager method that touches the database -- -``User.objects.create_user()``. Because ``create_user()`` is a -manager method, not a ``QuerySet`` method, you can't do -``User.objects.using('new_users').create_user()``. (The ``create_user()`` method -is only available on ``User.objects``, the manager, not on ``QuerySet`` objects -derived from the manager.) The solution is to use ``db_manager()``, like this:: +For example, say you have a custom manager method that touches the +database -- ``User.objects.create_user()``. Because ``create_user()`` +is a manager method, not a ``QuerySet`` method, you can't do +``User.objects.using('new_users').create_user()``. (The +``create_user()`` method is only available on ``User.objects``, the +manager, not on ``QuerySet`` objects derived from the manager.) The +solution is to use ``db_manager()``, like this:: User.objects.db_manager('new_users').create_user(...) ``db_manager()`` returns a copy of the manager bound to the database you specify. Using ``get_query_set()`` with multiple databases -------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you're overriding ``get_query_set()`` on your manager, be sure to either -call the method on the parent (using ``super()``) or do the appropriate -handling of the ``_db`` attribute on the manager (a string containing the name -of the database to use). +If you're overriding ``get_query_set()`` on your manager, be sure to +either call the method on the parent (using ``super()``) or do the +appropriate handling of the ``_db`` attribute on the manager (a string +containing the name of the database to use). -For example, if you want to return a custom ``QuerySet`` class from the -``get_query_set`` method, you could do this:: +For example, if you want to return a custom ``QuerySet`` class from +the ``get_query_set`` method, you could do this:: class MyManager(models.Manager): def get_query_set(self): @@ -228,9 +422,9 @@ Exposing multiple databases in Django's admin interface Django's admin doesn't have any explicit support for multiple databases. If you want to provide an admin interface for a model on a -database other than ``default``, you'll need to write custom -:class:`~django.contrib.admin.ModelAdmin` classes that will direct the -admin to use a specific database for content. +database other than that that specified by your router chain, you'll +need to write custom :class:`~django.contrib.admin.ModelAdmin` classes +that will direct the admin to use a specific database for content. ``ModelAdmin`` objects have four methods that require customization for multiple-database support:: @@ -257,11 +451,11 @@ multiple-database support:: # on the 'other' database. return super(MultiDBModelAdmin, self).formfield_for_manytomany(db_field, request=request, using=self.using, **kwargs) -The implementation provided here implements a multi-database strategy where -all objects of a given type are stored on a specific database (e.g., -all ``User`` objects are in the ``other`` database). If your usage of -multiple databases is more complex, your ``ModelAdmin`` will need to reflect -that strategy. +The implementation provided here implements a multi-database strategy +where all objects of a given type are stored on a specific database +(e.g., all ``User`` objects are in the ``other`` database). If your +usage of multiple databases is more complex, your ``ModelAdmin`` will +need to reflect that strategy. Inlines can be handled in a similar fashion. They require three customized methods:: @@ -282,8 +476,8 @@ Inlines can be handled in a similar fashion. They require three customized metho # on the 'other' database. return super(MultiDBTabularInline, self).formfield_for_manytomany(db_field, request=request, using=self.using, **kwargs) -Once you've written your model admin definitions, they can be registered with -any ``Admin`` instance:: +Once you've written your model admin definitions, they can be +registered with any ``Admin`` instance:: from django.contrib import admin diff --git a/tests/regressiontests/multiple_database/models.py b/tests/regressiontests/multiple_database/models.py index 33cb7fb745..318ff85ea0 100644 --- a/tests/regressiontests/multiple_database/models.py +++ b/tests/regressiontests/multiple_database/models.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic -from django.db import models, DEFAULT_DB_ALIAS +from django.db import models class Review(models.Model): source = models.CharField(max_length=100) @@ -36,6 +36,7 @@ class Book(models.Model): authors = models.ManyToManyField(Person) editor = models.ForeignKey(Person, null=True, related_name='edited') reviews = generic.GenericRelation(Review) + pages = models.IntegerField(default=100) def __unicode__(self): return self.title diff --git a/tests/regressiontests/multiple_database/tests.py b/tests/regressiontests/multiple_database/tests.py index 300ed5e0a6..a7c22c0cdb 100644 --- a/tests/regressiontests/multiple_database/tests.py +++ b/tests/regressiontests/multiple_database/tests.py @@ -3,7 +3,8 @@ import pickle from django.conf import settings from django.contrib.auth.models import User -from django.db import connections +from django.db import connections, router, DEFAULT_DB_ALIAS +from django.db.utils import ConnectionRouter from django.test import TestCase from models import Book, Person, Review, UserProfile @@ -18,6 +19,16 @@ except ImportError: class QueryTestCase(TestCase): multi_db = True + def test_db_selection(self): + "Check that querysets will use the default databse by default" + self.assertEquals(Book.objects.db, DEFAULT_DB_ALIAS) + self.assertEquals(Book.objects.all().db, DEFAULT_DB_ALIAS) + + self.assertEquals(Book.objects.using('other').db, 'other') + + self.assertEquals(Book.objects.db_manager('other').db, 'other') + self.assertEquals(Book.objects.db_manager('other').all().db, 'other') + def test_default_creation(self): "Objects created on the default database don't leak onto other databases" # Create a book on the default database using create() @@ -259,53 +270,53 @@ class QueryTestCase(TestCase): self.assertEquals(list(Person.objects.using('other').filter(book__title='Dive into HTML5').values_list('name', flat=True)), [u'Mark Pilgrim']) -# def test_m2m_cross_database_protection(self): -# "Operations that involve sharing M2M objects across databases raise an error" -# # Create a book and author on the default database -# pro = Book.objects.create(title="Pro Django", -# published=datetime.date(2008, 12, 16)) + def test_m2m_cross_database_protection(self): + "Operations that involve sharing M2M objects across databases raise an error" + # Create a book and author on the default database + pro = Book.objects.create(title="Pro Django", + published=datetime.date(2008, 12, 16)) -# marty = Person.objects.create(name="Marty Alchin") + marty = Person.objects.create(name="Marty Alchin") -# # Create a book and author on the other database -# dive = Book.objects.using('other').create(title="Dive into Python", -# published=datetime.date(2009, 5, 4)) + # Create a book and author on the other database + dive = Book.objects.using('other').create(title="Dive into Python", + published=datetime.date(2009, 5, 4)) -# mark = Person.objects.using('other').create(name="Mark Pilgrim") -# # Set a foreign key set with an object from a different database -# try: -# marty.book_set = [pro, dive] -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + mark = Person.objects.using('other').create(name="Mark Pilgrim") + # Set a foreign key set with an object from a different database + try: + marty.book_set = [pro, dive] + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Add to an m2m with an object from a different database -# try: -# marty.book_set.add(dive) -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Add to an m2m with an object from a different database + try: + marty.book_set.add(dive) + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Set a m2m with an object from a different database -# try: -# marty.book_set = [pro, dive] -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Set a m2m with an object from a different database + try: + marty.book_set = [pro, dive] + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Add to a reverse m2m with an object from a different database -# try: -# dive.authors.add(marty) -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Add to a reverse m2m with an object from a different database + try: + dive.authors.add(marty) + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Set a reverse m2m with an object from a different database -# try: -# dive.authors = [mark, marty] -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Set a reverse m2m with an object from a different database + try: + dive.authors = [mark, marty] + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass def test_foreign_key_separation(self): "FK fields are constrained to a single database" @@ -401,88 +412,88 @@ class QueryTestCase(TestCase): self.assertEquals(list(Person.objects.using('other').filter(edited__title='Dive into Python').values_list('name', flat=True)), []) -# def test_foreign_key_cross_database_protection(self): -# "Operations that involve sharing FK objects across databases raise an error" -# # Create a book and author on the default database -# pro = Book.objects.create(title="Pro Django", -# published=datetime.date(2008, 12, 16)) + def test_foreign_key_cross_database_protection(self): + "Operations that involve sharing FK objects across databases raise an error" + # Create a book and author on the default database + pro = Book.objects.create(title="Pro Django", + published=datetime.date(2008, 12, 16)) -# marty = Person.objects.create(name="Marty Alchin") + marty = Person.objects.create(name="Marty Alchin") -# # Create a book and author on the other database -# dive = Book.objects.using('other').create(title="Dive into Python", -# published=datetime.date(2009, 5, 4)) + # Create a book and author on the other database + dive = Book.objects.using('other').create(title="Dive into Python", + published=datetime.date(2009, 5, 4)) -# mark = Person.objects.using('other').create(name="Mark Pilgrim") + mark = Person.objects.using('other').create(name="Mark Pilgrim") -# # Set a foreign key with an object from a different database -# try: -# dive.editor = marty -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Set a foreign key with an object from a different database + try: + dive.editor = marty + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Set a foreign key set with an object from a different database -# try: -# marty.edited = [pro, dive] -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Set a foreign key set with an object from a different database + try: + marty.edited = [pro, dive] + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Add to a foreign key set with an object from a different database -# try: -# marty.edited.add(dive) -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Add to a foreign key set with an object from a different database + try: + marty.edited.add(dive) + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # BUT! if you assign a FK object when the base object hasn't -# # been saved yet, you implicitly assign the database for the -# # base object. -# chris = Person(name="Chris Mills") -# html5 = Book(title="Dive into HTML5", published=datetime.date(2010, 3, 15)) -# # initially, no db assigned -# self.assertEquals(chris._state.db, None) -# self.assertEquals(html5._state.db, None) + # BUT! if you assign a FK object when the base object hasn't + # been saved yet, you implicitly assign the database for the + # base object. + chris = Person(name="Chris Mills") + html5 = Book(title="Dive into HTML5", published=datetime.date(2010, 3, 15)) + # initially, no db assigned + self.assertEquals(chris._state.db, None) + self.assertEquals(html5._state.db, None) -# # old object comes from 'other', so the new object is set to use 'other'... -# dive.editor = chris -# html5.editor = mark -# # self.assertEquals(chris._state.db, 'other') -# self.assertEquals(html5._state.db, 'other') -# # ... but it isn't saved yet -# self.assertEquals(list(Person.objects.using('other').values_list('name',flat=True)), -# [u'Mark Pilgrim']) -# self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), -# [u'Dive into Python']) + # old object comes from 'other', so the new object is set to use 'other'... + dive.editor = chris + html5.editor = mark + self.assertEquals(chris._state.db, 'other') + self.assertEquals(html5._state.db, 'other') + # ... but it isn't saved yet + self.assertEquals(list(Person.objects.using('other').values_list('name',flat=True)), + [u'Mark Pilgrim']) + self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), + [u'Dive into Python']) -# # When saved (no using required), new objects goes to 'other' -# chris.save() -# html5.save() -# self.assertEquals(list(Person.objects.using('default').values_list('name',flat=True)), -# [u'Marty Alchin']) -# self.assertEquals(list(Person.objects.using('other').values_list('name',flat=True)), -# [u'Chris Mills', u'Mark Pilgrim']) -# self.assertEquals(list(Book.objects.using('default').values_list('title',flat=True)), -# [u'Pro Django']) -# self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), -# [u'Dive into HTML5', u'Dive into Python']) + # When saved (no using required), new objects goes to 'other' + chris.save() + html5.save() + self.assertEquals(list(Person.objects.using('default').values_list('name',flat=True)), + [u'Marty Alchin']) + self.assertEquals(list(Person.objects.using('other').values_list('name',flat=True)), + [u'Chris Mills', u'Mark Pilgrim']) + self.assertEquals(list(Book.objects.using('default').values_list('title',flat=True)), + [u'Pro Django']) + self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), + [u'Dive into HTML5', u'Dive into Python']) -# # This also works if you assign the FK in the constructor -# water = Book(title="Dive into Water", published=datetime.date(2001, 1, 1), editor=mark) -# self.assertEquals(water._state.db, 'other') -# # ... but it isn't saved yet -# self.assertEquals(list(Book.objects.using('default').values_list('title',flat=True)), -# [u'Pro Django']) -# self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), -# [u'Dive into HTML5', u'Dive into Python']) + # This also works if you assign the FK in the constructor + water = Book(title="Dive into Water", published=datetime.date(2001, 1, 1), editor=mark) + self.assertEquals(water._state.db, 'other') + # ... but it isn't saved yet + self.assertEquals(list(Book.objects.using('default').values_list('title',flat=True)), + [u'Pro Django']) + self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), + [u'Dive into HTML5', u'Dive into Python']) -# # When saved, the new book goes to 'other' -# water.save() -# self.assertEquals(list(Book.objects.using('default').values_list('title',flat=True)), -# [u'Pro Django']) -# self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), -# [u'Dive into HTML5', u'Dive into Python', u'Dive into Water']) + # When saved, the new book goes to 'other' + water.save() + self.assertEquals(list(Book.objects.using('default').values_list('title',flat=True)), + [u'Pro Django']) + self.assertEquals(list(Book.objects.using('other').values_list('title',flat=True)), + [u'Dive into HTML5', u'Dive into Python', u'Dive into Water']) def test_generic_key_separation(self): "Generic fields are constrained to a single database" @@ -555,56 +566,56 @@ class QueryTestCase(TestCase): self.assertEquals(list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source', flat=True)), [u'Python Daily']) -# def test_generic_key_cross_database_protection(self): -## "Operations that involve sharing FK objects across databases raise an error" -## # Create a book and author on the default database -## pro = Book.objects.create(title="Pro Django", -## published=datetime.date(2008, 12, 16)) + def test_generic_key_cross_database_protection(self): + "Operations that involve sharing generic key objects across databases raise an error" + # Create a book and author on the default database + pro = Book.objects.create(title="Pro Django", + published=datetime.date(2008, 12, 16)) -## review1 = Review.objects.create(source="Python Monthly", content_object=pro) + review1 = Review.objects.create(source="Python Monthly", content_object=pro) -## # Create a book and author on the other database -## dive = Book.objects.using('other').create(title="Dive into Python", -## published=datetime.date(2009, 5, 4)) + # Create a book and author on the other database + dive = Book.objects.using('other').create(title="Dive into Python", + published=datetime.date(2009, 5, 4)) -## review2 = Review.objects.using('other').create(source="Python Weekly", content_object=dive) + review2 = Review.objects.using('other').create(source="Python Weekly", content_object=dive) -## # Set a foreign key with an object from a different database -## try: -## review1.content_object = dive -## self.fail("Shouldn't be able to assign across databases") -## except ValueError: -## pass + # Set a foreign key with an object from a different database + try: + review1.content_object = dive + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # Add to a foreign key set with an object from a different database -# try: -# dive.reviews.add(review1) -# self.fail("Shouldn't be able to assign across databases") -# except ValueError: -# pass + # Add to a foreign key set with an object from a different database + try: + dive.reviews.add(review1) + self.fail("Shouldn't be able to assign across databases") + except ValueError: + pass -# # BUT! if you assign a FK object when the base object hasn't -# # been saved yet, you implicitly assign the database for the -# # base object. -# review3 = Review(source="Python Daily") -# # initially, no db assigned -# self.assertEquals(review3._state.db, None) + # BUT! if you assign a FK object when the base object hasn't + # been saved yet, you implicitly assign the database for the + # base object. + review3 = Review(source="Python Daily") + # initially, no db assigned + self.assertEquals(review3._state.db, None) -# # Dive comes from 'other', so review3 is set to use 'other'... -# review3.content_object = dive -# self.assertEquals(review3._state.db, 'other') -# # ... but it isn't saved yet -# self.assertEquals(list(Review.objects.using('default').filter(object_id=pro.pk).values_list('source', flat=True)), -# [u'Python Monthly']) -# self.assertEquals(list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source',flat=True)), -# [u'Python Weekly']) + # Dive comes from 'other', so review3 is set to use 'other'... + review3.content_object = dive + self.assertEquals(review3._state.db, 'other') + # ... but it isn't saved yet + self.assertEquals(list(Review.objects.using('default').filter(object_id=pro.pk).values_list('source', flat=True)), + [u'Python Monthly']) + self.assertEquals(list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source',flat=True)), + [u'Python Weekly']) -# # When saved, John goes to 'other' -# review3.save() -# self.assertEquals(list(Review.objects.using('default').filter(object_id=pro.pk).values_list('source', flat=True)), -# [u'Python Monthly']) -# self.assertEquals(list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source',flat=True)), -# [u'Python Daily', u'Python Weekly']) + # When saved, John goes to 'other' + review3.save() + self.assertEquals(list(Review.objects.using('default').filter(object_id=pro.pk).values_list('source', flat=True)), + [u'Python Monthly']) + self.assertEquals(list(Review.objects.using('other').filter(object_id=dive.pk).values_list('source',flat=True)), + [u'Python Daily', u'Python Weekly']) def test_ordering(self): "get_next_by_XXX commands stick to a single database" @@ -630,6 +641,388 @@ class QueryTestCase(TestCase): val = Book.objects.raw('SELECT id FROM "multiple_database_book"').using('other') self.assertEqual(map(lambda o: o.pk, val), [dive.pk]) +class TestRouter(object): + # A test router. The behaviour is vaguely master/slave, but the + # databases aren't assumed to propagate changes. + def db_for_read(self, model, instance=None, **hints): + if instance: + return instance._state.db or 'other' + return 'other' + + def db_for_write(self, model, **hints): + return DEFAULT_DB_ALIAS + + def allow_relation(self, obj1, obj2, **hints): + return obj1._state.db in ('default', 'other') and obj2._state.db in ('default', 'other') + +class RouterTestCase(TestCase): + multi_db = True + + def setUp(self): + # Make the 'other' database appear to be a slave of the 'default' + self.old_routers = router.routers + router.routers = [TestRouter()] + + def tearDown(self): + # Restore the 'other' database as an independent database + router.routers = self.old_routers + + def test_db_selection(self): + "Check that querysets obey the router for db suggestions" + self.assertEquals(Book.objects.db, 'other') + self.assertEquals(Book.objects.all().db, 'other') + + self.assertEquals(Book.objects.using('default').db, 'default') + + self.assertEquals(Book.objects.db_manager('default').db, 'default') + self.assertEquals(Book.objects.db_manager('default').all().db, 'default') + + def test_database_routing(self): + marty = Person.objects.using('default').create(name="Marty Alchin") + pro = Book.objects.using('default').create(title="Pro Django", + published=datetime.date(2008, 12, 16), + editor=marty) + pro.authors = [marty] + + # Create a book and author on the other database + dive = Book.objects.using('other').create(title="Dive into Python", + published=datetime.date(2009, 5, 4)) + + # An update query will be routed to the default database + Book.objects.filter(title='Pro Django').update(pages=200) + + try: + # By default, the get query will be directed to 'other' + Book.objects.get(title='Pro Django') + self.fail("Shouldn't be able to find the book") + except Book.DoesNotExist: + pass + + # But the same query issued explicitly at a database will work. + pro = Book.objects.using('default').get(title='Pro Django') + + # Check that the update worked. + self.assertEquals(pro.pages, 200) + + # An update query with an explicit using clause will be routed + # to the requested database. + Book.objects.using('other').filter(title='Dive into Python').update(pages=300) + self.assertEquals(Book.objects.get(title='Dive into Python').pages, 300) + + # Related object queries stick to the same database + # as the original object, regardless of the router + self.assertEquals(list(pro.authors.values_list('name', flat=True)), [u'Marty Alchin']) + self.assertEquals(pro.editor.name, u'Marty Alchin') + + # get_or_create is a special case. The get needs to be targetted at + # the write database in order to avoid potential transaction + # consistency problems + book, created = Book.objects.get_or_create(title="Pro Django") + self.assertFalse(created) + + book, created = Book.objects.get_or_create(title="Dive Into Python", + defaults={'published':datetime.date(2009, 5, 4)}) + self.assertTrue(created) + + # Check the head count of objects + self.assertEquals(Book.objects.using('default').count(), 2) + self.assertEquals(Book.objects.using('other').count(), 1) + # If a database isn't specified, the read database is used + self.assertEquals(Book.objects.count(), 1) + + # A delete query will also be routed to the default database + Book.objects.filter(pages__gt=150).delete() + + # The default database has lost the book. + self.assertEquals(Book.objects.using('default').count(), 1) + self.assertEquals(Book.objects.using('other').count(), 1) + + def test_foreign_key_cross_database_protection(self): + "Foreign keys can cross databases if they two databases have a common source" + # Create a book and author on the default database + pro = Book.objects.using('default').create(title="Pro Django", + published=datetime.date(2008, 12, 16)) + + marty = Person.objects.using('default').create(name="Marty Alchin") + + # Create a book and author on the other database + dive = Book.objects.using('other').create(title="Dive into Python", + published=datetime.date(2009, 5, 4)) + + mark = Person.objects.using('other').create(name="Mark Pilgrim") + + # Set a foreign key with an object from a different database + try: + dive.editor = marty + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments of original objects haven't changed... + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(mark._state.db, 'other') + + # ... but they will when the affected object is saved. + dive.save() + self.assertEquals(dive._state.db, 'default') + + # ...and the source database now has a copy of any object saved + try: + Book.objects.using('default').get(title='Dive into Python').delete() + except Book.DoesNotExist: + self.fail('Source database should have a copy of saved object') + + # This isn't a real master-slave database, so restore the original from other + dive = Book.objects.using('other').get(title='Dive into Python') + self.assertEquals(dive._state.db, 'other') + + # Set a foreign key set with an object from a different database + try: + marty.edited = [pro, dive] + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Assignment implies a save, so database assignments of original objects have changed... + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'default') + self.assertEquals(mark._state.db, 'other') + + # ...and the source database now has a copy of any object saved + try: + Book.objects.using('default').get(title='Dive into Python').delete() + except Book.DoesNotExist: + self.fail('Source database should have a copy of saved object') + + # This isn't a real master-slave database, so restore the original from other + dive = Book.objects.using('other').get(title='Dive into Python') + self.assertEquals(dive._state.db, 'other') + + # Add to a foreign key set with an object from a different database + try: + marty.edited.add(dive) + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Add implies a save, so database assignments of original objects have changed... + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'default') + self.assertEquals(mark._state.db, 'other') + + # ...and the source database now has a copy of any object saved + try: + Book.objects.using('default').get(title='Dive into Python').delete() + except Book.DoesNotExist: + self.fail('Source database should have a copy of saved object') + + # This isn't a real master-slave database, so restore the original from other + dive = Book.objects.using('other').get(title='Dive into Python') + + # If you assign a FK object when the base object hasn't + # been saved yet, you implicitly assign the database for the + # base object. + chris = Person(name="Chris Mills") + html5 = Book(title="Dive into HTML5", published=datetime.date(2010, 3, 15)) + # initially, no db assigned + self.assertEquals(chris._state.db, None) + self.assertEquals(html5._state.db, None) + + # old object comes from 'other', so the new object is set to use the + # source of 'other'... + self.assertEquals(dive._state.db, 'other') + dive.editor = chris + html5.editor = mark + + self.assertEquals(dive._state.db, 'other') + self.assertEquals(mark._state.db, 'other') + self.assertEquals(chris._state.db, 'default') + self.assertEquals(html5._state.db, 'default') + + # This also works if you assign the FK in the constructor + water = Book(title="Dive into Water", published=datetime.date(2001, 1, 1), editor=mark) + self.assertEquals(water._state.db, 'default') + + def test_m2m_cross_database_protection(self): + "M2M relations can cross databases if the database share a source" + # Create books and authors on the inverse to the usual database + pro = Book.objects.using('other').create(pk=1, title="Pro Django", + published=datetime.date(2008, 12, 16)) + + marty = Person.objects.using('other').create(pk=1, name="Marty Alchin") + + dive = Book.objects.using('default').create(pk=2, title="Dive into Python", + published=datetime.date(2009, 5, 4)) + + mark = Person.objects.using('default').create(pk=2, name="Mark Pilgrim") + + # Now save back onto the usual databse. + # This simulates master/slave - the objects exist on both database, + # but the _state.db is as it is for all other tests. + pro.save(using='default') + marty.save(using='default') + dive.save(using='other') + mark.save(using='other') + + # Check that we have 2 of both types of object on both databases + self.assertEquals(Book.objects.using('default').count(), 2) + self.assertEquals(Book.objects.using('other').count(), 2) + self.assertEquals(Person.objects.using('default').count(), 2) + self.assertEquals(Person.objects.using('other').count(), 2) + + # Set a m2m set with an object from a different database + try: + marty.book_set = [pro, dive] + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments don't change + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(mark._state.db, 'other') + + # All m2m relations should be saved on the default database + self.assertEquals(Book.authors.through.objects.using('default').count(), 2) + self.assertEquals(Book.authors.through.objects.using('other').count(), 0) + + # Reset relations + Book.authors.through.objects.using('default').delete() + + # Add to an m2m with an object from a different database + try: + marty.book_set.add(dive) + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments don't change + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(mark._state.db, 'other') + + # All m2m relations should be saved on the default database + self.assertEquals(Book.authors.through.objects.using('default').count(), 1) + self.assertEquals(Book.authors.through.objects.using('other').count(), 0) + + # Reset relations + Book.authors.through.objects.using('default').delete() + + # Set a reverse m2m with an object from a different database + try: + dive.authors = [mark, marty] + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments don't change + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(mark._state.db, 'other') + + # All m2m relations should be saved on the default database + self.assertEquals(Book.authors.through.objects.using('default').count(), 2) + self.assertEquals(Book.authors.through.objects.using('other').count(), 0) + + # Reset relations + Book.authors.through.objects.using('default').delete() + + self.assertEquals(Book.authors.through.objects.using('default').count(), 0) + self.assertEquals(Book.authors.through.objects.using('other').count(), 0) + + # Add to a reverse m2m with an object from a different database + try: + dive.authors.add(marty) + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments don't change + self.assertEquals(marty._state.db, 'default') + self.assertEquals(pro._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(mark._state.db, 'other') + + # All m2m relations should be saved on the default database + self.assertEquals(Book.authors.through.objects.using('default').count(), 1) + self.assertEquals(Book.authors.through.objects.using('other').count(), 0) + + def test_generic_key_cross_database_protection(self): + "Generic Key operations can span databases if they share a source" + # Create a book and author on the default database + pro = Book.objects.using('default' + ).create(title="Pro Django", published=datetime.date(2008, 12, 16)) + + review1 = Review.objects.using('default' + ).create(source="Python Monthly", content_object=pro) + + # Create a book and author on the other database + dive = Book.objects.using('other' + ).create(title="Dive into Python", published=datetime.date(2009, 5, 4)) + + review2 = Review.objects.using('other' + ).create(source="Python Weekly", content_object=dive) + + # Set a generic foreign key with an object from a different database + try: + review1.content_object = dive + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments of original objects haven't changed... + self.assertEquals(pro._state.db, 'default') + self.assertEquals(review1._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(review2._state.db, 'other') + + # ... but they will when the affected object is saved. + dive.save() + self.assertEquals(review1._state.db, 'default') + self.assertEquals(dive._state.db, 'default') + + # ...and the source database now has a copy of any object saved + try: + Book.objects.using('default').get(title='Dive into Python').delete() + except Book.DoesNotExist: + self.fail('Source database should have a copy of saved object') + + # This isn't a real master-slave database, so restore the original from other + dive = Book.objects.using('other').get(title='Dive into Python') + self.assertEquals(dive._state.db, 'other') + + # Add to a generic foreign key set with an object from a different database + try: + dive.reviews.add(review1) + except ValueError: + self.fail("Assignment across master/slave databases with a common source should be ok") + + # Database assignments of original objects haven't changed... + self.assertEquals(pro._state.db, 'default') + self.assertEquals(review1._state.db, 'default') + self.assertEquals(dive._state.db, 'other') + self.assertEquals(review2._state.db, 'other') + + # ... but they will when the affected object is saved. + dive.save() + self.assertEquals(dive._state.db, 'default') + + # ...and the source database now has a copy of any object saved + try: + Book.objects.using('default').get(title='Dive into Python').delete() + except Book.DoesNotExist: + self.fail('Source database should have a copy of saved object') + + # BUT! if you assign a FK object when the base object hasn't + # been saved yet, you implicitly assign the database for the + # base object. + review3 = Review(source="Python Daily") + # initially, no db assigned + self.assertEquals(review3._state.db, None) + + # Dive comes from 'other', so review3 is set to use the source of 'other'... + review3.content_object = dive + self.assertEquals(review3._state.db, 'default') + class UserProfileTestCase(TestCase): def setUp(self):