Added TransactionTestCase.available_apps.
This can be used to make Django's test suite significantly faster by reducing the number of models for which content types and permissions must be created and tables must be flushed in each non-transactional test. It's documented for Django contributors and committers but it's branded as a private API to preserve our freedom to change it in the future. Most of the credit goes to Anssi. He got the idea and did the research. Fixed #20483.
This commit is contained in:
Aymeric Augustin
parent
13b7f299de
commit
4daf570b98
|
|
@ -11,7 +11,7 @@ from django.contrib.auth import models as auth_app, get_user_model
|
|||
from django.core import exceptions
|
||||
from django.core.management.base import CommandError
|
||||
from django.db import DEFAULT_DB_ALIAS, router
|
||||
from django.db.models import get_models, signals
|
||||
from django.db.models import get_model, get_models, signals, UnavailableApp
|
||||
from django.utils.encoding import DEFAULT_LOCALE_ENCODING
|
||||
from django.utils import six
|
||||
from django.utils.six.moves import input
|
||||
|
|
@ -60,6 +60,11 @@ def _check_permission_clashing(custom, builtin, ctype):
|
|||
pool.add(codename)
|
||||
|
||||
def create_permissions(app, created_models, verbosity, db=DEFAULT_DB_ALIAS, **kwargs):
|
||||
try:
|
||||
get_model('auth', 'Permission')
|
||||
except UnavailableApp:
|
||||
return
|
||||
|
||||
if not router.allow_syncdb(db, auth_app.Permission):
|
||||
return
|
||||
|
||||
|
|
@ -101,9 +106,13 @@ def create_permissions(app, created_models, verbosity, db=DEFAULT_DB_ALIAS, **kw
|
|||
|
||||
|
||||
def create_superuser(app, created_models, verbosity, db, **kwargs):
|
||||
from django.core.management import call_command
|
||||
try:
|
||||
get_model('auth', 'Permission')
|
||||
UserModel = get_user_model()
|
||||
except UnavailableApp:
|
||||
return
|
||||
|
||||
UserModel = get_user_model()
|
||||
from django.core.management import call_command
|
||||
|
||||
if UserModel in created_models and kwargs.get('interactive', True):
|
||||
msg = ("\nYou just installed Django's auth system, which means you "
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db import DEFAULT_DB_ALIAS, router
|
||||
from django.db.models import get_apps, get_models, signals
|
||||
from django.db.models import get_apps, get_model, get_models, signals, UnavailableApp
|
||||
from django.utils.encoding import smart_text
|
||||
from django.utils import six
|
||||
from django.utils.six.moves import input
|
||||
|
|
@ -11,6 +11,11 @@ def update_contenttypes(app, created_models, verbosity=2, db=DEFAULT_DB_ALIAS, *
|
|||
Creates content types for models in the given app, removing any model
|
||||
entries that no longer have a matching model class.
|
||||
"""
|
||||
try:
|
||||
get_model('contenttypes', 'ContentType')
|
||||
except UnavailableApp:
|
||||
return
|
||||
|
||||
if not router.allow_syncdb(db, ContentType):
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from functools import wraps
|
||||
|
||||
from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured
|
||||
from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models
|
||||
from django.db.models.loading import get_apps, get_app_paths, get_app, get_models, get_model, register_models, UnavailableApp
|
||||
from django.db.models.query import Q
|
||||
from django.db.models.expressions import F
|
||||
from django.db.models.manager import Manager
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import os
|
|||
__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models',
|
||||
'load_app', 'app_cache_ready')
|
||||
|
||||
class UnavailableApp(Exception):
|
||||
pass
|
||||
|
||||
class AppCache(object):
|
||||
"""
|
||||
|
|
@ -43,6 +45,7 @@ class AppCache(object):
|
|||
postponed=[],
|
||||
nesting_level=0,
|
||||
_get_models_cache={},
|
||||
available_apps=None,
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
|
|
@ -135,12 +138,17 @@ class AppCache(object):
|
|||
"""
|
||||
self._populate()
|
||||
|
||||
apps = self.app_store.items()
|
||||
if self.available_apps is not None:
|
||||
apps = [elt for elt in apps
|
||||
if self._label_for(elt[0]) in self.available_apps]
|
||||
|
||||
# Ensure the returned list is always in the same order (with new apps
|
||||
# added at the end). This avoids unstable ordering on the admin app
|
||||
# list page, for example.
|
||||
apps = [(v, k) for k, v in self.app_store.items()]
|
||||
apps.sort()
|
||||
return [elt[1] for elt in apps]
|
||||
apps = sorted(apps, key=lambda elt: elt[1])
|
||||
|
||||
return [elt[0] for elt in apps]
|
||||
|
||||
def get_app_paths(self):
|
||||
"""
|
||||
|
|
@ -161,8 +169,12 @@ class AppCache(object):
|
|||
|
||||
def get_app(self, app_label, emptyOK=False):
|
||||
"""
|
||||
Returns the module containing the models for the given app_label. If
|
||||
the app has no models in it and 'emptyOK' is True, returns None.
|
||||
Returns the module containing the models for the given app_label.
|
||||
|
||||
Returns None if the app has no models in it and emptyOK is True.
|
||||
|
||||
Raises UnavailableApp when set_available_apps() in in effect and
|
||||
doesn't include app_label.
|
||||
"""
|
||||
self._populate()
|
||||
imp.acquire_lock()
|
||||
|
|
@ -170,12 +182,11 @@ class AppCache(object):
|
|||
for app_name in settings.INSTALLED_APPS:
|
||||
if app_label == app_name.split('.')[-1]:
|
||||
mod = self.load_app(app_name, False)
|
||||
if mod is None:
|
||||
if emptyOK:
|
||||
return None
|
||||
if mod is None and not emptyOK:
|
||||
raise ImproperlyConfigured("App with label %s is missing a models.py module." % app_label)
|
||||
else:
|
||||
return mod
|
||||
if self.available_apps is not None and app_label not in self.available_apps:
|
||||
raise UnavailableApp("App with label %s isn't available." % app_label)
|
||||
return mod
|
||||
raise ImproperlyConfigured("App with label %s could not be found" % app_label)
|
||||
finally:
|
||||
imp.release_lock()
|
||||
|
|
@ -209,8 +220,13 @@ class AppCache(object):
|
|||
include_swapped, they will be.
|
||||
"""
|
||||
cache_key = (app_mod, include_auto_created, include_deferred, only_installed, include_swapped)
|
||||
model_list = None
|
||||
try:
|
||||
return self._get_models_cache[cache_key]
|
||||
model_list = self._get_models_cache[cache_key]
|
||||
if self.available_apps is not None and only_installed:
|
||||
model_list = [m for m in model_list
|
||||
if m._meta.app_label in self.available_apps]
|
||||
return model_list
|
||||
except KeyError:
|
||||
pass
|
||||
self._populate()
|
||||
|
|
@ -235,6 +251,9 @@ class AppCache(object):
|
|||
(not model._meta.swapped or include_swapped))
|
||||
)
|
||||
self._get_models_cache[cache_key] = model_list
|
||||
if self.available_apps is not None and only_installed:
|
||||
model_list = [m for m in model_list
|
||||
if m._meta.app_label in self.available_apps]
|
||||
return model_list
|
||||
|
||||
def get_model(self, app_label, model_name,
|
||||
|
|
@ -244,12 +263,21 @@ class AppCache(object):
|
|||
model_name.
|
||||
|
||||
Returns None if no model is found.
|
||||
|
||||
Raises UnavailableApp when set_available_apps() in in effect and
|
||||
doesn't include app_label.
|
||||
"""
|
||||
if seed_cache:
|
||||
self._populate()
|
||||
if only_installed and app_label not in self.app_labels:
|
||||
return None
|
||||
return self.app_models.get(app_label, SortedDict()).get(model_name.lower())
|
||||
if (self.available_apps is not None and only_installed
|
||||
and app_label not in self.available_apps):
|
||||
raise UnavailableApp("App with label %s isn't available." % app_label)
|
||||
try:
|
||||
return self.app_models[app_label][model_name.lower()]
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
def register_models(self, app_label, *models):
|
||||
"""
|
||||
|
|
@ -274,6 +302,16 @@ class AppCache(object):
|
|||
model_dict[model_name] = model
|
||||
self._get_models_cache.clear()
|
||||
|
||||
def set_available_apps(self, available):
|
||||
if not set(available).issubset(set(settings.INSTALLED_APPS)):
|
||||
extra = set(available) - set(settings.INSTALLED_APPS)
|
||||
raise ValueError("Available apps isn't a subset of installed "
|
||||
"apps, extra apps: " + ", ".join(extra))
|
||||
self.available_apps = set(app.rsplit('.', 1)[-1] for app in available)
|
||||
|
||||
def unset_available_apps(self):
|
||||
self.available_apps = None
|
||||
|
||||
cache = AppCache()
|
||||
|
||||
# These methods were always module level, so are kept that way for backwards
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from django.core.servers.basehttp import (WSGIRequestHandler, WSGIServer,
|
|||
WSGIServerException)
|
||||
from django.core.urlresolvers import clear_url_caches, set_urlconf
|
||||
from django.db import connection, connections, DEFAULT_DB_ALIAS, transaction
|
||||
from django.db.models.loading import cache
|
||||
from django.forms.fields import CharField
|
||||
from django.http import QueryDict
|
||||
from django.test.client import Client
|
||||
|
|
@ -725,6 +726,9 @@ class TransactionTestCase(SimpleTestCase):
|
|||
# test case
|
||||
reset_sequences = False
|
||||
|
||||
# Subclasses can enable only a subset of apps for faster tests
|
||||
available_apps = None
|
||||
|
||||
def _pre_setup(self):
|
||||
"""Performs any pre-test setup. This includes:
|
||||
|
||||
|
|
@ -733,7 +737,14 @@ class TransactionTestCase(SimpleTestCase):
|
|||
named fixtures.
|
||||
"""
|
||||
super(TransactionTestCase, self)._pre_setup()
|
||||
self._fixture_setup()
|
||||
if self.available_apps is not None:
|
||||
cache.set_available_apps(self.available_apps)
|
||||
try:
|
||||
self._fixture_setup()
|
||||
except Exception:
|
||||
if self.available_apps is not None:
|
||||
cache.unset_available_apps()
|
||||
raise
|
||||
|
||||
def _databases_names(self, include_mirrors=True):
|
||||
# If the test case has a multi_db=True flag, act on all databases,
|
||||
|
|
@ -775,22 +786,27 @@ class TransactionTestCase(SimpleTestCase):
|
|||
* Force closing the connection, so that the next test gets
|
||||
a clean cursor.
|
||||
"""
|
||||
self._fixture_teardown()
|
||||
super(TransactionTestCase, self)._post_teardown()
|
||||
# Some DB cursors include SQL statements as part of cursor
|
||||
# creation. If you have a test that does rollback, the effect
|
||||
# of these statements is lost, which can effect the operation
|
||||
# of tests (e.g., losing a timezone setting causing objects to
|
||||
# be created with the wrong time).
|
||||
# To make sure this doesn't happen, get a clean connection at the
|
||||
# start of every test.
|
||||
for conn in connections.all():
|
||||
conn.close()
|
||||
try:
|
||||
self._fixture_teardown()
|
||||
super(TransactionTestCase, self)._post_teardown()
|
||||
# Some DB cursors include SQL statements as part of cursor
|
||||
# creation. If you have a test that does rollback, the effect of
|
||||
# these statements is lost, which can effect the operation of
|
||||
# tests (e.g., losing a timezone setting causing objects to be
|
||||
# created with the wrong time). To make sure this doesn't happen,
|
||||
# get a clean connection at the start of every test.
|
||||
for conn in connections.all():
|
||||
conn.close()
|
||||
finally:
|
||||
cache.unset_available_apps()
|
||||
|
||||
def _fixture_teardown(self):
|
||||
# Allow TRUNCATE ... CASCADE when flushing only a subset of the apps
|
||||
allow_cascade = self.available_apps is not None
|
||||
for db_name in self._databases_names(include_mirrors=False):
|
||||
call_command('flush', verbosity=0, interactive=False, database=db_name,
|
||||
skip_validation=True, reset_sequences=False)
|
||||
call_command('flush', verbosity=0, interactive=False,
|
||||
database=db_name, skip_validation=True,
|
||||
reset_sequences=False, allow_cascade=allow_cascade)
|
||||
|
||||
def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True):
|
||||
items = six.moves.map(transform, qs)
|
||||
|
|
|
|||
|
|
@ -985,6 +985,34 @@ to test the effects of commit and rollback:
|
|||
Using ``reset_sequences = True`` will slow down the test, since the primary
|
||||
key reset is an relatively expensive database operation.
|
||||
|
||||
.. attribute:: TransactionTestCase.available_apps
|
||||
|
||||
.. warning::
|
||||
|
||||
This attribute is a private API. It may be changed or removed without
|
||||
a deprecation period in the future, for instance to accomodate changes
|
||||
in application loading.
|
||||
|
||||
It's used to optimize Django's own test suite, which contains hundreds
|
||||
of models but no relations between models in different applications.
|
||||
|
||||
.. versionadded:: 1.6
|
||||
|
||||
By default, ``available_apps`` is set to ``None`` and has no effect.
|
||||
Setting it to a list of applications tells Django to behave as if only the
|
||||
models from these applications were available:
|
||||
|
||||
- Before each test, Django creates content types and permissions only for
|
||||
these models.
|
||||
- After each test, Django flushes only the corresponding tables. However,
|
||||
at the database level, truncation may cascade to other related models,
|
||||
even if they aren't in ``available_apps``.
|
||||
|
||||
Since the database isn't fully flushed, if a test creates instances of
|
||||
models not included in ``available_apps``, they will leak and they may
|
||||
cause unrelated tests to fail. Be careful with tests that use sessions;
|
||||
the default session engine stores them in the database.
|
||||
|
||||
TestCase
|
||||
~~~~~~~~
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue