p15693087
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #22487: Optional rollback emulation for migrated apps

This commit is contained in:
Andrew Godwin 2014-06-08 19:30:15 -07:00
parent 8721adcbfb
commit 8c12d51ea2
17 changed files with 246 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
django/conf/global_settings.py
View File

@ -578,6 +578,10 @@ DEFAULT_EXCEPTION_REPORTER_FILTER = 'django.views.debug.SafeExceptionReporterFil
# The name of the class to use to run the test suite
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
# Apps that don't need to be serialized at test database creation time
# (only apps with migrations are to start with)
TEST_NON_SERIALIZED_APPS = []
############
# FIXTURES #
############

9
django/core/management/commands/flush.py
View File

@ -22,10 +22,9 @@ class Command(NoArgsCommand):
make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True,
help='Tells Django not to load any initial data after database synchronization.'),
)
help = ('Returns the database to the state it was in immediately after '
'migrate was first executed. This means that all data will be removed '
'from the database, any post-migration handlers will be '
're-executed, and the initial_data fixture will be re-installed.')
help = ('Removes ALL DATA from the database, including data added during '
'migrations. Unmigrated apps will also have their initial_data '
'fixture reloaded. Does not achieve a "fresh install" state.')
def handle_noargs(self, **options):
database = options.get('database')
@ -54,7 +53,7 @@ class Command(NoArgsCommand):
if interactive:
confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to a fresh state.
and return each table to an empty state.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])

2
django/core/management/commands/testserver.py
View File

@ -27,7 +27,7 @@ class Command(BaseCommand):
addrport = options.get('addrport')
# Create a test database.
db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive)
db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False)
# Import the fixture data into the test database.
call_command('loaddata', *fixture_labels, **{'verbosity': verbosity})

75
django/db/backends/creation.py
View File

@ -7,6 +7,11 @@ from django.db.utils import load_backend
from django.utils.encoding import force_bytes
from django.utils.functional import cached_property
from django.utils.six.moves import input
from django.utils.six import StringIO
from django.core.management.commands.dumpdata import sort_dependencies
from django.db import router
from django.apps import apps
from django.core import serializers
from .utils import truncate_name
@ -332,7 +337,7 @@ class BaseDatabaseCreation(object):
";",
]
def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False, serialize=True):
"""
Creates a test database, prompting the user for confirmation if the
database already exists. Returns the name of the test database created.
@ -364,25 +369,31 @@ class BaseDatabaseCreation(object):
settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
self.connection.settings_dict["NAME"] = test_database_name
# Report migrate messages at one level lower than that requested.
# We report migrate messages at one level lower than that requested.
# This ensures we don't get flooded with messages during testing
# (unless you really ask to be flooded)
call_command('migrate',
# (unless you really ask to be flooded).
call_command(
'migrate',
verbosity=max(verbosity - 1, 0),
interactive=False,
database=self.connection.alias,
load_initial_data=False,
test_database=True)
test_database=True,
)
# We need to then do a flush to ensure that any data installed by
# custom SQL has been removed. The only test data should come from
# test fixtures, or autogenerated from post_migrate triggers.
# This has the side effect of loading initial data (which was
# intentionally skipped in the syncdb).
call_command('flush',
# We then serialize the current state of the database into a string
# and store it on the connection. This slightly horrific process is so people
# who are testing on databases without transactions or who are using
# a TransactionTestCase still get a clean database on every test run.
if serialize:
self.connection._test_serialized_contents = self.serialize_db_to_string()
# Finally, we flush the database to clean
call_command(
'flush',
verbosity=max(verbosity - 1, 0),
interactive=False,
database=self.connection.alias)
database=self.connection.alias
)
call_command('createcachetable', database=self.connection.alias)
@ -391,6 +402,44 @@ class BaseDatabaseCreation(object):
return test_database_name
def serialize_db_to_string(self):
"""
Serializes all data in the database into a JSON string.
Designed only for test runner usage; will not handle large
amounts of data.
"""
# Build list of all apps to serialize
from django.db.migrations.loader import MigrationLoader
loader = MigrationLoader(self.connection)
app_list = []
for app_config in apps.get_app_configs():
if (
app_config.models_module is not None and
app_config.label in loader.migrated_apps and
app_config.name not in settings.TEST_NON_SERIALIZED_APPS
):
app_list.append((app_config, None))
# Make a function to iteratively return every object
def get_objects():
for model in sort_dependencies(app_list):
if not model._meta.proxy and router.allow_migrate(self.connection.alias, model):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
# Serialise to a string
out = StringIO()
serializers.serialize("json", get_objects(), indent=None, stream=out)
return out.getvalue()
def deserialize_db_from_string(self, data):
"""
Reloads the database with data from a string generated by
the serialize_db_to_string method.
"""
data = StringIO(data)
for obj in serializers.deserialize("json", data, using=self.connection.alias):
obj.save()
def _get_test_db_name(self):
"""
Internal implementation - returns the name of the test DB that will be

6
django/test/runner.py
View File

@ -298,7 +298,11 @@ def setup_databases(verbosity, interactive, keepdb=False, **kwargs):
connection = connections[alias]
if test_db_name is None:
test_db_name = connection.creation.create_test_db(
verbosity, autoclobber=not interactive, keepdb=keepdb)
verbosity,
autoclobber=not interactive,
keepdb=keepdb,
serialize=connection.settings_dict.get("TEST_SERIALIZE", True),
)
destroy = True
else:
connection.settings_dict['NAME'] = test_db_name

19
django/test/testcases.py
View File

@ -753,6 +753,12 @@ class TransactionTestCase(SimpleTestCase):
# Subclasses can define fixtures which will be automatically installed.
fixtures = None
# If transactions aren't available, Django will serialize the database
# contents into a fixture during setup and flush and reload them
# during teardown (as flush does not restore data from migrations).
# This can be slow; this flag allows enabling on a per-case basis.
serialized_rollback = False
def _pre_setup(self):
"""Performs any pre-test setup. This includes:
@ -808,6 +814,17 @@ class TransactionTestCase(SimpleTestCase):
if self.reset_sequences:
self._reset_sequences(db_name)
# If we need to provide replica initial data from migrated apps,
# then do so.
if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
if self.available_apps is not None:
apps.unset_available_apps()
connections[db_name].creation.deserialize_db_from_string(
connections[db_name]._test_serialized_contents
)
if self.available_apps is not None:
apps.set_available_apps(self.available_apps)
if self.fixtures:
# We have to use this slightly awkward syntax due to the fact
# that we're using *args and **kwargs together.
@ -844,12 +861,14 @@ class TransactionTestCase(SimpleTestCase):
# Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
# when flushing only a subset of the apps
for db_name in self._databases_names(include_mirrors=False):
# Flush the database
call_command('flush', verbosity=0, interactive=False,
database=db_name, skip_checks=True,
reset_sequences=False,
allow_cascade=self.available_apps is not None,
inhibit_post_migrate=self.available_apps is not None)
def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
items = six.moves.map(transform, qs)
if not ordered:

13
docs/ref/migration-operations.txt
View File

@ -199,8 +199,9 @@ model::
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
Country = apps.get_model("myapp", "Country")
Country.objects.create(name="USA", code="us")
Country.objects.create(name="France", code="fr")
db_alias = schema_editor.connection.alias
Country.objects.create(name="USA", code="us", using=db_alias)
Country.objects.create(name="France", code="fr", using=db_alias)
class Migration(migrations.Migration):
@ -236,6 +237,14 @@ Oracle). This should be safe, but may cause a crash if you attempt to use
the ``schema_editor`` provided on these backends; in this case, please
set ``atomic=False``.
.. warning::
RunPython does not magically alter the connection of the models for you;
any model methods you call will go to the default database unless you
give them the current database alias (available from
``schema_editor.connection.alias``, where ``schema_editor`` is the second
argument to your function).
SeparateDatabaseAndState
------------------------

18
docs/ref/settings.txt
View File

@ -2078,6 +2078,24 @@ Default: ``'django.test.runner.DiscoverRunner'``
The name of the class to use for starting the test suite. See
:ref:`other-testing-frameworks`.
.. setting:: TEST_NON_SERIALIZED_APPS
TEST_NON_SERIALIZED_APPS
------------------------
Default: ``[]``
In order to restore the database state between tests for TransactionTestCases
and database backends without transactions, Django will :ref:`serialize the
contents of all apps with migrations <test-case-serialized-rollback>` when it
starts the test run so it can then reload from that copy before tests that
need it.
This slows down the startup time of the test runner; if you have apps that
you know don't need this feature, you can add their full names in here (e.g.
``django.contrib.contenttypes``) to exclude them from this serialization
process.
.. setting:: THOUSAND_SEPARATOR
THOUSAND_SEPARATOR

4
docs/releases/1.7.txt
View File

@ -63,6 +63,10 @@ but a few of the key features are:
* ``initial_data`` fixtures are no longer loaded for apps with migrations; if
you want to load initial data for an app, we suggest you do it in a migration.
* Test rollback behaviour is different for apps with migrations; in particular,
Django will no longer emulate rollbacks on non-transactional databases or
inside ``TransactionTestCase`` :ref:`unless specifically asked <test-case-serialized-rollback>`.
App-loading refactor
~~~~~~~~~~~~~~~~~~~~

8
docs/topics/testing/advanced.txt
View File

@ -485,7 +485,7 @@ django.db.connection.creation
The creation module of the database backend also provides some utilities that
can be useful during testing.
.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False])
.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False, serialize=True])
Creates a new test database and runs ``migrate`` against it.
@ -507,6 +507,12 @@ can be useful during testing.
a new database will be created, prompting the user to remove
the existing one, if present.
``serialize`` determines if Django serializes the database into an
in-memory JSON string before running tests (used to restore the database
state between tests if you don't have transactions). You can set this to
False to significantly speed up creation time if you know you don't need
data persistance outside of test fixtures.
Returns the name of the test database that it created.
``create_test_db()`` has the side effect of modifying the value of

28
docs/topics/testing/overview.txt
View File

@ -234,6 +234,33 @@ the Django test runner reorders tests in the following way:
database by a given :class:`~django.test.TransactionTestCase` test, they
must be updated to be able to run independently.
.. _test-case-serialized-rollback:
Rollback emulation
------------------
Any initial data loaded in migrations will only be available in ``TestCase``
tests and not in ``TransactionTestCase`` tests, and additionally only on
backends where transactions are supported (the most important exception being
MyISAM).
Django can re-load that data for you on a per-testcase basis by
setting the ``serialized_rollback`` option to ``True`` in the body of the
``TestCase`` or ``TransactionTestCase``, but note that this will slow down
that test suite by approximately 3x.
Third-party apps or those developing against MyISAM will need to set this;
in general, however, you should be developing your own projects against a
transactional database and be using ``TestCase`` for most tests, and thus
not need this setting.
The initial serialization is usually very quick, but if you wish to exclude
some apps from this process (and speed up test runs slightly), you may add
those apps to :setting:`TEST_NON_SERIALIZED_APPS`.
Apps without migrations are not affected; ``initial_data`` fixtures are
reloaded as usual.
Other test conditions
---------------------
@ -249,6 +276,7 @@ used. This behavior `may change`_ in the future.
.. _may change: https://code.djangoproject.com/ticket/11505
Understanding the test output
-----------------------------

14
docs/topics/testing/tools.txt
View File

@ -600,9 +600,17 @@ to test the effects of commit and rollback:
guarantees that the rollback at the end of the test restores the database to
its initial state.
When running on a database that does not support rollback (e.g. MySQL with the
MyISAM storage engine), ``TestCase`` falls back to initializing the database
by truncating tables and reloading initial data.
.. warning::
``TestCase`` running on a database that does not support rollback (e.g. MySQL with the
MyISAM storage engine), and all instances of ``TransactionTestCase``, will
roll back at the end of the test by deleting all data from the test database
and reloading initial data for apps without migrations.
Apps with migrations :ref:`will not see their data reloaded <test-case-serialized-rollback>`;
if you need this functionality (for example, third-party apps should enable
this) you can set ``serialized_rollback = True`` inside the
``TestCase`` body.
.. warning::

0
tests/migration_test_data_persistence/__init__.py Normal file
View File

34
tests/migration_test_data_persistence/migrations/0001_initial.py Normal file
View File

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
def add_book(apps, schema_editor):
apps.get_model("migration_test_data_persistence", "Book").objects.using(
schema_editor.connection.alias,
).create(
title="I Love Django",
)
class Migration(migrations.Migration):
dependencies = [
]
operations = [
migrations.CreateModel(
name='Book',
fields=[
('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
('title', models.CharField(max_length=100)),
],
options={
},
bases=(models.Model,),
),
migrations.RunPython(
add_book,
),
]

0
tests/migration_test_data_persistence/migrations/__init__.py Normal file
View File

5
tests/migration_test_data_persistence/models.py Normal file
View File

@ -0,0 +1,5 @@
from django.db import models
class Book(models.Model):
title = models.CharField(max_length=100)

33
tests/migration_test_data_persistence/tests.py Normal file
View File

@ -0,0 +1,33 @@
from django.test import TransactionTestCase
from .models import Book
class MigrationDataPersistenceTestCase(TransactionTestCase):
"""
Tests that data loaded in migrations is available if we set
serialized_rollback = True.
"""
available_apps = ["migration_test_data_persistence"]
serialized_rollback = True
def test_persistence(self):
self.assertEqual(
Book.objects.count(),
1,
)
class MigrationDataNoPersistenceTestCase(TransactionTestCase):
"""
Tests the failure case
"""
available_apps = ["migration_test_data_persistence"]
serialized_rollback = False
def test_no_persistence(self):
self.assertEqual(
Book.objects.count(),
0,
)