From 8c99b7920e8187f98cf4d7dbd9918bd6c6da1238 Mon Sep 17 00:00:00 2001 From: Andriy Sokolovskiy Date: Thu, 4 Dec 2014 01:17:59 +0200 Subject: [PATCH] Fixed #12118 -- Added shared cache support to SQLite in-memory testing. --- django/db/backends/sqlite3/base.py | 16 +++++++++++++++- django/db/backends/sqlite3/creation.py | 17 +++++++++++++---- django/test/testcases.py | 8 +++----- docs/releases/1.8.txt | 4 ++++ docs/topics/testing/overview.txt | 8 ++++++++ tests/backends/tests.py | 19 +++++++++++++++++++ 6 files changed, 62 insertions(+), 10 deletions(-) diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 73fc99ac07..3560d4498e 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -9,6 +9,7 @@ from __future__ import unicode_literals import datetime import decimal import re +import sys import uuid import warnings @@ -123,6 +124,14 @@ class DatabaseFeatures(BaseDatabaseFeatures): def can_release_savepoints(self): return self.uses_savepoints + @cached_property + def can_share_in_memory_db(self): + return ( + sys.version_info[:2] >= (3, 4) and + Database.__name__ == 'sqlite3.dbapi2' and + Database.sqlite_version_info >= (3, 7, 13) + ) + @cached_property def supports_stddev(self): """Confirm support for STDDEV and related stats functions @@ -405,6 +414,8 @@ class DatabaseWrapper(BaseDatabaseWrapper): RuntimeWarning ) kwargs.update({'check_same_thread': False}) + if self.features.can_share_in_memory_db: + kwargs.update({'uri': True}) return kwargs def get_new_connection(self, conn_params): @@ -429,7 +440,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): # If database is in memory, closing the connection destroys the # database. To prevent accidental data loss, ignore close requests on # an in-memory db. - if self.settings_dict['NAME'] != ":memory:": + if not self.is_in_memory_db(self.settings_dict['NAME']): BaseDatabaseWrapper.close(self) def _savepoint_allowed(self): @@ -505,6 +516,9 @@ class DatabaseWrapper(BaseDatabaseWrapper): """ self.cursor().execute("BEGIN") + def is_in_memory_db(self, name): + return name == ":memory:" or "mode=memory" in name + FORMAT_QMARK_REGEX = re.compile(r'(?= 1: print("Destroying old test database '%s'..." % self.connection.alias) @@ -80,7 +89,7 @@ class DatabaseCreation(BaseDatabaseCreation): return test_database_name def _destroy_test_db(self, test_database_name, verbosity): - if test_database_name and test_database_name != ":memory:": + if test_database_name and not self.connection.is_in_memory_db(test_database_name): # Remove the SQLite database file os.remove(test_database_name) @@ -92,8 +101,8 @@ class DatabaseCreation(BaseDatabaseCreation): SQLite since the databases will be distinct despite having the same TEST NAME. See http://www.sqlite.org/inmemorydb.html """ - test_dbname = self._get_test_db_name() + test_database_name = self._get_test_db_name() sig = [self.connection.settings_dict['NAME']] - if test_dbname == ':memory:': + if self.connection.is_in_memory_db(test_database_name): sig.append(self.connection.alias) return tuple(sig) diff --git a/django/test/testcases.py b/django/test/testcases.py index 38bbd40946..a7c89fe854 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -1212,8 +1212,7 @@ class LiveServerTestCase(TransactionTestCase): for conn in connections.all(): # If using in-memory sqlite databases, pass the connections to # the server thread. - if (conn.vendor == 'sqlite' - and conn.settings_dict['NAME'] == ':memory:'): + if conn.vendor == 'sqlite' and conn.is_in_memory_db(conn.settings_dict['NAME']): # Explicitly enable thread-shareability for this connection conn.allow_thread_sharing = True connections_override[conn.alias] = conn @@ -1267,10 +1266,9 @@ class LiveServerTestCase(TransactionTestCase): cls.server_thread.terminate() cls.server_thread.join() - # Restore sqlite connections' non-shareability + # Restore sqlite in-memory database connections' non-shareability for conn in connections.all(): - if (conn.vendor == 'sqlite' - and conn.settings_dict['NAME'] == ':memory:'): + if conn.vendor == 'sqlite' and conn.is_in_memory_db(conn.settings_dict['NAME']): conn.allow_thread_sharing = False @classmethod diff --git a/docs/releases/1.8.txt b/docs/releases/1.8.txt index 507aed6584..a34f3bf5b8 100644 --- a/docs/releases/1.8.txt +++ b/docs/releases/1.8.txt @@ -573,6 +573,10 @@ Tests * Added test client support for file uploads with file-like objects. +* A shared cache is now used when testing with a SQLite in-memory database when + using Python 3.4+ and SQLite 3.7.13+. This allows sharing the database + between threads. + Validators ^^^^^^^^^^ diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt index 913820adf5..bd4216e029 100644 --- a/docs/topics/testing/overview.txt +++ b/docs/topics/testing/overview.txt @@ -185,12 +185,20 @@ control the particular collation used by the test database. See the :doc:`settings documentation ` for details of these and other advanced settings. +If using a SQLite in-memory database with Python 3.4+ and SQLite 3.7.13+, +`shared cache `_ will be enabled, so +you can write tests with ability to share the database between threads. + .. versionchanged:: 1.7 The different options in the :setting:`TEST ` database setting used to be separate options in the database settings dictionary, prefixed with ``TEST_``. +.. versionadded:: 1.8 + + The ability to use SQLite with a shared cache as described above was added. + .. admonition:: Finding data from your production database when running tests? If your code attempts to access the database when its modules are compiled, diff --git a/tests/backends/tests.py b/tests/backends/tests.py index 3eae1ac4d4..606441b407 100644 --- a/tests/backends/tests.py +++ b/tests/backends/tests.py @@ -6,6 +6,7 @@ import copy import datetime from decimal import Decimal, Rounded import re +import sys import threading import unittest import warnings @@ -1201,3 +1202,21 @@ class DBTestSettingsRenamedTests(IgnoreAllDeprecationWarningsMixin, TestCase): def test_empty_settings(self): with override_settings(DATABASES=self.db_settings): self.handler.prepare_test_settings('default') + + +@unittest.skipUnless(connection.vendor == 'sqlite', 'SQLite specific test.') +@skipUnlessDBFeature('can_share_in_memory_db') +class TestSqliteThreadSharing(TransactionTestCase): + available_apps = ['backends'] + + def test_database_sharing_in_threads(self): + def create_object(): + models.Object.objects.create() + + create_object() + + thread = threading.Thread(target=create_object) + thread.start() + thread.join() + + self.assertEqual(models.Object.objects.count(), 2)