Implemented persistent database connections.

Thanks Anssi Kääriäinen and Karen Tracey for their inputs.

django/contrib/auth/handlers/modwsgi.py

@@ -25,7 +25,7 @@ def check_password(environ, username, password):
             return None
         return user.check_password(password)
     finally:
-        db.close_connection()
+        db.close_old_connections()
 
 def groups_for_user(environ, username):
     """
@@ -44,4 +44,4 @@ def groups_for_user(environ, username):
             return []
         return [force_bytes(group.name) for group in user.groups.all()]
     finally:
-        db.close_connection()
+        db.close_old_connections()

django/db/__init__.py

@@ -42,9 +42,10 @@ class DefaultConnectionProxy(object):
 connection = DefaultConnectionProxy()
 backend = load_backend(connection.settings_dict['ENGINE'])
 
-# Register an event that closes the database connection
-# when a Django request is finished.
 def close_connection(**kwargs):
+    warnings.warn(
+        "close_connection is superseded by close_old_connections.",
+        PendingDeprecationWarning, stacklevel=2)
     # Avoid circular imports
     from django.db import transaction
     for conn in connections:
@@ -53,15 +54,25 @@ def close_connection(**kwargs):
         # connection state will be cleaned up.
         transaction.abort(conn)
         connections[conn].close()
-signals.request_finished.connect(close_connection)
 
-# Register an event that resets connection.queries
+# Register an event to reset saved queries when a Django request is started.
-# when a Django request is started.
 def reset_queries(**kwargs):
     for conn in connections.all():
         conn.queries = []
 signals.request_started.connect(reset_queries)
 
+# Register an event to reset transaction state and close connections past
+# their lifetime. NB: abort() doesn't do anything outside of a transaction.
+def close_old_connections(**kwargs):
+    for conn in connections.all():
+        try:
+            conn.abort()
+        except DatabaseError:
+            pass
+        conn.close_if_unusable_or_obsolete()
+signals.request_started.connect(close_old_connections)
+signals.request_finished.connect(close_old_connections)
+
 # Register an event that rolls back the connections
 # when a Django request has an exception.
 def _rollback_on_exception(**kwargs):

django/db/backends/__init__.py

@@ -1,4 +1,5 @@
 import datetime
+import time
 
 from django.db.utils import DatabaseError
 
@@ -49,6 +50,10 @@ class BaseDatabaseWrapper(object):
         self._thread_ident = thread.get_ident()
         self.allow_thread_sharing = allow_thread_sharing
 
+        # Connection termination related attributes
+        self.close_at = None
+        self.errors_occurred = False
+
     def __eq__(self, other):
         return self.alias == other.alias
 
@@ -59,7 +64,7 @@ class BaseDatabaseWrapper(object):
         return hash(self.alias)
 
     def wrap_database_errors(self):
-        return DatabaseErrorWrapper(self.Database)
+        return DatabaseErrorWrapper(self)
 
     def get_connection_params(self):
         raise NotImplementedError
@@ -76,6 +81,11 @@ class BaseDatabaseWrapper(object):
     def _cursor(self):
         with self.wrap_database_errors():
             if self.connection is None:
+                # Reset parameters defining when to close the connection
+                max_age = self.settings_dict['CONN_MAX_AGE']
+                self.close_at = None if max_age is None else time.time() + max_age
+                self.errors_occurred = False
+                # Establish the connection
                 conn_params = self.get_connection_params()
                 self.connection = self.get_new_connection(conn_params)
                 self.init_connection_state()
@@ -351,6 +361,26 @@ class BaseDatabaseWrapper(object):
             self.connection = None
         self.set_clean()
 
+    def close_if_unusable_or_obsolete(self):
+        if self.connection is not None:
+            if self.errors_occurred:
+                if self.is_usable():
+                    self.errors_occurred = False
+                else:
+                    self.close()
+                    return
+            if self.close_at is not None and time.time() >= self.close_at:
+                self.close()
+                return
+
+    def is_usable(self):
+        """
+        Test if the database connection is usable.
+
+        This function may assume that self.connection is not None.
+        """
+        raise NotImplementedError
+
     def cursor(self):
         self.validate_thread_sharing()
         if (self.use_debug_cursor or

django/db/backends/mysql/base.py

@@ -439,6 +439,14 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         cursor = self.connection.cursor()
         return CursorWrapper(cursor)
 
+    def is_usable(self):
+        try:
+            self.connection.ping()
+        except DatabaseError:
+            return False
+        else:
+            return True
+
     def _rollback(self):
         try:
             BaseDatabaseWrapper._rollback(self)

django/db/backends/oracle/base.py

@@ -598,6 +598,18 @@ class DatabaseWrapper(BaseDatabaseWrapper):
             # stmtcachesize is available only in 4.3.2 and up.
             pass
 
+    def is_usable(self):
+        try:
+            if hasattr(self.connection, 'ping'):    # Oracle 10g R2 and higher
+                self.connection.ping()
+            else:
+                # Use a cx_Oracle cursor directly, bypassing Django's utilities.
+                self.connection.cursor().execute("SELECT 1 FROM DUAL")
+        except DatabaseError:
+            return False
+        else:
+            return True
+
     # Oracle doesn't support savepoint commits.  Ignore them.
     def _savepoint_commit(self, sid):
         pass

django/db/backends/postgresql_psycopg2/base.py

@@ -177,6 +177,15 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         cursor.tzinfo_factory = utc_tzinfo_factory if settings.USE_TZ else None
         return cursor
 
+    def is_usable(self):
+        try:
+            # Use a psycopg cursor directly, bypassing Django's utilities.
+            self.connection.cursor().execute("SELECT 1")
+        except DatabaseError:
+            return False
+        else:
+            return True
+
     def _enter_transaction_management(self, managed):
         """
         Switch the isolation level when needing transaction support, so that

django/db/backends/sqlite3/base.py

@@ -347,6 +347,9 @@ class DatabaseWrapper(BaseDatabaseWrapper):
     def create_cursor(self):
         return self.connection.cursor(factory=SQLiteCursorWrapper)
 
+    def is_usable(self):
+        return True
+
     def check_constraints(self, table_names=None):
         """
         Checks each table name in `table_names` for rows with invalid foreign key references. This method is

django/db/utils.py

@@ -56,11 +56,13 @@ class DatabaseErrorWrapper(object):
     exceptions using Django's common wrappers.
     """
 
-    def __init__(self, database):
+    def __init__(self, wrapper):
         """
-        database is a module defining PEP-249 exceptions.
+        wrapper is a database wrapper.
+
+        It must have a Database attribute defining PEP-249 exceptions.
         """
-        self.database = database
+        self.wrapper = wrapper
 
     def __enter__(self):
         pass
@@ -79,7 +81,7 @@ class DatabaseErrorWrapper(object):
                 InterfaceError,
                 Error,
             ):
-            db_exc_type = getattr(self.database, dj_exc_type.__name__)
+            db_exc_type = getattr(self.wrapper.Database, dj_exc_type.__name__)
             if issubclass(exc_type, db_exc_type):
                 # Under Python 2.6, exc_value can still be a string.
                 try:
@@ -89,6 +91,10 @@ class DatabaseErrorWrapper(object):
                 dj_exc_value = dj_exc_type(*args)
                 if six.PY3:
                     dj_exc_value.__cause__ = exc_value
+                # Only set the 'errors_occurred' flag for errors that may make
+                # the connection unusable.
+                if dj_exc_type not in (DataError, IntegrityError):
+                    self.wrapper.errors_occurred = True
                 six.reraise(dj_exc_type, dj_exc_value, traceback)
 
     def __call__(self, func):
@@ -155,6 +161,7 @@ class ConnectionHandler(object):
         conn.setdefault('ENGINE', 'django.db.backends.dummy')
         if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']:
             conn['ENGINE'] = 'django.db.backends.dummy'
+        conn.setdefault('CONN_MAX_AGE', 600)
         conn.setdefault('OPTIONS', {})
         conn.setdefault('TIME_ZONE', 'UTC' if settings.USE_TZ else settings.TIME_ZONE)
         for setting in ['NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']:

django/test/client.py

@@ -18,7 +18,7 @@ from django.core.handlers.base import BaseHandler
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.signals import (request_started, request_finished,
     got_request_exception)
-from django.db import close_connection
+from django.db import close_old_connections
 from django.http import SimpleCookie, HttpRequest, QueryDict
 from django.template import TemplateDoesNotExist
 from django.test import signals
@@ -78,9 +78,9 @@ def closing_iterator_wrapper(iterable, close):
         for item in iterable:
             yield item
     finally:
-        request_finished.disconnect(close_connection)
+        request_finished.disconnect(close_old_connections)
         close()                                 # will fire request_finished
-        request_finished.connect(close_connection)
+        request_finished.connect(close_old_connections)
 
 
 class ClientHandler(BaseHandler):
@@ -101,7 +101,9 @@ class ClientHandler(BaseHandler):
         if self._request_middleware is None:
             self.load_middleware()
 
+        request_started.disconnect(close_old_connections)
         request_started.send(sender=self.__class__)
+        request_started.connect(close_old_connections)
         request = WSGIRequest(environ)
         # sneaky little hack so that we can easily get round
         # CsrfViewMiddleware.  This makes life easier, and is probably
@@ -115,9 +117,9 @@ class ClientHandler(BaseHandler):
             response.streaming_content = closing_iterator_wrapper(
                 response.streaming_content, response.close)
         else:
-            request_finished.disconnect(close_connection)
+            request_finished.disconnect(close_old_connections)
             response.close()                    # will fire request_finished
-            request_finished.connect(close_connection)
+            request_finished.connect(close_old_connections)
 
         return response
 

docs/internals/deprecation.txt

@@ -339,6 +339,8 @@ these changes.
 
 * ``Model._meta.module_name`` was renamed to ``model_name``.
 
+* The private API ``django.db.close_connection`` will be removed.
+
 2.0
 ---
 

docs/ref/databases.txt

@@ -11,6 +11,68 @@ This file describes some of the features that might be relevant to Django
 usage. Of course, it is not intended as a replacement for server-specific
 documentation or reference manuals.
 
+General notes
+=============
+
+.. _persistent-database-connections:
+
+Persistent connections
+----------------------
+
+.. versionadded:: 1.6
+
+Persistent connections avoid the overhead of re-establishing a connection to
+the database in each request. By default, connections are kept open for up 10
+minutes — if not specified, :setting:`CONN_MAX_AGE` defaults to 600 seconds.
+
+Django 1.5 and earlier didn't have persistent connections. To restore the
+legacy behavior of closing the connection at the end of every request, set
+:setting:`CONN_MAX_AGE` to ``0``.
+
+For unlimited persistent connections, set :setting:`CONN_MAX_AGE` to ``None``.
+
+Connection management
+~~~~~~~~~~~~~~~~~~~~~
+
+Django opens a connection to the database when it first makes a database
+query. It keeps this connection open and reuses it in subsequent requests.
+Django closes the connection once it exceeds the maximum age defined by
+:setting:`CONN_MAX_AGE` or when it isn't usable any longer.
+
+In detail, Django automatically opens a connection to the database whenever it
+needs one and doesn't have one already — either because this is the first
+connection, or because the previous connection was closed.
+
+At the beginning of each request, Django closes the connection if it has
+reached its maximum age. If your database terminates idle connections after
+some time, you should set :setting:`CONN_MAX_AGE` to a lower value, so that
+Django doesn't attempt to use a connection that has been terminated by the
+database server. (This problem may only affect very low traffic sites.)
+
+At the end of each request, Django closes the connection if it has reached its
+maximum age or if it is in an unrecoverable error state. If any database
+errors have occurred while processing the requests, Django checks whether the
+connection still works, and closes it if it doesn't. Thus, database errors
+affect at most one request; if the connection becomes unusable, the next
+request gets a fresh connection.
+
+Caveats
+~~~~~~~
+
+Since each thread maintains its own connection, your database must support at
+least as many simultaneous connections as you have worker threads.
+
+Sometimes a database won't be accessed by the majority of your views, for
+example because it's the database of an external system, or thanks to caching.
+In such cases, you should set :setting:`CONN_MAX_AGE` to a lower value, or
+even ``0``, because it doesn't make sense to maintain a connection that's
+unlikely to be reused. This will help keep the number of simultaneous
+connections to this database small.
+
+
+The development server creates a new thread for each request it handles,
+negating the effect of persistent connections.
+
 .. _postgresql-notes:
 
 PostgreSQL notes

docs/ref/settings.txt

@@ -464,6 +464,19 @@ The name of the database to use. For SQLite, it's the full path to the database
 file. When specifying the path, always use forward slashes, even on Windows
 (e.g. ``C:/homes/user/mysite/sqlite3.db``).
 
+.. setting:: CONN_MAX_AGE
+
+CONN_MAX_AGE
+~~~~~~~~~~~~
+
+.. versionadded:: 1.6
+
+Default: ``600``
+
+The lifetime of a database connection, in seconds. Use ``0`` to close database
+connections at the end of each request — Django's historical behavior — and
+``None`` for unlimited persistent connections.
+
 .. setting:: OPTIONS
 
 OPTIONS

docs/releases/1.6.txt

@@ -30,6 +30,19 @@ prevention <clickjacking-prevention>` are turned on.
 If the default templates don't suit your tastes, you can use :ref:`custom
 project and app templates <custom-app-and-project-templates>`.
 
+Persistent database connections
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Django now supports reusing the same database connection for several requests.
+This avoids the overhead of re-establishing a connection at the beginning of
+each request.
+
+By default, database connections will kept open for 10 minutes. This behavior
+is controlled by the :setting:`CONN_MAX_AGE` setting. To restore the previous
+behavior of closing the connection at the end of each request, set
+:setting:`CONN_MAX_AGE` to ``0``. See :ref:`persistent-database-connections`
+for details.
+
 Time zone aware aggregation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -136,6 +149,14 @@ Backwards incompatible changes in 1.6
 * Model fields named ``hour``, ``minute`` or ``second`` may clash with the new
   lookups. Append an explicit :lookup:`exact` lookup if this is an issue.
 
+* When Django establishes a connection to the database, it sets up appropriate
+  parameters, depending on the backend being used. Since `persistent database
+  connections <persistent-database-connections>`_ are enabled by default in
+  Django 1.6, this setup isn't repeated at every request any more. If you
+  modifiy parameters such as the connection's isolation level or time zone,
+  you should either restore Django's defaults at the end of each request, or
+  force an appropriate value at the beginning of each request.
+
 * If your CSS/Javascript code used to access HTML input widgets by type, you
   should review it as ``type='text'`` widgets might be now output as
   ``type='email'``, ``type='url'`` or ``type='number'`` depending on their

tests/handlers/tests.py

@@ -1,5 +1,6 @@
 from django.core.handlers.wsgi import WSGIHandler
-from django.core import signals
+from django.core.signals import request_started, request_finished
+from django.db import close_old_connections
 from django.test import RequestFactory, TestCase
 from django.test.utils import override_settings
 from django.utils import six
@@ -7,6 +8,12 @@ from django.utils import six
 
 class HandlerTests(TestCase):
 
+    def setUp(self):
+        request_started.disconnect(close_old_connections)
+
+    def tearDown(self):
+        request_started.connect(close_old_connections)
+
     # Mangle settings so the handler will fail
     @override_settings(MIDDLEWARE_CLASSES=42)
     def test_lock_safety(self):
@@ -35,12 +42,12 @@ class SignalsTests(TestCase):
 
     def setUp(self):
         self.signals = []
-        signals.request_started.connect(self.register_started)
+        request_started.connect(self.register_started)
-        signals.request_finished.connect(self.register_finished)
+        request_finished.connect(self.register_finished)
 
     def tearDown(self):
-        signals.request_started.disconnect(self.register_started)
+        request_started.disconnect(self.register_started)
-        signals.request_finished.disconnect(self.register_finished)
+        request_finished.disconnect(self.register_finished)
 
     def register_started(self, **kwargs):
         self.signals.append('started')

tests/httpwrappers/tests.py

@@ -8,7 +8,7 @@ import warnings
 
 from django.core.exceptions import SuspiciousOperation
 from django.core.signals import request_finished
-from django.db import close_connection
+from django.db import close_old_connections
 from django.http import (QueryDict, HttpResponse, HttpResponseRedirect,
                          HttpResponsePermanentRedirect, HttpResponseNotAllowed,
                          HttpResponseNotModified, StreamingHttpResponse,
@@ -490,10 +490,10 @@ class FileCloseTests(TestCase):
     def setUp(self):
         # Disable the request_finished signal during this test
         # to avoid interfering with the database connection.
-        request_finished.disconnect(close_connection)
+        request_finished.disconnect(close_old_connections)
 
     def tearDown(self):
-        request_finished.connect(close_connection)
+        request_finished.connect(close_old_connections)
 
     def test_response(self):
         filename = os.path.join(os.path.dirname(upath(__file__)), 'abc.txt')

tests/wsgi/tests.py

@@ -2,7 +2,9 @@ from __future__ import unicode_literals
 
 from django.core.exceptions import ImproperlyConfigured
 from django.core.servers.basehttp import get_internal_wsgi_application
+from django.core.signals import request_started
 from django.core.wsgi import get_wsgi_application
+from django.db import close_old_connections
 from django.test import TestCase
 from django.test.client import RequestFactory
 from django.test.utils import override_settings
@@ -12,6 +14,12 @@ from django.utils import six, unittest
 class WSGITest(TestCase):
     urls = "wsgi.urls"
 
+    def setUp(self):
+        request_started.disconnect(close_old_connections)
+
+    def tearDown(self):
+        request_started.connect(close_old_connections)
+
     def test_get_wsgi_application(self):
         """
         Verify that ``get_wsgi_application`` returns a functioning WSGI