From 2ee21d9f0d9eaed0494f3b9cd4b5bc9beffffae5 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 18 Feb 2013 11:37:26 +0100 Subject: [PATCH] Implemented persistent database connections. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks Anssi Kääriäinen and Karen Tracey for their inputs. --- django/contrib/auth/handlers/modwsgi.py | 4 +- django/db/__init__.py | 21 +++++-- django/db/backends/__init__.py | 32 +++++++++- django/db/backends/mysql/base.py | 8 +++ django/db/backends/oracle/base.py | 12 ++++ .../db/backends/postgresql_psycopg2/base.py | 9 +++ django/db/backends/sqlite3/base.py | 3 + django/db/utils.py | 15 +++-- django/test/client.py | 12 ++-- docs/internals/deprecation.txt | 2 + docs/ref/databases.txt | 62 +++++++++++++++++++ docs/ref/settings.txt | 13 ++++ docs/releases/1.6.txt | 21 +++++++ tests/handlers/tests.py | 17 +++-- tests/httpwrappers/tests.py | 6 +- tests/wsgi/tests.py | 8 +++ 16 files changed, 220 insertions(+), 25 deletions(-) diff --git a/django/contrib/auth/handlers/modwsgi.py b/django/contrib/auth/handlers/modwsgi.py index df20f1283a..f14afcf290 100644 --- a/django/contrib/auth/handlers/modwsgi.py +++ b/django/contrib/auth/handlers/modwsgi.py @@ -25,7 +25,7 @@ def check_password(environ, username, password): return None return user.check_password(password) finally: - db.close_connection() + db.close_old_connections() def groups_for_user(environ, username): """ @@ -44,4 +44,4 @@ def groups_for_user(environ, username): return [] return [force_bytes(group.name) for group in user.groups.all()] finally: - db.close_connection() + db.close_old_connections() diff --git a/django/db/__init__.py b/django/db/__init__.py index e76c6c3268..5e630392e7 100644 --- a/django/db/__init__.py +++ b/django/db/__init__.py @@ -42,9 +42,10 @@ class DefaultConnectionProxy(object): connection = DefaultConnectionProxy() backend = load_backend(connection.settings_dict['ENGINE']) -# Register an event that closes the database connection -# when a Django request is finished. def close_connection(**kwargs): + warnings.warn( + "close_connection is superseded by close_old_connections.", + PendingDeprecationWarning, stacklevel=2) # Avoid circular imports from django.db import transaction for conn in connections: @@ -53,15 +54,25 @@ def close_connection(**kwargs): # connection state will be cleaned up. transaction.abort(conn) connections[conn].close() -signals.request_finished.connect(close_connection) -# Register an event that resets connection.queries -# when a Django request is started. +# Register an event to reset saved queries when a Django request is started. def reset_queries(**kwargs): for conn in connections.all(): conn.queries = [] signals.request_started.connect(reset_queries) +# Register an event to reset transaction state and close connections past +# their lifetime. NB: abort() doesn't do anything outside of a transaction. +def close_old_connections(**kwargs): + for conn in connections.all(): + try: + conn.abort() + except DatabaseError: + pass + conn.close_if_unusable_or_obsolete() +signals.request_started.connect(close_old_connections) +signals.request_finished.connect(close_old_connections) + # Register an event that rolls back the connections # when a Django request has an exception. def _rollback_on_exception(**kwargs): diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 49e07cfa9e..9fb2b23644 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -1,4 +1,5 @@ import datetime +import time from django.db.utils import DatabaseError @@ -49,6 +50,10 @@ class BaseDatabaseWrapper(object): self._thread_ident = thread.get_ident() self.allow_thread_sharing = allow_thread_sharing + # Connection termination related attributes + self.close_at = None + self.errors_occurred = False + def __eq__(self, other): return self.alias == other.alias @@ -59,7 +64,7 @@ class BaseDatabaseWrapper(object): return hash(self.alias) def wrap_database_errors(self): - return DatabaseErrorWrapper(self.Database) + return DatabaseErrorWrapper(self) def get_connection_params(self): raise NotImplementedError @@ -76,6 +81,11 @@ class BaseDatabaseWrapper(object): def _cursor(self): with self.wrap_database_errors(): if self.connection is None: + # Reset parameters defining when to close the connection + max_age = self.settings_dict['CONN_MAX_AGE'] + self.close_at = None if max_age is None else time.time() + max_age + self.errors_occurred = False + # Establish the connection conn_params = self.get_connection_params() self.connection = self.get_new_connection(conn_params) self.init_connection_state() @@ -351,6 +361,26 @@ class BaseDatabaseWrapper(object): self.connection = None self.set_clean() + def close_if_unusable_or_obsolete(self): + if self.connection is not None: + if self.errors_occurred: + if self.is_usable(): + self.errors_occurred = False + else: + self.close() + return + if self.close_at is not None and time.time() >= self.close_at: + self.close() + return + + def is_usable(self): + """ + Test if the database connection is usable. + + This function may assume that self.connection is not None. + """ + raise NotImplementedError + def cursor(self): self.validate_thread_sharing() if (self.use_debug_cursor or diff --git a/django/db/backends/mysql/base.py b/django/db/backends/mysql/base.py index 4bfd3c4481..6b2ecaead1 100644 --- a/django/db/backends/mysql/base.py +++ b/django/db/backends/mysql/base.py @@ -439,6 +439,14 @@ class DatabaseWrapper(BaseDatabaseWrapper): cursor = self.connection.cursor() return CursorWrapper(cursor) + def is_usable(self): + try: + self.connection.ping() + except DatabaseError: + return False + else: + return True + def _rollback(self): try: BaseDatabaseWrapper._rollback(self) diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index d35e814d1f..478124f5df 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -598,6 +598,18 @@ class DatabaseWrapper(BaseDatabaseWrapper): # stmtcachesize is available only in 4.3.2 and up. pass + def is_usable(self): + try: + if hasattr(self.connection, 'ping'): # Oracle 10g R2 and higher + self.connection.ping() + else: + # Use a cx_Oracle cursor directly, bypassing Django's utilities. + self.connection.cursor().execute("SELECT 1 FROM DUAL") + except DatabaseError: + return False + else: + return True + # Oracle doesn't support savepoint commits. Ignore them. def _savepoint_commit(self, sid): pass diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index fb04072494..db4b5ade05 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -177,6 +177,15 @@ class DatabaseWrapper(BaseDatabaseWrapper): cursor.tzinfo_factory = utc_tzinfo_factory if settings.USE_TZ else None return cursor + def is_usable(self): + try: + # Use a psycopg cursor directly, bypassing Django's utilities. + self.connection.cursor().execute("SELECT 1") + except DatabaseError: + return False + else: + return True + def _enter_transaction_management(self, managed): """ Switch the isolation level when needing transaction support, so that diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index 6bf1ffc469..ad54af46ad 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -347,6 +347,9 @@ class DatabaseWrapper(BaseDatabaseWrapper): def create_cursor(self): return self.connection.cursor(factory=SQLiteCursorWrapper) + def is_usable(self): + return True + def check_constraints(self, table_names=None): """ Checks each table name in `table_names` for rows with invalid foreign key references. This method is diff --git a/django/db/utils.py b/django/db/utils.py index 0c98cc23fd..cc17b3e7a3 100644 --- a/django/db/utils.py +++ b/django/db/utils.py @@ -56,11 +56,13 @@ class DatabaseErrorWrapper(object): exceptions using Django's common wrappers. """ - def __init__(self, database): + def __init__(self, wrapper): """ - database is a module defining PEP-249 exceptions. + wrapper is a database wrapper. + + It must have a Database attribute defining PEP-249 exceptions. """ - self.database = database + self.wrapper = wrapper def __enter__(self): pass @@ -79,7 +81,7 @@ class DatabaseErrorWrapper(object): InterfaceError, Error, ): - db_exc_type = getattr(self.database, dj_exc_type.__name__) + db_exc_type = getattr(self.wrapper.Database, dj_exc_type.__name__) if issubclass(exc_type, db_exc_type): # Under Python 2.6, exc_value can still be a string. try: @@ -89,6 +91,10 @@ class DatabaseErrorWrapper(object): dj_exc_value = dj_exc_type(*args) if six.PY3: dj_exc_value.__cause__ = exc_value + # Only set the 'errors_occurred' flag for errors that may make + # the connection unusable. + if dj_exc_type not in (DataError, IntegrityError): + self.wrapper.errors_occurred = True six.reraise(dj_exc_type, dj_exc_value, traceback) def __call__(self, func): @@ -155,6 +161,7 @@ class ConnectionHandler(object): conn.setdefault('ENGINE', 'django.db.backends.dummy') if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']: conn['ENGINE'] = 'django.db.backends.dummy' + conn.setdefault('CONN_MAX_AGE', 600) conn.setdefault('OPTIONS', {}) conn.setdefault('TIME_ZONE', 'UTC' if settings.USE_TZ else settings.TIME_ZONE) for setting in ['NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']: diff --git a/django/test/client.py b/django/test/client.py index 2506437023..46f55d7cdc 100644 --- a/django/test/client.py +++ b/django/test/client.py @@ -18,7 +18,7 @@ from django.core.handlers.base import BaseHandler from django.core.handlers.wsgi import WSGIRequest from django.core.signals import (request_started, request_finished, got_request_exception) -from django.db import close_connection +from django.db import close_old_connections from django.http import SimpleCookie, HttpRequest, QueryDict from django.template import TemplateDoesNotExist from django.test import signals @@ -78,9 +78,9 @@ def closing_iterator_wrapper(iterable, close): for item in iterable: yield item finally: - request_finished.disconnect(close_connection) + request_finished.disconnect(close_old_connections) close() # will fire request_finished - request_finished.connect(close_connection) + request_finished.connect(close_old_connections) class ClientHandler(BaseHandler): @@ -101,7 +101,9 @@ class ClientHandler(BaseHandler): if self._request_middleware is None: self.load_middleware() + request_started.disconnect(close_old_connections) request_started.send(sender=self.__class__) + request_started.connect(close_old_connections) request = WSGIRequest(environ) # sneaky little hack so that we can easily get round # CsrfViewMiddleware. This makes life easier, and is probably @@ -115,9 +117,9 @@ class ClientHandler(BaseHandler): response.streaming_content = closing_iterator_wrapper( response.streaming_content, response.close) else: - request_finished.disconnect(close_connection) + request_finished.disconnect(close_old_connections) response.close() # will fire request_finished - request_finished.connect(close_connection) + request_finished.connect(close_old_connections) return response diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index f1ae1338df..3a9cbd195d 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -339,6 +339,8 @@ these changes. * ``Model._meta.module_name`` was renamed to ``model_name``. +* The private API ``django.db.close_connection`` will be removed. + 2.0 --- diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index e933ee350d..34f60e99ac 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -11,6 +11,68 @@ This file describes some of the features that might be relevant to Django usage. Of course, it is not intended as a replacement for server-specific documentation or reference manuals. +General notes +============= + +.. _persistent-database-connections: + +Persistent connections +---------------------- + +.. versionadded:: 1.6 + +Persistent connections avoid the overhead of re-establishing a connection to +the database in each request. By default, connections are kept open for up 10 +minutes — if not specified, :setting:`CONN_MAX_AGE` defaults to 600 seconds. + +Django 1.5 and earlier didn't have persistent connections. To restore the +legacy behavior of closing the connection at the end of every request, set +:setting:`CONN_MAX_AGE` to ``0``. + +For unlimited persistent connections, set :setting:`CONN_MAX_AGE` to ``None``. + +Connection management +~~~~~~~~~~~~~~~~~~~~~ + +Django opens a connection to the database when it first makes a database +query. It keeps this connection open and reuses it in subsequent requests. +Django closes the connection once it exceeds the maximum age defined by +:setting:`CONN_MAX_AGE` or when it isn't usable any longer. + +In detail, Django automatically opens a connection to the database whenever it +needs one and doesn't have one already — either because this is the first +connection, or because the previous connection was closed. + +At the beginning of each request, Django closes the connection if it has +reached its maximum age. If your database terminates idle connections after +some time, you should set :setting:`CONN_MAX_AGE` to a lower value, so that +Django doesn't attempt to use a connection that has been terminated by the +database server. (This problem may only affect very low traffic sites.) + +At the end of each request, Django closes the connection if it has reached its +maximum age or if it is in an unrecoverable error state. If any database +errors have occurred while processing the requests, Django checks whether the +connection still works, and closes it if it doesn't. Thus, database errors +affect at most one request; if the connection becomes unusable, the next +request gets a fresh connection. + +Caveats +~~~~~~~ + +Since each thread maintains its own connection, your database must support at +least as many simultaneous connections as you have worker threads. + +Sometimes a database won't be accessed by the majority of your views, for +example because it's the database of an external system, or thanks to caching. +In such cases, you should set :setting:`CONN_MAX_AGE` to a lower value, or +even ``0``, because it doesn't make sense to maintain a connection that's +unlikely to be reused. This will help keep the number of simultaneous +connections to this database small. + + +The development server creates a new thread for each request it handles, +negating the effect of persistent connections. + .. _postgresql-notes: PostgreSQL notes diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index bba936d837..baeb02c32d 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -464,6 +464,19 @@ The name of the database to use. For SQLite, it's the full path to the database file. When specifying the path, always use forward slashes, even on Windows (e.g. ``C:/homes/user/mysite/sqlite3.db``). +.. setting:: CONN_MAX_AGE + +CONN_MAX_AGE +~~~~~~~~~~~~ + +.. versionadded:: 1.6 + +Default: ``600`` + +The lifetime of a database connection, in seconds. Use ``0`` to close database +connections at the end of each request — Django's historical behavior — and +``None`` for unlimited persistent connections. + .. setting:: OPTIONS OPTIONS diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt index c6a4fb2d5d..34fa687290 100644 --- a/docs/releases/1.6.txt +++ b/docs/releases/1.6.txt @@ -30,6 +30,19 @@ prevention ` are turned on. If the default templates don't suit your tastes, you can use :ref:`custom project and app templates `. +Persistent database connections +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Django now supports reusing the same database connection for several requests. +This avoids the overhead of re-establishing a connection at the beginning of +each request. + +By default, database connections will kept open for 10 minutes. This behavior +is controlled by the :setting:`CONN_MAX_AGE` setting. To restore the previous +behavior of closing the connection at the end of each request, set +:setting:`CONN_MAX_AGE` to ``0``. See :ref:`persistent-database-connections` +for details. + Time zone aware aggregation ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -136,6 +149,14 @@ Backwards incompatible changes in 1.6 * Model fields named ``hour``, ``minute`` or ``second`` may clash with the new lookups. Append an explicit :lookup:`exact` lookup if this is an issue. +* When Django establishes a connection to the database, it sets up appropriate + parameters, depending on the backend being used. Since `persistent database + connections `_ are enabled by default in + Django 1.6, this setup isn't repeated at every request any more. If you + modifiy parameters such as the connection's isolation level or time zone, + you should either restore Django's defaults at the end of each request, or + force an appropriate value at the beginning of each request. + * If your CSS/Javascript code used to access HTML input widgets by type, you should review it as ``type='text'`` widgets might be now output as ``type='email'``, ``type='url'`` or ``type='number'`` depending on their diff --git a/tests/handlers/tests.py b/tests/handlers/tests.py index f647bf199b..6eb9bd23fe 100644 --- a/tests/handlers/tests.py +++ b/tests/handlers/tests.py @@ -1,5 +1,6 @@ from django.core.handlers.wsgi import WSGIHandler -from django.core import signals +from django.core.signals import request_started, request_finished +from django.db import close_old_connections from django.test import RequestFactory, TestCase from django.test.utils import override_settings from django.utils import six @@ -7,6 +8,12 @@ from django.utils import six class HandlerTests(TestCase): + def setUp(self): + request_started.disconnect(close_old_connections) + + def tearDown(self): + request_started.connect(close_old_connections) + # Mangle settings so the handler will fail @override_settings(MIDDLEWARE_CLASSES=42) def test_lock_safety(self): @@ -35,12 +42,12 @@ class SignalsTests(TestCase): def setUp(self): self.signals = [] - signals.request_started.connect(self.register_started) - signals.request_finished.connect(self.register_finished) + request_started.connect(self.register_started) + request_finished.connect(self.register_finished) def tearDown(self): - signals.request_started.disconnect(self.register_started) - signals.request_finished.disconnect(self.register_finished) + request_started.disconnect(self.register_started) + request_finished.disconnect(self.register_finished) def register_started(self, **kwargs): self.signals.append('started') diff --git a/tests/httpwrappers/tests.py b/tests/httpwrappers/tests.py index 2964a86034..194232e92f 100644 --- a/tests/httpwrappers/tests.py +++ b/tests/httpwrappers/tests.py @@ -8,7 +8,7 @@ import warnings from django.core.exceptions import SuspiciousOperation from django.core.signals import request_finished -from django.db import close_connection +from django.db import close_old_connections from django.http import (QueryDict, HttpResponse, HttpResponseRedirect, HttpResponsePermanentRedirect, HttpResponseNotAllowed, HttpResponseNotModified, StreamingHttpResponse, @@ -490,10 +490,10 @@ class FileCloseTests(TestCase): def setUp(self): # Disable the request_finished signal during this test # to avoid interfering with the database connection. - request_finished.disconnect(close_connection) + request_finished.disconnect(close_old_connections) def tearDown(self): - request_finished.connect(close_connection) + request_finished.connect(close_old_connections) def test_response(self): filename = os.path.join(os.path.dirname(upath(__file__)), 'abc.txt') diff --git a/tests/wsgi/tests.py b/tests/wsgi/tests.py index 9b7ee68afd..a66258d4eb 100644 --- a/tests/wsgi/tests.py +++ b/tests/wsgi/tests.py @@ -2,7 +2,9 @@ from __future__ import unicode_literals from django.core.exceptions import ImproperlyConfigured from django.core.servers.basehttp import get_internal_wsgi_application +from django.core.signals import request_started from django.core.wsgi import get_wsgi_application +from django.db import close_old_connections from django.test import TestCase from django.test.client import RequestFactory from django.test.utils import override_settings @@ -12,6 +14,12 @@ from django.utils import six, unittest class WSGITest(TestCase): urls = "wsgi.urls" + def setUp(self): + request_started.disconnect(close_old_connections) + + def tearDown(self): + request_started.connect(close_old_connections) + def test_get_wsgi_application(self): """ Verify that ``get_wsgi_application`` returns a functioning WSGI