From 88336fdbb5e101fa25825b737169c0d6af2faa93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Freitag?= Date: Fri, 5 May 2017 19:19:34 -0700 Subject: [PATCH] Fixed #28062 -- Added a setting to disable server-side cursors on PostgreSQL. When a connection pooler is set up in transaction pooling mode, queries relying on server-side cursors fail. The DISABLE_SERVER_SIDE_CURSORS setting in DATABASES disables server-side cursors for this use case. --- django/db/models/query.py | 3 ++- docs/ref/databases.txt | 35 +++++++++++++++++++++++++++++++ docs/ref/models/querysets.txt | 5 +++++ docs/ref/settings.txt | 15 +++++++++++++ docs/releases/1.11.1.txt | 12 ++++++++++- docs/spelling_wordlist | 2 ++ tests/backends/test_postgresql.py | 28 +++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 2 deletions(-) diff --git a/django/db/models/query.py b/django/db/models/query.py index 1cc5217d0e..ae0b44cf2a 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -306,7 +306,8 @@ class QuerySet: An iterator over the results from applying this QuerySet to the database. """ - return iter(self._iterable_class(self, chunked_fetch=True)) + use_chunked_fetch = not connections[self.db].settings_dict.get('DISABLE_SERVER_SIDE_CURSORS') + return iter(self._iterable_class(self, chunked_fetch=use_chunked_fetch)) def aggregate(self, *args, **kwargs): """ diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 665a1c102a..587c1ebd65 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -189,6 +189,41 @@ cursor query is controlled with the `cursor_tuple_fraction`_ option. .. _cursor_tuple_fraction: https://www.postgresql.org/docs/current/static/runtime-config-query.html#GUC-CURSOR-TUPLE-FRACTION +.. _transaction-pooling-server-side-cursors: + +Transaction pooling and server-side cursors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 1.11.1 + +Using a connection pooler in transaction pooling mode (e.g. `pgBouncer`_) +requires disabling server-side cursors for that connection. + +Server-side cursors are local to a connection and remain open at the end of a +transaction when :setting:`AUTOCOMMIT ` is ``True``. A +subsequent transaction may attempt to fetch more results from a server-side +cursor. In transaction pooling mode, there's no guarantee that subsequent +transactions will use the same connection. If a different connection is used, +an error is raised when the transaction references the server-side cursor, +because server-side cursors are only accessible in the connection in which they +were created. + +One solution is to disable server-side cursors for a connection in +:setting:`DATABASES` by setting :setting:`DISABLE_SERVER_SIDE_CURSORS +` to ``True``. + +To benefit from server-side cursors in transaction pooling mode, you could set +up :doc:`another connection to the database ` in order to +perform queries that use server-side cursors. This connection needs to either +be directly to the database or to a connection pooler in session pooling mode. + +Another option is to wrap each ``QuerySet`` using server-side cursors in an +:func:`~django.db.transaction.atomic` block, because it disables ``autocommit`` +for the duration of the transaction. This way, the server-side cursor will only +live for the duration of the transaction. + +.. _pgBouncer: https://pgbouncer.github.io/ + Test database templates ----------------------- diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index 867ce47ce3..38c6d84b0b 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -2026,6 +2026,11 @@ won't cache results after iterating over them. Oracle and :ref:`PostgreSQL ` use server-side cursors to stream results from the database without loading the entire result set into memory. +On PostgreSQL, server-side cursors will only be used when the +:setting:`DISABLE_SERVER_SIDE_CURSORS ` +setting is ``False``. Read :ref:`transaction-pooling-server-side-cursors` if +you're using a connection pooler configured in transaction pooling mode. + .. versionchanged:: 1.11 PostgreSQL support for server-side cursors was added. diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 7b9283c1b2..c0d19a59db 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -641,6 +641,21 @@ PostgreSQL), it is an error to set this option. When :setting:`USE_TZ` is ``False``, it is an error to set this option. +.. setting:: DATABASE-DISABLE_SERVER_SIDE_CURSORS + +``DISABLE_SERVER_SIDE_CURSORS`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 1.11.1 + +Default: ``False`` + +Set this to ``True`` if you want to disable the use of server-side cursors with +:meth:`.QuerySet.iterator`. :ref:`transaction-pooling-server-side-cursors` +describes the use case. + +This is a PostgreSQL-specific setting. + .. setting:: USER ``USER`` diff --git a/docs/releases/1.11.1.txt b/docs/releases/1.11.1.txt index 7b1b887fa9..a93f2e8295 100644 --- a/docs/releases/1.11.1.txt +++ b/docs/releases/1.11.1.txt @@ -4,7 +4,17 @@ Django 1.11.1 release notes *Under development* -Django 1.11.1 fixes several bugs in 1.11. +Django 1.11.1 adds a minor feature and fixes several bugs in 1.11. + +Allowed disabling server-side cursors on PostgreSQL +=================================================== + +The change in Django 1.11 to make :meth:`.QuerySet.iterator()` use server-side +cursors on PostgreSQL prevents running Django with `pgBouncer` in transaction +pooling mode. To reallow that, use the :setting:`DISABLE_SERVER_SIDE_CURSORS +` setting in :setting:`DATABASES`. + +See :ref:`transaction-pooling-server-side-cursors` for more discussion. Bugfixes ======== diff --git a/docs/spelling_wordlist b/docs/spelling_wordlist index 2a75f0d01e..de01fa314c 100644 --- a/docs/spelling_wordlist +++ b/docs/spelling_wordlist @@ -496,6 +496,8 @@ plugins pluralizations po podcast +pooler +pooling popup postfix postgis diff --git a/tests/backends/test_postgresql.py b/tests/backends/test_postgresql.py index 024fc1add3..f4020f4e5c 100644 --- a/tests/backends/test_postgresql.py +++ b/tests/backends/test_postgresql.py @@ -1,5 +1,7 @@ +import operator import unittest from collections import namedtuple +from contextlib import contextmanager from django.db import connection from django.test import TestCase @@ -23,6 +25,18 @@ class ServerSideCursorsPostgres(TestCase): cursors = cursor.fetchall() return [self.PostgresCursor._make(cursor) for cursor in cursors] + @contextmanager + def override_db_setting(self, **kwargs): + for setting, value in kwargs.items(): + original_value = connection.settings_dict.get(setting) + if setting in connection.settings_dict: + self.addCleanup(operator.setitem, connection.settings_dict, setting, original_value) + else: + self.addCleanup(operator.delitem, connection.settings_dict, setting) + + connection.settings_dict[setting] = kwargs[setting] + yield + def test_server_side_cursor(self): persons = Person.objects.iterator() next(persons) # Open a server-side cursor @@ -52,3 +66,17 @@ class ServerSideCursorsPostgres(TestCase): del persons cursors = self.inspect_cursors() self.assertEqual(len(cursors), 0) + + def test_server_side_cursors_setting(self): + with self.override_db_setting(DISABLE_SERVER_SIDE_CURSORS=False): + persons = Person.objects.iterator() + next(persons) # Open a server-side cursor + cursors = self.inspect_cursors() + self.assertEqual(len(cursors), 1) + del persons # Close server-side cursor + + with self.override_db_setting(DISABLE_SERVER_SIDE_CURSORS=True): + persons = Person.objects.iterator() + next(persons) # Should not open a server-side cursor + cursors = self.inspect_cursors() + self.assertEqual(len(cursors), 0)