From c0e3c65b9d1b26cfc38137b7666ef0e108aab77f Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Fri, 20 Jul 2018 14:57:17 -0400 Subject: [PATCH] Fixed #29563 -- Added result streaming for QuerySet.iterator() on SQLite. --- django/db/backends/sqlite3/features.py | 9 ++++----- docs/ref/databases.txt | 13 +++++++++++++ docs/ref/models/querysets.txt | 17 +++++++++++++---- docs/releases/2.2.txt | 2 +- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py index 4d9f4985d4..770b40a1ba 100644 --- a/django/db/backends/sqlite3/features.py +++ b/django/db/backends/sqlite3/features.py @@ -4,11 +4,10 @@ from django.utils.functional import cached_property class DatabaseFeatures(BaseDatabaseFeatures): - # SQLite cannot handle us only partially reading from a cursor's result set - # and then writing the same rows to the database in another cursor. This - # setting ensures we always read result sets fully into memory all in one - # go. - can_use_chunked_reads = False + # SQLite can read from a cursor since SQLite 3.6.5, subject to the caveat + # that statements within a connection aren't isolated from each other. See + # https://sqlite.org/isolation.html. + can_use_chunked_reads = True test_db_allows_multiple_connections = False supports_unspecified_pk = True supports_timezones = False diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index a0f7fa059d..31853f785c 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -710,6 +710,19 @@ can use the "pyformat" parameter style, where placeholders in the query are given as ``'%(name)s'`` and the parameters are passed as a dictionary rather than a list. SQLite does not support this. +.. _sqlite-isolation: + +Isolation when using ``QuerySet.iterator()`` +-------------------------------------------- + +There are special considerations described in `Isolation In SQLite`_ when +modifying a table while iterating over it using :meth:`.QuerySet.iterator`. If +a row is added, changed, or deleted within the loop, then that row may or may +not appear, or may appear twice, in subsequent results fetched from the +iterator. Your code must handle this. + +.. _`Isolation in SQLite`: https://sqlite.org/isolation.html + .. _oracle-notes: Oracle notes diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index b190ff02b0..a5a0787d4c 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -2178,10 +2178,15 @@ don't support server-side cursors. Without server-side cursors ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -MySQL and SQLite don't support streaming results, hence the Python database -drivers load the entire result set into memory. The result set is then -transformed into Python row objects by the database adapter using the -``fetchmany()`` method defined in :pep:`249`. +MySQL doesn't support streaming results, hence the Python database driver loads +the entire result set into memory. The result set is then transformed into +Python row objects by the database adapter using the ``fetchmany()`` method +defined in :pep:`249`. + +SQLite can fetch results in batches using ``fetchmany()``, but since SQLite +doesn't provide isolation between queries within a connection, be careful when +writing to the table being iterated over. See :ref:`sqlite-isolation` for +more information. The ``chunk_size`` parameter controls the size of batches Django retrieves from the database driver. Larger batches decrease the overhead of communicating with @@ -2195,6 +2200,10 @@ psycopg mailing list