From d97f026a7ab5212192426e45121f7a52751a2044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dra=C5=BEen=20Odoba=C5=A1i=C4=87?= Date: Sun, 19 Nov 2017 10:13:10 -0500 Subject: [PATCH] Fixed #28817 -- Made QuerySet.iterator() use server-side cursors after values() and values_list(). --- django/db/models/query.py | 11 +++++++---- django/db/models/sql/compiler.py | 5 +++-- docs/releases/1.11.8.txt | 3 +++ .../postgresql/test_server_side_cursors.py | 15 ++++++++++++++- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/django/db/models/query.py b/django/db/models/query.py index b05497db7a..c1d57e4c61 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -105,7 +105,7 @@ class ValuesIterable(BaseIterable): names = extra_names + field_names + annotation_names indexes = range(len(names)) - for row in compiler.results_iter(): + for row in compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size): yield {names[i]: row[i] for i in indexes} @@ -133,8 +133,11 @@ class ValuesListIterable(BaseIterable): # Reorder according to fields. index_map = {name: idx for idx, name in enumerate(names)} rowfactory = operator.itemgetter(*[index_map[f] for f in fields]) - return map(rowfactory, compiler.results_iter()) - return compiler.results_iter(tuple_expected=True) + return map( + rowfactory, + compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size) + ) + return compiler.results_iter(tuple_expected=True, chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size) class NamedValuesListIterable(ValuesListIterable): @@ -174,7 +177,7 @@ class FlatValuesListIterable(BaseIterable): def __iter__(self): queryset = self.queryset compiler = queryset.query.get_compiler(queryset.db) - return chain.from_iterable(compiler.results_iter()) + return chain.from_iterable(compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)) class QuerySet: diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 59f5ac6491..2b0590de3a 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -1002,10 +1002,11 @@ class SQLCompiler: row[pos] = value yield row - def results_iter(self, results=None, tuple_expected=False): + def results_iter(self, results=None, tuple_expected=False, chunked_fetch=False, + chunk_size=GET_ITERATOR_CHUNK_SIZE): """Return an iterator over the results from executing this query.""" if results is None: - results = self.execute_sql(MULTI) + results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size) fields = [s[0] for s in self.select[0:self.col_count]] converters = self.get_converters(fields) rows = chain.from_iterable(results) diff --git a/docs/releases/1.11.8.txt b/docs/releases/1.11.8.txt index 426b6d92b2..89b6e5ed52 100644 --- a/docs/releases/1.11.8.txt +++ b/docs/releases/1.11.8.txt @@ -18,3 +18,6 @@ Bugfixes * Fixed incorrect index name truncation when using a namespaced ``db_table`` (:ticket:`28792`). + +* Made ``QuerySet.iterator()`` use server-side cursors on PostgreSQL after + ``values()`` and ``values_list()`` (:ticket:`28817`). diff --git a/tests/backends/postgresql/test_server_side_cursors.py b/tests/backends/postgresql/test_server_side_cursors.py index 53d5afce90..ff06318e6c 100644 --- a/tests/backends/postgresql/test_server_side_cursors.py +++ b/tests/backends/postgresql/test_server_side_cursors.py @@ -3,7 +3,7 @@ import unittest from collections import namedtuple from contextlib import contextmanager -from django.db import connection +from django.db import connection, models from django.test import TestCase from ..models import Person @@ -53,6 +53,19 @@ class ServerSideCursorsPostgres(TestCase): def test_server_side_cursor(self): self.assertUsesCursor(Person.objects.iterator()) + def test_values(self): + self.assertUsesCursor(Person.objects.values('first_name').iterator()) + + def test_values_list(self): + self.assertUsesCursor(Person.objects.values_list('first_name').iterator()) + + def test_values_list_flat(self): + self.assertUsesCursor(Person.objects.values_list('first_name', flat=True).iterator()) + + def test_values_list_fields_not_equal_to_names(self): + expr = models.Count('id') + self.assertUsesCursor(Person.objects.annotate(id__count=expr).values_list(expr, 'id__count').iterator()) + def test_server_side_cursor_many_cursors(self): persons = Person.objects.iterator() persons2 = Person.objects.iterator()