From 02983c5242ee71925a7a9ca15ea7cab49e81396b Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Fri, 18 Oct 2019 10:08:50 +0100 Subject: [PATCH] Fixed #30943 -- Added BloomIndex to django.contrib.postgres. --- django/contrib/postgres/indexes.py | 52 +++++++++++- django/contrib/postgres/operations.py | 6 ++ django/db/backends/postgresql/features.py | 1 + docs/ref/contrib/postgres/indexes.txt | 23 ++++++ docs/ref/contrib/postgres/operations.txt | 9 ++ docs/releases/3.1.txt | 5 +- .../migrations/0001_setup_extensions.py | 4 +- tests/postgres_tests/test_indexes.py | 82 ++++++++++++++++++- 8 files changed, 177 insertions(+), 5 deletions(-) diff --git a/django/contrib/postgres/indexes.py b/django/contrib/postgres/indexes.py index 526e729f2a..8f0c77aa52 100644 --- a/django/contrib/postgres/indexes.py +++ b/django/contrib/postgres/indexes.py @@ -3,8 +3,8 @@ from django.db.utils import NotSupportedError from django.utils.functional import cached_property __all__ = [ - 'BrinIndex', 'BTreeIndex', 'GinIndex', 'GistIndex', 'HashIndex', - 'SpGistIndex', + 'BloomIndex', 'BrinIndex', 'BTreeIndex', 'GinIndex', 'GistIndex', + 'HashIndex', 'SpGistIndex', ] @@ -36,6 +36,54 @@ class PostgresIndex(Index): return [] +class BloomIndex(PostgresIndex): + suffix = 'bloom' + + def __init__(self, *, length=None, columns=(), **kwargs): + super().__init__(**kwargs) + if len(self.fields) > 32: + raise ValueError('Bloom indexes support a maximum of 32 fields.') + if not isinstance(columns, (list, tuple)): + raise ValueError('BloomIndex.columns must be a list or tuple.') + if len(columns) > len(self.fields): + raise ValueError( + 'BloomIndex.columns cannot have more values than fields.' + ) + if not all(0 < col <= 4095 for col in columns): + raise ValueError( + 'BloomIndex.columns must contain integers from 1 to 4095.', + ) + if length is not None and not 0 < length <= 4096: + raise ValueError( + 'BloomIndex.length must be None or an integer from 1 to 4096.', + ) + self.length = length + self.columns = columns + + def deconstruct(self): + path, args, kwargs = super().deconstruct() + if self.length is not None: + kwargs['length'] = self.length + if self.columns: + kwargs['columns'] = self.columns + return path, args, kwargs + + def check_supported(self, schema_editor): + if not schema_editor.connection.features.has_bloom_index: + raise NotSupportedError('Bloom indexes require PostgreSQL 9.6+.') + + def get_with_params(self): + with_params = [] + if self.length is not None: + with_params.append('length = %d' % self.length) + if self.columns: + with_params.extend( + 'col%d = %d' % (i, v) + for i, v in enumerate(self.columns, start=1) + ) + return with_params + + class BrinIndex(PostgresIndex): suffix = 'brin' diff --git a/django/contrib/postgres/operations.py b/django/contrib/postgres/operations.py index 9e417725ec..52776324c5 100644 --- a/django/contrib/postgres/operations.py +++ b/django/contrib/postgres/operations.py @@ -37,6 +37,12 @@ class CreateExtension(Operation): return "Creates extension %s" % self.name +class BloomExtension(CreateExtension): + + def __init__(self): + self.name = 'bloom' + + class BtreeGinExtension(CreateExtension): def __init__(self): diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index 5e7c99510d..58b16d2b48 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -68,6 +68,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): def is_postgresql_12(self): return self.connection.pg_version >= 120000 + has_bloom_index = property(operator.attrgetter('is_postgresql_9_6')) has_brin_autosummarize = property(operator.attrgetter('is_postgresql_10')) has_phraseto_tsquery = property(operator.attrgetter('is_postgresql_9_6')) supports_table_partitions = property(operator.attrgetter('is_postgresql_10')) diff --git a/docs/ref/contrib/postgres/indexes.txt b/docs/ref/contrib/postgres/indexes.txt index 3c5ff89c67..f9c0fb2f97 100644 --- a/docs/ref/contrib/postgres/indexes.txt +++ b/docs/ref/contrib/postgres/indexes.txt @@ -7,6 +7,29 @@ PostgreSQL specific model indexes The following are PostgreSQL specific :doc:`indexes ` available from the ``django.contrib.postgres.indexes`` module. +``BloomIndex`` +============== + +.. class:: BloomIndex(length=None, columns=(), **options) + + .. versionadded:: 3.1 + + Creates a bloom_ index. + + To use this index access you need to activate the bloom_ extension on + PostgreSQL. You can install it using the + :class:`~django.contrib.postgres.operations.BloomExtension` migration + operation. + + Provide an integer number of bits from 1 to 4096 to the ``length`` + parameter to specify the length of each index entry. PostgreSQL's default + is 80. + + The ``columns`` argument takes a tuple or list of up to 32 values that are + integer number of bits from 1 to 4095. + + .. _bloom: https://www.postgresql.org/docs/current/bloom.html + ``BrinIndex`` ============= diff --git a/docs/ref/contrib/postgres/operations.txt b/docs/ref/contrib/postgres/operations.txt index 1329ff1f9e..620e5db8b7 100644 --- a/docs/ref/contrib/postgres/operations.txt +++ b/docs/ref/contrib/postgres/operations.txt @@ -49,6 +49,15 @@ run the query ``CREATE EXTENSION IF NOT EXISTS hstore;``. This is a required argument. The name of the extension to be installed. +``BloomExtension`` +================== + +.. class:: BloomExtension() + + .. versionadded:: 3.1 + + Install the ``bloom`` extension. + ``BtreeGinExtension`` ===================== diff --git a/docs/releases/3.1.txt b/docs/releases/3.1.txt index d7c05e88df..a3b4f30d63 100644 --- a/docs/releases/3.1.txt +++ b/docs/releases/3.1.txt @@ -71,7 +71,10 @@ Minor features :mod:`django.contrib.postgres` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* ... +* The new :class:`~django.contrib.postgres.indexes.BloomIndex` class allows + creating ``bloom`` indexes in the database. The new + :class:`~django.contrib.postgres.operations.BloomExtension` migration + operation installs the ``bloom`` extension to add support for this index. :mod:`django.contrib.redirects` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/postgres_tests/migrations/0001_setup_extensions.py b/tests/postgres_tests/migrations/0001_setup_extensions.py index 067c21849e..5064f7ff99 100644 --- a/tests/postgres_tests/migrations/0001_setup_extensions.py +++ b/tests/postgres_tests/migrations/0001_setup_extensions.py @@ -4,11 +4,12 @@ from django.db import migrations try: from django.contrib.postgres.operations import ( - BtreeGinExtension, BtreeGistExtension, CITextExtension, + BloomExtension, BtreeGinExtension, BtreeGistExtension, CITextExtension, CreateExtension, CryptoExtension, HStoreExtension, TrigramExtension, UnaccentExtension, ) except ImportError: + BloomExtension = mock.Mock() BtreeGinExtension = mock.Mock() BtreeGistExtension = mock.Mock() CITextExtension = mock.Mock() @@ -22,6 +23,7 @@ except ImportError: class Migration(migrations.Migration): operations = [ + BloomExtension(), BtreeGinExtension(), BtreeGistExtension(), CITextExtension(), diff --git a/tests/postgres_tests/test_indexes.py b/tests/postgres_tests/test_indexes.py index 5772069d25..91cac631e2 100644 --- a/tests/postgres_tests/test_indexes.py +++ b/tests/postgres_tests/test_indexes.py @@ -1,7 +1,8 @@ from unittest import mock from django.contrib.postgres.indexes import ( - BrinIndex, BTreeIndex, GinIndex, GistIndex, HashIndex, SpGistIndex, + BloomIndex, BrinIndex, BTreeIndex, GinIndex, GistIndex, HashIndex, + SpGistIndex, ) from django.db import connection from django.db.models import CharField @@ -30,6 +31,50 @@ class IndexTestMixin: self.assertEqual(kwargs, {'fields': ['title'], 'name': 'test_title_%s' % self.index_class.suffix}) +class BloomIndexTests(IndexTestMixin, PostgreSQLSimpleTestCase): + index_class = BloomIndex + + def test_suffix(self): + self.assertEqual(BloomIndex.suffix, 'bloom') + + def test_deconstruction(self): + index = BloomIndex(fields=['title'], name='test_bloom', length=80, columns=[4]) + path, args, kwargs = index.deconstruct() + self.assertEqual(path, 'django.contrib.postgres.indexes.BloomIndex') + self.assertEqual(args, ()) + self.assertEqual(kwargs, { + 'fields': ['title'], + 'name': 'test_bloom', + 'length': 80, + 'columns': [4], + }) + + def test_invalid_fields(self): + msg = 'Bloom indexes support a maximum of 32 fields.' + with self.assertRaisesMessage(ValueError, msg): + BloomIndex(fields=['title'] * 33, name='test_bloom') + + def test_invalid_columns(self): + msg = 'BloomIndex.columns must be a list or tuple.' + with self.assertRaisesMessage(ValueError, msg): + BloomIndex(fields=['title'], name='test_bloom', columns='x') + msg = 'BloomIndex.columns cannot have more values than fields.' + with self.assertRaisesMessage(ValueError, msg): + BloomIndex(fields=['title'], name='test_bloom', columns=[4, 3]) + + def test_invalid_columns_value(self): + msg = 'BloomIndex.columns must contain integers from 1 to 4095.' + for length in (0, 4096): + with self.subTest(length), self.assertRaisesMessage(ValueError, msg): + BloomIndex(fields=['title'], name='test_bloom', columns=[length]) + + def test_invalid_length(self): + msg = 'BloomIndex.length must be None or an integer from 1 to 4096.' + for length in (0, 4097): + with self.subTest(length), self.assertRaisesMessage(ValueError, msg): + BloomIndex(fields=['title'], name='test_bloom', length=length) + + class BrinIndexTests(IndexTestMixin, PostgreSQLSimpleTestCase): index_class = BrinIndex @@ -217,6 +262,41 @@ class SchemaTests(PostgreSQLTestCase): editor.remove_index(IntegerArrayModel, index) self.assertNotIn(index_name, self.get_constraints(IntegerArrayModel._meta.db_table)) + @skipUnlessDBFeature('has_bloom_index') + def test_bloom_index(self): + index_name = 'char_field_model_field_bloom' + index = BloomIndex(fields=['field'], name=index_name) + with connection.schema_editor() as editor: + editor.add_index(CharFieldModel, index) + constraints = self.get_constraints(CharFieldModel._meta.db_table) + self.assertEqual(constraints[index_name]['type'], BloomIndex.suffix) + with connection.schema_editor() as editor: + editor.remove_index(CharFieldModel, index) + self.assertNotIn(index_name, self.get_constraints(CharFieldModel._meta.db_table)) + + @skipUnlessDBFeature('has_bloom_index') + def test_bloom_parameters(self): + index_name = 'char_field_model_field_bloom_params' + index = BloomIndex(fields=['field'], name=index_name, length=512, columns=[3]) + with connection.schema_editor() as editor: + editor.add_index(CharFieldModel, index) + constraints = self.get_constraints(CharFieldModel._meta.db_table) + self.assertEqual(constraints[index_name]['type'], BloomIndex.suffix) + self.assertEqual(constraints[index_name]['options'], ['length=512', 'col1=3']) + with connection.schema_editor() as editor: + editor.remove_index(CharFieldModel, index) + self.assertNotIn(index_name, self.get_constraints(CharFieldModel._meta.db_table)) + + def test_bloom_index_not_supported(self): + index_name = 'bloom_index_exception' + index = BloomIndex(fields=['field'], name=index_name) + msg = 'Bloom indexes require PostgreSQL 9.6+.' + with self.assertRaisesMessage(NotSupportedError, msg): + with mock.patch('django.db.backends.postgresql.features.DatabaseFeatures.has_bloom_index', False): + with connection.schema_editor() as editor: + editor.add_index(CharFieldModel, index) + self.assertNotIn(index_name, self.get_constraints(CharFieldModel._meta.db_table)) + def test_brin_index(self): index_name = 'char_field_model_field_brin' index = BrinIndex(fields=['field'], name=index_name, pages_per_range=4)