test0908
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #26709 -- Added class-based indexes. 

Added the AddIndex and RemoveIndex operations to use them in migrations.

Thanks markush, mjtamlyn, timgraham, and charettes for review and advice.
This commit is contained in:
Akshesh 2016-06-25 22:02:56 +05:30 committed by Tim Graham
parent c962b9104a
commit 156e2d59cf
18 changed files with 473 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
django/db/backends/base/schema.py
View File

@ -316,6 +316,18 @@ class BaseDatabaseSchemaEditor(object):
"table": self.quote_name(model._meta.db_table),
})
def add_index(self, index):
"""
Add an index on a model.
"""
self.execute(index.create_sql(self))
def remove_index(self, index):
"""
Remove an index from a model.
"""
self.execute(index.remove_sql(self))
def alter_unique_together(self, model, old_unique_together, new_unique_together):
"""
Deals with a model changing its unique_together.
@ -836,12 +848,7 @@ class BaseDatabaseSchemaEditor(object):
index_name = "D%s" % index_name[:-1]
return index_name
def _create_index_sql(self, model, fields, suffix="", sql=None):
"""
Return the SQL statement to create the index for one or several fields.
`sql` can be specified if the syntax differs from the standard (GIS
indexes, ...).
"""
def _get_index_tablespace_sql(self, model, fields):
if len(fields) == 1 and fields[0].db_tablespace:
tablespace_sql = self.connection.ops.tablespace_sql(fields[0].db_tablespace)
elif model._meta.db_tablespace:
@ -850,7 +857,15 @@ class BaseDatabaseSchemaEditor(object):
tablespace_sql = ""
if tablespace_sql:
tablespace_sql = " " + tablespace_sql
return tablespace_sql
def _create_index_sql(self, model, fields, suffix="", sql=None):
"""
Return the SQL statement to create the index for one or several fields.
`sql` can be specified if the syntax differs from the standard (GIS
indexes, ...).
"""
tablespace_sql = self._get_index_tablespace_sql(model, fields)
columns = [field.column for field in fields]
sql_create_index = sql or self.sql_create_index
return sql_create_index % {

10
django/db/migrations/operations/__init__.py
View File

@ -1,15 +1,15 @@
from .fields import AddField, AlterField, RemoveField, RenameField
from .models import (
AlterIndexTogether, AlterModelManagers, AlterModelOptions, AlterModelTable,
AlterOrderWithRespectTo, AlterUniqueTogether, CreateModel, DeleteModel,
RenameModel,
AddIndex, AlterIndexTogether, AlterModelManagers, AlterModelOptions,
AlterModelTable, AlterOrderWithRespectTo, AlterUniqueTogether, CreateModel,
DeleteModel, RemoveIndex, RenameModel,
)
from .special import RunPython, RunSQL, SeparateDatabaseAndState
__all__ = [
'CreateModel', 'DeleteModel', 'AlterModelTable', 'AlterUniqueTogether',
'RenameModel', 'AlterIndexTogether', 'AlterModelOptions',
'AddField', 'RemoveField', 'AlterField', 'RenameField',
'RenameModel', 'AlterIndexTogether', 'AlterModelOptions', 'AddIndex',
'RemoveIndex', 'AddField', 'RemoveField', 'AlterField', 'RenameField',
'SeparateDatabaseAndState', 'RunSQL', 'RunPython',
'AlterOrderWithRespectTo', 'AlterModelManagers',
]

77
django/db/migrations/operations/models.py
View File

@ -742,3 +742,80 @@ class AlterModelManagers(ModelOptionOperation):
def describe(self):
return "Change managers on %s" % (self.name, )
class AddIndex(Operation):
"""
Add an index on a model.
"""
def __init__(self, model_name, index):
self.model_name = model_name
self.index = index
def state_forwards(self, app_label, state):
model_state = state.models[app_label, self.model_name.lower()]
self.index.model = state.apps.get_model(app_label, self.model_name)
model_state.options['indexes'].append(self.index)
def database_forwards(self, app_label, schema_editor, from_state, to_state):
schema_editor.add_index(self.index)
def database_backwards(self, app_label, schema_editor, from_state, to_state):
schema_editor.remove_index(self.index)
def deconstruct(self):
kwargs = {
'model_name': self.model_name,
'index': self.index,
}
return (
self.__class__.__name__,
[],
kwargs,
)
def describe(self):
return 'Create index on field(s) %s of model %s' % (
', '.join(self.index.fields),
self.model_name,
)
class RemoveIndex(Operation):
"""
Remove an index from a model.
"""
def __init__(self, model_name, name):
self.model_name = model_name
self.name = name
def state_forwards(self, app_label, state):
model_state = state.models[app_label, self.model_name.lower()]
indexes = model_state.options['indexes']
model_state.options['indexes'] = [idx for idx in indexes if idx.name != self.name]
def database_forwards(self, app_label, schema_editor, from_state, to_state):
from_model_state = from_state.models[app_label, self.model_name.lower()]
index = from_model_state.get_index_by_name(self.name)
schema_editor.remove_index(index)
def database_backwards(self, app_label, schema_editor, from_state, to_state):
to_model_state = to_state.models[app_label, self.model_name.lower()]
index = to_model_state.get_index_by_name(self.name)
schema_editor.add_index(index)
def deconstruct(self):
kwargs = {
'model_name': self.model_name,
'name': self.name,
}
return (
self.__class__.__name__,
[],
kwargs,
)
def describe(self):
return 'Remove index %s from %s' % (self.name, self.model_name)

7
django/db/migrations/state.py
View File

@ -330,6 +330,7 @@ class ModelState(object):
self.name = force_text(name)
self.fields = fields
self.options = options or {}
self.options.setdefault('indexes', [])
self.bases = bases or (models.Model, )
self.managers = managers or []
# Sanity-check that fields is NOT a dict. It must be ordered.
@ -557,6 +558,12 @@ class ModelState(object):
return field
raise ValueError("No field called %s on model %s" % (name, self.name))
def get_index_by_name(self, name):
for index in self.options['indexes']:
if index.name == name:
return index
raise ValueError("No index named %s on model %s" % (name, self.name))
def __repr__(self):
return "<ModelState: '%s.%s'>" % (self.app_label, self.name)

1
django/db/models/__init__.py
View File

@ -12,6 +12,7 @@ from django.db.models.expressions import ( # NOQA
from django.db.models.fields import * # NOQA
from django.db.models.fields.files import FileField, ImageField # NOQA
from django.db.models.fields.proxy import OrderWrt # NOQA
from django.db.models.indexes import * # NOQA
from django.db.models.lookups import Lookup, Transform # NOQA
from django.db.models.manager import Manager # NOQA
from django.db.models.query import ( # NOQA

113
django/db/models/indexes.py Normal file
View File

@ -0,0 +1,113 @@
from __future__ import unicode_literals
import hashlib
from django.utils.encoding import force_bytes
from django.utils.functional import cached_property
__all__ = ['Index']
# The max length of the names of the indexes (restricted to 30 due to Oracle)
MAX_NAME_LENGTH = 30
class Index(object):
suffix = 'idx'
def __init__(self, fields=[], name=None):
if not fields:
raise ValueError('At least one field is required to define an index.')
self.fields = fields
self._name = name or ''
if self._name:
errors = self.check_name()
if len(self._name) > MAX_NAME_LENGTH:
errors.append('Index names cannot be longer than %s characters.' % MAX_NAME_LENGTH)
if errors:
raise ValueError(errors)
@cached_property
def name(self):
if not self._name:
self._name = self.get_name()
self.check_name()
return self._name
def check_name(self):
errors = []
# Name can't start with an underscore on Oracle; prepend D if needed.
if self._name[0] == '_':
errors.append('Index names cannot start with an underscore (_).')
self._name = 'D%s' % self._name[1:]
# Name can't start with a number on Oracle; prepend D if needed.
elif self._name[0].isdigit():
errors.append('Index names cannot start with a number (0-9).')
self._name = 'D%s' % self._name[1:]
return errors
def create_sql(self, schema_editor):
fields = [self.model._meta.get_field(field) for field in self.fields]
tablespace_sql = schema_editor._get_index_tablespace_sql(self.model, fields)
columns = [field.column for field in fields]
quote_name = schema_editor.quote_name
return schema_editor.sql_create_index % {
'table': quote_name(self.model._meta.db_table),
'name': quote_name(self.name),
'columns': ', '.join(quote_name(column) for column in columns),
'extra': tablespace_sql,
}
def remove_sql(self, schema_editor):
quote_name = schema_editor.quote_name
return schema_editor.sql_delete_index % {
'table': quote_name(self.model._meta.db_table),
'name': quote_name(self.name),
}
def deconstruct(self):
path = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)
path = path.replace('django.db.models.indexes', 'django.db.models')
return (path, (), {'fields': self.fields})
@staticmethod
def _hash_generator(*args):
"""
Generate a 32-bit digest of a set of arguments that can be used to
shorten identifying names.
"""
h = hashlib.md5()
for arg in args:
h.update(force_bytes(arg))
return h.hexdigest()[:6]
def get_name(self):
"""
Generate a unique name for the index.
The name is divided into 3 parts - table name (12 chars), field name
(8 chars) and unique hash + suffix (10 chars). Each part is made to
fit its size by truncating the excess length.
"""
table_name = self.model._meta.db_table
column_names = [self.model._meta.get_field(field).column for field in self.fields]
hash_data = [table_name] + column_names + [self.suffix]
index_name = '%s_%s_%s' % (
table_name[:11],
column_names[0][:7],
'%s_%s' % (self._hash_generator(*hash_data), self.suffix),
)
assert len(index_name) <= 30, (
'Index too long for multiple database support. Is self.suffix '
'longer than 3 characters?'
)
return index_name
def __repr__(self):
return "<%s: fields='%s'>" % (self.__class__.__name__, ', '.join(self.fields))
def __eq__(self, other):
return (self.__class__ == other.__class__) and (self.deconstruct() == other.deconstruct())
def __ne__(self, other):
return not (self == other)

2
django/db/models/options.py
View File

@ -43,7 +43,7 @@ DEFAULT_NAMES = (
'auto_created', 'index_together', 'apps', 'default_permissions',
'select_on_save', 'default_related_name', 'required_db_features',
'required_db_vendor', 'base_manager_name', 'default_manager_name',
'manager_inheritance_from_future',
'manager_inheritance_from_future', 'indexes',
)

1
docs/index.txt
View File

@ -82,6 +82,7 @@ manipulating the data of your Web application. Learn more about it below:
* **Models:**
:doc:`Introduction to models <topics/db/models>` |
:doc:`Field types <ref/models/fields>` |
:doc:`Indexes <ref/models/indexes>` |
:doc:`Meta options <ref/models/options>` |
:doc:`Model class <ref/models/class>`

37
docs/ref/migration-operations.txt
View File

@ -192,6 +192,43 @@ field like ``models.IntegerField()`` on most databases.
Changes a field's name (and, unless :attr:`~django.db.models.Field.db_column`
is set, its column name).
``AddIndex``
------------
.. class:: AddIndex(model_name, index)
.. versionadded:: 1.11
Creates an index in the database table for the model with ``model_name``.
``index`` is an instance of the :class:`~django.db.models.Index` class.
For example, to add an index on the ``title`` and ``author`` fields of the
``Book`` model::
from django.db import migrations, models
class Migration(migrations.Migration):
operations = [
migrations.AddIndex(
'Book',
models.Index(fields=['title', 'author'], name='my_index_name'),
),
]
If you're writing your own migration to add an index, it's recommended to pass
a ``name`` to the ``index`` as done above so that you can reference it if you
later want to remove it. Otherwise, a name will be autogenerated and you'll
have to inspect the database to find the index name if you want to remove it.
``RemoveIndex``
---------------
.. class:: RemoveIndex(model_name, name)
.. versionadded:: 1.11
Removes the index named ``name`` from the model with ``model_name``.
Special Operations
==================

1
docs/ref/models/index.txt
View File

@ -8,6 +8,7 @@ Model API reference. For introductory material, see :doc:`/topics/db/models`.
:maxdepth: 1
fields
indexes
meta
relations
class

48
docs/ref/models/indexes.txt Normal file
View File

@ -0,0 +1,48 @@
=====================
Model index reference
=====================
.. module:: django.db.models.indexes
.. currentmodule:: django.db.models
.. versionadded:: 1.11
Index classes ease creating database indexes. This document explains the API
references of :class:`Index` which includes the `index options`_.
.. admonition:: Referencing built-in indexes
Indexes are defined in ``django.db.models.indexes``, but for convenience
they're imported into :mod:`django.db.models`. The standard convention is
to use ``from django.db import models`` and refer to the indexes as
``models.<IndexClass>``.
``Index`` options
=================
.. class:: Index(fields=[], name=None)
Creates an index (B-Tree) in the database.
``fields``
-----------
.. attribute:: Index.fields
A list of the name of the fields on which the index is desired.
``name``
--------
.. attribute:: Index.name
The name of the index. If ``name`` isn't provided Django will auto-generate a
name. For compatibility with different databases, index names cannot be longer
than 30 characters and shouldn't start with a number (0-9) or underscore (_).
.. seealso::
Use the :class:`~django.db.migrations.operations.AddIndex` and
:class:`~django.db.migrations.operations.RemoveIndex` operations to add
and remove indexes.

10
tests/migrations/test_autodetector.py
View File

@ -1263,7 +1263,9 @@ class AutodetectorTests(TestCase):
# Right number/type of migrations?
self.assertNumberMigrations(changes, "testapp", 1)
self.assertOperationTypes(changes, "testapp", 0, ["CreateModel"])
self.assertOperationAttributes(changes, "testapp", 0, 0, name="AuthorProxy", options={"proxy": True})
self.assertOperationAttributes(
changes, "testapp", 0, 0, name="AuthorProxy", options={"proxy": True, "indexes": []}
)
# Now, we test turning a proxy model into a non-proxy model
# It should delete the proxy then make the real one
changes = self.get_changes(
@ -1273,7 +1275,7 @@ class AutodetectorTests(TestCase):
self.assertNumberMigrations(changes, "testapp", 1)
self.assertOperationTypes(changes, "testapp", 0, ["DeleteModel", "CreateModel"])
self.assertOperationAttributes(changes, "testapp", 0, 0, name="AuthorProxy")
self.assertOperationAttributes(changes, "testapp", 0, 1, name="AuthorProxy", options={})
self.assertOperationAttributes(changes, "testapp", 0, 1, name="AuthorProxy", options={"indexes": []})
def test_proxy_custom_pk(self):
"""
@ -1296,7 +1298,9 @@ class AutodetectorTests(TestCase):
# Right number/type of migrations?
self.assertNumberMigrations(changes, 'testapp', 1)
self.assertOperationTypes(changes, 'testapp', 0, ["CreateModel"])
self.assertOperationAttributes(changes, 'testapp', 0, 0, name="AuthorUnmanaged", options={"managed": False})
self.assertOperationAttributes(
changes, 'testapp', 0, 0, name="AuthorUnmanaged", options={"managed": False, "indexes": []}
)
def test_unmanaged_to_managed(self):
# Now, we test turning an unmanaged model into a managed model

61
tests/migrations/test_operations.py
View File

@ -51,7 +51,7 @@ class OperationTestBase(MigrationTestBase):
return project_state, new_state
def set_up_test_model(
self, app_label, second_model=False, third_model=False,
self, app_label, second_model=False, third_model=False, multicol_index=False,
related_model=False, mti_model=False, proxy_model=False, manager_model=False,
unique_together=False, options=False, db_table=None, index_together=False):
"""
@ -96,6 +96,11 @@ class OperationTestBase(MigrationTestBase):
],
options=model_options,
)]
if multicol_index:
operations.append(migrations.AddIndex(
"Pony",
models.Index(fields=["pink", "weight"], name="pony_test_idx")
))
if second_model:
operations.append(migrations.CreateModel(
"Stable",
@ -1375,6 +1380,60 @@ class OperationTests(OperationTestBase):
operation = migrations.AlterUniqueTogether("Pony", None)
self.assertEqual(operation.describe(), "Alter unique_together for Pony (0 constraint(s))")
def test_add_index(self):
"""
Test the AddIndex operation.
"""
project_state = self.set_up_test_model("test_adin")
index = models.Index(fields=["pink"])
operation = migrations.AddIndex("Pony", index)
self.assertEqual(operation.describe(), "Create index on field(s) pink of model Pony")
new_state = project_state.clone()
operation.state_forwards("test_adin", new_state)
# Test the database alteration
self.assertEqual(len(new_state.models["test_adin", "pony"].options['indexes']), 1)
self.assertIndexNotExists("test_adin_pony", ["pink"])
with connection.schema_editor() as editor:
operation.database_forwards("test_adin", editor, project_state, new_state)
self.assertIndexExists("test_adin_pony", ["pink"])
# And test reversal
with connection.schema_editor() as editor:
operation.database_backwards("test_adin", editor, new_state, project_state)
self.assertIndexNotExists("test_adin_pony", ["pink"])
# And deconstruction
definition = operation.deconstruct()
self.assertEqual(definition[0], "AddIndex")
self.assertEqual(definition[1], [])
self.assertEqual(definition[2], {'model_name': "Pony", 'index': index})
def test_remove_index(self):
"""
Test the RemoveIndex operation.
"""
project_state = self.set_up_test_model("test_rmin", multicol_index=True)
self.assertTableExists("test_rmin_pony")
self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
operation = migrations.RemoveIndex("Pony", "pony_test_idx")
self.assertEqual(operation.describe(), "Remove index pony_test_idx from Pony")
new_state = project_state.clone()
operation.state_forwards("test_rmin", new_state)
# Test the state alteration
self.assertEqual(len(new_state.models["test_rmin", "pony"].options['indexes']), 0)
self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
# Test the database alteration
with connection.schema_editor() as editor:
operation.database_forwards("test_rmin", editor, project_state, new_state)
self.assertIndexNotExists("test_rmin_pony", ["pink", "weight"])
# And test reversal
with connection.schema_editor() as editor:
operation.database_backwards("test_rmin", editor, new_state, project_state)
self.assertIndexExists("test_rmin_pony", ["pink", "weight"])
# And deconstruction
definition = operation.deconstruct()
self.assertEqual(definition[0], "RemoveIndex")
self.assertEqual(definition[1], [])
self.assertEqual(definition[2], {'model_name': "Pony", 'name': "pony_test_idx"})
def test_alter_index_together(self):
"""
Tests the AlterIndexTogether operation.

8
tests/migrations/test_state.py
View File

@ -125,7 +125,7 @@ class StateTests(SimpleTestCase):
self.assertIs(author_state.fields[3][1].null, True)
self.assertEqual(
author_state.options,
{"unique_together": {("name", "bio")}, "index_together": {("bio", "age")}}
{"unique_together": {("name", "bio")}, "index_together": {("bio", "age")}, "indexes": []}
)
self.assertEqual(author_state.bases, (models.Model, ))
@ -135,13 +135,13 @@ class StateTests(SimpleTestCase):
self.assertEqual(book_state.fields[1][1].max_length, 1000)
self.assertIs(book_state.fields[2][1].null, False)
self.assertEqual(book_state.fields[3][1].__class__.__name__, "ManyToManyField")
self.assertEqual(book_state.options, {"verbose_name": "tome", "db_table": "test_tome"})
self.assertEqual(book_state.options, {"verbose_name": "tome", "db_table": "test_tome", "indexes": []})
self.assertEqual(book_state.bases, (models.Model, ))
self.assertEqual(author_proxy_state.app_label, "migrations")
self.assertEqual(author_proxy_state.name, "AuthorProxy")
self.assertEqual(author_proxy_state.fields, [])
self.assertEqual(author_proxy_state.options, {"proxy": True, "ordering": ["name"]})
self.assertEqual(author_proxy_state.options, {"proxy": True, "ordering": ["name"], "indexes": []})
self.assertEqual(author_proxy_state.bases, ("migrations.author", ))
self.assertEqual(sub_author_state.app_label, "migrations")
@ -960,7 +960,7 @@ class ModelStateTests(SimpleTestCase):
self.assertEqual(author_state.fields[1][1].max_length, 255)
self.assertIs(author_state.fields[2][1].null, False)
self.assertIs(author_state.fields[3][1].null, True)
self.assertEqual(author_state.options, {'swappable': 'TEST_SWAPPABLE_MODEL'})
self.assertEqual(author_state.options, {'swappable': 'TEST_SWAPPABLE_MODEL', 'indexes': []})
self.assertEqual(author_state.bases, (models.Model, ))
self.assertEqual(author_state.managers, [])

0
tests/model_indexes/__init__.py Normal file
View File

7
tests/model_indexes/models.py Normal file
View File

@ -0,0 +1,7 @@
from django.db import models
class Book(models.Model):
title = models.CharField(max_length=50)
author = models.CharField(max_length=50)
pages = models.IntegerField(db_column='page_count')

62
tests/model_indexes/tests.py Normal file
View File

@ -0,0 +1,62 @@
from django.db import models
from django.test import TestCase
from .models import Book
class IndexesTests(TestCase):
def test_repr(self):
index = models.Index(fields=['title'])
multi_col_index = models.Index(fields=['title', 'author'])
self.assertEqual(repr(index), "<Index: fields='title'>")
self.assertEqual(repr(multi_col_index), "<Index: fields='title, author'>")
def test_eq(self):
index = models.Index(fields=['title'])
same_index = models.Index(fields=['title'])
another_index = models.Index(fields=['title', 'author'])
self.assertEqual(index, same_index)
self.assertNotEqual(index, another_index)
def test_raises_error_without_field(self):
msg = 'At least one field is required to define an index.'
with self.assertRaisesMessage(ValueError, msg):
models.Index()
def test_max_name_length(self):
msg = 'Index names cannot be longer than 30 characters.'
with self.assertRaisesMessage(ValueError, msg):
models.Index(fields=['title'], name='looooooooooooong_index_name_idx')
def test_name_constraints(self):
msg = 'Index names cannot start with an underscore (_).'
with self.assertRaisesMessage(ValueError, msg):
models.Index(fields=['title'], name='_name_starting_with_underscore')
msg = 'Index names cannot start with a number (0-9).'
with self.assertRaisesMessage(ValueError, msg):
models.Index(fields=['title'], name='5name_starting_with_number')
def test_name_auto_generation(self):
index = models.Index(fields=['author'])
index.model = Book
self.assertEqual(index.name, 'model_index_author_0f5565_idx')
# fields may be truncated in the name. db_column is used for naming.
long_field_index = models.Index(fields=['pages'])
long_field_index.model = Book
self.assertEqual(long_field_index.name, 'model_index_page_co_69235a_idx')
# suffix can't be longer than 3 characters.
long_field_index.suffix = 'suff'
msg = 'Index too long for multiple database support. Is self.suffix longer than 3 characters?'
with self.assertRaisesMessage(AssertionError, msg):
long_field_index.get_name()
def test_deconstruction(self):
index = models.Index(fields=['title'])
path, args, kwargs = index.deconstruct()
self.assertEqual(path, 'django.db.models.Index')
self.assertEqual(args, ())
self.assertEqual(kwargs, {'fields': ['title']})

21
tests/schema/tests.py
View File

@ -16,6 +16,7 @@ from django.db.models.fields import (
from django.db.models.fields.related import (
ForeignKey, ForeignObject, ManyToManyField, OneToOneField,
)
from django.db.models.indexes import Index
from django.db.transaction import atomic
from django.test import (
TransactionTestCase, mock, skipIfDBFeature, skipUnlessDBFeature,
@ -1443,6 +1444,26 @@ class SchemaTests(TransactionTestCase):
columns = self.column_classes(Author)
self.assertEqual(columns['name'][0], "CharField")
def test_add_remove_index(self):
"""
Tests index addition and removal
"""
# Create the table
with connection.schema_editor() as editor:
editor.create_model(Author)
# Ensure the table is there and has no index
self.assertNotIn('title', self.get_indexes(Author._meta.db_table))
# Add the index
index = Index(fields=['name'], name='author_title_idx')
index.model = Author
with connection.schema_editor() as editor:
editor.add_index(index)
self.assertIn('name', self.get_indexes(Author._meta.db_table))
# Drop the index
with connection.schema_editor() as editor:
editor.remove_index(index)
self.assertNotIn('name', self.get_indexes(Author._meta.db_table))
def test_indexes(self):
"""
Tests creation/altering of indexes