Fixed #31777 -- Added support for database collations to Char/TextFields.

Thanks Simon Charette and Mariusz Felisiak for reviews.
This commit is contained in:
Tom Carrick 2020-07-18 13:17:39 +02:00 committed by Mariusz Felisiak
parent ba6b32e5ef
commit e387f191f7
25 changed files with 544 additions and 30 deletions

View File

@ -247,6 +247,9 @@ class Command(BaseCommand):
if field_type == 'CharField' and row.internal_size: if field_type == 'CharField' and row.internal_size:
field_params['max_length'] = int(row.internal_size) field_params['max_length'] = int(row.internal_size)
if field_type in {'CharField', 'TextField'} and row.collation:
field_params['db_collation'] = row.collation
if field_type == 'DecimalField': if field_type == 'DecimalField':
if row.precision is None or row.scale is None: if row.precision is None or row.scale is None:
field_notes.append( field_notes.append(

View File

@ -302,10 +302,17 @@ class BaseDatabaseFeatures:
# {'d': [{'f': 'g'}]}? # {'d': [{'f': 'g'}]}?
json_key_contains_list_matching_requires_list = False json_key_contains_list_matching_requires_list = False
# Does the backend support column collations?
supports_collation_on_charfield = True
supports_collation_on_textfield = True
# Does the backend support non-deterministic collations?
supports_non_deterministic_collations = True
# Collation names for use by the Django test suite. # Collation names for use by the Django test suite.
test_collations = { test_collations = {
'ci': None, # Case-insensitive. 'ci': None, # Case-insensitive.
'cs': None, # Case-sensitive. 'cs': None, # Case-sensitive.
'non_default': None, # Non-default.
'swedish_ci': None # Swedish case-insensitive. 'swedish_ci': None # Swedish case-insensitive.
} }

View File

@ -4,7 +4,11 @@ from collections import namedtuple
TableInfo = namedtuple('TableInfo', ['name', 'type']) TableInfo = namedtuple('TableInfo', ['name', 'type'])
# Structure returned by the DB-API cursor.description interface (PEP 249) # Structure returned by the DB-API cursor.description interface (PEP 249)
FieldInfo = namedtuple('FieldInfo', 'name type_code display_size internal_size precision scale null_ok default') FieldInfo = namedtuple(
'FieldInfo',
'name type_code display_size internal_size precision scale null_ok '
'default collation'
)
class BaseDatabaseIntrospection: class BaseDatabaseIntrospection:

View File

@ -61,6 +61,7 @@ class BaseDatabaseSchemaEditor:
sql_alter_column_not_null = "ALTER COLUMN %(column)s SET NOT NULL" sql_alter_column_not_null = "ALTER COLUMN %(column)s SET NOT NULL"
sql_alter_column_default = "ALTER COLUMN %(column)s SET DEFAULT %(default)s" sql_alter_column_default = "ALTER COLUMN %(column)s SET DEFAULT %(default)s"
sql_alter_column_no_default = "ALTER COLUMN %(column)s DROP DEFAULT" sql_alter_column_no_default = "ALTER COLUMN %(column)s DROP DEFAULT"
sql_alter_column_collate = "ALTER COLUMN %(column)s TYPE %(type)s%(collation)s"
sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s CASCADE" sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s CASCADE"
sql_rename_column = "ALTER TABLE %(table)s RENAME COLUMN %(old_column)s TO %(new_column)s" sql_rename_column = "ALTER TABLE %(table)s RENAME COLUMN %(old_column)s TO %(new_column)s"
sql_update_with_default = "UPDATE %(table)s SET %(column)s = %(default)s WHERE %(column)s IS NULL" sql_update_with_default = "UPDATE %(table)s SET %(column)s = %(default)s WHERE %(column)s IS NULL"
@ -215,6 +216,10 @@ class BaseDatabaseSchemaEditor:
# Check for fields that aren't actually columns (e.g. M2M) # Check for fields that aren't actually columns (e.g. M2M)
if sql is None: if sql is None:
return None, None return None, None
# Collation.
collation = getattr(field, 'db_collation', None)
if collation:
sql += self._collate_sql(collation)
# Work out nullability # Work out nullability
null = field.null null = field.null
# If we were told to include a default value, do so # If we were told to include a default value, do so
@ -676,8 +681,15 @@ class BaseDatabaseSchemaEditor:
actions = [] actions = []
null_actions = [] null_actions = []
post_actions = [] post_actions = []
# Collation change?
old_collation = getattr(old_field, 'db_collation', None)
new_collation = getattr(new_field, 'db_collation', None)
if old_collation != new_collation:
# Collation change handles also a type change.
fragment = self._alter_column_collation_sql(model, new_field, new_type, new_collation)
actions.append(fragment)
# Type change? # Type change?
if old_type != new_type: elif old_type != new_type:
fragment, other_actions = self._alter_column_type_sql(model, old_field, new_field, new_type) fragment, other_actions = self._alter_column_type_sql(model, old_field, new_field, new_type)
actions.append(fragment) actions.append(fragment)
post_actions.extend(other_actions) post_actions.extend(other_actions)
@ -895,6 +907,16 @@ class BaseDatabaseSchemaEditor:
[], [],
) )
def _alter_column_collation_sql(self, model, new_field, new_type, new_collation):
return (
self.sql_alter_column_collate % {
'column': self.quote_name(new_field.column),
'type': new_type,
'collation': self._collate_sql(new_collation) if new_collation else '',
},
[],
)
def _alter_many_to_many(self, model, old_field, new_field, strict): def _alter_many_to_many(self, model, old_field, new_field, strict):
"""Alter M2Ms to repoint their to= endpoints.""" """Alter M2Ms to repoint their to= endpoints."""
# Rename the through table # Rename the through table
@ -1274,6 +1296,9 @@ class BaseDatabaseSchemaEditor:
def _delete_primary_key_sql(self, model, name): def _delete_primary_key_sql(self, model, name):
return self._delete_constraint_sql(self.sql_delete_pk, model, name) return self._delete_constraint_sql(self.sql_delete_pk, model, name)
def _collate_sql(self, collation):
return ' COLLATE ' + self.quote_name(collation)
def remove_procedure(self, procedure_name, param_types=()): def remove_procedure(self, procedure_name, param_types=()):
sql = self.sql_delete_procedure % { sql = self.sql_delete_procedure % {
'procedure': self.quote_name(procedure_name), 'procedure': self.quote_name(procedure_name),

View File

@ -46,6 +46,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
order_by_nulls_first = True order_by_nulls_first = True
test_collations = { test_collations = {
'ci': 'utf8_general_ci', 'ci': 'utf8_general_ci',
'non_default': 'utf8_esperanto_ci',
'swedish_ci': 'utf8_swedish_ci', 'swedish_ci': 'utf8_swedish_ci',
} }

View File

@ -10,7 +10,11 @@ from django.db.models import Index
from django.utils.datastructures import OrderedSet from django.utils.datastructures import OrderedSet
FieldInfo = namedtuple('FieldInfo', BaseFieldInfo._fields + ('extra', 'is_unsigned', 'has_json_constraint')) FieldInfo = namedtuple('FieldInfo', BaseFieldInfo._fields + ('extra', 'is_unsigned', 'has_json_constraint'))
InfoLine = namedtuple('InfoLine', 'col_name data_type max_len num_prec num_scale extra column_default is_unsigned') InfoLine = namedtuple(
'InfoLine',
'col_name data_type max_len num_prec num_scale extra column_default '
'collation is_unsigned'
)
class DatabaseIntrospection(BaseDatabaseIntrospection): class DatabaseIntrospection(BaseDatabaseIntrospection):
@ -84,6 +88,15 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
c.constraint_schema = DATABASE() c.constraint_schema = DATABASE()
""", [table_name]) """, [table_name])
json_constraints = {row[0] for row in cursor.fetchall()} json_constraints = {row[0] for row in cursor.fetchall()}
# A default collation for the given table.
cursor.execute("""
SELECT table_collation
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name = %s
""", [table_name])
row = cursor.fetchone()
default_column_collation = row[0] if row else ''
# information_schema database gives more accurate results for some figures: # information_schema database gives more accurate results for some figures:
# - varchar length returned by cursor.description is an internal length, # - varchar length returned by cursor.description is an internal length,
# not visible length (#5725) # not visible length (#5725)
@ -93,12 +106,17 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
SELECT SELECT
column_name, data_type, character_maximum_length, column_name, data_type, character_maximum_length,
numeric_precision, numeric_scale, extra, column_default, numeric_precision, numeric_scale, extra, column_default,
CASE
WHEN collation_name = %s THEN NULL
ELSE collation_name
END AS collation_name,
CASE CASE
WHEN column_type LIKE '%% unsigned' THEN 1 WHEN column_type LIKE '%% unsigned' THEN 1
ELSE 0 ELSE 0
END AS is_unsigned END AS is_unsigned
FROM information_schema.columns FROM information_schema.columns
WHERE table_name = %s AND table_schema = DATABASE()""", [table_name]) WHERE table_name = %s AND table_schema = DATABASE()
""", [default_column_collation, table_name])
field_info = {line[0]: InfoLine(*line) for line in cursor.fetchall()} field_info = {line[0]: InfoLine(*line) for line in cursor.fetchall()}
cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name)) cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
@ -116,6 +134,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
to_int(info.num_scale) or line[5], to_int(info.num_scale) or line[5],
line[6], line[6],
info.column_default, info.column_default,
info.collation,
info.extra, info.extra,
info.is_unsigned, info.is_unsigned,
line[0] in json_constraints, line[0] in json_constraints,

View File

@ -9,6 +9,7 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
sql_alter_column_null = "MODIFY %(column)s %(type)s NULL" sql_alter_column_null = "MODIFY %(column)s %(type)s NULL"
sql_alter_column_not_null = "MODIFY %(column)s %(type)s NOT NULL" sql_alter_column_not_null = "MODIFY %(column)s %(type)s NOT NULL"
sql_alter_column_type = "MODIFY %(column)s %(type)s" sql_alter_column_type = "MODIFY %(column)s %(type)s"
sql_alter_column_collate = "MODIFY %(column)s %(type)s%(collation)s"
# No 'CASCADE' which works as a no-op in MySQL but is undocumented # No 'CASCADE' which works as a no-op in MySQL but is undocumented
sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s" sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"

View File

@ -1,4 +1,4 @@
from django.db import InterfaceError from django.db import DatabaseError, InterfaceError
from django.db.backends.base.features import BaseDatabaseFeatures from django.db.backends.base.features import BaseDatabaseFeatures
from django.utils.functional import cached_property from django.utils.functional import cached_property
@ -61,9 +61,11 @@ class DatabaseFeatures(BaseDatabaseFeatures):
supports_boolean_expr_in_select_clause = False supports_boolean_expr_in_select_clause = False
supports_primitives_in_json_field = False supports_primitives_in_json_field = False
supports_json_field_contains = False supports_json_field_contains = False
supports_collation_on_textfield = False
test_collations = { test_collations = {
'ci': 'BINARY_CI', 'ci': 'BINARY_CI',
'cs': 'BINARY', 'cs': 'BINARY',
'non_default': 'SWEDISH_CI',
'swedish_ci': 'SWEDISH_CI', 'swedish_ci': 'SWEDISH_CI',
} }
@ -78,3 +80,14 @@ class DatabaseFeatures(BaseDatabaseFeatures):
'SmallIntegerField': 'IntegerField', 'SmallIntegerField': 'IntegerField',
'TimeField': 'DateTimeField', 'TimeField': 'DateTimeField',
} }
@cached_property
def supports_collation_on_charfield(self):
with self.connection.cursor() as cursor:
try:
cursor.execute("SELECT CAST('a' AS VARCHAR2(4001)) FROM dual")
except DatabaseError as e:
if e.args[0].code == 910:
return False
raise
return True

View File

@ -95,14 +95,20 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
# user_tab_columns gives data default for columns # user_tab_columns gives data default for columns
cursor.execute(""" cursor.execute("""
SELECT SELECT
column_name, user_tab_cols.column_name,
data_default, user_tab_cols.data_default,
CASE CASE
WHEN char_used IS NULL THEN data_length WHEN user_tab_cols.collation = user_tables.default_collation
ELSE char_length THEN NULL
ELSE user_tab_cols.collation
END collation,
CASE
WHEN user_tab_cols.char_used IS NULL
THEN user_tab_cols.data_length
ELSE user_tab_cols.char_length
END as internal_size, END as internal_size,
CASE CASE
WHEN identity_column = 'YES' THEN 1 WHEN user_tab_cols.identity_column = 'YES' THEN 1
ELSE 0 ELSE 0
END as is_autofield, END as is_autofield,
CASE CASE
@ -117,10 +123,13 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
ELSE 0 ELSE 0
END as is_json END as is_json
FROM user_tab_cols FROM user_tab_cols
WHERE table_name = UPPER(%s)""", [table_name]) LEFT OUTER JOIN
user_tables ON user_tables.table_name = user_tab_cols.table_name
WHERE user_tab_cols.table_name = UPPER(%s)
""", [table_name])
field_map = { field_map = {
column: (internal_size, default if default != 'NULL' else None, is_autofield, is_json) column: (internal_size, default if default != 'NULL' else None, collation, is_autofield, is_json)
for column, default, internal_size, is_autofield, is_json in cursor.fetchall() for column, default, collation, internal_size, is_autofield, is_json in cursor.fetchall()
} }
self.cache_bust_counter += 1 self.cache_bust_counter += 1
cursor.execute("SELECT * FROM {} WHERE ROWNUM < 2 AND {} > 0".format( cursor.execute("SELECT * FROM {} WHERE ROWNUM < 2 AND {} > 0".format(
@ -129,11 +138,11 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
description = [] description = []
for desc in cursor.description: for desc in cursor.description:
name = desc[0] name = desc[0]
internal_size, default, is_autofield, is_json = field_map[name] internal_size, default, collation, is_autofield, is_json = field_map[name]
name = name % {} # cx_Oracle, for some reason, doubles percent signs. name = name % {} # cx_Oracle, for some reason, doubles percent signs.
description.append(FieldInfo( description.append(FieldInfo(
self.identifier_converter(name), *desc[1:3], internal_size, desc[4] or 0, self.identifier_converter(name), *desc[1:3], internal_size, desc[4] or 0,
desc[5] or 0, *desc[6:], default, is_autofield, is_json, desc[5] or 0, *desc[6:], default, collation, is_autofield, is_json,
)) ))
return description return description

View File

@ -14,6 +14,8 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
sql_alter_column_not_null = "MODIFY %(column)s NOT NULL" sql_alter_column_not_null = "MODIFY %(column)s NOT NULL"
sql_alter_column_default = "MODIFY %(column)s DEFAULT %(default)s" sql_alter_column_default = "MODIFY %(column)s DEFAULT %(default)s"
sql_alter_column_no_default = "MODIFY %(column)s DEFAULT NULL" sql_alter_column_no_default = "MODIFY %(column)s DEFAULT NULL"
sql_alter_column_collate = "MODIFY %(column)s %(type)s%(collation)s"
sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s" sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"
sql_create_column_inline_fk = 'CONSTRAINT %(name)s REFERENCES %(to_table)s(%(to_column)s)%(deferrable)s' sql_create_column_inline_fk = 'CONSTRAINT %(name)s REFERENCES %(to_table)s(%(to_column)s)%(deferrable)s'
sql_delete_table = "DROP TABLE %(table)s CASCADE CONSTRAINTS" sql_delete_table = "DROP TABLE %(table)s CASCADE CONSTRAINTS"
@ -181,3 +183,15 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
'table': self.quote_name(table_name), 'table': self.quote_name(table_name),
'column': self.quote_name(column_name), 'column': self.quote_name(column_name),
}) })
def _get_default_collation(self, table_name):
with self.connection.cursor() as cursor:
cursor.execute("""
SELECT default_collation FROM user_tables WHERE table_name = %s
""", [self.normalize_name(table_name)])
return cursor.fetchone()[0]
def _alter_column_collation_sql(self, model, new_field, new_type, new_collation):
if new_collation is None:
new_collation = self._get_default_collation(model._meta.db_table)
return super()._alter_column_collation_sql(model, new_field, new_type, new_collation)

View File

@ -59,6 +59,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
has_json_operators = True has_json_operators = True
json_key_contains_list_matching_requires_list = True json_key_contains_list_matching_requires_list = True
test_collations = { test_collations = {
'non_default': 'sv-x-icu',
'swedish_ci': 'sv-x-icu', 'swedish_ci': 'sv-x-icu',
} }
@ -92,3 +93,4 @@ class DatabaseFeatures(BaseDatabaseFeatures):
supports_table_partitions = property(operator.attrgetter('is_postgresql_10')) supports_table_partitions = property(operator.attrgetter('is_postgresql_10'))
supports_covering_indexes = property(operator.attrgetter('is_postgresql_11')) supports_covering_indexes = property(operator.attrgetter('is_postgresql_11'))
supports_covering_gist_indexes = property(operator.attrgetter('is_postgresql_12')) supports_covering_gist_indexes = property(operator.attrgetter('is_postgresql_12'))
supports_non_deterministic_collations = property(operator.attrgetter('is_postgresql_12'))

View File

@ -69,9 +69,11 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
SELECT SELECT
a.attname AS column_name, a.attname AS column_name,
NOT (a.attnotnull OR (t.typtype = 'd' AND t.typnotnull)) AS is_nullable, NOT (a.attnotnull OR (t.typtype = 'd' AND t.typnotnull)) AS is_nullable,
pg_get_expr(ad.adbin, ad.adrelid) AS column_default pg_get_expr(ad.adbin, ad.adrelid) AS column_default,
CASE WHEN collname = 'default' THEN NULL ELSE collname END AS collation
FROM pg_attribute a FROM pg_attribute a
LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum
LEFT JOIN pg_collation co ON a.attcollation = co.oid
JOIN pg_type t ON a.atttypid = t.oid JOIN pg_type t ON a.atttypid = t.oid
JOIN pg_class c ON a.attrelid = c.oid JOIN pg_class c ON a.attrelid = c.oid
JOIN pg_namespace n ON c.relnamespace = n.oid JOIN pg_namespace n ON c.relnamespace = n.oid

View File

@ -47,6 +47,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
test_collations = { test_collations = {
'ci': 'nocase', 'ci': 'nocase',
'cs': 'binary', 'cs': 'binary',
'non_default': 'nocase',
} }
@cached_property @cached_property

View File

@ -84,6 +84,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
""" """
cursor.execute('PRAGMA table_info(%s)' % self.connection.ops.quote_name(table_name)) cursor.execute('PRAGMA table_info(%s)' % self.connection.ops.quote_name(table_name))
table_info = cursor.fetchall() table_info = cursor.fetchall()
collations = self._get_column_collations(cursor, table_name)
json_columns = set() json_columns = set()
if self.connection.features.can_introspect_json_field: if self.connection.features.can_introspect_json_field:
for line in table_info: for line in table_info:
@ -102,7 +103,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
return [ return [
FieldInfo( FieldInfo(
name, data_type, None, get_field_size(data_type), None, None, name, data_type, None, get_field_size(data_type), None, None,
not notnull, default, pk == 1, name in json_columns not notnull, default, collations.get(name), pk == 1, name in json_columns
) )
for cid, name, data_type, notnull, default, pk in table_info for cid, name, data_type, notnull, default, pk in table_info
] ]
@ -435,3 +436,27 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
} }
constraints.update(self._get_foreign_key_constraints(cursor, table_name)) constraints.update(self._get_foreign_key_constraints(cursor, table_name))
return constraints return constraints
def _get_column_collations(self, cursor, table_name):
row = cursor.execute("""
SELECT sql
FROM sqlite_master
WHERE type = 'table' AND name = %s
""", [table_name]).fetchone()
if not row:
return {}
sql = row[0]
columns = str(sqlparse.parse(sql)[0][-1]).strip('()').split(', ')
collations = {}
for column in columns:
tokens = column[1:].split()
column_name = tokens[0].strip('"')
for index, token in enumerate(tokens):
if token == 'COLLATE':
collation = tokens[index + 1]
break
else:
collation = None
collations[column_name] = collation
return collations

View File

@ -429,3 +429,6 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
super().remove_constraint(model, constraint) super().remove_constraint(model, constraint)
else: else:
self._remake_table(model) self._remake_table(model)
def _collate_sql(self, collation):
return ' COLLATE ' + collation

View File

@ -1002,13 +1002,16 @@ class BooleanField(Field):
class CharField(Field): class CharField(Field):
description = _("String (up to %(max_length)s)") description = _("String (up to %(max_length)s)")
def __init__(self, *args, **kwargs): def __init__(self, *args, db_collation=None, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.db_collation = db_collation
self.validators.append(validators.MaxLengthValidator(self.max_length)) self.validators.append(validators.MaxLengthValidator(self.max_length))
def check(self, **kwargs): def check(self, **kwargs):
databases = kwargs.get('databases') or []
return [ return [
*super().check(**kwargs), *super().check(**kwargs),
*self._check_db_collation(databases),
*self._check_max_length_attribute(**kwargs), *self._check_max_length_attribute(**kwargs),
] ]
@ -1033,6 +1036,27 @@ class CharField(Field):
else: else:
return [] return []
def _check_db_collation(self, databases):
errors = []
for db in databases:
if not router.allow_migrate_model(db, self.model):
continue
connection = connections[db]
if not (
self.db_collation is None or
'supports_collation_on_charfield' in self.model._meta.required_db_features or
connection.features.supports_collation_on_charfield
):
errors.append(
checks.Error(
'%s does not support a database collation on '
'CharFields.' % connection.display_name,
obj=self,
id='fields.E190',
),
)
return errors
def cast_db_type(self, connection): def cast_db_type(self, connection):
if self.max_length is None: if self.max_length is None:
return connection.ops.cast_char_field_without_max_length return connection.ops.cast_char_field_without_max_length
@ -1061,6 +1085,12 @@ class CharField(Field):
defaults.update(kwargs) defaults.update(kwargs)
return super().formfield(**defaults) return super().formfield(**defaults)
def deconstruct(self):
name, path, args, kwargs = super().deconstruct()
if self.db_collation:
kwargs['db_collation'] = self.db_collation
return name, path, args, kwargs
class CommaSeparatedIntegerField(CharField): class CommaSeparatedIntegerField(CharField):
default_validators = [validators.validate_comma_separated_integer_list] default_validators = [validators.validate_comma_separated_integer_list]
@ -2074,6 +2104,38 @@ class SmallIntegerField(IntegerField):
class TextField(Field): class TextField(Field):
description = _("Text") description = _("Text")
def __init__(self, *args, db_collation=None, **kwargs):
super().__init__(*args, **kwargs)
self.db_collation = db_collation
def check(self, **kwargs):
databases = kwargs.get('databases') or []
return [
*super().check(**kwargs),
*self._check_db_collation(databases),
]
def _check_db_collation(self, databases):
errors = []
for db in databases:
if not router.allow_migrate_model(db, self.model):
continue
connection = connections[db]
if not (
self.db_collation is None or
'supports_collation_on_textfield' in self.model._meta.required_db_features or
connection.features.supports_collation_on_textfield
):
errors.append(
checks.Error(
'%s does not support a database collation on '
'TextFields.' % connection.display_name,
obj=self,
id='fields.E190',
),
)
return errors
def get_internal_type(self): def get_internal_type(self):
return "TextField" return "TextField"
@ -2096,6 +2158,12 @@ class TextField(Field):
**kwargs, **kwargs,
}) })
def deconstruct(self):
name, path, args, kwargs = super().deconstruct()
if self.db_collation:
kwargs['db_collation'] = self.db_collation
return name, path, args, kwargs
class TimeField(DateTimeCheckMixin, Field): class TimeField(DateTimeCheckMixin, Field):
empty_strings_allowed = False empty_strings_allowed = False

View File

@ -196,6 +196,8 @@ Model fields
* **fields.E170**: ``BinaryField``s ``default`` cannot be a string. Use bytes * **fields.E170**: ``BinaryField``s ``default`` cannot be a string. Use bytes
content instead. content instead.
* **fields.E180**: ``<database>`` does not support ``JSONField``\s. * **fields.E180**: ``<database>`` does not support ``JSONField``\s.
* **fields.E190**: ``<database>`` does not support a database collation on
``<field_type>``\s.
* **fields.E900**: ``IPAddressField`` has been removed except for support in * **fields.E900**: ``IPAddressField`` has been removed except for support in
historical migrations. historical migrations.
* **fields.W900**: ``IPAddressField`` has been deprecated. Support for it * **fields.W900**: ``IPAddressField`` has been deprecated. Support for it

View File

@ -593,20 +593,36 @@ For large amounts of text, use :class:`~django.db.models.TextField`.
The default form widget for this field is a :class:`~django.forms.TextInput`. The default form widget for this field is a :class:`~django.forms.TextInput`.
:class:`CharField` has one extra required argument: :class:`CharField` has two extra arguments:
.. attribute:: CharField.max_length .. attribute:: CharField.max_length
The maximum length (in characters) of the field. The max_length is enforced Required. The maximum length (in characters) of the field. The max_length
at the database level and in Django's validation using is enforced at the database level and in Django's validation using
:class:`~django.core.validators.MaxLengthValidator`. :class:`~django.core.validators.MaxLengthValidator`.
.. note:: .. note::
If you are writing an application that must be portable to multiple If you are writing an application that must be portable to multiple
database backends, you should be aware that there are restrictions on database backends, you should be aware that there are restrictions on
``max_length`` for some backends. Refer to the :doc:`database backend ``max_length`` for some backends. Refer to the :doc:`database backend
notes </ref/databases>` for details. notes </ref/databases>` for details.
.. attribute:: CharField.db_collation
.. versionadded:: 3.2
Optional. The database collation name of the field.
.. note::
Collation names are not standardized. As such, this will not be
portable across multiple database backends.
.. admonition:: Oracle
Oracle supports collations only when the ``MAX_STRING_SIZE`` database
initialization parameter is set to ``EXTENDED``.
``DateField`` ``DateField``
------------- -------------
@ -1329,6 +1345,21 @@ If you specify a ``max_length`` attribute, it will be reflected in the
However it is not enforced at the model or database level. Use a However it is not enforced at the model or database level. Use a
:class:`CharField` for that. :class:`CharField` for that.
.. attribute:: TextField.db_collation
.. versionadded:: 3.2
The database collation name of the field.
.. note::
Collation names are not standardized. As such, this will not be
portable across multiple database backends.
.. admonition:: Oracle
Oracle does not support collations for a ``TextField``.
``TimeField`` ``TimeField``
------------- -------------

View File

@ -308,6 +308,11 @@ Models
:class:`~django.db.models.functions.TruncTime` database functions allows :class:`~django.db.models.functions.TruncTime` database functions allows
truncating datetimes in a specific timezone. truncating datetimes in a specific timezone.
* The new ``db_collation`` argument for
:attr:`CharField <django.db.models.CharField.db_collation>` and
:attr:`TextField <django.db.models.TextField.db_collation>` allows setting a
database collation for the field.
Pagination Pagination
~~~~~~~~~~ ~~~~~~~~~~
@ -431,6 +436,13 @@ backends.
unique constraints (:attr:`.UniqueConstraint.include`), set unique constraints (:attr:`.UniqueConstraint.include`), set
``DatabaseFeatures.supports_covering_indexes`` to ``True``. ``DatabaseFeatures.supports_covering_indexes`` to ``True``.
* Third-party database backends must implement support for column database
collations on ``CharField``\s and ``TextField``\s or set
``DatabaseFeatures.supports_collation_on_charfield`` and
``DatabaseFeatures.supports_collation_on_textfield`` to ``False``. If
non-deterministic collations are not supported, set
``supports_non_deterministic_collations`` to ``False``.
:mod:`django.contrib.admin` :mod:`django.contrib.admin`
--------------------------- ---------------------------

View File

@ -1,4 +1,4 @@
from django.db import models from django.db import connection, models
class People(models.Model): class People(models.Model):
@ -79,6 +79,23 @@ class JSONFieldColumnType(models.Model):
} }
test_collation = connection.features.test_collations.get('non_default')
class CharFieldDbCollation(models.Model):
char_field = models.CharField(max_length=10, db_collation=test_collation)
class Meta:
required_db_features = {'supports_collation_on_charfield'}
class TextFieldDbCollation(models.Model):
text_field = models.TextField(db_collation=test_collation)
class Meta:
required_db_features = {'supports_collation_on_textfield'}
class UniqueTogether(models.Model): class UniqueTogether(models.Model):
field1 = models.IntegerField() field1 = models.IntegerField()
field2 = models.CharField(max_length=10) field2 = models.CharField(max_length=10)

View File

@ -8,7 +8,7 @@ from django.db import connection
from django.db.backends.base.introspection import TableInfo from django.db.backends.base.introspection import TableInfo
from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature
from .models import PeopleMoreData from .models import PeopleMoreData, test_collation
def inspectdb_tables_only(table_name): def inspectdb_tables_only(table_name):
@ -104,6 +104,41 @@ class InspectDBTestCase(TestCase):
self.assertIn('json_field = models.JSONField()', output) self.assertIn('json_field = models.JSONField()', output)
self.assertIn('null_json_field = models.JSONField(blank=True, null=True)', output) self.assertIn('null_json_field = models.JSONField(blank=True, null=True)', output)
@skipUnlessDBFeature('supports_collation_on_charfield')
def test_char_field_db_collation(self):
out = StringIO()
call_command('inspectdb', 'inspectdb_charfielddbcollation', stdout=out)
output = out.getvalue()
if not connection.features.interprets_empty_strings_as_nulls:
self.assertIn(
"char_field = models.CharField(max_length=10, "
"db_collation='%s')" % test_collation,
output,
)
else:
self.assertIn(
"char_field = models.CharField(max_length=10, "
"db_collation='%s', blank=True, null=True)" % test_collation,
output,
)
@skipUnlessDBFeature('supports_collation_on_textfield')
def test_text_field_db_collation(self):
out = StringIO()
call_command('inspectdb', 'inspectdb_textfielddbcollation', stdout=out)
output = out.getvalue()
if not connection.features.interprets_empty_strings_as_nulls:
self.assertIn(
"text_field = models.TextField(db_collation='%s')" % test_collation,
output,
)
else:
self.assertIn(
"text_field = models.TextField(db_collation='%s, blank=True, "
"null=True)" % test_collation,
output,
)
def test_number_field_types(self): def test_number_field_types(self):
"""Test introspection of various Django field types""" """Test introspection of various Django field types"""
assertFieldType = self.make_field_type_asserter() assertFieldType = self.make_field_type_asserter()

View File

@ -86,7 +86,7 @@ class BinaryFieldTests(SimpleTestCase):
@isolate_apps('invalid_models_tests') @isolate_apps('invalid_models_tests')
class CharFieldTests(SimpleTestCase): class CharFieldTests(TestCase):
def test_valid_field(self): def test_valid_field(self):
class Model(models.Model): class Model(models.Model):
@ -387,6 +387,30 @@ class CharFieldTests(SimpleTestCase):
) )
]) ])
def test_db_collation(self):
class Model(models.Model):
field = models.CharField(max_length=100, db_collation='anything')
field = Model._meta.get_field('field')
error = Error(
'%s does not support a database collation on CharFields.'
% connection.display_name,
id='fields.E190',
obj=field,
)
expected = [] if connection.features.supports_collation_on_charfield else [error]
self.assertEqual(field.check(databases=self.databases), expected)
def test_db_collation_required_db_features(self):
class Model(models.Model):
field = models.CharField(max_length=100, db_collation='anything')
class Meta:
required_db_features = {'supports_collation_on_charfield'}
field = Model._meta.get_field('field')
self.assertEqual(field.check(databases=self.databases), [])
@isolate_apps('invalid_models_tests') @isolate_apps('invalid_models_tests')
class DateFieldTests(SimpleTestCase): class DateFieldTests(SimpleTestCase):
@ -779,6 +803,30 @@ class TextFieldTests(TestCase):
) )
]) ])
def test_db_collation(self):
class Model(models.Model):
field = models.TextField(db_collation='anything')
field = Model._meta.get_field('field')
error = Error(
'%s does not support a database collation on TextFields.'
% connection.display_name,
id='fields.E190',
obj=field,
)
expected = [] if connection.features.supports_collation_on_textfield else [error]
self.assertEqual(field.check(databases=self.databases), expected)
def test_db_collation_required_db_features(self):
class Model(models.Model):
field = models.TextField(db_collation='anything')
class Meta:
required_db_features = {'supports_collation_on_textfield'}
field = Model._meta.get_field('field')
self.assertEqual(field.check(databases=self.databases), [])
@isolate_apps('invalid_models_tests') @isolate_apps('invalid_models_tests')
class UUIDFieldTests(TestCase): class UUIDFieldTests(TestCase):

View File

@ -44,6 +44,16 @@ class TestCharField(TestCase):
self.assertEqual(p2.title, Event.C) self.assertEqual(p2.title, Event.C)
class TestMethods(SimpleTestCase):
def test_deconstruct(self):
field = models.CharField()
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {})
field = models.CharField(db_collation='utf8_esperanto_ci')
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {'db_collation': 'utf8_esperanto_ci'})
class ValidationTests(SimpleTestCase): class ValidationTests(SimpleTestCase):
class Choices(models.TextChoices): class Choices(models.TextChoices):

View File

@ -2,7 +2,7 @@ from unittest import skipIf
from django import forms from django import forms
from django.db import connection, models from django.db import connection, models
from django.test import TestCase from django.test import SimpleTestCase, TestCase
from .models import Post from .models import Post
@ -37,3 +37,13 @@ class TextFieldTests(TestCase):
p = Post.objects.create(title='Whatever', body='Smile 😀.') p = Post.objects.create(title='Whatever', body='Smile 😀.')
p.refresh_from_db() p.refresh_from_db()
self.assertEqual(p.body, 'Smile 😀.') self.assertEqual(p.body, 'Smile 😀.')
class TestMethods(SimpleTestCase):
def test_deconstruct(self):
field = models.TextField()
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {})
field = models.TextField(db_collation='utf8_esperanto_ci')
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {'db_collation': 'utf8_esperanto_ci'})

View File

@ -185,6 +185,14 @@ class SchemaTests(TransactionTestCase):
counts['indexes'] += 1 counts['indexes'] += 1
return counts return counts
def get_column_collation(self, table, column):
with connection.cursor() as cursor:
return next(
f.collation
for f in connection.introspection.get_table_description(cursor, table)
if f.name == column
)
def assertIndexOrder(self, table, index, order): def assertIndexOrder(self, table, index, order):
constraints = self.get_constraints(table) constraints = self.get_constraints(table)
self.assertIn(index, constraints) self.assertIn(index, constraints)
@ -3224,3 +3232,147 @@ class SchemaTests(TransactionTestCase):
with connection.schema_editor(atomic=True) as editor: with connection.schema_editor(atomic=True) as editor:
editor.alter_db_table(Foo, Foo._meta.db_table, 'renamed_table') editor.alter_db_table(Foo, Foo._meta.db_table, 'renamed_table')
Foo._meta.db_table = 'renamed_table' Foo._meta.db_table = 'renamed_table'
@isolate_apps('schema')
@skipUnlessDBFeature('supports_collation_on_charfield')
def test_db_collation_charfield(self):
collation = connection.features.test_collations['non_default']
class Foo(Model):
field = CharField(max_length=255, db_collation=collation)
class Meta:
app_label = 'schema'
self.isolated_local_models = [Foo]
with connection.schema_editor() as editor:
editor.create_model(Foo)
self.assertEqual(
self.get_column_collation(Foo._meta.db_table, 'field'),
collation,
)
@isolate_apps('schema')
@skipUnlessDBFeature('supports_collation_on_textfield')
def test_db_collation_textfield(self):
collation = connection.features.test_collations['non_default']
class Foo(Model):
field = TextField(db_collation=collation)
class Meta:
app_label = 'schema'
self.isolated_local_models = [Foo]
with connection.schema_editor() as editor:
editor.create_model(Foo)
self.assertEqual(
self.get_column_collation(Foo._meta.db_table, 'field'),
collation,
)
@skipUnlessDBFeature('supports_collation_on_charfield')
def test_add_field_db_collation(self):
with connection.schema_editor() as editor:
editor.create_model(Author)
collation = connection.features.test_collations['non_default']
new_field = CharField(max_length=255, db_collation=collation)
new_field.set_attributes_from_name('alias')
with connection.schema_editor() as editor:
editor.add_field(Author, new_field)
columns = self.column_classes(Author)
self.assertEqual(
columns['alias'][0],
connection.features.introspected_field_types['CharField'],
)
self.assertEqual(columns['alias'][1][8], collation)
@skipUnlessDBFeature('supports_collation_on_charfield')
def test_alter_field_db_collation(self):
with connection.schema_editor() as editor:
editor.create_model(Author)
collation = connection.features.test_collations['non_default']
old_field = Author._meta.get_field('name')
new_field = CharField(max_length=255, db_collation=collation)
new_field.set_attributes_from_name('name')
new_field.model = Author
with connection.schema_editor() as editor:
editor.alter_field(Author, old_field, new_field, strict=True)
self.assertEqual(
self.get_column_collation(Author._meta.db_table, 'name'),
collation,
)
with connection.schema_editor() as editor:
editor.alter_field(Author, new_field, old_field, strict=True)
self.assertIsNone(self.get_column_collation(Author._meta.db_table, 'name'))
@skipUnlessDBFeature('supports_collation_on_charfield')
def test_alter_field_type_and_db_collation(self):
with connection.schema_editor() as editor:
editor.create_model(Note)
collation = connection.features.test_collations['non_default']
old_field = Note._meta.get_field('info')
new_field = CharField(max_length=255, db_collation=collation)
new_field.set_attributes_from_name('info')
new_field.model = Note
with connection.schema_editor() as editor:
editor.alter_field(Note, old_field, new_field, strict=True)
columns = self.column_classes(Note)
self.assertEqual(
columns['info'][0],
connection.features.introspected_field_types['CharField'],
)
self.assertEqual(columns['info'][1][8], collation)
with connection.schema_editor() as editor:
editor.alter_field(Note, new_field, old_field, strict=True)
columns = self.column_classes(Note)
self.assertEqual(columns['info'][0], 'TextField')
self.assertIsNone(columns['info'][1][8])
@skipUnlessDBFeature(
'supports_collation_on_charfield',
'supports_non_deterministic_collations',
)
def test_ci_cs_db_collation(self):
cs_collation = connection.features.test_collations.get('cs')
ci_collation = connection.features.test_collations.get('ci')
try:
if connection.vendor == 'mysql':
cs_collation = 'latin1_general_cs'
elif connection.vendor == 'postgresql':
cs_collation = 'en-x-icu'
with connection.cursor() as cursor:
cursor.execute(
"CREATE COLLATION IF NOT EXISTS case_insensitive "
"(provider = icu, locale = 'und-u-ks-level2', "
"deterministic = false)"
)
ci_collation = 'case_insensitive'
# Create the table.
with connection.schema_editor() as editor:
editor.create_model(Author)
# Case-insensitive collation.
old_field = Author._meta.get_field('name')
new_field_ci = CharField(max_length=255, db_collation=ci_collation)
new_field_ci.set_attributes_from_name('name')
new_field_ci.model = Author
with connection.schema_editor() as editor:
editor.alter_field(Author, old_field, new_field_ci, strict=True)
Author.objects.create(name='ANDREW')
self.assertIs(Author.objects.filter(name='Andrew').exists(), True)
# Case-sensitive collation.
new_field_cs = CharField(max_length=255, db_collation=cs_collation)
new_field_cs.set_attributes_from_name('name')
new_field_cs.model = Author
with connection.schema_editor() as editor:
editor.alter_field(Author, new_field_ci, new_field_cs, strict=True)
self.assertIs(Author.objects.filter(name='Andrew').exists(), False)
finally:
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
cursor.execute('DROP COLLATION IF EXISTS case_insensitive')