p15693087
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #24604 -- Added JSONField to contrib.postgres.

This commit is contained in:
Marc Tamlyn 2015-05-30 22:13:58 +01:00
parent 74fe4428e5
commit 33ea472f69
11 changed files with 540 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
django/contrib/postgres/fields/__init__.py
View File

@ -1,3 +1,4 @@
from .array import * # NOQA
from .hstore import * # NOQA
from .jsonb import * # NOQA
from .ranges import * # NOQA

99
django/contrib/postgres/fields/jsonb.py Normal file
View File

@ -0,0 +1,99 @@
import json
from psycopg2.extras import Json
from django.contrib.postgres import forms, lookups
from django.core import exceptions
from django.db.models import Field, Transform
from django.utils.translation import ugettext_lazy as _
__all__ = ['JSONField']
class JSONField(Field):
empty_strings_allowed = False
description = _('A JSON object')
default_error_messages = {
'invalid': _("Value must be valid JSON."),
}
def db_type(self, connection):
return 'jsonb'
def get_transform(self, name):
transform = super(JSONField, self).get_transform(name)
if transform:
return transform
return KeyTransformFactory(name)
def get_prep_value(self, value):
if value is not None:
return Json(value)
return value
def get_prep_lookup(self, lookup_type, value):
if lookup_type in ('has_key', 'has_keys', 'has_any_keys'):
return value
if isinstance(value, (dict, list)):
return Json(value)
return super(JSONField, self).get_prep_lookup(lookup_type, value)
def validate(self, value, model_instance):
super(JSONField, self).validate(value, model_instance)
try:
json.dumps(value)
except TypeError:
raise exceptions.ValidationError(
self.error_messages['invalid'],
code='invalid',
params={'value': value},
)
def value_to_string(self, obj):
value = self._get_val_from_obj(obj)
return value
def formfield(self, **kwargs):
defaults = {'form_class': forms.JSONField}
defaults.update(kwargs)
return super(JSONField, self).formfield(**defaults)
JSONField.register_lookup(lookups.DataContains)
JSONField.register_lookup(lookups.ContainedBy)
JSONField.register_lookup(lookups.HasKey)
JSONField.register_lookup(lookups.HasKeys)
JSONField.register_lookup(lookups.HasAnyKeys)
class KeyTransform(Transform):
def __init__(self, key_name, *args, **kwargs):
super(KeyTransform, self).__init__(*args, **kwargs)
self.key_name = key_name
def as_sql(self, compiler, connection):
key_transforms = [self.key_name]
previous = self.lhs
while isinstance(previous, KeyTransform):
key_transforms.insert(0, previous.key_name)
previous = previous.lhs
lhs, params = compiler.compile(previous)
if len(key_transforms) > 1:
return "{} #> %s".format(lhs), [key_transforms] + params
try:
int(self.key_name)
except ValueError:
lookup = "'%s'" % self.key_name
else:
lookup = "%s" % self.key_name
return "%s -> %s" % (lhs, lookup), params
class KeyTransformFactory(object):
def __init__(self, key_name):
self.key_name = key_name
def __call__(self, *args, **kwargs):
return KeyTransform(self.key_name, *args, **kwargs)

1
django/contrib/postgres/forms/__init__.py
View File

@ -1,3 +1,4 @@
from .array import * # NOQA
from .hstore import * # NOQA
from .jsonb import * # NOQA
from .ranges import * # NOQA

31
django/contrib/postgres/forms/jsonb.py Normal file
View File

@ -0,0 +1,31 @@
import json
from django import forms
from django.utils.translation import ugettext_lazy as _
__all__ = ['JSONField']
class JSONField(forms.CharField):
default_error_messages = {
'invalid': _("'%(value)s' value must be valid JSON."),
}
def __init__(self, **kwargs):
kwargs.setdefault('widget', forms.Textarea)
super(JSONField, self).__init__(**kwargs)
def to_python(self, value):
if value in self.empty_values:
return None
try:
return json.loads(value)
except ValueError:
raise forms.ValidationError(
self.error_messages['invalid'],
code='invalid',
params={'value': value},
)
def prepare_value(self, value):
return json.dumps(value)

105
docs/ref/contrib/postgres/fields.txt
View File

@ -450,6 +450,111 @@ using in conjunction with lookups on
>>> Dog.objects.filter(data__values__contains=['collie'])
[<Dog: Meg>]
JSONField
---------
.. versionadded:: 1.9
.. class:: JSONField(**options)
A field for storing JSON encoded data. In Python the data is represented in
its Python native format: dictionaries, lists, strings, numbers, booleans
and ``None``.
.. note::
PostgreSQL has two native JSON based data types: ``json`` and ``jsonb``.
The main difference between them is how they are stored and how they can be
queried. PostgreSQL's ``json`` field is stored as the original string
representation of the JSON and must be decoded on the fly when queried
based on keys. The ``jsonb`` field is stored based on the actual structure
of the JSON which allows indexing. The trade-off is a small additional cost
on writing to the ``jsonb`` field. ``JSONField`` uses ``jsonb``.
**As a result, the usage of this field is only supported on PostgreSQL
versions at least 9.4**.
Querying JSONField
^^^^^^^^^^^^^^^^^^
We will use the following example model::
from django.contrib.postgres.fields import JSONField
from django.db import models
class Dog(models.Model):
name = models.CharField(max_length=200)
data = JSONField()
def __str__(self): # __unicode__ on Python 2
return self.name
.. fieldlookup:: jsonfield.key
Key, index, and path lookups
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To query based on a given dictionary key, simply use that key as the lookup
name::
>>> Dog.objects.create(name='Rufus', data={
... 'breed': 'labrador',
... 'owner': {
... 'name': 'Bob',
... 'other_pets': [{
... 'name': 'Fishy',
... }],
... },
... })
>>> Dog.objects.create(name='Meg', data={'breed': 'collie'})
>>> Dog.objects.filter(data__breed='collie')
[<Dog: Meg>]
Multiple keys can be chained together to form a path lookup::
>>> Dog.objects.filter(data__owner__name='Bob')
[<Dog: Rufus>]
If the key is an integer, it will be interpreted as an index lookup in an
array::
>>> Dog.objects.filter(data__owner__other_pets__0__name='Fishy')
[<Dog: Rufus>]
If the key you wish to query by clashes with the name of another lookup, use
the :lookup:`jsonfield.contains` lookup instead.
If only one key or index is used, the SQL operator ``->`` is used. If multiple
operators are used then the ``#>`` operator is used.
.. warning::
Since any string could be a key in a JSON object, any lookup other than
those listed below will be interpreted as a key lookup. No errors are
raised. Be extra careful for typing mistakes, and always check your queries
work as you intend.
Containment and key operations
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. fieldlookup:: jsonfield.contains
.. fieldlookup:: jsonfield.contained_by
.. fieldlookup:: jsonfield.has_key
.. fieldlookup:: jsonfield.has_any_keys
.. fieldlookup:: jsonfield.has_keys
:class:`~django.contrib.postgres.fields.JSONField` shares lookups relating to
containment and keys with :class:`~django.contrib.postgres.fields.HStoreField`.
- :lookup:`contains <hstorefield.contains>` (accepts any JSON rather than
just a dictionary of strings)
- :lookup:`contained_by <hstorefield.contained_by>` (accepts any JSON
rather than just a dictionary of strings)
- :lookup:`has_key <hstorefield.has_key>`
- :lookup:`has_any_keys <hstorefield.has_any_keys>`
- :lookup:`has_keys <hstorefield.has_keys>`
.. _range-fields:
Range Fields

15
docs/ref/contrib/postgres/forms.txt
View File

@ -155,6 +155,21 @@ HStoreField
valid for a given field. This can be done using the
:class:`~django.contrib.postgres.validators.KeysValidator`.
JSONField
---------
.. class:: JSONField
A field which accepts JSON encoded data for a
:class:`~django.contrib.postgres.fields.JSONField`. It is represented by an
HTML ``<textarea>``.
.. admonition:: User friendly forms
``JSONField`` is not particularly user friendly in most cases, however
it is a useful way to format data from a client-side widget for
submission to the server.
Range Fields
------------

1
docs/releases/1.9.txt
View File

@ -91,6 +91,7 @@ Minor features
:mod:`django.contrib.postgres`
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
* Added :class:`~django.contrib.postgres.fields.JSONField`.
* Added :doc:`/ref/contrib/postgres/aggregates`.
:mod:`django.contrib.redirects`

3
tests/postgres_tests/fields.py
View File

@ -7,7 +7,7 @@ from django.db import models
try:
from django.contrib.postgres.fields import (
ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
FloatRangeField, HStoreField, IntegerRangeField,
FloatRangeField, HStoreField, IntegerRangeField, JSONField,
)
except ImportError:
class DummyArrayField(models.Field):
@ -29,3 +29,4 @@ except ImportError:
FloatRangeField = models.Field
HStoreField = models.Field
IntegerRangeField = models.Field
JSONField = models.Field

15
tests/postgres_tests/migrations/0002_create_test_models.py
View File

@ -150,6 +150,19 @@ class Migration(migrations.Migration):
),
]
pg_94_operations = [
migrations.CreateModel(
name='JSONModel',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('field', JSONField(null=True, blank=True)),
],
options={
},
bases=(models.Model,),
),
]
def apply(self, project_state, schema_editor, collect_sql=False):
try:
PG_VERSION = schema_editor.connection.pg_version
@ -158,4 +171,6 @@ class Migration(migrations.Migration):
else:
if PG_VERSION >= 90200:
self.operations = self.operations + self.pg_92_operations
if PG_VERSION >= 90400:
self.operations = self.operations + self.pg_94_operations
return super(Migration, self).apply(project_state, schema_editor, collect_sql)

14
tests/postgres_tests/models.py
View File

@ -2,7 +2,7 @@ from django.db import connection, models
from .fields import (
ArrayField, BigIntegerRangeField, DateRangeField, DateTimeRangeField,
FloatRangeField, HStoreField, IntegerRangeField,
FloatRangeField, HStoreField, IntegerRangeField, JSONField,
)
@ -52,7 +52,7 @@ class TextFieldModel(models.Model):
field = models.TextField()
# Only create this model for databases which support it
# Only create this model for postgres >= 9.2
if connection.vendor == 'postgresql' and connection.pg_version >= 90200:
class RangesModel(PostgreSQLModel):
ints = IntegerRangeField(blank=True, null=True)
@ -66,6 +66,16 @@ else:
pass
# Only create this model for postgres >= 9.4
if connection.vendor == 'postgresql' and connection.pg_version >= 90400:
class JSONModel(models.Model):
field = JSONField(blank=True, null=True)
else:
# create an object with this name so we don't have failing imports
class JSONModel(object):
pass
class ArrayFieldSubclass(ArrayField):
def __init__(self, *args, **kwargs):
super(ArrayFieldSubclass, self).__init__(models.IntegerField())

258
tests/postgres_tests/test_json.py Normal file
View File

@ -0,0 +1,258 @@
import datetime
import unittest
from django.core import exceptions, serializers
from django.db import connection
from django.test import TestCase
from . import PostgresSQLTestCase
from .models import JSONModel
try:
from django.contrib.postgres import forms
from django.contrib.postgres.fields import JSONField
except ImportError:
pass
def skipUnlessPG94(test):
try:
PG_VERSION = connection.pg_version
except AttributeError:
PG_VERSION = 0
if PG_VERSION < 90400:
return unittest.skip('PostgreSQL >= 9.4 required')(test)
return test
@skipUnlessPG94
class TestSaveLoad(TestCase):
def test_null(self):
instance = JSONModel()
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, None)
def test_empty_object(self):
instance = JSONModel(field={})
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, {})
def test_empty_list(self):
instance = JSONModel(field=[])
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, [])
def test_boolean(self):
instance = JSONModel(field=True)
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, True)
def test_string(self):
instance = JSONModel(field='why?')
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, 'why?')
def test_number(self):
instance = JSONModel(field=1)
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, 1)
def test_realistic_object(self):
obj = {
'a': 'b',
'c': 1,
'd': ['e', {'f': 'g'}],
'h': True,
'i': False,
'j': None,
}
instance = JSONModel(field=obj)
instance.save()
loaded = JSONModel.objects.get()
self.assertEqual(loaded.field, obj)
@skipUnlessPG94
class TestQuerying(TestCase):
@classmethod
def setUpTestData(cls):
cls.objs = [
JSONModel.objects.create(field=None),
JSONModel.objects.create(field=True),
JSONModel.objects.create(field=False),
JSONModel.objects.create(field='yes'),
JSONModel.objects.create(field=7),
JSONModel.objects.create(field=[]),
JSONModel.objects.create(field={}),
JSONModel.objects.create(field={
'a': 'b',
'c': 1,
}),
JSONModel.objects.create(field={
'a': 'b',
'c': 1,
'd': ['e', {'f': 'g'}],
'h': True,
'i': False,
'j': None,
'k': {'l': 'm'},
}),
JSONModel.objects.create(field=[1, [2]]),
JSONModel.objects.create(field={
'k': True,
'l': False,
}),
]
def test_exact(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__exact={}),
[self.objs[6]]
)
def test_exact_complex(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__exact={'a': 'b', 'c': 1}),
[self.objs[7]]
)
def test_isnull(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__isnull=True),
[self.objs[0]]
)
def test_contains(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__contains={'a': 'b'}),
[self.objs[7], self.objs[8]]
)
def test_contained_by(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__contained_by={'a': 'b', 'c': 1, 'h': True}),
[self.objs[6], self.objs[7]]
)
def test_has_key(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__has_key='a'),
[self.objs[7], self.objs[8]]
)
def test_has_keys(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__has_keys=['a', 'c', 'h']),
[self.objs[8]]
)
def test_has_any_keys(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__has_any_keys=['c', 'l']),
[self.objs[7], self.objs[8], self.objs[10]]
)
def test_shallow_list_lookup(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__0=1),
[self.objs[9]]
)
def test_shallow_obj_lookup(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__a='b'),
[self.objs[7], self.objs[8]]
)
def test_deep_lookup_objs(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__k__l='m'),
[self.objs[8]]
)
def test_shallow_lookup_obj_target(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__k={'l': 'm'}),
[self.objs[8]]
)
def test_deep_lookup_array(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__1__0=2),
[self.objs[9]]
)
def test_deep_lookup_mixed(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__d__1__f='g'),
[self.objs[8]]
)
def test_deep_lookup_transform(self):
self.assertSequenceEqual(
JSONModel.objects.filter(field__c__gt=1),
[]
)
self.assertSequenceEqual(
JSONModel.objects.filter(field__c__lt=5),
[self.objs[7], self.objs[8]]
)
@skipUnlessPG94
class TestSerialization(TestCase):
test_data = '[{"fields": {"field": {"a": "b"}}, "model": "postgres_tests.jsonmodel", "pk": null}]'
def test_dumping(self):
instance = JSONModel(field={'a': 'b'})
data = serializers.serialize('json', [instance])
self.assertJSONEqual(data, self.test_data)
def test_loading(self):
instance = list(serializers.deserialize('json', self.test_data))[0].object
self.assertEqual(instance.field, {'a': 'b'})
class TestValidation(PostgresSQLTestCase):
def test_not_serializable(self):
field = JSONField()
with self.assertRaises(exceptions.ValidationError) as cm:
field.clean(datetime.timedelta(days=1), None)
self.assertEqual(cm.exception.code, 'invalid')
self.assertEqual(cm.exception.message % cm.exception.params, "Value must be valid JSON.")
class TestFormField(PostgresSQLTestCase):
def test_valid(self):
field = forms.JSONField()
value = field.clean('{"a": "b"}')
self.assertEqual(value, {'a': 'b'})
def test_valid_empty(self):
field = forms.JSONField(required=False)
value = field.clean('')
self.assertEqual(value, None)
def test_invalid(self):
field = forms.JSONField()
with self.assertRaises(exceptions.ValidationError) as cm:
field.clean('{some badly formed: json}')
self.assertEqual(cm.exception.messages[0], "'{some badly formed: json}' value must be valid JSON.")
def test_formfield(self):
model_field = JSONField()
form_field = model_field.formfield()
self.assertIsInstance(form_field, forms.JSONField)
def test_prepare_value(self):
field = forms.JSONField()
self.assertEqual(field.prepare_value({'a': 'b'}), '{"a": "b"}')
self.assertEqual(field.prepare_value(None), 'null')