From 78c811334c9e5477b86fd113fa2c4a261e167d15 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 17 Jun 2020 07:59:40 +0200 Subject: [PATCH] Refs #30190 -- Minor edits to JSONL serializer. Follow up to e29637681be07606674cdccb47d1e53acb930f5b. --- django/core/serializers/jsonl.py | 2 +- docs/topics/serialization.txt | 8 +- tests/serializers/test_jsonl.py | 147 ++++++++++--------------------- 3 files changed, 53 insertions(+), 104 deletions(-) diff --git a/django/core/serializers/jsonl.py b/django/core/serializers/jsonl.py index ff0d9eb605..4b3e46ed8e 100644 --- a/django/core/serializers/jsonl.py +++ b/django/core/serializers/jsonl.py @@ -50,7 +50,7 @@ def Deserializer(stream_or_string, **options): if not line.strip(): continue try: - yield list(PythonDeserializer([json.loads(line), ], **options))[0] + yield from PythonDeserializer([json.loads(line)], **options) except (GeneratorExit, DeserializationError): raise except Exception as exc: diff --git a/docs/topics/serialization.txt b/docs/topics/serialization.txt index 7cc4905da3..9ef851368a 100644 --- a/docs/topics/serialization.txt +++ b/docs/topics/serialization.txt @@ -318,12 +318,12 @@ JSONL .. versionadded:: 3.2 *JSONL* stands for *JSON Lines*. With this format, objects are separated by new -lines, and each line contains a valid JSON object. JSONL serialized data look +lines, and each line contains a valid JSON object. JSONL serialized data looks like this:: - { "pk": "4b678b301dfd8a4e0dad910de3ae245b", "model": "sessions.session", "fields": { ... }} - { "pk": "88bea72c02274f3c9bf1cb2bb8cee4fc", "model": "sessions.session", "fields": { ... }} - { "pk": "9cf0e26691b64147a67e2a9f06ad7a53", "model": "sessions.session", "fields": { ... }} + {"pk": "4b678b301dfd8a4e0dad910de3ae245b", "model": "sessions.session", "fields": {...}} + {"pk": "88bea72c02274f3c9bf1cb2bb8cee4fc", "model": "sessions.session", "fields": {...}} + {"pk": "9cf0e26691b64147a67e2a9f06ad7a53", "model": "sessions.session", "fields": {...}} JSONL can be useful for populating large databases, since the data can be processed line by line, rather than being loaded into memory all at once. diff --git a/tests/serializers/test_jsonl.py b/tests/serializers/test_jsonl.py index 19557e6c29..e4f1a3e80f 100644 --- a/tests/serializers/test_jsonl.py +++ b/tests/serializers/test_jsonl.py @@ -15,32 +15,20 @@ from .tests import SerializersTestBase, SerializersTransactionTestBase class JsonlSerializerTestCase(SerializersTestBase, TestCase): serializer_name = "jsonl" pkless_str = [ - """{ - "pk": null, - "model": "serializers.category", - "fields": {"name": "Reference"} - }""", - """{ - "model": "serializers.category", - "fields": {"name": "Non-fiction"} - }""" + '{"pk": null,"model": "serializers.category","fields": {"name": "Reference"}}', + '{"model": "serializers.category","fields": {"name": "Non-fiction"}}', ] pkless_str = "\n".join([s.replace("\n", "") for s in pkless_str]) - mapping_ordering_str = """{ -"model": "serializers.article", -"pk": %(article_pk)s, -"fields": { -"author": %(author_pk)s, -"headline": "Poker has no place on ESPN", -"pub_date": "2006-06-16T11:00:00", -"categories": [ -%(first_category_pk)s, -%(second_category_pk)s -], -"meta_data": [] -} -}""".replace("\n", "") + "\n" + mapping_ordering_str = ( + '{"model": "serializers.article","pk": %(article_pk)s,' + '"fields": {' + '"author": %(author_pk)s,' + '"headline": "Poker has no place on ESPN",' + '"pub_date": "2006-06-16T11:00:00",' + '"categories": [%(first_category_pk)s,%(second_category_pk)s],' + '"meta_data": []}}\n' + ) @staticmethod def _validate_output(serial_str): @@ -82,7 +70,7 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): s = serializers.jsonl.Serializer() json_data = s.serialize( - [ScoreDecimal(score=decimal.Decimal(1.0))], cls=CustomJSONEncoder + [ScoreDecimal(score=decimal.Decimal(1.0))], cls=CustomJSONEncoder, ) self.assertIn('"fields": {"score": "1"}', json_data) @@ -93,55 +81,40 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): def test_helpful_error_message_invalid_pk(self): """ - If there is an invalid primary key, the error message should contain - the model associated with it. + If there is an invalid primary key, the error message contains the + model associated with it. """ - test_string = """{ - "pk": "badpk", - "model": "serializers.player", - "fields": { - "name": "Bob", - "rank": 1, - "team": "Team" - } - }""".replace("\n", "") + test_string = ( + '{"pk": "badpk","model": "serializers.player",' + '"fields": {"name": "Bob","rank": 1,"team": "Team"}}' + ) with self.assertRaisesMessage(DeserializationError, "(serializers.player:pk=badpk)"): list(serializers.deserialize('jsonl', test_string)) def test_helpful_error_message_invalid_field(self): """ - If there is an invalid field value, the error message should contain - the model associated with it. + If there is an invalid field value, the error message contains the + model associated with it. """ - test_string = """{ - "pk": "1", - "model": "serializers.player", - "fields": { - "name": "Bob", - "rank": "invalidint", - "team": "Team" - } - }""".replace("\n", "") + test_string = ( + '{"pk": "1","model": "serializers.player",' + '"fields": {"name": "Bob","rank": "invalidint","team": "Team"}}' + ) expected = "(serializers.player:pk=1) field_value was 'invalidint'" with self.assertRaisesMessage(DeserializationError, expected): list(serializers.deserialize('jsonl', test_string)) def test_helpful_error_message_for_foreign_keys(self): """ - Invalid foreign keys with a natural key should throw a helpful error - message, such as what the failing key is. + Invalid foreign keys with a natural key throws a helpful error message, + such as what the failing key is. """ - test_string = """{ - "pk": 1, - "model": "serializers.category", - "fields": { - "name": "Unknown foreign key", - "meta_data": [ - "doesnotexist", - "metadata" - ] - } - }""".replace("\n", "") + test_string = ( + '{"pk": 1, "model": "serializers.category",' + '"fields": {' + '"name": "Unknown foreign key",' + '"meta_data": ["doesnotexist","metadata"]}}' + ) key = ["doesnotexist", "metadata"] expected = "(serializers.category:pk=1) field_value was '%r'" % key with self.assertRaisesMessage(DeserializationError, expected): @@ -149,7 +122,7 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): def test_helpful_error_message_for_many2many_non_natural(self): """ - Invalid many-to-many keys should throw a helpful error message. + Invalid many-to-many keys throws a helpful error message. """ test_strings = [ """{ @@ -165,16 +138,12 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): """{ "pk": 1, "model": "serializers.author", - "fields": { - "name": "Agnes" - } + "fields": {"name": "Agnes"} }""", """{ "pk": 1, "model": "serializers.category", - "fields": { - "name": "Reference" - } + "fields": {"name": "Reference"} }""" ] test_string = "\n".join([s.replace("\n", "") for s in test_strings]) @@ -184,18 +153,14 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): def test_helpful_error_message_for_many2many_natural1(self): """ - Invalid many-to-many keys should throw a helpful error message. - This tests the code path where one of a list of natural keys is invalid. + Invalid many-to-many keys throws a helpful error message where one of a + list of natural keys is invalid. """ test_strings = [ """{ "pk": 1, "model": "serializers.categorymetadata", - "fields": { - "kind": "author", - "name": "meta1", - "value": "Agnes" - } + "fields": {"kind": "author","name": "meta1","value": "Agnes"} }""", """{ "pk": 1, @@ -214,9 +179,7 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): """{ "pk": 1, "model": "serializers.author", - "fields": { - "name": "Agnes" - } + "fields": {"name": "Agnes"} }""" ] test_string = "\n".join([s.replace("\n", "") for s in test_strings]) @@ -228,9 +191,8 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): def test_helpful_error_message_for_many2many_natural2(self): """ - Invalid many-to-many keys should throw a helpful error message. This - tests the code path where a natural many-to-many key has only a single - value. + Invalid many-to-many keys throws a helpful error message where a + natural many-to-many key has only a single value. """ test_strings = [ """{ @@ -246,18 +208,12 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): """{ "pk": 1, "model": "serializers.categorymetadata", - "fields": { - "kind": "author", - "name": "meta1", - "value": "Agnes" - } + "fields": {"kind": "author","name": "meta1","value": "Agnes"} }""", """{ "pk": 1, "model": "serializers.author", - "fields": { - "name": "Agnes" - } + "fields": {"name": "Agnes"} }""" ] test_string = "\n".join([s.replace("\n", "") for s in test_strings]) @@ -270,12 +226,9 @@ class JsonlSerializerTestCase(SerializersTestBase, TestCase): """ Not iterable many-to-many field value throws a helpful error message. """ - test_string = """{ - "pk": 1, - "model": "serializers.m2mdata", - "fields": {"data": null} - }""".replace("\n", "") - + test_string = ( + '{"pk": 1,"model": "serializers.m2mdata","fields": {"data": null}}' + ) expected = "(serializers.m2mdata:pk=1) field_value was 'None'" with self.assertRaisesMessage(DeserializationError, expected): next(serializers.deserialize('jsonl', test_string, ignore=False)) @@ -297,16 +250,12 @@ class JsonSerializerTransactionTestCase(SerializersTransactionTestBase, Transact """{ "pk": 1, "model": "serializers.category", - "fields": { - "name": "Reference" - } + "fields": {"name": "Reference"} }""", """{ "pk": 1, "model": "serializers.author", - "fields": { - "name": "Agnes" - } + "fields": {"name": "Agnes"} }""" ] fwd_ref_str = "\n".join([s.replace("\n", "") for s in fwd_ref_str])