Repository: django/django
Issue category: Feature Request

Issue:
Postgresql: ArrayField with Unicode characters gets serialized as string of "\u XXXX" characters
Description
	 
In ArrayField.value_to_string(self, obj) the value is encoded using JSON.dumps(values), which produces escaped Unicode \u XXXX by default.
For example, an ArrayField with 3 elements ["один", "два", "три"] (1,2,3 in Russian) will produce
["\u043e\u0434\u0438\u043d", "\u0434\u0432\u0430", "\u0442\u0440\u0438"]
While this is not a bug per se, this becomes a nuisance when viewing on result of "dumpdata" management command:
The ArrayField fields will be encoded differently from other text fields.
Perhaps there should be an option to turn on/off the ensure_ascii parameter in JSON.dumps(values, ensure_ascii=option)) ?
The option can be enabled by default, as we do for 'hstore' field, or perhaps enabled conditionally:
in the field settings ArrayField(name='numbers', ascii_only=False)
in settings.py ( ARRAY_FIELD_ENSURE_ASCII )
I will be glad to submit a patch.


Hints:
["Given we made the decision to have JSON serialization default to ensure_ascii=False when dealing with Unicode in #29249 (68fc21b3784aa34c7ba5515ab02ef0c7b6ee856d) I think we should use the same approach here and use ensure_ascii=False for any usage of json.dumps in Field.value_to_string for fields that might include text which includes ArrayField, and HStoreField. I don't think an additional field option to control this behavior and certainly not a setting is warranted here as it should be possible to subclass either field class to override value_to_string and ensure_ascii=False does constitute a more coherent default. Feel free to assign the issue to you and submit a PR with tests for ArrayField and HStoreField.", 1732638340.0]
['\u200bhttps://github.com/django/django/pull/18854', 1732680151.0]
['In ded48546: Fixed #35944 -- Handled serialization of Unicode values in ArrayField and HStoreField.', 1733387518.0]

Patch:
diff --git a/django/contrib/postgres/fields/array.py b/django/contrib/postgres/fields/array.py
index 4171af82f9d2..a7e40703a3f8 100644
--- a/django/contrib/postgres/fields/array.py
+++ b/django/contrib/postgres/fields/array.py
@@ -169,7 +169,7 @@ def value_to_string(self, obj):
             else:
                 obj = AttributeSetter(base_field.attname, val)
                 values.append(base_field.value_to_string(obj))
-        return json.dumps(values)
+        return json.dumps(values, ensure_ascii=False)
 
     def get_transform(self, name):
         transform = super().get_transform(name)
diff --git a/django/contrib/postgres/fields/hstore.py b/django/contrib/postgres/fields/hstore.py
index cfc156ab596b..300458c0b118 100644
--- a/django/contrib/postgres/fields/hstore.py
+++ b/django/contrib/postgres/fields/hstore.py
@@ -43,7 +43,7 @@ def to_python(self, value):
         return value
 
     def value_to_string(self, obj):
-        return json.dumps(self.value_from_object(obj))
+        return json.dumps(self.value_from_object(obj), ensure_ascii=False)
 
     def formfield(self, **kwargs):
         return super().formfield(


Test patch:
diff --git a/tests/postgres_tests/test_array.py b/tests/postgres_tests/test_array.py
index ea7807687ea2..ba7151d4a2f5 100644
--- a/tests/postgres_tests/test_array.py
+++ b/tests/postgres_tests/test_array.py
@@ -1008,6 +1008,32 @@ def test_loading(self):
         self.assertEqual(instance.field, [1, 2, None])
 
 
+class TestStringSerialization(PostgreSQLSimpleTestCase):
+    field_values = [["Django", "Python", None], ["Джанго", "פייתון", None, "król"]]
+
+    @staticmethod
+    def create_json_data(array_field_value):
+        fields = {"field": json.dumps(array_field_value, ensure_ascii=False)}
+        return json.dumps(
+            [{"model": "postgres_tests.chararraymodel", "pk": None, "fields": fields}]
+        )
+
+    def test_encode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                instance = CharArrayModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
+
+    def test_decode(self):
+        for field_value in self.field_values:
+            with self.subTest(field_value=field_value):
+                json_data = self.create_json_data(field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+
+
 class TestValidation(PostgreSQLSimpleTestCase):
     def test_unbounded(self):
         field = ArrayField(models.IntegerField())
diff --git a/tests/postgres_tests/test_hstore.py b/tests/postgres_tests/test_hstore.py
index cac3eb742af0..2d19364736be 100644
--- a/tests/postgres_tests/test_hstore.py
+++ b/tests/postgres_tests/test_hstore.py
@@ -297,39 +297,53 @@ class MyModel(PostgreSQLModel):
 
 
 class TestSerialization(PostgreSQLSimpleTestCase):
-    test_data = json.dumps(
-        [
-            {
-                "model": "postgres_tests.hstoremodel",
-                "pk": None,
-                "fields": {
-                    "field": json.dumps({"a": "b"}),
-                    "array_field": json.dumps(
-                        [
-                            json.dumps({"a": "b"}),
-                            json.dumps({"b": "a"}),
-                        ]
-                    ),
-                },
-            }
-        ]
-    )
+    field_values = [
+        ({"a": "b"}, [{"a": "b"}, {"b": "a"}]),
+        (
+            {"все": "Трурль и Клапауций"},
+            [{"Трурль": "Клапауций"}, {"Клапауций": "Трурль"}],
+        ),
+    ]
+
+    @staticmethod
+    def create_json_data(field_value, array_field_value):
+        fields = {
+            "field": json.dumps(field_value, ensure_ascii=False),
+            "array_field": json.dumps(
+                [json.dumps(item, ensure_ascii=False) for item in array_field_value],
+                ensure_ascii=False,
+            ),
+        }
+        return json.dumps(
+            [{"model": "postgres_tests.hstoremodel", "pk": None, "fields": fields}]
+        )
 
     def test_dumping(self):
-        instance = HStoreModel(field={"a": "b"}, array_field=[{"a": "b"}, {"b": "a"}])
-        data = serializers.serialize("json", [instance])
-        self.assertEqual(json.loads(data), json.loads(self.test_data))
+        for field_value, array_field_value in self.field_values:
+            with self.subTest(field_value=field_value, array_value=array_field_value):
+                instance = HStoreModel(field=field_value, array_field=array_field_value)
+                data = serializers.serialize("json", [instance])
+                json_data = self.create_json_data(field_value, array_field_value)
+                self.assertEqual(json.loads(data), json.loads(json_data))
 
     def test_loading(self):
-        instance = list(serializers.deserialize("json", self.test_data))[0].object
-        self.assertEqual(instance.field, {"a": "b"})
-        self.assertEqual(instance.array_field, [{"a": "b"}, {"b": "a"}])
+        for field_value, array_field_value in self.field_values:
+            with self.subTest(field_value=field_value, array_value=array_field_value):
+                json_data = self.create_json_data(field_value, array_field_value)
+                instance = list(serializers.deserialize("json", json_data))[0].object
+                self.assertEqual(instance.field, field_value)
+                self.assertEqual(instance.array_field, array_field_value)
 
     def test_roundtrip_with_null(self):
-        instance = HStoreModel(field={"a": "b", "c": None})
-        data = serializers.serialize("json", [instance])
-        new_instance = list(serializers.deserialize("json", data))[0].object
-        self.assertEqual(instance.field, new_instance.field)
+        for field_value in [
+            {"a": "b", "c": None},
+            {"Енеїда": "Ти знаєш, він який суціга", "Зефір": None},
+        ]:
+            with self.subTest(field_value=field_value):
+                instance = HStoreModel(field=field_value)
+                data = serializers.serialize("json", [instance])
+                new_instance = list(serializers.deserialize("json", data))[0].object
+                self.assertEqual(instance.field, new_instance.field)
 
 
 class TestValidation(PostgreSQLSimpleTestCase):