Repository: frictionlessdata/frictionless-py

Issue:
Incorrect export of a row to JSON
# Overview

Null values are cast to empty strings. Also, we need to re-check the whole writing system regarding type-handling for similar issues.


Discussion:

Patch:
diff --git a/frictionless/field.py b/frictionless/field.py
index 9160d5ae..7de27241 100644
--- a/frictionless/field.py
+++ b/frictionless/field.py
@@ -213,7 +213,7 @@ class Field(Metadata):
     # Read
 
     def read_cell(self, cell):
-        """Read cell (cast)
+        """Read cell
 
         Parameters:
             cell (any): cell
@@ -237,8 +237,8 @@ class Field(Metadata):
                     notes[name] = f'constraint "{name}" is "{self.constraints[name]}"'
         return cell, notes
 
-    def read_cell_cast(self, cell):
-        """Read cell low-level (cast)
+    def read_cell_convert(self, cell):
+        """Read cell (convert only)
 
         Parameters:
             cell (any): cell
@@ -251,7 +251,7 @@ class Field(Metadata):
 
     @Metadata.property(write=False)
     def read_cell_checks(self):
-        """Read cell low-level (cast)
+        """Read cell (checks only)
 
         Returns:
             OrderedDict: dictionlary of check function by a constraint name
@@ -262,19 +262,20 @@ class Field(Metadata):
             constraint = self.constraints.get(name)
             if constraint is not None:
                 if name in ["minimum", "maximum"]:
-                    constraint = self.read_cell_cast(constraint)
+                    constraint = self.__type.read_cell(constraint)
                 if name == "enum":
-                    constraint = list(map(self.read_cell_cast, constraint))
+                    constraint = list(map(self.__type.read_cell, constraint))
                 checks[name] = partial(globals().get(f"check_{name}"), constraint)
         return checks
 
     # Write
 
-    def write_cell(self, cell):
-        """Write cell (cast)
+    def write_cell(self, cell, *, ignore_missing=False):
+        """Write cell
 
         Parameters:
-            cell (any): cell
+            cell (any): cell to convert
+            ignore_missing? (bool): don't convert None values
 
         Returns:
             (any, OrderedDict): processed cell and dict of notes
@@ -282,17 +283,16 @@ class Field(Metadata):
         """
         notes = None
         if cell is None:
-            cell = ""
-        if cell is not None:
-            cell = self.__type.write_cell(cell)
+            missing_value = cell if ignore_missing else self.write_cell_missing_value
+            return missing_value, notes
+        cell = self.__type.write_cell(cell)
         if cell is None:
             notes = notes or OrderedDict()
             notes["type"] = f'type is "{self.type}/{self.format}"'
         return cell, notes
 
-    # NOTE: rename to convert everywhere like in storage?
-    def write_cell_cast(self, cell):
-        """Write cell low-level (cast)
+    def write_cell_convert(self, cell):
+        """Write cell (convert only)
 
         Parameters:
             cell (any): cell
@@ -303,6 +303,18 @@ class Field(Metadata):
         """
         return self.__type.write_cell(cell)
 
+    @Metadata.property(write=False)
+    def write_cell_missing_value(self):
+        """Write cell (missing value only)
+
+        Returns:
+            str: a value to replace None cells
+
+        """
+        if self.missing_values:
+            return self.missing_values[0]
+        return config.DEFAULT_MISSING_VALUES[0]
+
     # Metadata
 
     def metadata_process(self):
diff --git a/frictionless/row.py b/frictionless/row.py
index 1f71c4f5..fa54ce24 100644
--- a/frictionless/row.py
+++ b/frictionless/row.py
@@ -221,7 +221,7 @@ class Row(OrderedDict):
                 if field.name in self:
                     cell = self[field.name]
                     if field.type not in JsonParser.native_types:
-                        cell, notes = field.write_cell(cell)
+                        cell, notes = field.write_cell(cell, ignore_missing=True)
                     result[field.name] = cell
             return result
         return dict(self)
@@ -240,7 +240,7 @@ class Row(OrderedDict):
                 if field.name in self:
                     cell = self[field.name]
                     if field.type not in JsonParser.native_types:
-                        cell, notes = field.write_cell(cell)
+                        cell, notes = field.write_cell(cell, ignore_missing=True)
                     result.append(cell)
             return result
         return list(self.values())


Test patch:
diff --git a/tests/test_row.py b/tests/test_row.py
index f5d46d18..2466bf80 100644
--- a/tests/test_row.py
+++ b/tests/test_row.py
@@ -1,4 +1,4 @@
-from frictionless import Row, Field, Schema
+from frictionless import Row, Field, Schema, extract
 
 
 # General
@@ -15,6 +15,31 @@ def test_basic():
     assert row == {"field1": 1, "field2": 2, "field3": 3}
 
 
+# Import/Export
+
+
+def test_to_dict_with_json_null_values_issue_519():
+    source = "text://value\n2020-01-01\n\n2020-03-03"
+    process = lambda row: row.to_dict(json=True)
+    data = extract(source, format="csv", process=process)
+    assert data == [
+        {"value": "2020-01-01"},
+        {"value": None},
+        {"value": "2020-03-03"},
+    ]
+
+
+def test_to_list_with_json_null_values_issue_519():
+    source = "text://value\n2020-01-01\n\n2020-03-03"
+    process = lambda row: row.to_list(json=True)
+    data = extract(source, format="csv", process=process)
+    assert data == [
+        ["2020-01-01"],
+        [None],
+        ["2020-03-03"],
+    ]
+
+
 # Helpers