Repository: frictionlessdata/framework
Issue: JSON Data Resource Validation of Python Frictionless Package

# Problem Description

The python implementation of frictionlessdata package only successfully validates a json data resource if the first array initiates all properties defined in the schema. If properties are missing in the first array validation errors occur and all values of the affected property are set to null/None.
This is somewhat related to [Table Schemas and JSON data](https://github.com/frictionlessdata/specs/issues/656#issuecomment-574386328) .

# Expected Behaviour

JSON allows missing keys within their data schema and does not expect all possible keys to be present in each array.
The python implementation should not validate only against the first array to check the schema.
Workable solutions are to either weaken the validation in a sense that it allows missing keys also in the first array or to collect a list of all observed keys across the whole json resource and validate this against the schema definition of the tabular data resource.

# Known workarounds

1. Instantiate all schema-defined keys in the first array with value "null". This way validation of frictionless-py is passed and all values are there. See valid datapackage.json sample.
2. Use profile "data-resource" instead of "tabular-data-resource", but then no validation of data resources occur.
3. Do not use json data resources with frictionless-py.

# Minimal sample datapackage.json

## Valid datapackage.json

```json
{
    "profile" : "tabular-data-package",
    "name" : "some-json-datapackage",
    "id" : "hello there",
    "resources" : [ {
        "profile": "tabular-data-resource",
        "name": "some-json-resource",
        "data": [
          {
            "prop1": 1,
            "prop2": null
          },
          {
            "prop1": 2,
            "prop2": {
              "a" : 10,
              "b" : 11
            }
          }
        ],
        "schema": {
          "fields": [
            {
              "name": "prop1",
              "type": "integer"
            },
            {
              "name": "prop2",
              "type": "object"
            }
          ]
        }
      } ]
  }
```

## Invalid datapackage.json

```json
{
    "profile" : "tabular-data-package",
    "name" : "some-json-datapackage",
    "id" : "hello there again",
    "resources" : [ {
        "profile": "tabular-data-resource",
        "name": "some-json-resource",
        "data": [
          {
            "prop1": 1
          },
          {
            "prop1": 2,
            "prop2": {
              "a" : 10,
              "b" : 11
            }
          }
        ],
        "schema": {
          "fields": [
            {
              "name": "prop1",
              "type": "integer"
            },
            {
              "name": "prop2",
              "type": "object"
            }
          ]
        }
      } ]
  }
```

## Validation code

```py
import frictionless

path_valid = 'datapackages/json-datapackage-spec-sample/datapackage-valid.json'
package_valid = frictionless. Package(path_valid)
package_valid.validate()


path_invalid = 'datapackages/json-datapackage-spec-sample/datapackage-invalid.json'
package_invalid = frictionless. Package(path_invalid)
package_invalid.validate()

package_valid.get_resource('some-json-resource').read_rows()
package_invalid.get_resource('some-json-resource').read_rows()
```

# Output

## Output valid datapackage.json

```py
>>> package_valid.get_resource('some-json-resource').read_rows()
[{'prop1': 1, 'prop2': None}, {'prop1': 2, 'prop2': {'a': 10, 'b': 11}}]
```

```py
>>> package_valid.validate()
{'valid': True,
 'stats': {'tasks': 1, 'warnings': 0, 'errors': 0, 'seconds': 0.005},
 'warnings': [],
 'errors': [],
 'tasks': [{'valid': True,
            'name': 'some-json-resource',
            'type': 'table',
            'place': '<memory>',
            'labels': ['prop1', 'prop2'],
            'stats': {'fields': 2,
                      'rows': 2,
                      'warnings': 0,
                      'errors': 0,
                      'seconds': 0.003},
            'warnings': [],
            'errors': []}]}
```

## Output invalid datapackage.json

```py
>>> package_invalid.get_resource('some-json-resource').read_rows()
[{'prop1': 1, 'prop2': None}, {'prop1': 2, 'prop2': None}]
```

```py
>>> package_invalid.validate()
{'valid': False,
 'stats': {'tasks': 1, 'warnings': 0, 'errors': 3, 'seconds': 0.005},
 'warnings': [],
 'errors': [],
 'tasks': [{'valid': False,
            'name': 'some-json-resource',
            'type': 'table',
            'place': '<memory>',
            'labels': ['prop1'],
            'stats': {'fields': 2,
                      'rows': 2,
                      'warnings': 0,
                      'errors': 3,
                      'seconds': 0.003},
            'warnings': [],
            'errors': [{'type': 'missing-label',
                        'title': 'Missing Label',
                        'description': 'Based on the schema there should be a '
                                       "label that is missing in the data's "
                                       'header.',
                        'message': "There is a missing label in the header's "
                                   'field "prop2" at position "2"',
                        'tags': ['#table', '#header', '#label'],
                        'note': '',
                        'labels': ['prop1'],
                        'rowNumbers': [1],
                        'label': '',
                        'fieldName': 'prop2',
                        'fieldNumber': 2},
                       {'type': 'missing-cell',
                        'title': 'Missing Cell',
                        'description': 'This row has less values compared to '
                                       'the header row (the first row in the '
                                       'data source). A key concept is that '
                                       'all the rows in tabular data must have '
                                       'the same number of columns.',
                        'message': 'Row at position "2" has a missing cell in '
                                   'field "prop2" at position "2"',
                        'tags': ['#table', '#row', '#cell'],
                        'note': '',
                        'cells': ['1'],
                        'rowNumber': 2,
                        'cell': '',
                        'fieldName': 'prop2',
                        'fieldNumber': 2},
                       {'type': 'missing-cell',
                        'title': 'Missing Cell',
                        'description': 'This row has less values compared to '
                                       'the header row (the first row in the '
                                       'data source). A key concept is that '
                                       'all the rows in tabular data must have '
                                       'the same number of columns.',
                        'message': 'Row at position "3" has a missing cell in '
                                   'field "prop2" at position "2"',
                        'tags': ['#table', '#row', '#cell'],
                        'note': '',
                        'cells': ['2'],
                        'rowNumber': 3,
                        'cell': '',
                        'fieldName': 'prop2',
                        'fieldNumber': 2}]}]}
```

Changed files:
frictionless/formats/inline/parser.py, tests/formats/inline/test_parser.py

diff --git a/frictionless/formats/inline/parser.py b/frictionless/formats/inline/parser.py
index a54f6c0c..e0548192 100644
--- a/frictionless/formats/inline/parser.py
+++ b/frictionless/formats/inline/parser.py
@@ -56,6 +56,8 @@ class InlineParser(Parser):
         elif isinstance(item, dict):
             control.keyed = True
             headers = control.keys or list(item.keys())
+            if self.resource.schema:
+                headers = [field.name for field in self.resource.schema.fields]
             yield headers
             yield [item.get(header) for header in headers]
             for item in data:
diff --git a/tests/formats/inline/test_parser.py b/tests/formats/inline/test_parser.py
index 39d28d60..e6f4bd88 100644
--- a/tests/formats/inline/test_parser.py
+++ b/tests/formats/inline/test_parser.py
@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
 import pytest
-from frictionless import formats
+from frictionless import formats, Schema
 from frictionless.resources import TableResource
 
 
@@ -95,6 +95,18 @@ def test_inline_parser_from_ordered_dict():
         assert rows[1].cells == ["中国人", "2"]
 
 
+def test_inline_parser_read_inline_data_with_missing_column_values():
+    data = [{"id": 1}, {"id": 2, "name": "中国人"}]
+    schema = {
+        "fields": [{"name": "id", "type": "integer"}, {"name": "name", "type": "string"}]
+    }
+    with TableResource(data=data, schema=Schema.from_descriptor(schema)) as resource:
+        assert resource.read_rows() == [
+            {"id": 1, "name": None},
+            {"id": 2, "name": "中国人"},
+        ]
+
+
 # Write