Repository: inveniosoftware/invenio-rdm-records (language: python)

Issue:
sciebo: deserialize ro-crate payload to rdm
> note: serializers depend on data model, therefore they cannot go lower in the module chain

For now we integrate the RO-Crate in rdm-records, but we also need a way in the future to make these configurable. 

Schema at https://www.researchobject.org/ro-crate/

Mini-RFC: https://codimd.web.cern.ch/5RF5ylH3TTuCTkZ08RJQ_g?edit




Discussion / hints:

Fix patch:
diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index e8ec2c530..98173db4a 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2022 CERN.
 # Copyright (C) 2020-2021 Northwestern University.
 # Copyright (C) 2022 Universität Hamburg.
 #
@@ -14,7 +14,9 @@
 from flask_babelex import lazy_gettext as _
 from flask_resources import (
     HTTPJSONException,
+    JSONDeserializer,
     JSONSerializer,
+    RequestBodyParser,
     ResourceConfig,
     ResponseHandler,
     create_error_handler,
@@ -31,6 +33,8 @@
     ReviewStateError,
 )
 from .args import RDMSearchRequestArgsSchema
+from .deserializers import ROCrateJSONDeserializer
+from .deserializers.errors import DeserializerError
 from .serializers import (
     CSLJSONSerializer,
     DataCite43JSONSerializer,
@@ -96,11 +100,24 @@ class RDMRecordResourceConfig(RecordResourceConfig):
         "locale": ma.fields.Str(),
     }
 
+    request_body_parsers = {
+        "application/json": RequestBodyParser(JSONDeserializer()),
+        'application/ld+json;profile="https://w3id.org/ro/crate/1.1"': RequestBodyParser(
+            ROCrateJSONDeserializer()
+        ),
+    }
+
     request_search_args = RDMSearchRequestArgsSchema
 
     response_handlers = record_serializers
 
     error_handlers = {
+        DeserializerError: create_error_handler(
+            lambda exc: HTTPJSONException(
+                code=400,
+                description=exc.args[0],
+            )
+        ),
         StyleNotFoundError: create_error_handler(
             HTTPJSONException(
                 code=400,
diff --git a/invenio_rdm_records/resources/deserializers/__init__.py b/invenio_rdm_records/resources/deserializers/__init__.py
new file mode 100644
index 000000000..d9f0ef09f
--- /dev/null
+++ b/invenio_rdm_records/resources/deserializers/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Deserializers."""
+
+from .rocrate import ROCrateJSONDeserializer
+
+__all__ = ("ROCrateJSONDeserializer",)
diff --git a/invenio_rdm_records/resources/deserializers/errors.py b/invenio_rdm_records/resources/deserializers/errors.py
new file mode 100644
index 000000000..14efd663f
--- /dev/null
+++ b/invenio_rdm_records/resources/deserializers/errors.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Errors for deserializers."""
+
+
+class DeserializerError(Exception):
+    """Base class for deserializer errors."""
diff --git a/invenio_rdm_records/resources/deserializers/rocrate/__init__.py b/invenio_rdm_records/resources/deserializers/rocrate/__init__.py
new file mode 100644
index 000000000..c3ed71bb6
--- /dev/null
+++ b/invenio_rdm_records/resources/deserializers/rocrate/__init__.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""RO-Crate deserializer."""
+
+from flask_babelex import lazy_gettext as _
+from flask_resources import JSONDeserializer
+
+from ..errors import DeserializerError
+from .schema import ROCrateSchema
+
+
+class ROCrateJSONDeserializer(JSONDeserializer):
+    """RO-Crate metadata deserializer."""
+
+    def __init__(self, schema=ROCrateSchema):
+        """Deserializer initialization."""
+        self.schema = schema
+
+    def deserialize(self, data):
+        """Deserialize the RO-Crate payload."""
+        data = super().deserialize(data)
+        graph = data.get("@graph")
+        if not graph:
+            raise DeserializerError(
+                _("Invalid RO-Crate metadata format, missing '@graph' key.")
+            )
+
+        dataset = self._dereference(graph)
+        metadata = self.schema().load(dataset)
+        return {"metadata": metadata}
+
+    @staticmethod
+    def _dereference(data: list):
+        entities = {}
+        for item in data:
+            if "@id" in item:
+                entities[item["@id"]] = item
+            item.pop("@reverse", None)
+
+        dataset = entities.pop("./")
+
+        def _walk(node):
+            if isinstance(node, dict):
+                for key in list(node.keys()):
+                    value = node[key]
+                    if key == "@id" and value in entities:
+                        node.update(entities[value])
+                    if isinstance(value, (list, dict)):
+                        _walk(value)
+            elif isinstance(node, list):
+                for item in node:
+                    _walk(item)
+
+        for value in entities.values():
+            _walk(value)
+        _walk(dataset)
+
+        return dataset
diff --git a/invenio_rdm_records/resources/deserializers/rocrate/schema.py b/invenio_rdm_records/resources/deserializers/rocrate/schema.py
new file mode 100644
index 000000000..b9f56684a
--- /dev/null
+++ b/invenio_rdm_records/resources/deserializers/rocrate/schema.py
@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""RO-Crate schema."""
+
+from distutils.log import error
+
+from flask_babelex import lazy_gettext as _
+from marshmallow import EXCLUDE, Schema, ValidationError, fields, pre_load, validate
+from marshmallow_utils.fields import SanitizedHTML, SanitizedUnicode
+
+
+def _list_value(lst):
+    """Return the first value from single-value lists."""
+    if isinstance(lst, list) and len(lst) == 1:
+        return lst[0]
+    return lst
+
+
+class ROCrateSchema(Schema):
+    """Schema for the loading RO-Crate into record metadata."""
+
+    class Meta:
+        """Meta attributes for the schema."""
+
+        unknown = EXCLUDE
+
+    error_messages = {
+        "required": _("Missing data for required field."),
+    }
+
+    LIST_VALUE_FIELDS = [
+        "license",
+        "description",
+        "datePublished",
+    ]
+
+    @pre_load
+    def list_to_value(self, data, **kwargs):
+        """Extract top-level single-item list field values."""
+        for field in self.LIST_VALUE_FIELDS:
+            if field in data:
+                data[field] = _list_value(data[field])
+        return data
+
+    resource_type = fields.Constant({"id": "dataset"})
+    title = SanitizedUnicode(data_key="name", required=True)
+    description = SanitizedHTML()
+    publication_date = fields.Method(
+        deserialize="load_publication_date",
+        data_key="datePublished",
+        required=True,
+    )
+
+    def load_publication_date(self, value):
+        """Load publication date."""
+        if value:
+            return value[:10]
+
+    creators = fields.Method(
+        deserialize="load_creators",
+        required=True,
+        data_key="author",
+        validate=validate.Length(min=1),
+    )
+
+    def load_creators(self, value):
+        """Load creators."""
+        errors = {}
+        creators = []
+        for c_idx, obj in enumerate(value):
+            person_or_org = {}
+            affiliations = []
+
+            if obj.get("@type") == "Person":
+                person_or_org["type"] = "personal"
+                family_name = _list_value(obj.get("familyName"))
+                if not family_name:
+                    errors[f"{c_idx}.familyName"] = self.error_messages["required"]
+                person_or_org["family_name"] = family_name
+                given_name = _list_value(obj.get("givenName"))
+                if not given_name:
+                    errors[f"{c_idx}.givenName"] = self.error_messages["required"]
+                person_or_org["given_name"] = given_name
+                if obj.get("affiliation"):
+                    for a_idx, a in enumerate(obj.get("affiliation")):
+                        if not a.get("name"):
+                            errors[
+                                f"{c_idx}.affiliation.{a_idx}.name"
+                            ] = self.error_messages["required"]
+                        affiliations.append({"name": a.get("name")})
+            elif obj.get("@type") == "Organization":
+                person_or_org["type"] = "organizational"
+                if not obj.get("name"):
+                    errors[f"{c_idx}.name"] = self.error_messages["required"]
+                person_or_org["name"] = obj.get("name")
+            else:
+                errors[f"{c_idx}.@type"] = _(
+                    "'@type' must be 'Person' or 'Organization'"
+                )
+
+            creators.append(
+                {
+                    "person_or_org": person_or_org,
+                    "affiliations": affiliations,
+                }
+            )
+
+        if errors:
+            raise ValidationError(errors)
+        return creators
+
+    rights = fields.Method(deserialize="load_rights", data_key="license", required=True)
+
+    def load_rights(self, value):
+        """Load rights."""
+        if value:
+            ret = {}
+            if value.get("name"):
+                ret["title"] = {"en": value["name"]}
+            else:
+                raise ValidationError({"name": self.error_messages["required"]})
+            if value.get("description"):
+                ret["description"] = {"en": _list_value(value["description"])}
+
+            return [ret]
+
+    subjects = fields.Method(deserialize="load_subjects", data_key="keywords")
+
+    def load_subjects(self, value):
+        """Load subjects."""
+        if value and isinstance(value, list):
+            return [{"subject": s} for s in value]
+        else:
+            raise ValidationError(_("Format must be a list of strings."))


Test patch:
diff --git a/tests/resources/test_rocrate.py b/tests/resources/test_rocrate.py
new file mode 100644
index 000000000..fd95d4f4d
--- /dev/null
+++ b/tests/resources/test_rocrate.py
@@ -0,0 +1,220 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""RO-Crate resource tests."""
+
+import pytest
+
+
+@pytest.fixture
+def headers(headers):
+    """RO-Crate Content-Type headers."""
+    return {
+        **headers,
+        "content-type": 'application/ld+json;profile="https://w3id.org/ro/crate/1.1"',
+    }
+
+
+def test_rocrate_content_type(running_app, client_with_login, headers, search_clear):
+    """Test draft creation via RO-Crate metadata payload."""
+    client = client_with_login
+    response = client.post(
+        "/records",
+        json={
+            "@context": [
+                "https://w3id.org/ro/crate/1.0/context",
+                {"@vocab": "https://schema.org/"},
+            ],
+            "@graph": [
+                {
+                    "@id": "ro-crate-metadata.json",
+                    "@type": "CreativeWork",
+                    "about": {"@id": "./"},
+                    "identifier": "ro-crate-metadata.json",
+                    "conformsTo": {"@id": "https://w3id.org/ro/crate/1.0"},
+                },
+                {
+                    "@id": "./",
+                    "@type": "Dataset",
+                    "name": "Test dataset",
+                    "description": ["This is a test dataset."],
+                    "license": [{"@id": "#f96b4aaf-7cdc-4d21-9c13-d26546d50fd4"}],
+                    "datePublished": ["2022-10-12T22:00:00.000Z"],
+                    "author": [
+                        {"@id": "#98ccae13-bb8a-41e5-9bac-1db0652100d5"},
+                        {"@id": "#575492d0-8743-410c-bb79-25b4bb9de9d3"},
+                    ],
+                    "keywords": ["machine learning", "nlp"],
+                },
+                {
+                    "@id": "#f96b4aaf-7cdc-4d21-9c13-d26546d50fd4",
+                    "@type": "CreativeWork",
+                    "name": "MIT",
+                    "description": ["MIT License"],
+                },
+                {
+                    "@id": "#98ccae13-bb8a-41e5-9bac-1db0652100d5",
+                    "@type": "Person",
+                    "familyName": ["Smith"],
+                    "givenName": ["John"],
+                    "affiliation": [
+                        {"@id": "https://ror.org/01ggx4157"},
+                    ],
+                },
+                {
+                    "@id": "#575492d0-8743-410c-bb79-25b4bb9de9d3",
+                    "@type": "Organization",
+                    "name": "Research Org",
+                },
+                {
+                    "@id": "https://ror.org/01ggx4157",
+                    "@type": "Organization",
+                    "name": "European Organization for Nuclear Research",
+                },
+            ],
+        },
+        headers=headers,
+    )
+
+    assert response.status_code == 201
+    assert response.json["metadata"] == {
+        "title": "Test dataset",
+        "description": "This is a test dataset.",
+        "publication_date": "2022-10-12",
+        "resource_type": {"id": "dataset", "title": {"en": "Dataset"}},
+        "creators": [
+            {
+                "affiliations": [
+                    {"name": "European Organization for Nuclear Research"}
+                ],
+                "person_or_org": {
+                    "family_name": "Smith",
+                    "given_name": "John",
+                    "name": "Smith, John",
+                    "type": "personal",
+                },
+            },
+            {
+                "person_or_org": {
+                    "name": "Research Org",
+                    "type": "organizational",
+                },
+            },
+        ],
+        "rights": [{"description": {"en": "MIT License"}, "title": {"en": "MIT"}}],
+        "subjects": [{"subject": "machine learning"}, {"subject": "nlp"}],
+    }
+
+
+def test_invalid_rocrate(running_app, client_with_login, headers, search_clear):
+    """Test invalid RO-Crate metadata payload."""
+    client = client_with_login
+
+    # Check missing "@graph"
+    response = client.post("/records", json={}, headers=headers)
+    assert response.status_code == 400
+    assert response.json == {
+        "message": "Invalid RO-Crate metadata format, missing '@graph' key.",
+        "status": 400,
+    }
+
+    # Check required fields
+    response = client.post(
+        "/records",
+        json={
+            "@context": [
+                "https://w3id.org/ro/crate/1.0/context",
+                {"@vocab": "https://schema.org/"},
+            ],
+            "@graph": [
+                {
+                    "@id": "ro-crate-metadata.json",
+                    "@type": "CreativeWork",
+                    "about": {"@id": "./"},
+                    "identifier": "ro-crate-metadata.json",
+                    "conformsTo": {"@id": "https://w3id.org/ro/crate/1.0"},
+                },
+                {
+                    "@id": "./",
+                    "@type": "Dataset",
+                },
+            ],
+        },
+        headers=headers,
+    )
+    assert response.status_code == 400
+    assert {(e["field"], e["messages"][0]) for e in response.json["errors"]} == {
+        ("license", "Missing data for required field."),
+        ("name", "Missing data for required field."),
+        ("datePublished", "Missing data for required field."),
+        ("author", "Missing data for required field."),
+    }
+
+    # Check required fields
+    response = client.post(
+        "/records",
+        json={
+            "@context": [
+                "https://w3id.org/ro/crate/1.0/context",
+                {"@vocab": "https://schema.org/"},
+            ],
+            "@graph": [
+                {
+                    "@id": "ro-crate-metadata.json",
+                    "@type": "CreativeWork",
+                    "about": {"@id": "./"},
+                    "identifier": "ro-crate-metadata.json",
+                    "conformsTo": {"@id": "https://w3id.org/ro/crate/1.0"},
+                },
+                {
+                    "@id": "./",
+                    "@type": "Dataset",
+                    "name": "Test dataset",
+                    "license": [{"@id": "#f96b4aaf-7cdc-4d21-9c13-d26546d50fd4"}],
+                    "datePublished": ["2022-10-12T22:00:00.000Z"],
+                    "author": [
+                        {"@id": "#98ccae13-bb8a-41e5-9bac-1db0652100d5"},
+                        {"@id": "#575492d0-8743-410c-bb79-25b4bb9de9d3"},
+                        {"@id": "#41f1b05c-aad9-42f2-8d35-13495e7452d2"},
+                    ],
+                },
+                {
+                    "@id": "#f96b4aaf-7cdc-4d21-9c13-d26546d50fd4",
+                    "@type": "CreativeWork",
+                },
+                {
+                    "@id": "#98ccae13-bb8a-41e5-9bac-1db0652100d5",
+                    "@type": "Person",
+                    "affiliation": [
+                        {"@id": "https://ror.org/01ggx4157"},
+                    ],
+                },
+                {
+                    "@id": "#575492d0-8743-410c-bb79-25b4bb9de9d3",
+                    "@type": "Organization",
+                },
+                {
+                    "@id": "https://ror.org/01ggx4157",
+                    "@type": "Organization",
+                },
+                {
+                    "@id": "#41f1b05c-aad9-42f2-8d35-13495e7452d2",
+                    "@type": "invalid",
+                },
+            ],
+        },
+        headers=headers,
+    )
+    assert response.status_code == 400
+    assert {(e["field"], e["messages"][0]) for e in response.json["errors"]} == {
+        ("author.0.familyName", "Missing data for required field."),
+        ("author.0.givenName", "Missing data for required field."),
+        ("author.0.affiliation.0.name", "Missing data for required field."),
+        ("author.1.name", "Missing data for required field."),
+        ("author.2.@type", "'@type' must be 'Person' or 'Organization'"),
+        ("license.name", "Missing data for required field."),
+    }