Repository: ResearchObject/ro-crate-py (python)

Issue:
Can't read recent WorkflowHub crates
Reported by @lrodrin and @jmfernandez at yesterday's community meeting. One such example is https://workflowhub.eu/workflows/244.

```pycon
>>> from rocrate.rocrate import ROCrate
>>> crate = ROCrate("/tmp/workflow-244-3")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/simleo/git/ro-crate-py/rocrate/rocrate.py", line 90, in __init__
    source = self.__read(source, gen_preview=gen_preview)
  File "/home/simleo/git/ro-crate-py/rocrate/rocrate.py", line 132, in __read
    self.__read_data_entities(entities, source, gen_preview)
  File "/home/simleo/git/ro-crate-py/rocrate/rocrate.py", line 177, in __read_data_entities
    metadata_id, root_id = self.find_root_entity_id(entities)
  File "/home/simleo/git/ro-crate-py/rocrate/rocrate.py", line 150, in find_root_entity_id
    if conformsTo and conformsTo.startswith("https://w3id.org/ro/crate/"):
AttributeError: 'list' object has no attribute 'startswith'

```

Pull request:
Support for latest Workflow RO-Crate profile
Fixes #99.

Patch:
diff --git a/rocrate/model/computerlanguage.py b/rocrate/model/computerlanguage.py
index 171d864..6571cd9 100644
--- a/rocrate/model/computerlanguage.py
+++ b/rocrate/model/computerlanguage.py
@@ -75,7 +75,6 @@ CWL_DEFAULT_VERSION = "1.2"
 # https://github.com/galaxyproject/gxformat2 has some info on gxformat2 versions
 # version can probably be simply ignored for "native" *.ga workflows
 GALAXY_DEFAULT_VERSION = "v19_09"
-COMPSS_DEFAULT_VERSION = "2.10"
 
 
 def cwl(crate, version=CWL_DEFAULT_VERSION):
@@ -105,20 +104,9 @@ def galaxy(crate, version=GALAXY_DEFAULT_VERSION):
     })
 
 
-def compss(crate, version=COMPSS_DEFAULT_VERSION):
-    return ComputerLanguage(crate, identifier="#compss", properties={
-        "name": "COMPSs Programming Model",
-        "alternateName": "COMPSs",
-        "url": "http://compss.bsc.es/",
-        "citation": "https://doi.org/10.1007/s10723-013-9272-5",
-        "version": version
-    })
-
-
 LANG_MAP = {
     "cwl": cwl,
     "galaxy": galaxy,
-    "compss": compss,
 }
 
 
diff --git a/rocrate/model/metadata.py b/rocrate/model/metadata.py
index 711c0f1..7a4d85c 100644
--- a/rocrate/model/metadata.py
+++ b/rocrate/model/metadata.py
@@ -31,8 +31,15 @@ class Metadata(File):
     BASENAME = "ro-crate-metadata.json"
     PROFILE = "https://w3id.org/ro/crate/1.1"
 
-    def __init__(self, crate):
-        super().__init__(crate, None, self.BASENAME, False, None)
+    def __init__(self, crate, properties=None):
+        super().__init__(
+            crate,
+            source=None,
+            dest_path=self.BASENAME,
+            fetch_remote=False,
+            validate_url=False,
+            properties=properties
+        )
         # https://www.researchobject.org/ro-crate/1.1/appendix/jsonld.html#extending-ro-crate
         self.extra_terms = {}
 
@@ -88,3 +95,12 @@ TESTING_EXTRA_TERMS = {
     "definition": "https://w3id.org/ro/terms/test#definition",
     "engineVersion": "https://w3id.org/ro/terms/test#engineVersion"
 }
+
+
+def metadata_class(descriptor_id):
+    if descriptor_id == Metadata.BASENAME:
+        return Metadata
+    elif descriptor_id == LegacyMetadata.BASENAME:
+        return LegacyMetadata
+    else:
+        return ValueError("Invalid metadata descriptor ID: {descriptor_id!r}")
diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py
index af7a1cb..ebb8614 100644
--- a/rocrate/rocrate.py
+++ b/rocrate/rocrate.py
@@ -37,7 +37,7 @@ from .model.data_entity import DataEntity
 from .model.file_or_dir import FileOrDir
 from .model.file import File
 from .model.dataset import Dataset
-from .model.metadata import Metadata, LegacyMetadata, TESTING_EXTRA_TERMS
+from .model.metadata import Metadata, LegacyMetadata, TESTING_EXTRA_TERMS, metadata_class
 from .model.preview import Preview
 from .model.testdefinition import TestDefinition
 from .model.computationalworkflow import ComputationalWorkflow, galaxy_to_abstract_cwl
@@ -47,7 +47,7 @@ from .model.testservice import TestService, get_service
 from .model.softwareapplication import SoftwareApplication, get_app, PLANEMO_DEFAULT_VERSION
 from .model.testsuite import TestSuite
 
-from .utils import is_url, subclasses
+from .utils import is_url, subclasses, get_norm_value
 
 
 def read_metadata(metadata_path):
@@ -67,6 +67,18 @@ def read_metadata(metadata_path):
     return context, {_["@id"]: _ for _ in graph}
 
 
+def pick_type(json_entity, type_map, fallback=None):
+    try:
+        t = json_entity["@type"]
+    except KeyError:
+        raise ValueError(f'entity {json_entity["@id"]!r} has no @type')
+    types = {_.strip() for _ in set(t if isinstance(t, list) else [t])}
+    for name, c in type_map.items():
+        if name in types:
+            return c
+    return fallback
+
+
 class ROCrate():
 
     def __init__(self, source=None, gen_preview=False, init=False):
@@ -121,13 +133,10 @@ class ROCrate():
                 zf.extractall(zip_path)
             source = Path(zip_path)
         metadata_path = source / Metadata.BASENAME
-        MetadataClass = Metadata
         if not metadata_path.is_file():
             metadata_path = source / LegacyMetadata.BASENAME
-            MetadataClass = LegacyMetadata
         if not metadata_path.is_file():
             raise ValueError(f"Not a valid RO-Crate: missing {Metadata.BASENAME}")
-        self.add(MetadataClass(self))
         _, entities = read_metadata(metadata_path)
         self.__read_data_entities(entities, source, gen_preview)
         self.__read_contextual_entities(entities)
@@ -144,12 +153,11 @@ class ROCrate():
         # First let's try conformsTo algorithm in
         # <https://www.researchobject.org/ro-crate/1.1/root-data-entity.html#finding-the-root-data-entity>
         for entity in entities.values():
-            conformsTo = entity.get("conformsTo")
-            if conformsTo and "@id" in conformsTo:
-                conformsTo = conformsTo["@id"]
-            if conformsTo and conformsTo.startswith("https://w3id.org/ro/crate/"):
-                if "about" in entity:
-                    return (entity["@id"], entity["about"]["@id"])
+            about = get_norm_value(entity, "about")
+            if about:
+                for id_ in get_norm_value(entity, "conformsTo"):
+                    if id_.startswith("https://w3id.org/ro/crate/"):
+                        return(entity["@id"], about[0])
         # ..fall back to a generous look up by filename,
         for candidate in (
                 Metadata.BASENAME, LegacyMetadata.BASENAME,
@@ -175,7 +183,10 @@ class ROCrate():
 
     def __read_data_entities(self, entities, source, gen_preview):
         metadata_id, root_id = self.find_root_entity_id(entities)
-        entities.pop(metadata_id)  # added previously
+        MetadataClass = metadata_class(metadata_id)
+        metadata_properties = entities.pop(metadata_id)
+        self.add(MetadataClass(self, properties=metadata_properties))
+
         root_entity = entities.pop(root_id)
         assert root_id == root_entity.pop('@id')
         parts = root_entity.pop('hasPart', [])
@@ -188,17 +199,7 @@ class ROCrate():
             id_ = data_entity_ref['@id']
             entity = entities.pop(id_)
             assert id_ == entity.pop('@id')
-            try:
-                t = entity["@type"]
-            except KeyError:
-                raise ValueError(f'entity "{id_}" has no @type')
-            types = {_.strip() for _ in set(t if isinstance(t, list) else [t])}
-            # pick the most specific type (order guaranteed by subclasses)
-            cls = DataEntity
-            for name, c in type_map.items():
-                if name in types:
-                    cls = c
-                    break
+            cls = pick_type(entity, type_map, fallback=DataEntity)
             if cls is DataEntity:
                 instance = DataEntity(self, identifier=id_, properties=entity)
             else:
@@ -212,10 +213,7 @@ class ROCrate():
         type_map = {_.__name__: _ for _ in subclasses(ContextEntity)}
         for identifier, entity in entities.items():
             assert identifier == entity.pop('@id')
-            # https://github.com/ResearchObject/ro-crate/issues/83
-            if isinstance(entity['@type'], list):
-                raise RuntimeError(f"multiple types for '{identifier}'")
-            cls = type_map.get(entity['@type'], ContextEntity)
+            cls = pick_type(entity, type_map, fallback=ContextEntity)
             self.add(cls(self, identifier, entity))
 
     @property
diff --git a/rocrate/utils.py b/rocrate/utils.py
index 9cd22c5..40a2b14 100644
--- a/rocrate/utils.py
+++ b/rocrate/utils.py
@@ -65,3 +65,16 @@ def subclasses(cls):
         for c in subclasses(d):
             yield c
         yield d
+
+
+def get_norm_value(json_entity, prop):
+    """\
+    Get a normalized value for a property (always as a list of strings).
+    """
+    value = json_entity.get(prop, [])
+    if not isinstance(value, list):
+        value = [value]
+    try:
+        return [_ if isinstance(_, str) else _["@id"] for _ in value]
+    except (TypeError, KeyError):
+        raise ValueError(f"Malformed value for {prop!r}: {json_entity.get(prop)!r}")


Test patch:
diff --git a/test/test_read.py b/test/test_read.py
index ac173f1..b2fffe1 100644
--- a/test/test_read.py
+++ b/test/test_read.py
@@ -368,3 +368,65 @@ def test_generic_data_entity(tmpdir):
     crate = ROCrate(out_crate_dir)
 
     check_rc()
+
+
+def test_root_conformsto(tmpdir):
+    # actually not a valid workflow ro-crate, but here it does not matter
+    metadata = {
+        "@context": "https://w3id.org/ro/crate/1.1/context",
+        "@graph": [
+            {
+                "@id": "ro-crate-metadata.json",
+                "@type": "CreativeWork",
+                "about": {"@id": "./"},
+                "conformsTo": [
+                    {"@id": "https://w3id.org/ro/crate/1.1"},
+                    {"@id": "https://about.workflowhub.eu/Workflow-RO-Crate/"}
+                ]
+            },
+            {
+                "@id": "./",
+                "@type": "Dataset",
+            },
+        ]
+    }
+    crate_dir = tmpdir / "test_root_conformsto_crate"
+    crate_dir.mkdir()
+    with open(crate_dir / "ro-crate-metadata.json", "wt") as f:
+        json.dump(metadata, f, indent=4)
+    crate = ROCrate(crate_dir)
+    assert crate.metadata["conformsTo"] == [
+        "https://w3id.org/ro/crate/1.1",
+        "https://about.workflowhub.eu/Workflow-RO-Crate/"
+    ]
+
+
+def test_multi_type_context_entity(tmpdir):
+    id_, type_ = "#xyz", ["Project", "Organization"]
+    metadata = {
+        "@context": "https://w3id.org/ro/crate/1.1/context",
+        "@graph": [
+            {
+                "@id": "ro-crate-metadata.json",
+                "@type": "CreativeWork",
+                "about": {"@id": "./"},
+                "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}
+            },
+            {
+                "@id": "./",
+                "@type": "Dataset",
+            },
+            {
+                "@id": id_,
+                "@type": type_,
+            }
+        ]
+    }
+    crate_dir = tmpdir / "test_multi_type_context_entity_crate"
+    crate_dir.mkdir()
+    with open(crate_dir / "ro-crate-metadata.json", "wt") as f:
+        json.dump(metadata, f, indent=4)
+    crate = ROCrate(crate_dir)
+    entity = crate.dereference(id_)
+    assert entity in crate.contextual_entities
+    assert set(entity.type) == set(type_)
diff --git a/test/test_utils.py b/test/test_utils.py
index e230d0f..ba220b9 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -15,7 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from rocrate.utils import subclasses
+import pytest
+
+from rocrate.utils import subclasses, get_norm_value
 
 
 class Pet:
@@ -38,3 +40,15 @@ def test_subclasses():
     pet_subclasses = list(subclasses(Pet))
     assert set(pet_subclasses) == {Cat, Dog, Beagle}
     assert pet_subclasses.index(Beagle) < pet_subclasses.index(Dog)
+
+
+def test_get_norm_value():
+    for value in {"@id": "foo"}, "foo", ["foo"], [{"@id": "foo"}]:
+        entity = {"@id": "#xyz", "name": value}
+        assert get_norm_value(entity, "name") == ["foo"]
+    for value in [{"@id": "foo"}, "bar"], ["foo", {"@id": "bar"}]:
+        entity = {"@id": "#xyz", "name": value}
+        assert get_norm_value(entity, "name") == ["foo", "bar"]
+    assert get_norm_value({"@id": "#xyz"}, "name") == []
+    with pytest.raises(ValueError):
+        get_norm_value({"@id": "#xyz", "name": [["foo"]]}, "name")