Repository: ResearchObject/ro-crate-py

Issue:
Support indirect data entity linking from root
Currently we detect data entities only if they are *directly* linked to from the root data entity:

```json
{
    "@id": "./",
    "@type": "Dataset",
    "hasPart": [
        {"@id": "spam"},
        {"@id": "spam/foo.txt"}
    ]
},
{
    "@id": "spam",
    "@type": "Dataset",
    "hasPart": [
        {"@id": "spam/foo.txt"}
    ]
},
{
    "@id": "spam/foo.txt",
    "@type": "File"
}
```

We should also support indirect linking as specified [in the specs](https://www.researchobject.org/ro-crate/1.1/data-entities.html#referencing-files-and-folders-from-the-root-data-entity). In the above example, for instance, we should treat `spam/foo.txt` as a data entity even if it wasn't listed in the root data entity's `hasPart`.

Discussion:

Patch:
diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py
index d6b12a6..57bcd39 100644
--- a/rocrate/rocrate.py
+++ b/rocrate/rocrate.py
@@ -147,10 +147,16 @@ class ROCrate():
         preview_entity = entities.pop(Preview.BASENAME, None)
         if preview_entity and not gen_preview:
             self.add(Preview(self, source / Preview.BASENAME, properties=preview_entity))
+        self.__add_parts(parts, entities, source)
+
+    def __add_parts(self, parts, entities, source):
         type_map = OrderedDict((_.__name__, _) for _ in subclasses(FileOrDir))
         for data_entity_ref in parts:
             id_ = data_entity_ref['@id']
-            entity = entities.pop(id_)
+            try:
+                entity = entities.pop(id_)
+            except KeyError:
+                continue
             assert id_ == entity.pop('@id')
             cls = pick_type(entity, type_map, fallback=DataEntity)
             if cls is DataEntity:
@@ -161,6 +167,7 @@ class ROCrate():
                 else:
                     instance = cls(self, source / id_, id_, properties=entity)
             self.add(instance)
+            self.__add_parts(entity.get("hasPart", []), entities, source)
 
     def __read_contextual_entities(self, entities):
         type_map = {_.__name__: _ for _ in subclasses(ContextEntity)}


Test patch:
diff --git a/test/test_read.py b/test/test_read.py
index c7843f7..9aa3e96 100644
--- a/test/test_read.py
+++ b/test/test_read.py
@@ -484,3 +484,53 @@ def test_multi_type_context_entity(tmpdir):
     entity = crate.dereference(id_)
     assert entity in crate.contextual_entities
     assert set(entity.type) == set(type_)
+
+
+def test_indirect_data_entity(tmpdir):
+    metadata = {
+        "@context": "https://w3id.org/ro/crate/1.1/context",
+        "@graph": [
+            {
+                "@id": "ro-crate-metadata.json",
+                "@type": "CreativeWork",
+                "about": {"@id": "./"},
+                "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}
+            },
+            {
+                "@id": "./",
+                "@type": "Dataset",
+                "hasPart": [{"@id": "d1"}]
+            },
+            {
+                "@id": "d1",
+                "@type": "Dataset",
+                "hasPart": [{"@id": "d1/d2"}]
+            },
+            {
+                "@id": "d1/d2",
+                "@type": "Dataset",
+                "hasPart": [{"@id": "d1/d2/f1"}]
+            },
+            {
+                "@id": "d1/d2/f1",
+                "@type": "File"
+            }
+        ]
+    }
+    crate_dir = tmpdir / "test_indirect_data_entity"
+    crate_dir.mkdir()
+    with open(crate_dir / "ro-crate-metadata.json", "wt") as f:
+        json.dump(metadata, f, indent=4)
+    d1 = crate_dir / "d1"
+    d1.mkdir()
+    d2 = d1 / "d2"
+    d2.mkdir()
+    f1 = d2 / "f1"
+    f1.touch()
+    crate = ROCrate(crate_dir)
+    d1_e = crate.dereference("d1")
+    assert d1_e
+    assert d1_e in crate.data_entities
+    d2_e = crate.dereference("d1/d2")
+    assert d2_e
+    assert d2_e in crate.data_entities