Repository: ResearchObject/ro-crate-py

Issue:
Track files/dirs not listed in the metadata file
From the spec:

> Payload files may appear directly in the RO-Crate Root alongside the RO-Crate Metadata File, and/or appear in sub-directories of the RO-Crate Root. Each file and directory MAY be represented as Data Entities in the RO-Crate Metadata File.

> It is important to note that the RO-Crate Metadata File is not an exhaustive manifest or inventory, that is, it does not necessarily list or describe all files in the package.

Currently, when reading an ro-crate, files or directories not listed in the metadata file are ignored. Thus, when the crate is written out to some other location, those files / directories are not kept.

Comments:

Patch:
diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py
index 4a47c41..2944dff 100644
--- a/rocrate/model/dataset.py
+++ b/rocrate/model/dataset.py
@@ -79,4 +79,5 @@ class Dataset(DataEntity):
         # iterate over the entries
         for file_src, rel_path in self.directory_entries():
             dest_path = os.path.join(out_path, rel_path)
-            zip_out.write(file_src, dest_path)
+            if dest_path not in zip_out.namelist():
+                zip_out.write(file_src, dest_path)
diff --git a/rocrate/model/file.py b/rocrate/model/file.py
index 9f2e5a6..4b5cc50 100644
--- a/rocrate/model/file.py
+++ b/rocrate/model/file.py
@@ -124,4 +124,5 @@ class File(DataEntity):
                         shutil.copyfileobj(response, out_file)
 
     def write_zip(self, zip_out):
-        zip_out.write(self.source, self.id)
+        if self.id not in zip_out.namelist():
+            zip_out.write(self.source, self.id)
diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py
index 0e652c7..74e0ab6 100644
--- a/rocrate/rocrate.py
+++ b/rocrate/rocrate.py
@@ -87,6 +87,8 @@ class ROCrate():
             entities = self.entities_from_metadata(metadata_path)
             self.build_crate(entities, source_path, load_preview)
             # TODO: load root dataset properties
+        # in the zip case, self.source_path is the extracted dir
+        self.source_path = source_path
 
     def __init_from_tree(self, top_dir, load_preview=False):
         top_dir = Path(top_dir)
@@ -427,10 +429,26 @@ class ROCrate():
 
     # write crate to local dir
     def write_crate(self, base_path):
-        Path(base_path).mkdir(parents=True, exist_ok=True)
+        base_path = Path(base_path)
+        base_path.mkdir(parents=True, exist_ok=True)
+        # copy unlisted files and directories
+        if self.source_path:
+            top = self.source_path
+            for root, dirs, files in os.walk(top):
+                root = Path(root)
+                for name in dirs:
+                    source = root / name
+                    dest = base_path / source.relative_to(top)
+                    dest.mkdir(parents=True, exist_ok=True)
+                for name in files:
+                    source = root / name
+                    rel = source.relative_to(top)
+                    if not self.dereference(str(rel)):
+                        dest = base_path / rel
+                        shutil.copyfile(source, dest)
         # write data entities
         for writable_entity in self.data_entities + self.default_entities:
-            writable_entity.write(base_path)
+            writable_entity.write(str(base_path))
 
     def write_zip(self, out_zip):
         if str(out_zip).endswith('.zip'):
@@ -441,6 +459,16 @@ class ROCrate():
             out_file_path, 'w', compression=zipfile.ZIP_DEFLATED,
             allowZip64=True
         )
+        # copy unlisted files and directories
+        if self.source_path:
+            top = self.source_path
+            for root, dirs, files in os.walk(top):
+                root = Path(root)
+                for name in files:
+                    source = root / name
+                    dest = source.relative_to(top)
+                    if not self.dereference(str(dest)):
+                        zf.write(str(source), str(dest))
         for writable_entity in self.data_entities + self.default_entities:
             writable_entity.write_zip(zf)
         zf.close()


Test patch:
diff --git a/test/test-data/read_extra/listed.txt b/test/test-data/read_extra/listed.txt
new file mode 100644
index 0000000..f6174c1
--- /dev/null
+++ b/test/test-data/read_extra/listed.txt
@@ -0,0 +1,1 @@
+LISTED
diff --git a/test/test-data/read_extra/listed/listed.txt b/test/test-data/read_extra/listed/listed.txt
new file mode 100644
index 0000000..f6174c1
--- /dev/null
+++ b/test/test-data/read_extra/listed/listed.txt
@@ -0,0 +1,1 @@
+LISTED
diff --git a/test/test-data/read_extra/listed/not_listed.txt b/test/test-data/read_extra/listed/not_listed.txt
new file mode 100644
index 0000000..821cc78
--- /dev/null
+++ b/test/test-data/read_extra/listed/not_listed.txt
@@ -0,0 +1,1 @@
+NOT_LISTED
diff --git a/test/test-data/read_extra/not_listed.txt b/test/test-data/read_extra/not_listed.txt
new file mode 100644
index 0000000..821cc78
--- /dev/null
+++ b/test/test-data/read_extra/not_listed.txt
@@ -0,0 +1,1 @@
+NOT_LISTED
diff --git a/test/test-data/read_extra/not_listed/not_listed.txt b/test/test-data/read_extra/not_listed/not_listed.txt
new file mode 100644
index 0000000..821cc78
--- /dev/null
+++ b/test/test-data/read_extra/not_listed/not_listed.txt
@@ -0,0 +1,1 @@
+NOT_LISTED
diff --git a/test/test-data/read_extra/ro-crate-metadata.json b/test/test-data/read_extra/ro-crate-metadata.json
new file mode 100644
index 0000000..90e2682
--- /dev/null
+++ b/test/test-data/read_extra/ro-crate-metadata.json
@@ -0,0 +1,43 @@
+{
+    "@context": "https://w3id.org/ro/crate/1.1/context",
+    "@graph": [
+        {
+            "@id": "./",
+            "@type": "Dataset",
+            "datePublished": "2021-02-26T09:46:41.236862",
+            "hasPart": [
+                {
+                    "@id": "listed/"
+                },
+                {
+                    "@id": "listed.txt"
+                },
+                {
+                    "@id": "listed/listed.txt"
+                }
+            ]
+        },
+        {
+            "@id": "ro-crate-metadata.json",
+            "@type": "CreativeWork",
+            "about": {
+                "@id": "./"
+            },
+            "conformsTo": {
+                "@id": "https://w3id.org/ro/crate/1.1"
+            }
+        },
+        {
+            "@id": "listed/",
+            "@type": "Dataset"
+        },
+        {
+            "@id": "listed.txt",
+            "@type": "File"
+        },
+        {
+            "@id": "listed/listed.txt",
+            "@type": "File"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/test/test_read.py b/test/test_read.py
index 4db2864..eea335a 100644
--- a/test/test_read.py
+++ b/test/test_read.py
@@ -18,6 +18,7 @@
 import pytest
 import shutil
 import uuid
+import zipfile
 from pathlib import Path
 
 from rocrate.rocrate import ROCrate
@@ -120,7 +121,10 @@ def test_crate_dir_loading(test_data_dir, tmpdir, helpers, load_preview, from_zi
     metadata_path = out_path / helpers.METADATA_FILE_NAME
     assert metadata_path.exists()
     legacy_metadata_path = out_path / helpers.LEGACY_METADATA_FILE_NAME
-    assert not legacy_metadata_path.exists()
+    # legacy metadata file should just be copied over as a regular file
+    assert legacy_metadata_path.exists()
+    with open(crate_dir / helpers.LEGACY_METADATA_FILE_NAME) as f1, open(legacy_metadata_path) as f2:
+        assert f1.read() == f2.read()
     preview_path = out_path / helpers.PREVIEW_FILE_NAME
     assert preview_path.exists()
     if load_preview:
@@ -241,3 +245,28 @@ def test_no_parts(tmpdir):
 
     crate = ROCrate(out_path)
     assert not crate.root_dataset["hasPart"]
+
+
+@pytest.mark.parametrize("to_zip", [False, True])
+def test_extra_data(test_data_dir, tmpdir, to_zip):
+    crate_dir = test_data_dir / 'read_extra'
+    crate = ROCrate(crate_dir)
+    out_path = tmpdir / 'read_extra_out'
+    if to_zip:
+        zip_path = tmpdir / 'ro_crate_out.crate.zip'
+        crate.write_zip(zip_path)
+        with zipfile.ZipFile(zip_path, "r") as zf:
+            zf.extractall(out_path)
+    else:
+        out_path.mkdir()
+        crate.write_crate(out_path)
+    for rel in {
+            "listed.txt",
+            "listed/listed.txt",
+            "listed/not_listed.txt",
+            "not_listed.txt",
+            "not_listed/not_listed.txt",
+    }:
+        assert (out_path / rel).is_file()
+        with open(crate_dir / rel) as f1, open(out_path / rel) as f2:
+            assert f1.read() == f2.read()