Repository: ResearchObject/ro-crate-py
Pull request #46 (2021-02-12 13:40:06)

Crate init
Adds an "init" mode that allows to initialize an RO-Crate from an existing directory tree, i.e., build an RO-Crate object that references all files and directories in the tree.

Also adds an "rocrate" command with a nested interface, with `init` as its first implemented subcommand. `rocrate init top_dir` initializes the directory tree rooted at `top_dir`, adding an `ro-crate-metadata.json` that references files and directories.

```
$ docker run --rm -it --name ro-crate-py ro-crate-py bash
root@ea40838f1c80:/ro-crate-py# cd /tmp/
root@ea40838f1c80:/tmp# rocrate 
Usage: rocrate [OPTIONS] COMMAND [ARGS]...

Options:
  --help  Show this message and exit.

Commands:
  add
  init
root@ea40838f1c80:/tmp# rocrate init --help
Usage: rocrate init [OPTIONS] TOP_DIR

Options:
  --help  Show this message and exit.
root@ea40838f1c80:/tmp# cp -rf /ro-crate-py/test/test-data/ro-crate-galaxy-sortchangecase crate_dir
root@ea40838f1c80:/tmp# rm -fv crate_dir/ro-crate-metadata.json 
removed 'crate_dir/ro-crate-metadata.json'
root@ea40838f1c80:/tmp# cp -a crate_dir{,.bak}
root@ea40838f1c80:/tmp# rocrate init crate_dir
root@ea40838f1c80:/tmp# diff -r --brief crate_dir.bak crate_dir
Only in crate_dir: ro-crate-metadata.json
root@ea40838f1c80:/tmp# cat crate_dir/ro-crate-metadata.json 
{
    "@context": "https://w3id.org/ro/crate/1.1/context",
    "@graph": [
        {
            "@id": "./",
            "@type": "Dataset",
            "datePublished": "2021-02-12T12:11:20.426930",
            "hasPart": [
                {
                    "@id": "test/"
                },
                {
                    "@id": "LICENSE"
                },
                {
                    "@id": "sort-and-change-case.ga"
                },
                {
                    "@id": "README.md"
                },
                {
                    "@id": "test/test1/"
                },
                {
                    "@id": "test/test1/sort-and-change-case-test.yml"
                },
                {
                    "@id": "test/test1/input.bed"
                },
                {
                    "@id": "test/test1/output_exp.bed"
                }
            ]
        },
        {
            "@id": "ro-crate-metadata.json",
            "@type": "CreativeWork",
            "about": {
                "@id": "./"
            },
            "conformsTo": {
                "@id": "https://w3id.org/ro/crate/1.1"
            }
        },
        {
            "@id": "test/",
            "@type": "Dataset"
        },
        {
            "@id": "LICENSE",
            "@type": "File"
        },
        {
            "@id": "sort-and-change-case.ga",
            "@type": "File"
        },
        {
            "@id": "README.md",
            "@type": "File"
        },
        {
            "@id": "test/test1/",
            "@type": "Dataset"
        },
        {
            "@id": "test/test1/sort-and-change-case-test.yml",
            "@type": "File"
        },
        {
            "@id": "test/test1/input.bed",
            "@type": "File"
        },
        {
            "@id": "test/test1/output_exp.bed",
            "@type": "File"
        }
    ]
}
```

Note that the `add` command is not implemented yet.



Patch:
diff --git a/requirements.txt b/requirements.txt
index 9cf1bff..69b8447 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ arcp==0.2.1
 galaxy2cwl
 jinja2
 python-dateutil
+click
diff --git a/rocrate/cli.py b/rocrate/cli.py
new file mode 100644
index 0000000..799064e
--- /dev/null
+++ b/rocrate/cli.py
@@ -0,0 +1,28 @@
+import click
+
+from .rocrate import ROCrate
+
+
+@click.group()
+def cli():
+    pass
+
+
+@click.command()
+@click.argument('top_dir')
+def init(top_dir):
+    crate = ROCrate(top_dir, init=True, load_preview=True)
+    crate.metadata.write(top_dir)
+
+
+@click.command()
+def add():
+    click.echo('Not implemented yet')
+
+
+cli.add_command(init)
+cli.add_command(add)
+
+
+if __name__ == '__main__':
+    cli()
diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py
index cc22fb0..9fd378e 100644
--- a/rocrate/model/dataset.py
+++ b/rocrate/model/dataset.py
@@ -32,7 +32,7 @@ class Dataset(DataEntity):
         if not dest_path:
             identifier = os.path.dirname(source)
         else:
-            identifier = dest_path
+            identifier = str(dest_path)
         super().__init__(crate, identifier, properties)
 
     def _empty(self):
diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py
index 9a5edc3..81b5858 100644
--- a/rocrate/rocrate.py
+++ b/rocrate/rocrate.py
@@ -49,7 +49,7 @@ from .utils import is_url
 
 class ROCrate():
 
-    def __init__(self, source_path=None, load_preview=False):
+    def __init__(self, source_path=None, load_preview=False, init=False):
         self.__entity_map = {}
         self.default_entities = []
         self.data_entities = []
@@ -61,11 +61,12 @@ class ROCrate():
 
         # TODO: default_properties must include name, description,
         # datePublished, license
-        if not source_path or not load_preview:
-            self.add(Preview(self))
         if not source_path:
             # create a new ro-crate
-            self.add(RootDataset(self), Metadata(self))
+            self.add(RootDataset(self), Metadata(self), Preview(self))
+        elif init:
+            # initialize an ro-crate from a directory tree
+            self.__init_from_tree(source_path, load_preview=load_preview)
         else:
             # load an existing ro-crate
             if zipfile.is_zipfile(source_path):
@@ -87,6 +88,27 @@ class ROCrate():
             self.build_crate(entities, source_path, load_preview)
             # TODO: load root dataset properties
 
+    def __init_from_tree(self, top_dir, load_preview=False):
+        top_dir = Path(top_dir)
+        if not top_dir.is_dir():
+            raise ValueError(f"{top_dir} is not a valid directory path")
+        self.add(RootDataset(self), Metadata(self))
+        if not load_preview:
+            self.add(Preview(self))
+        for root, dirs, files in os.walk(top_dir):
+            root = Path(root)
+            for name in dirs:
+                source = root / name
+                self.add_dataset(source, source.relative_to(top_dir))
+            for name in files:
+                source = root / name
+                if source == top_dir / Metadata.BASENAME or source == top_dir / LegacyMetadata.BASENAME:
+                    continue
+                if source != top_dir / Preview.BASENAME:
+                    self.add_file(source, source.relative_to(top_dir))
+                elif load_preview:
+                    self.add(Preview(self, source))
+
     def entities_from_metadata(self, metadata_path):
         # Creates a dictionary {id: entity} from the metadata file
         with open(metadata_path) as metadata_file:
@@ -157,6 +179,8 @@ class ROCrate():
         if Preview.BASENAME in entities.keys() and load_preview:
             preview_source = os.path.join(source, Preview.BASENAME)
             self.add(Preview(self, preview_source))
+        else:
+            self.add(Preview(self))
 
         added_entities = []
         # iterate over data entities
diff --git a/setup.py b/setup.py
index 2ecf914..899e641 100755
--- a/setup.py
+++ b/setup.py
@@ -81,4 +81,7 @@ setup(
         'Topic :: System :: Archiving',
         'Topic :: System :: Archiving :: Packaging',
     ],
+    entry_points={
+        "console_scripts": ["rocrate=rocrate.cli:cli"],
+    },
 )


Test patch:
diff --git a/test/test_read.py b/test/test_read.py
index d98af60..f844987 100644
--- a/test/test_read.py
+++ b/test/test_read.py
@@ -17,9 +17,12 @@
 
 import pytest
 import shutil
+import uuid
 from pathlib import Path
 
 from rocrate.rocrate import ROCrate
+from rocrate.model.file import File
+from rocrate.model.dataset import Dataset
 
 _URL = ('https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/'
         'test/test-data/sample_file.txt')
@@ -148,3 +151,82 @@ def test_legacy_crate(test_data_dir, tmpdir, helpers):
     main_wf = crate.dereference('test_galaxy_wf.ga')
     wf_prop = main_wf.properties()
     assert set(wf_prop['@type']) == helpers.LEGACY_WORKFLOW_TYPES
+
+
+def test_bad_crate(test_data_dir, tmpdir):
+    # nonexistent dir
+    crate_dir = test_data_dir / uuid.uuid4().hex
+    with pytest.raises(ValueError):
+        ROCrate(crate_dir)
+    with pytest.raises(ValueError):
+        ROCrate(crate_dir, init=True)
+    # no metadata file
+    crate_dir = tmpdir / uuid.uuid4().hex
+    crate_dir.mkdir()
+    with pytest.raises(ValueError):
+        ROCrate(crate_dir)
+
+
+@pytest.mark.parametrize("override", [False, True])
+def test_init(test_data_dir, tmpdir, helpers, override):
+    crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
+    if not override:
+        (crate_dir / helpers.METADATA_FILE_NAME).unlink()
+    crate = ROCrate(crate_dir, init=True)
+    assert crate.dereference("./") is not None
+    assert crate.dereference(helpers.METADATA_FILE_NAME) is not None
+    fpaths = [
+        "LICENSE",
+        "README.md",
+        "sort-and-change-case.ga",
+        "test/test1/input.bed",
+        "test/test1/output_exp.bed",
+        "test/test1/sort-and-change-case-test.yml"
+    ]
+    dpaths = [
+        "test/",
+        "test/test1/",
+    ]
+    for p in fpaths:
+        assert isinstance(crate.dereference(p), File)
+    for p in dpaths:
+        assert isinstance(crate.dereference(p), Dataset)
+
+    out_path = tmpdir / 'ro_crate_out'
+    out_path.mkdir()
+    crate.write_crate(out_path)
+
+    assert (out_path / helpers.METADATA_FILE_NAME).exists()
+    json_entities = helpers.read_json_entities(out_path)
+    data_entity_ids = fpaths + dpaths
+    helpers.check_crate(json_entities, data_entity_ids=data_entity_ids)
+    for p in fpaths:
+        with open(crate_dir / p) as f1, open(out_path / p) as f2:
+            assert f1.read() == f2.read()
+
+
+@pytest.mark.parametrize("load_preview,preview_exists", [(False, False), (False, True), (True, False), (True, True)])
+def test_init_preview(test_data_dir, tmpdir, helpers, load_preview, preview_exists):
+    crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
+    dummy_prev_content = "foo\nbar\n"
+    if preview_exists:
+        with open(crate_dir / helpers.PREVIEW_FILE_NAME, "wt") as f:
+            f.write(dummy_prev_content)
+    crate = ROCrate(crate_dir, load_preview=load_preview, init=True)
+    prev = crate.dereference(helpers.PREVIEW_FILE_NAME)
+    if load_preview and not preview_exists:
+        assert prev is None
+    else:
+        assert prev is not None
+
+    out_path = tmpdir / 'ro_crate_out'
+    out_path.mkdir()
+    crate.write_crate(out_path)
+
+    out_prev_path = out_path / helpers.PREVIEW_FILE_NAME
+    if load_preview and not preview_exists:
+        assert not out_prev_path.exists()
+    else:
+        assert out_prev_path.is_file()
+        if load_preview:
+            assert out_prev_path.open().read() == dummy_prev_content