Repository: activeloopai/deeplake

Issue:
[FEATURE] HuggingFace to Hub converter
## 🚨🚨 Feature Request

- [ ] Related to an existing [Issue](../issues) 
- [x] A new implementation (Improvement, Extension) 

### If your feature will improve `HUB`

Add a converter from HuggingFace Datasets to Hub format. 
Can be added once text support is ready.

Difficulty: Medium-hard


Discussion:

Resolving patch:
diff --git a/hub/integrations/huggingface/__init__.py b/hub/integrations/huggingface/__init__.py
new file mode 100644
--- /dev/null
+++ b/hub/integrations/huggingface/__init__.py
@@ -0,0 +1 @@
+from .huggingface import from_huggingface
diff --git a/hub/integrations/huggingface/huggingface.py b/hub/integrations/huggingface/huggingface.py
new file mode 100644
--- /dev/null
+++ b/hub/integrations/huggingface/huggingface.py
@@ -0,0 +1,118 @@
+from typing import Union, Set
+from hub.core.dataset import Dataset
+from datasets import Dataset as hfDataset  # type: ignore
+from datasets import ClassLabel, Sequence, DatasetDict  # type: ignore
+import posixpath
+import hub
+from tqdm import tqdm  # type: ignore
+
+
+def _is_seq_convertible(seq: Union[Sequence, list]):
+    if isinstance(seq, Sequence):
+        feature = seq.feature
+    else:
+        feature = seq[0]
+    if isinstance(feature, dict):
+        return False
+    if feature.dtype in (
+        "string",
+        "large_string",
+    ):  # no support for sequences of strings.
+        return False
+    if isinstance(feature, Sequence):
+        return _is_seq_convertible(feature)
+    return True
+
+
+def _create_tensor_from_feature(key, feature, src, ds):
+    curr = posixpath.split(key)[-1]
+    if isinstance(feature, (dict, Sequence, list)):
+        if isinstance(feature, dict):
+            for x in feature:
+                _create_tensor_from_feature(f"{key}/{x}", feature[x], src[curr], ds)
+            return True
+        elif isinstance(feature, (Sequence, list)):
+            if isinstance(feature, Sequence):
+                inner = feature.feature
+            else:
+                inner = feature[0]
+            if isinstance(inner, dict):
+                _create_tensor_from_feature(key, feature.feature, src, ds)
+                return True
+            elif _is_seq_convertible(feature):
+                ds.create_tensor(key)
+            else:
+                return False
+    elif feature.dtype in ("string", "large_string"):
+        ds.create_tensor(key, htype="text")
+    elif isinstance(feature, ClassLabel):
+        ds.create_tensor(key, htype="class_label", class_names=feature.names)
+    else:
+        ds.create_tensor(key)
+
+    if isinstance(src, (hfDataset, dict)):
+        ds[key].extend(src[curr])
+    else:
+        values = []
+        for i in range(len(src)):
+            values.extend(src[i][curr])
+        ds[key].extend(values)
+    return True
+
+
+def from_huggingface(
+    src: Union[hfDataset, DatasetDict],
+    dest: Union[Dataset, str],
+    use_progressbar: bool = True,
+) -> Dataset:
+    """Converts hugging face datasets to hub format.
+
+    Args:
+        src (hfDataset, DatasetDict): Hugging Face Dataset or DatasetDict to be converted. Data in different splits of a
+            DatasetDict will be stored under respective tensor groups.
+        dest (Dataset, str): Destination dataset or path to it.
+        use_progressbar (bool): Defines if progress bar should be used to show conversion progress.
+
+    Returns:
+        Dataset: The destination Hub dataset.
+
+    Example:
+        if DatasetDict looks like:
+            {
+                train: Dataset({
+                    features: ['data']
+                }),
+                validation: Dataset({
+                    features: ['data']
+                }),
+                test: Dataset({
+                    features: ['data']
+                }),
+            }
+
+        it will be converted to a Hub dataset with tensors ['train/data', 'validation/data', 'test/data'].
+
+    Note:
+        Features of the type Sequence(feature=Value(dtype='string')) are not supported. Columns of such type are skipped.
+    """
+    if isinstance(dest, str):
+        ds = hub.dataset(dest)
+    else:
+        ds = dest  # type: ignore
+
+    if isinstance(src, DatasetDict):
+        for split, src_ds in src.items():
+            for key, feature in src_ds.features.items():
+                _create_tensor_from_feature(f"{split}/{key}", feature, src_ds, ds)
+    else:
+        skipped_keys: Set[str] = set()
+        features = tqdm(
+            src.features.items(),
+            desc=f"Converting...({len(skipped_keys)} skipped)",
+            disable=not use_progressbar,
+        )
+        for key, feature in features:
+            if not _create_tensor_from_feature(key, feature, src, ds):
+                skipped_keys.add(key)
+                features.set_description(f"Converting...({len(skipped_keys)} skipped)")
+    return ds  # type: ignore


Test patch:
diff --git a/hub/integrations/tests/test_huggingface.py b/hub/integrations/tests/test_huggingface.py
new file mode 100644
--- /dev/null
+++ b/hub/integrations/tests/test_huggingface.py
@@ -0,0 +1,75 @@
+from datasets import load_dataset  # type: ignore
+from datasets import Dataset  # type: ignore
+from hub.integrations.huggingface import from_huggingface
+from hub.integrations.huggingface.huggingface import _is_seq_convertible
+from numpy.testing import assert_array_equal
+
+
+def test_before_split():
+    ds = load_dataset("glue", "mrpc")
+    hub_ds = from_huggingface(ds, "mem://xyz")
+
+    splits = ds.keys()
+    columns = ds["train"].column_names
+
+    assert set(hub_ds.meta.tensors) == {
+        f"{split}/{column}" for split in splits for column in columns
+    }
+
+    for split in splits:
+        for column in columns:
+            assert_array_equal(
+                hub_ds[f"{split}/{column}"].numpy().reshape(-1), ds[split][column]
+            )
+
+
+def test_split():
+    ds = load_dataset("glue", "mrpc", split="train[:5%]")
+    hub_ds = from_huggingface(ds, "mem://xyz")
+
+    assert hub_ds.meta.tensors == ds.column_names
+
+    for column in ds.column_names:
+        assert_array_equal(hub_ds[column].numpy().reshape(-1), ds[column])
+
+
+def test_seq_with_dict():
+    ds = load_dataset("squad", split="train[:5%]")
+    hub_ds = from_huggingface(ds, "mem://xyz")
+
+    keys = set(ds.column_names) - {"answers"} | {"answers/text", "answers/answer_start"}
+
+    assert set(hub_ds.meta.tensors) == keys
+
+    for key in ("id", "title", "context", "question"):
+        assert_array_equal(hub_ds[key].numpy().reshape(-1), ds[key])
+
+    answers = {"text": [], "answer_start": []}
+    for answer in ds["answers"]:
+        answers["text"].extend(answer["text"])
+        answers["answer_start"].extend(answer["answer_start"])
+
+    assert_array_equal(hub_ds["answers/text"].numpy().reshape(-1), answers["text"])
+    assert_array_equal(
+        hub_ds["answers/answer_start"].numpy().reshape(-1), answers["answer_start"]
+    )
+
+
+def test_seq():
+    arr1 = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
+    arr2 = [[[5, 6], [7, 8]], [[9, 10], [11, 12]]]
+
+    data = {"id": [0, 1], "seq": [arr1, arr2]}
+    ds = Dataset.from_dict(data)
+
+    hub_ds = from_huggingface(ds, "mem://xyz")
+
+    assert set(hub_ds.meta.tensors) == {"id", "seq"}
+    assert_array_equal(hub_ds["seq"], [arr1, arr2])
+
+    arr = [["abcd"], ["efgh"]]
+
+    data = {"id": [0, 1], "seq": arr}
+    ds = Dataset.from_dict(data)
+
+    assert not _is_seq_convertible(ds.features["seq"])