Repository: fgnt/lazy_dataset

Issue:
Support lazy_dataset.new(other_dataset)
It would be logical if `lazy_dataset.new` would accept another dataset, like this:
```python
import lazy_dataset
lazy_dataset.new(lazy_dataset.new([1, 2, 3, 4]))
```
But it is unclear how `new` should handle datasets. It could
 - return the input dataset
 - make a copy of the dataset (`return dataset.copy(freeze=??)`)
 - cache and transform into `ListDataset` or `DictDataset`, i.e., `lazy_dataset.new(list(lazy_dataset.new([1, 2, 3, 4])))` or `lazy_dataset.new(dict(lazy_dataset.new({'a': 1, 'b': 2})))`

Discussion:

Patch:
diff --git a/lazy_dataset/core.py b/lazy_dataset/core.py
index 6a55273..56cb6c9 100644
--- a/lazy_dataset/core.py
+++ b/lazy_dataset/core.py
@@ -27,7 +27,7 @@ def _get_serialize_and_deserialize(immutable_warranty):
 
 
 def new(
-        examples: Union[list, dict],
+        examples: Union[list, dict, 'Dataset'],
         immutable_warranty: str = 'pickle',
         name: str = None,
 ):
@@ -73,6 +73,9 @@ def new(
         >>> ds = lazy_dataset.new([1, 2, 3, 4, 5])
         >>> list(ds)
         [1, 2, 3, 4, 5]
+        >>> lazy_dataset.new(ds)
+          ListDataset(len=5)
+        MapDataset(_pickle.loads)
 
     """
     if isinstance(examples, dict):
@@ -81,6 +84,9 @@ def new(
     elif isinstance(examples, (tuple, list)):
         dataset = from_list(
             examples, immutable_warranty=immutable_warranty, name=name)
+    elif isinstance(examples, Dataset):
+        dataset = from_dataset(
+            examples, immutable_warranty=immutable_warranty, name=name)
     else:
         raise TypeError(type(examples), examples)
     return dataset
@@ -97,7 +103,7 @@ def from_dict(
 
 
 def from_list(
-        examples: list,
+        examples: [list, tuple],
         immutable_warranty: str = 'pickle',
         name: str = None,
 ):
@@ -107,6 +113,50 @@ def from_list(
     return ListDataset(examples, name=name).map(deserialize)
 
 
+def from_dataset(
+        examples: 'Dataset',
+        immutable_warranty: str = 'pickle',
+        name: str = None,
+):
+    """
+    Similar to caching. Iterates over the whole dataset and creates a new
+    dataset from the obtained examples. If the input dataset has `.items()`,
+    the new dataset will be indexable and have `.keys()` and `.items()` (even
+    if the input dataset did not have `.keys()`). If not, the new
+    dataset will not have `.keys()`.
+
+    Example:
+        >>> ds = from_dataset(new([1, 2, 3, 4]))
+        >>> ds
+          ListDataset(len=4)
+        MapDataset(_pickle.loads)
+        >>> list(ds)
+        [1, 2, 3, 4]
+        >>> ds = from_dataset(new({'a': 1, 'b': 2, 'c': 3, 'd': 4}).map(lambda x: x**2))
+        >>> ds
+          DictDataset(len=4)
+        MapDataset(_pickle.loads)
+        >>> dict(ds)
+        {'a': 1, 'b': 4, 'c': 9, 'd': 16}
+
+        Works with filtered datasets:
+        >>> ds = from_dataset(new([1, 2, 3, 4]).filter(lambda x: x%2))
+        >>> list(ds)
+        [1, 3]
+        >>> ds = from_dataset(new({'a': 1, 'b': 2, 'c': 3, 'd': 4}).filter(lambda x: x%2))
+        >>> dict(ds)
+        {'a': 1, 'c': 3}
+    """
+    try:
+        items = list(examples.items())
+    except ItemsNotDefined:
+        return from_list(list(examples),
+                         immutable_warranty=immutable_warranty, name=name)
+    else:
+        return from_dict(dict(items),
+                         immutable_warranty=immutable_warranty, name=name)
+
+
 def concatenate(*datasets):
     """
     Create a new `Dataset` by concatenation of all passed datasets.
@@ -666,7 +716,7 @@ class Dataset:
                 )
             idx = [i for i, e in enumerate(self) if filter_fn(e)]
             if len(self) > len(idx):
-                LOG.info(f'Filtered {len(self)-len(idx)} of {len(self)} examples.')
+                LOG.info(f'Filtered {len(self) - len(idx)} of {len(self)} examples.')
             return self[idx]
 
     def catch(self, exceptions=FilterException,
@@ -1415,13 +1465,7 @@ class Dataset:
             assert self.indexable or self.ordered, (
                 'Caching is only supported for indexable or ordered datasets.'
             )
-
-            try:
-                self.keys()
-            except NotImplementedError:
-                return from_list(list(self))
-            else:
-                return from_dict(dict(self))
+            return new(self)
 
     def diskcache(
             self,
@@ -3572,8 +3616,8 @@ class ProfilingDataset(Dataset):
             hits += f' ({self.hit_count[1]} filtered)'
 
         # Better alternative for the name "fetch duration"?
-        r += (f' (fetch duration = {datetime.timedelta(seconds=self.time[0])}, ' 
-             f'{hits})')
+        r += (f' (fetch duration = {datetime.timedelta(seconds=self.time[0])}, '
+              f'{hits})')
         return r
 
     def __len__(self):


Test patch:
diff --git a/tests/test_str.py b/tests/test_str.py
index 493b9cb..5c0660c 100644
--- a/tests/test_str.py
+++ b/tests/test_str.py
@@ -66,7 +66,7 @@ def test():
 
     ds = ds_dict.filter(lambda ex: True).cache(lazy=False)
     check(ds, 'MapDataset(_pickle.loads)',
-          '  ListDataset(len=4)\n'
+          '  DictDataset(len=4)\n'
           'MapDataset(_pickle.loads)')
 
     ds = ds_dict.diskcache()