Repository: pytorch/pytorch

Issue:
Support for memory-mapped files
In Lua torch tensor constructors accepted filenames, and could use memory mapped files as their data.

[Forum thread](https://discuss.pytorch.org/t/quickly-loading-raw-binary-files/797)


Discussion:
This functionality is implemented in the core. Am I right? Here:
https://github.com/pytorch/pytorch/blob/master/torch/lib/TH/generic/THStorage.c#L44
So, the only what is needed is to embed it into Python code somewhere?
Yes, you're right. The best way would be to add a `from_file` class method to every storage and tensor type (so you could do e.g. `torch.ByteTensor.from_file('x.wav')`).
Could you clarify where are the implementations of different storage and tensor types? Found only base classes. And should `from_file` be added to every storage? Why not to add this method just to the base classes?
The implementation should be analogous to [`form_buffer`](https://github.com/pytorch/pytorch/blob/master/torch/csrc/generic/StorageMethods.cpp#L88-L180)

Resolving patch:
diff --git a/torch/__init__.py b/torch/__init__.py
--- a/torch/__init__.py
+++ b/torch/__init__.py
@@ -343,5 +343,5 @@ def manager_path():
 _C._init_names(list(torch._tensor_classes) + list(torch._storage_classes))
 
 # attach docstrings to torch and tensor functions
-from . import _torch_docs, _tensor_docs
-del _torch_docs, _tensor_docs
+from . import _torch_docs, _tensor_docs, _storage_docs
+del _torch_docs, _tensor_docs, _storage_docs
diff --git a/torch/_storage_docs.py b/torch/_storage_docs.py
new file mode 100644
--- /dev/null
+++ b/torch/_storage_docs.py
@@ -0,0 +1,43 @@
+"""Adds docstrings to Storage functions"""
+
+import torch._C
+from torch._C import _add_docstr as add_docstr
+
+
+storage_classes = [
+    'DoubleStorageBase',
+    'FloatStorageBase',
+    'LongStorageBase',
+    'IntStorageBase',
+    'ShortStorageBase',
+    'CharStorageBase',
+    'ByteStorageBase',
+]
+
+
+def add_docstr_all(classes, method, docstr):
+    for cls_name in classes:
+        cls = getattr(torch._C, cls_name)
+        add_docstr(getattr(cls, method),
+                   docstr.format(
+                       cls_name.replace('Base', ''),
+                       cls_name.replace('StorageBase', '')))
+
+
+add_docstr_all(storage_classes, 'from_file',
+               """
+from_file(filename, shared=False, size=0) -> {0}
+
+If shared is True then memory is shared between all processes. All changes are
+written to the file. If shared is False then the changes on the storage do not
+affect the file.
+
+Size is the number of elements in the storage. If shared is False then the file
+must contain at least `size * sizeof({1})` bytes. If shared is True the file
+will be created if needed.
+
+Args:
+    filename (str): file name to map
+    shared (bool): whether to share memory
+    size (int): number of elements in the storage
+""")


Test patch:
diff --git a/test/test_torch.py b/test/test_torch.py
--- a/test/test_torch.py
+++ b/test/test_torch.py
@@ -3305,6 +3305,31 @@ def test_from_buffer(self):
         self.assertEqual(floats.size(), 1)
         self.assertEqual(floats[0], 2.25)
 
+    def test_from_file(self):
+        size = 10000
+        filename = 'testPytorchStorageFromFile'
+        try:
+            s1 = torch.FloatStorage.from_file(filename, True, size)
+            t1 = torch.FloatTensor(s1).copy_(torch.randn(size))
+
+            # check mapping
+            s2 = torch.FloatStorage.from_file(filename, True, size)
+            t2 = torch.FloatTensor(s2)
+            self.assertEqual(t1, t2, 0)
+
+            # check changes to t1 from t2
+            rnum = random.uniform(-1, 1)
+            t1.fill_(rnum)
+            self.assertEqual(t1, t2, 0)
+
+            # check changes to t2 from t1
+            rnum = random.uniform(-1, 1)
+            t2.fill_(rnum)
+            self.assertEqual(t1, t2, 0)
+        finally:
+            if os.path.exists(filename):
+                os.remove(filename)
+
     def test_print(self):
         for t in torch._tensor_classes:
             if t in torch.sparse._sparse_tensor_classes: