Repository: piskvorky/gensim

Issue:
MmCorpus file-like object support bug
**Into**
We have some "weird" behavior if a user passes a `file-like` object to `MmCorpus`, based on [this mailing list thread](https://groups.google.com/forum/#!topic/gensim/420pRBwexKQ)

**Demonstration**

```python
from gensim.corpora import MmCorpus
import bz2

f = bz2.BZ2File("testcorpus.mm.bz2")
print(f.closed)  # 0
corpus = MmCorpus(f)
print(f.closed)  # 1 ???
```

**What happens**
File-like object was closed when we call `MmReader`, problem located here

https://github.com/RaRe-Technologies/gensim/blob/5342153eb4f4b02bb45bfa3951eef8250ac9f6b6/gensim/matutils.py#L1274


`with` automatically close `file-like` when we out of scope, **this is OK if we open this file**, but we **shouldn't close file-like passed from user**.

Related PR #1867 


**UPD:** another problem here - call `IndexCopus.__init__`, that didn't support `file-like` object at all.


Discussion:

Resolving patch:
diff --git a/gensim/matutils.py b/gensim/matutils.py
--- a/gensim/matutils.py
+++ b/gensim/matutils.py
@@ -1360,7 +1360,7 @@ def __init__(self, input, transposed=True):
             """
             logger.info("initializing corpus reader from %s", input)
             self.input, self.transposed = input, transposed
-            with utils.file_or_filename(self.input) as lines:
+            with utils.open_file(self.input) as lines:
                 try:
                     header = utils.to_unicode(next(lines)).strip()
                     if not header.lower().startswith('%%matrixmarket matrix coordinate real general'):
diff --git a/gensim/utils.py b/gensim/utils.py
--- a/gensim/utils.py
+++ b/gensim/utils.py
@@ -7,7 +7,7 @@
 """This module contains various general utility functions."""
 
 from __future__ import with_statement
-
+from contextlib import contextmanager
 import collections
 import logging
 import warnings
@@ -139,6 +139,36 @@ def file_or_filename(input):
         return input
 
 
+@contextmanager
+def open_file(input):
+    """Provide "with-like" behaviour except closing the file object.
+
+    Parameters
+    ----------
+    input : str or file-like
+        Filename or file-like object.
+
+    Yields
+    -------
+    file
+        File-like object based on input (or input if this already file-like).
+
+    """
+    mgr = file_or_filename(input)
+    exc = False
+    try:
+        yield mgr
+    except Exception:
+        # Handling any unhandled exceptions from the code nested in 'with' statement.
+        exc = True
+        if not isinstance(input, string_types) or not mgr.__exit__(*sys.exc_info()):
+            raise
+        # Try to introspect and silence errors.
+    finally:
+        if not exc and isinstance(input, string_types):
+            mgr.__exit__(None, None, None)
+
+
 def deaccent(text):
     """Remove accentuation from the given string.
 


Test patch:
diff --git a/gensim/test/test_corpora.py b/gensim/test/test_corpora.py
--- a/gensim/test/test_corpora.py
+++ b/gensim/test/test_corpora.py
@@ -225,6 +225,14 @@ def test_serialize_compressed(self):
         # MmCorpus needs file write with seek => doesn't support compressed output (only input)
         pass
 
+    def test_closed_file_object(self):
+        file_obj = open(datapath('testcorpus.mm'))
+        f = file_obj.closed
+        mmcorpus.MmCorpus(file_obj)
+        s = file_obj.closed
+        self.assertEqual(f, 0)
+        self.assertEqual(s, 0)
+
     def test_load(self):
         self.assertEqual(self.corpus.num_docs, 9)
         self.assertEqual(self.corpus.num_terms, 12)
diff --git a/gensim/test/test_utils.py b/gensim/test/test_utils.py
--- a/gensim/test/test_utils.py
+++ b/gensim/test/test_utils.py
@@ -15,6 +15,7 @@
 from six import iteritems
 
 from gensim import utils
+from gensim.test.utils import datapath
 
 
 class TestIsCorpus(unittest.TestCase):
@@ -84,6 +85,28 @@ def test_decode_entities(self):
         expected = u'It\x92s the Year of the Horse. YES VIN DIESEL \U0001f64c \U0001f4af'
         self.assertEqual(utils.decode_htmlentities(body), expected)
 
+    def test_open_file_existent_file(self):
+        number_of_lines_in_file = 30
+        with utils.open_file(datapath('testcorpus.mm')) as infile:
+            self.assertEqual(sum(1 for _ in infile), number_of_lines_in_file)
+
+    def test_open_file_non_existent_file(self):
+        with self.assertRaises(Exception):
+            with utils.open_file('non_existent_file.txt'):
+                pass
+
+    def test_open_file_existent_file_object(self):
+        number_of_lines_in_file = 30
+        file_obj = open(datapath('testcorpus.mm'))
+        with utils.open_file(file_obj) as infile:
+            self.assertEqual(sum(1 for _ in infile), number_of_lines_in_file)
+
+    def test_open_file_non_existent_file_object(self):
+        file_obj = None
+        with self.assertRaises(Exception):
+            with utils.open_file(file_obj):
+                pass
+
 
 class TestSampleDict(unittest.TestCase):
     def test_sample_dict(self):