Repository: piskvorky/gensim

Issue:
'pattern' library getting automatically invoked if installed when using LdaModel.load()
While loading previously saved LDA model with LdaModel.load() 'pattern' library gets invoked automatically (if installed). 

Error message:

```
2015-09-21 18:11:23,052 : INFO : loading LdaModel object from /Users/jurica/Documents/workspace/eclipse/pubMedUBS/HR_Corpus/ldaModel.lda

2015-09-21 18:11:23,553 : INFO : loading id2word recursively from /Users/jurica/Documents/workspace/eclipse/pubMedUBS/HR_Corpus/ldaModel.lda.id2word.* with mmap=r

2015-09-21 18:11:23,553 : INFO : loading expElogbeta from /Users/jurica/Documents/workspace/eclipse/pubMedUBS/HR_Corpus/ldaModel.lda.expElogbeta.npy with mmap=r

Traceback (most recent call last):

  File "/Users/jurica/Documents/workspace/eclipse/pubMedUBS/UMLSdb/processMetaMappedMEDLINEBaseline.py", line 372, in <module>

    processResults.analyzeLDA()

  File "/Users/jurica/Documents/workspace/eclipse/pubMedUBS/UMLSdb/processMetaMappedMEDLINEBaseline.py", line 348, in analyzeLDA

    model = LdaModel.load(self.__destination+model,  mmap='r')

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/gensim/models/ldamodel.py", line 887, in load

    result = super(LdaModel, cls).load(fname, *args, **kwargs)

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/gensim/utils.py", line 256, in load

    obj._load_specials(fname, mmap, compress, subname)

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/gensim/utils.py", line 287, in _load_specials

    val = numpy.load(subname(fname, attrib), mmap_mode=mmap)

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/lib/npyio.py", line 391, in load

    return format.open_memmap(file, mode=mmap_mode)

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/lib/format.py", line 722, in open_memmap

    shape, fortran_order, dtype = _read_array_header(fp, version)

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/lib/format.py", line 479, in _read_array_header

    d = safe_eval(header)

  File "/Users/jurica/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/lib/utils.py", line 1168, in safe_eval

    ast = compiler.parse(source, mode="eval")

  File "/Applications/Canopy.app/appdata/canopy-1.5.5.3123.macosx-x86_64/Canopy.app/Contents/lib/python2.7/compiler/transformer.py", line 53, in parse

    return Transformer().parseexpr(buf)

  File "/Applications/Canopy.app/appdata/canopy-1.5.5.3123.macosx-x86_64/Canopy.app/Contents/lib/python2.7/compiler/transformer.py", line 132, in parseexpr

    return self.transform(parser.expr(text))

AttributeError: 'module' object has no attribute 'expr'¸
```

Environment: 
OS X Yosemite (10.10.5) 64bit,
Canopy Python distribution with: 
pattern 2.6
gensim 0.12.2
numpy 1.9.2
scipy 0.16
sklearn 0.16.1

with Python 2.7



Discussion:

Resolving patch:
diff --git a/gensim/corpora/wikicorpus.py b/gensim/corpora/wikicorpus.py
--- a/gensim/corpora/wikicorpus.py
+++ b/gensim/corpora/wikicorpus.py
@@ -249,7 +249,7 @@ class WikiCorpus(TextCorpus):
     >>> MmCorpus.serialize('wiki_en_vocab200k', wiki) # another 8h, creates a file in MatrixMarket format plus file with id->word
 
     """
-    def __init__(self, fname, processes=None, lemmatize=utils.HAS_PATTERN, dictionary=None, filter_namespaces=('0',)):
+    def __init__(self, fname, processes=None, lemmatize=utils.has_pattern(), dictionary=None, filter_namespaces=('0',)):
         """
         Initialize the corpus. Unless a dictionary is provided, this scans the
         corpus once, to determine its vocabulary.
diff --git a/gensim/utils.py b/gensim/utils.py
--- a/gensim/utils.py
+++ b/gensim/utils.py
@@ -76,14 +76,6 @@ def smart_open(fname, mode='rb'):
         return open(fname, mode)
 
 
-try:
-    from pattern.en import parse
-    logger.info("'pattern' package found; utils.lemmatize() is available for English")
-    HAS_PATTERN = True
-except ImportError:
-    HAS_PATTERN = False
-
-
 PAT_ALPHABETIC = re.compile('(((?![\d])\w)+)', re.UNICODE)
 RE_HTML_ENTITY = re.compile(r'&(#?)([xX]?)(\w{1,8});', re.UNICODE)
 
@@ -1022,48 +1014,64 @@ def pyro_daemon(name, obj, random_suffix=False, ip=None, port=None):
             daemon.requestLoop()
 
 
-if HAS_PATTERN:
-    def lemmatize(content, allowed_tags=re.compile('(NN|VB|JJ|RB)'), light=False,
-            stopwords=frozenset(), min_length=2, max_length=15):
-        """
-        This function is only available when the optional 'pattern' package is installed.
+def has_pattern():
+    """
+    Function to check if there is installed pattern library
+    """
+    pattern = False
+    try:
+        from pattern.en import parse
+        pattern = True
+    except ImportError:
+        logger.warning("Pattern library is not installed.")
+    return pattern
+
 
-        Use the English lemmatizer from `pattern` to extract UTF8-encoded tokens in
-        their base form=lemma, e.g. "are, is, being" -> "be" etc.
-        This is a smarter version of stemming, taking word context into account.
+def lemmatize(content, allowed_tags=re.compile('(NN|VB|JJ|RB)'), light=False,
+        stopwords=frozenset(), min_length=2, max_length=15):
+    """
+    This function is only available when the optional 'pattern' package is installed.
 
-        Only considers nouns, verbs, adjectives and adverbs by default (=all other lemmas are discarded).
+    Use the English lemmatizer from `pattern` to extract UTF8-encoded tokens in
+    their base form=lemma, e.g. "are, is, being" -> "be" etc.
+    This is a smarter version of stemming, taking word context into account.
 
-        >>> lemmatize('Hello World! How is it going?! Nonexistentword, 21')
-        ['world/NN', 'be/VB', 'go/VB', 'nonexistentword/NN']
+    Only considers nouns, verbs, adjectives and adverbs by default (=all other lemmas are discarded).
 
-        >>> lemmatize('The study ranks high.')
-        ['study/NN', 'rank/VB', 'high/JJ']
+    >>> lemmatize('Hello World! How is it going?! Nonexistentword, 21')
+    ['world/NN', 'be/VB', 'go/VB', 'nonexistentword/NN']
 
-        >>> lemmatize('The ranks study hard.')
-        ['rank/NN', 'study/VB', 'hard/RB']
+    >>> lemmatize('The study ranks high.')
+    ['study/NN', 'rank/VB', 'high/JJ']
 
-        """
-        if light:
-            import warnings
-            warnings.warn("The light flag is no longer supported by pattern.")
-
-        # tokenization in `pattern` is weird; it gets thrown off by non-letters,
-        # producing '==relate/VBN' or '**/NN'... try to preprocess the text a little
-        # FIXME this throws away all fancy parsing cues, including sentence structure,
-        # abbreviations etc.
-        content = u(' ').join(tokenize(content, lower=True, errors='ignore'))
-
-        parsed = parse(content, lemmata=True, collapse=False)
-        result = []
-        for sentence in parsed:
-            for token, tag, _, _, lemma in sentence:
-                if min_length <= len(lemma) <= max_length and not lemma.startswith('_') and lemma not in stopwords:
-                    if allowed_tags.match(tag):
-                        lemma += "/" + tag[:2]
-                        result.append(lemma.encode('utf8'))
-        return result
-#endif HAS_PATTERN
+    >>> lemmatize('The ranks study hard.')
+    ['rank/NN', 'study/VB', 'hard/RB']
+
+    """
+    if not has_pattern():
+        raise ImportError("Pattern library is not installed. Pattern library is needed in order  \
+         to use lemmatize function")
+    from pattern.en import parse
+
+    if light:
+        import warnings
+        warnings.warn("The light flag is no longer supported by pattern.")
+
+    # tokenization in `pattern` is weird; it gets thrown off by non-letters,
+    # producing '==relate/VBN' or '**/NN'... try to preprocess the text a little
+    # FIXME this throws away all fancy parsing cues, including sentence structure,
+    # abbreviations etc.
+    content = u(' ').join(tokenize(content, lower=True, errors='ignore'))
+
+    parsed = parse(content, lemmata=True, collapse=False)
+    result = []
+    for sentence in parsed:
+        for token, tag, _, _, lemma in sentence:
+            if min_length <= len(lemma) <= max_length and not lemma.startswith('_') and lemma not in stopwords:
+                if allowed_tags.match(tag):
+                    lemma += "/" + tag[:2]
+                    result.append(lemma.encode('utf8'))
+    return result
 
 
 def mock_data_row(dim=1000, prob_nnz=0.5, lam=1.0):


Test patch: