Repository: piskvorky/gensim

Issue:
Add tests for #837
This PR is almost done, just needs tests https://github.com/RaRe-Technologies/gensim/pull/837

Great issue to fix while getting familiar with Gensim



Discussion:

Resolving patch:
diff --git a/gensim/models/phrases.py b/gensim/models/phrases.py
--- a/gensim/models/phrases.py
+++ b/gensim/models/phrases.py
@@ -60,6 +60,7 @@
 import sys
 import os
 import logging
+import warnings
 from collections import defaultdict
 
 from six import iteritems, string_types
@@ -79,7 +80,7 @@ class Phrases(interfaces.TransformationABC):
 
     """
     def __init__(self, sentences=None, min_count=5, threshold=10.0,
-            max_vocab_size=40000000, delimiter=b'_'):
+                 max_vocab_size=40000000, delimiter=b'_', progress_per=10000):
         """
         Initialize the model from an iterable of `sentences`. Each sentence must be
         a list of words (unicode strings) that will be used for training.
@@ -119,6 +120,7 @@ def __init__(self, sentences=None, min_count=5, threshold=10.0,
         self.vocab = defaultdict(int)  # mapping between utf8 token => its count
         self.min_reduce = 1  # ignore any tokens with count smaller than this
         self.delimiter = delimiter
+        self.progress_per = progress_per
 
         if sentences is not None:
             self.add_vocab(sentences)
@@ -130,7 +132,7 @@ def __str__(self):
             self.threshold, self.max_vocab_size)
 
     @staticmethod
-    def learn_vocab(sentences, max_vocab_size, delimiter=b'_'):
+    def learn_vocab(sentences, max_vocab_size, delimiter=b'_', progress_per=10000):
         """Collect unigram/bigram counts from the `sentences` iterable."""
         sentence_no = -1
         total_words = 0
@@ -138,7 +140,7 @@ def learn_vocab(sentences, max_vocab_size, delimiter=b'_'):
         vocab = defaultdict(int)
         min_reduce = 1
         for sentence_no, sentence in enumerate(sentences):
-            if sentence_no % 10000 == 0:
+            if sentence_no % progress_per == 0:
                 logger.info("PROGRESS: at sentence #%i, processed %i words and %i word types" %
                             (sentence_no, total_words, len(vocab)))
             sentence = [utils.any2utf8(w) for w in sentence]
@@ -169,19 +171,23 @@ def add_vocab(self, sentences):
         # directly, but gives the new sentences a fighting chance to collect
         # sufficient counts, before being pruned out by the (large) accummulated
         # counts collected in previous learn_vocab runs.
-        min_reduce, vocab = self.learn_vocab(sentences, self.max_vocab_size, self.delimiter)
-
-        logger.info("merging %i counts into %s", len(vocab), self)
-        self.min_reduce = max(self.min_reduce, min_reduce)
-        for word, count in iteritems(vocab):
-            self.vocab[word] += count
-        if len(self.vocab) > self.max_vocab_size:
-            utils.prune_vocab(self.vocab, self.min_reduce)
-            self.min_reduce += 1
-
-        logger.info("merged %s", self)
-
-    def export_phrases(self, sentences):
+        min_reduce, vocab = self.learn_vocab(sentences, self.max_vocab_size, self.delimiter, self.progress_per)
+
+        if len(self.vocab) > 0:
+            logger.info("merging %i counts into %s", len(vocab), self)
+            self.min_reduce = max(self.min_reduce, min_reduce)
+            for word, count in iteritems(vocab):
+                self.vocab[word] += count
+            if len(self.vocab) > self.max_vocab_size:
+                utils.prune_vocab(self.vocab, self.min_reduce)
+                self.min_reduce += 1
+            logger.info("merged %s", self)
+        else:
+            # in common case, avoid doubling gigantic dict
+            logger.info("using %i counts as vocab in %s", len(vocab), self)
+            self.vocab = vocab
+
+    def export_phrases(self, sentences, out_delimiter=b' ', as_tuples=False):
         """
         Generate an iterator that contains all phrases in given 'sentences'
 
@@ -199,7 +205,7 @@ def export_phrases(self, sentences):
             last_bigram = False
             vocab = self.vocab
             threshold = self.threshold
-            delimiter = self.delimiter
+            delimiter = self.delimiter  # delimiter used for lookup
             min_count = self.min_count
             for word_a, word_b in zip(s, s[1:]):
                 if word_a in vocab and word_b in vocab:
@@ -212,7 +218,10 @@ def export_phrases(self, sentences):
                         # logger.debug("score for %s: (pab=%s - min_count=%s) / pa=%s / pb=%s * vocab_size=%s = %s",
                         #     bigram_word, pab, self.min_count, pa, pb, len(self.vocab), score)
                         if score > threshold:
-                            yield (b' '.join((word_a, word_b)), score)
+                            if as_tuples:
+                                yield ((word_a, word_b), score)
+                            else:
+                                yield (out_delimiter.join((word_a, word_b)), score)
                             last_bigram = True
                             continue
                         last_bigram = False
@@ -236,6 +245,7 @@ def __getitem__(self, sentence):
             nonviolence leo_tolstoy
 
         """
+        warnings.warn("For a faster implementation, use the gensim.models.phrases.Phraser class")
         try:
             is_single = not sentence or isinstance(sentence[0], string_types)
         except:
@@ -278,6 +288,90 @@ def __getitem__(self, sentence):
         return [utils.to_unicode(w) for w in new_s]
 
 
+def pseudocorpus(source_vocab, sep):
+    """Feeds source_vocab's compound keys back to it, to discover phrases"""
+    for k in source_vocab:
+        if sep not in k:
+            continue
+        unigrams = k.split(sep)
+        for i in range(1, len(unigrams)):
+            yield [sep.join(unigrams[:i]), sep.join(unigrams[i:])]
+
+
+class Phraser(interfaces.TransformationABC):
+    """
+    Minimal state & functionality to apply results of a Phrases model to tokens.
+
+    After the one-time initialization, a Phraser will be much smaller and
+    somewhat faster than using the full Phrases model.
+
+    Reflects the results of the source model's `min_count` and `threshold`
+    settings. (You can tamper with those & create a new Phraser to try
+    other values.)
+
+    """
+    def __init__(self, phrases_model):
+        self.threshold = phrases_model.threshold
+        self.min_count = phrases_model.min_count
+        self.delimiter = phrases_model.delimiter
+        self.phrasegrams = {}
+        corpus = pseudocorpus(phrases_model.vocab, phrases_model.delimiter)
+        logger.info('source_vocab length %i', len(phrases_model.vocab))
+        count = 0
+        for bigram, score in phrases_model.export_phrases(corpus, self.delimiter, as_tuples=True):
+            if bigram in self.phrasegrams:
+                logger.info('Phraser repeat %s', bigram)
+            self.phrasegrams[bigram] = (phrases_model.vocab[self.delimiter.join(bigram)], score)
+            count += 1
+            if not count % 50000:
+                logger.info('Phraser added %i phrasegrams', count)
+        logger.info('Phraser built with %i %i phrasegrams', count, len(self.phrasegrams))
+
+
+    def __getitem__(self, sentence):
+        """
+        Convert the input tokens `sentence` (=list of unicode strings) into phrase
+        tokens (=list of unicode strings, where detected phrases are joined by u'_'
+        (or other configured delimiter-character).
+
+        If `sentence` is an entire corpus (iterable of sentences rather than a single
+        sentence), return an iterable that converts each of the corpus' sentences
+        into phrases on the fly, one after another.
+
+        """
+        try:
+            is_single = not sentence or isinstance(sentence[0], string_types)
+        except:
+            is_single = False
+        if not is_single:
+            # if the input is an entire corpus (rather than a single sentence),
+            # return an iterable stream.
+            return self._apply(sentence)
+
+        s, new_s = [utils.any2utf8(w) for w in sentence], []
+        last_bigram = False
+        phrasegrams = self.phrasegrams
+        delimiter = self.delimiter
+        for word_a, word_b in zip(s, s[1:]):
+            bigram_tuple = (word_a, word_b)
+            if phrasegrams.get(bigram_tuple, (-1, -1))[1] > self.threshold and not last_bigram:
+                bigram_word = delimiter.join((word_a, word_b))
+                new_s.append(bigram_word)
+                last_bigram = True
+                continue
+
+            if not last_bigram:
+                new_s.append(word_a)
+            last_bigram = False
+
+        if s:  # add last word skipped by previous loop
+            last_token = s[-1]
+            if not last_bigram:
+                new_s.append(last_token)
+
+        return [utils.to_unicode(w) for w in new_s]
+
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(threadName)s : %(levelname)s : %(message)s', level=logging.INFO)
     logging.info("running %s" % " ".join(sys.argv))
diff --git a/gensim/utils.py b/gensim/utils.py
--- a/gensim/utils.py
+++ b/gensim/utils.py
@@ -247,6 +247,7 @@ def load(cls, fname, mmap=None):
 
         obj = unpickle(fname)
         obj._load_specials(fname, mmap, compress, subname)
+        logger.info("loaded %s", fname)
         return obj
 
 
@@ -354,6 +355,7 @@ def _smart_save(self, fname, separately=None, sep_limit=10 * 1024**2,
             for obj, asides in restores:
                 for attrib, val in iteritems(asides):
                     setattr(obj, attrib, val)
+        logger.info("saved %s", fname)
 
 
     def _save_specials(self, fname, separately, sep_limit, ignore, pickle_protocol, compress, subname):


Test patch:
diff --git a/gensim/test/test_phrases.py b/gensim/test/test_phrases.py
--- a/gensim/test/test_phrases.py
+++ b/gensim/test/test_phrases.py
@@ -14,7 +14,7 @@
 import sys
 
 from gensim import utils
-from gensim.models.phrases import Phrases
+from gensim.models.phrases import Phrases, Phraser
 
 if sys.version_info[0] >= 3:
     unicode = str
@@ -35,24 +35,29 @@
     ['graph', 'minors', 'survey'],
     ['graph', 'minors', 'survey','human','interface'] #test bigrams within same sentence
 ]
+unicode_sentences = [[utils.to_unicode(w) for w in sentence] for sentence in sentences]
 
 
-class TestPhrasesModel(unittest.TestCase):
+class TestPhrasesCommon(unittest.TestCase):
+    """ Tests that need to be run for both Prases and Phraser classes."""
+    def setUp(self):
+        self.bigram = Phrases(sentences, min_count=1, threshold=1)
+        self.bigram_default = Phrases(sentences)
+        self.bigram_utf8 = Phrases(sentences, min_count=1, threshold=1)
+        self.bigram_unicode = Phrases(unicode_sentences, min_count=1, threshold=1)
+
     def testSentenceGeneration(self):
         """Test basic bigram using a dummy corpus."""
-        bigram = Phrases(sentences)
         # test that we generate the same amount of sentences as the input
-        self.assertEqual(len(sentences), len(list(bigram[sentences])))
+        self.assertEqual(len(sentences), len(list(self.bigram_default[sentences])))
 
     def testBigramConstruction(self):
         """Test Phrases bigram construction building."""
-        bigram = Phrases(sentences, min_count=1, threshold=1)
-
         # with this setting we should get response_time and graph_minors
         bigram1_seen = False
         bigram2_seen = False
 
-        for s in bigram[sentences]:
+        for s in self.bigram[sentences]:
             if not bigram1_seen and u'response_time' in s:
                 bigram1_seen = True
             if not bigram2_seen and u'graph_minors' in s:
@@ -64,13 +69,24 @@ def testBigramConstruction(self):
 
         # check the same thing, this time using single doc transformation
         # last sentence should contain both graph_minors and human_interface
-        self.assertTrue(u'response_time' in bigram[sentences[1]])
-        self.assertTrue(u'response_time' in bigram[sentences[4]])
-        self.assertTrue(u'graph_minors' in bigram[sentences[-3]])
-        self.assertTrue(u'graph_minors' in bigram[sentences[-2]])
-        self.assertTrue(u'graph_minors' in bigram[sentences[-1]])
-        self.assertTrue(u'human_interface' in bigram[sentences[-1]])
+        self.assertTrue(u'response_time' in self.bigram[sentences[1]])
+        self.assertTrue(u'response_time' in self.bigram[sentences[4]])
+        self.assertTrue(u'graph_minors' in self.bigram[sentences[-2]])
+        self.assertTrue(u'graph_minors' in self.bigram[sentences[-1]])
+        self.assertTrue(u'human_interface' in self.bigram[sentences[-1]])
 
+    def testEncoding(self):
+        """Test that both utf8 and unicode input work; output must be unicode."""
+        expected = [u'survey', u'user', u'computer', u'system', u'response_time']
+
+        self.assertEqual(self.bigram_utf8[sentences[1]], expected)
+        self.assertEqual(self.bigram_unicode[sentences[1]], expected)
+
+        transformed = ' '.join(self.bigram_utf8[sentences[1]])
+        self.assertTrue(isinstance(transformed, unicode))
+
+
+class TestPhrasesModel(unittest.TestCase):
     def testExportPhrases(self):
         """Test Phrases bigram export_phrases functionality."""
         bigram = Phrases(sentences, min_count=1, threshold=1)
@@ -98,20 +114,6 @@ def testBadParameters(self):
         # threshold should be positive
         self.assertRaises(ValueError, Phrases, sentences, threshold=-1)
 
-    def testEncoding(self):
-        """Test that both utf8 and unicode input work; output must be unicode."""
-        expected = [u'survey', u'user', u'computer', u'system', u'response_time']
-
-        bigram_utf8 = Phrases(sentences, min_count=1, threshold=1)
-        self.assertEqual(bigram_utf8[sentences[1]], expected)
-
-        unicode_sentences = [[utils.to_unicode(w) for w in sentence] for sentence in sentences]
-        bigram_unicode = Phrases(unicode_sentences, min_count=1, threshold=1)
-        self.assertEqual(bigram_unicode[sentences[1]], expected)
-
-        transformed = ' '.join(bigram_utf8[sentences[1]])
-        self.assertTrue(isinstance(transformed, unicode))
-
     def testPruning(self):
         """Test that max_vocab_size parameter is respected."""
         bigram = Phrases(sentences, max_vocab_size=5)
@@ -119,6 +121,23 @@ def testPruning(self):
 #endclass TestPhrasesModel
 
 
+class TestPhraserModel(TestPhrasesCommon):
+    """ Test Phraser models."""
+    def setUp(self):
+        """Set up Phraser models for the tests."""
+        bigram_phrases = Phrases(sentences, min_count=1, threshold=1)
+        self.bigram = Phraser(bigram_phrases)
+
+        bigram_default_phrases = Phrases(sentences)
+        self.bigram_default = Phraser(bigram_default_phrases)
+
+        bigram_utf8_phrases = Phrases(sentences, min_count=1, threshold=1)
+        self.bigram_utf8 = Phraser(bigram_utf8_phrases)
+
+        bigram_unicode_phrases = Phrases(unicode_sentences, min_count=1, threshold=1)
+        self.bigram_unicode = Phraser(bigram_unicode_phrases)
+
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
     unittest.main()