Repository: nltk/nltk

Issue:
Loading corpora in concordance and collocations tools
<b>What steps will reproduce the problem?</b>

&gt;&gt;&gt; nltk.app.concordance()

and

&gt;&gt;&gt; nltk.app.collocations()

<b>What is the expected output? What do you see instead?</b>

out of stack space (infinite loop?)
Exception in thread English: Brown Corpus (Learned, simplified):
Traceback (most recent call last):
  File
&quot;/opt/local/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/threading.py&quot;,
line 525, in __bootstrap_inner
    self.run()
...
TclError: out of stack space (infinite loop?)

<p>Migrated from http://code.google.com/p/nltk/issues/detail?id=445</p><hr/><h2>earlier comments</h2>
<p><strong>sumukh.ghodke said, at 2009-09-19T02:06:59.000Z:</strong></p>

<p>I tested the latest nltk build on ubuntu 9.04 (python 2.6.2; Tcl/Tk 8.5.6) and Mac
OsX 10.5.8 (python 2.5.2; Tcl/Tk 8.4.7), but didn't encounter the above error.

This problem seems to happen during Tkinter initialisation from python code. Some say
that it occurs when Tcl/Tk is built without a parameter called --enable-threads.
[ref:http://bugs.python.org/issue3835] Can you please check if this happens even
after you recompile Tcl/Tk?

Some background on why threads are used in the concordance app:
The concordance app uses a Model-View architecture. the view is loaded using python's
mainloop thread, whereas data loading and search operations take place in independent
threads. In order to make sure that gui rendering always happen in the mainloop
thread I fire events from the data loading threads. The mainloop thread should be
informed of those events and accordingly render results.
This bug wouldn't have occurred if it was single threaded, but then the app would
have been less responsive too. If really required we could make it run in a single
thread.


  </p>



Discussion:

Resolving patch:
diff --git a/nltk/app/collocations_app.py b/nltk/app/collocations_app.py
--- a/nltk/app/collocations_app.py
+++ b/nltk/app/collocations_app.py
@@ -10,6 +10,10 @@
 import nltk.compat
 import threading
 import tkinter.font
+if nltk.compat.PY3:
+    import queue as q
+else:    
+    import Queue as q
 from tkinter import (Button, END, Frame, IntVar, LEFT, Label, Menu,
                      OptionMenu, SUNKEN, Scrollbar, StringVar,
                      Text, Tk)
@@ -22,6 +26,7 @@
 
 CORPUS_LOADED_EVENT = '<<CL_EVENT>>'
 ERROR_LOADING_CORPUS_EVENT = '<<ELC_EVENT>>'
+POLL_INTERVAL = 100
 
 _DEFAULT = 'English: Brown Corpus (Humor)'
 _CORPORA = {
@@ -65,19 +70,20 @@ class CollocationsView:
     _BACKGROUND_COLOUR='#FFF' #white
 
     def __init__(self):
-        self.model = CollocationsModel()
-        self.model.add_listener(self)
+        self.queue = q.Queue()
+        self.model = CollocationsModel(self.queue)
         self.top = Tk()
         self._init_top(self.top)
         self._init_menubar()
         self._init_widgets(self.top)
-        self._bind_event_handlers()
         self.load_corpus(self.model.DEFAULT_CORPUS)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
 
     def _init_top(self, top):
         top.geometry('550x650+50+50')
         top.title('NLTK Collocations List')
         top.bind('<Control-q>', self.destroy)
+        top.protocol('WM_DELETE_WINDOW', self.destroy)
         top.minsize(550,650)
 
     def _init_widgets(self, parent):
@@ -165,9 +171,17 @@ def _init_paging(self, parent):
     def reset_current_page(self):
         self.current_page = -1
 
-    def _bind_event_handlers(self):
-        self.top.bind(CORPUS_LOADED_EVENT, self.handle_corpus_loaded)
-        self.top.bind(ERROR_LOADING_CORPUS_EVENT, self.handle_error_loading_corpus)
+    def _poll(self):   
+        try:
+            event = self.queue.get(block=False)
+        except q.Empty:
+            pass
+        else:
+            if event == CORPUS_LOADED_EVENT:
+                self.handle_corpus_loaded(event)
+            elif event == ERROR_LOADING_CORPUS_EVENT:
+                self.handle_error_loading_corpus(event)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
 
     def handle_error_loading_corpus(self, event):
         self.status['text'] = 'Error in loading ' + self.var.get()
@@ -227,6 +241,7 @@ def fire_event(self, event):
 
     def destroy(self, *e):
         if self.top is None: return
+        self.top.after_cancel(self.after)
         self.top.destroy()
         self.top = None
 
@@ -256,26 +271,19 @@ def write_results(self, results):
         self.results_box['state'] = 'disabled'
 
 class CollocationsModel:
-    def __init__(self):
-        self.listeners = []
+    def __init__(self, queue):
         self.result_count = None
         self.selected_corpus = None
         self.collocations = None
         self.CORPORA = _CORPORA
         self.DEFAULT_CORPUS = _DEFAULT
+        self.queue = queue
         self.reset_results()
 
     def reset_results(self):
         self.result_pages = []
         self.results_returned = 0
 
-    def add_listener(self, listener):
-        self.listeners.append(listener)
-
-    def notify_listeners(self, event):
-        for each in self.listeners:
-            each.fire_event(event)
-
     def load_corpus(self, name):
         self.selected_corpus = name
         self.collocations = None
@@ -322,10 +330,10 @@ def run(self):
                 scored = [((w1,w2), fd[(w1,w2)] ** 3 / float(vocab[w1] * vocab[w2])) for w1, w2 in fd]
                 scored.sort(key=itemgetter(1), reverse=True)
                 self.model.collocations = list(map(itemgetter(0), scored))
-                self.model.notify_listeners(CORPUS_LOADED_EVENT)
+                self.model.queue.put(CORPUS_LOADED_EVENT)
             except Exception as e:
                 print(e)
-                self.model.notify_listeners(ERROR_LOADING_CORPUS_EVENT)
+                self.model.queue.put(ERROR_LOADING_CORPUS_EVENT)
 
 #def collocations():
 #    colloc_strings = [w1 + ' ' + w2 for w1, w2 in self._collocations[:num]]
diff --git a/nltk/app/concordance_app.py b/nltk/app/concordance_app.py
--- a/nltk/app/concordance_app.py
+++ b/nltk/app/concordance_app.py
@@ -9,6 +9,10 @@
 import nltk.compat
 import re
 import threading
+if nltk.compat.PY3:
+    import queue as q
+else:    
+    import Queue as q
 import tkinter.font
 from tkinter import (Tk, Button, END, Entry, Frame, IntVar, LEFT,
                      Label, Menu, OptionMenu, SUNKEN, Scrollbar,
@@ -27,6 +31,8 @@
 SEARCH_ERROR_EVENT = '<<SE_EVENT>>'
 ERROR_LOADING_CORPUS_EVENT = '<<ELC_EVENT>>'
 
+POLL_INTERVAL = 50
+
 # NB All corpora must be specified in a lambda expression so as not to be
 # loaded when the module is imported.
 
@@ -97,19 +103,20 @@ class ConcordanceSearchView(object):
     _FRACTION_LEFT_TEXT=0.30
 
     def __init__(self):
-        self.model = ConcordanceSearchModel()
-        self.model.add_listener(self)
+        self.queue = q.Queue()
+        self.model = ConcordanceSearchModel(self.queue)
         self.top = Tk()
         self._init_top(self.top)
         self._init_menubar()
         self._init_widgets(self.top)
-        self._bind_event_handlers()
         self.load_corpus(self.model.DEFAULT_CORPUS)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
 
     def _init_top(self, top):
         top.geometry('950x680+50+50')
         top.title('NLTK Concordance Search')
         top.bind('<Control-q>', self.destroy)
+        top.protocol('WM_DELETE_WINDOW', self.destroy)
         top.minsize(950,680)
 
     def _init_widgets(self, parent):
@@ -284,6 +291,22 @@ def _bind_event_handlers(self):
         self.top.bind(SEARCH_ERROR_EVENT, self.handle_search_error)
         self.top.bind(ERROR_LOADING_CORPUS_EVENT, self.handle_error_loading_corpus)
 
+    def _poll(self):   
+        try:
+            event = self.queue.get(block=False)
+        except q.Empty:
+            pass
+        else:
+            if event == CORPUS_LOADED_EVENT:
+                self.handle_corpus_loaded(event)
+            elif event == SEARCH_TERMINATED_EVENT:
+                self.handle_search_terminated(event)
+            elif event == SEARCH_ERROR_EVENT:
+                self.handle_search_error(event)
+            elif event == ERROR_LOADING_CORPUS_EVENT:
+                self.handle_error_loading_corpus(event)
+        self.after = self.top.after(POLL_INTERVAL, self._poll)
+
     def handle_error_loading_corpus(self, event):
         self.status['text'] = 'Error in loading ' + self.var.get()
         self.unfreeze_editable()
@@ -308,7 +331,6 @@ def handle_search_terminated(self, event):
         self.unfreeze_editable()
         self.results_box.xview_moveto(self._FRACTION_LEFT_TEXT)
 
-
     def handle_search_error(self, event):
         self.status['text'] = 'Error in query ' + self.model.query
         self.unfreeze_editable()
@@ -378,6 +400,7 @@ def pad(self, sent, hstart, hend):
 
     def destroy(self, *e):
         if self.top is None: return
+        self.top.after_cancel(self.after)
         self.top.destroy()
         self.top = None
 
@@ -421,8 +444,8 @@ def mainloop(self, *args, **kwargs):
         self.top.mainloop(*args, **kwargs)
 
 class ConcordanceSearchModel(object):
-    def __init__(self):
-        self.listeners = []
+    def __init__(self, queue):
+        self.queue = queue
         self.CORPORA = _CORPORA
         self.DEFAULT_CORPUS = _DEFAULT
         self.selected_corpus = None
@@ -454,18 +477,11 @@ def next(self, page):
         if len(self.results) < page:
             self.search(self.query, page)
         else:
-            self.notify_listeners(SEARCH_TERMINATED_EVENT)
+            self.queue.put(SEARCH_TERMINATED_EVENT)
 
     def prev(self, page):
         self.last_requested_page = page
-        self.notify_listeners(SEARCH_TERMINATED_EVENT)
-
-    def add_listener(self, listener):
-        self.listeners.append(listener)
-
-    def notify_listeners(self, event):
-        for each in self.listeners:
-            each.fire_event(event)
+        self.queue.put(SEARCH_TERMINATED_EVENT)
 
     def reset_results(self):
         self.last_sent_searched = 0
@@ -497,10 +513,10 @@ def run(self):
             try:
                 ts = self.model.CORPORA[self.name]()
                 self.model.tagged_sents = [' '.join(w+'/'+t for (w,t) in sent) for sent in ts]
-                self.model.notify_listeners(CORPUS_LOADED_EVENT)
+                self.model.queue.put(CORPUS_LOADED_EVENT)
             except Exception as e:
                 print(e)
-                self.model.notify_listeners(ERROR_LOADING_CORPUS_EVENT)
+                self.model.queue.put(ERROR_LOADING_CORPUS_EVENT)
 
     class SearchCorpus(threading.Thread):
         def __init__(self, model, page, count):
@@ -515,7 +531,7 @@ def run(self):
                     m = re.search(q, sent)
                 except re.error:
                     self.model.reset_results()
-                    self.model.notify_listeners(SEARCH_ERROR_EVENT)
+                    self.model.queue.put(SEARCH_ERROR_EVENT)
                     return
                 if m:
                     sent_pos.append((sent, m.start(), m.end()))
@@ -530,7 +546,7 @@ def run(self):
                 self.model.set_results(self.page, sent_pos)
             else:
                 self.model.set_results(self.page, sent_pos[:-1])
-            self.model.notify_listeners(SEARCH_TERMINATED_EVENT)
+            self.model.queue.put(SEARCH_TERMINATED_EVENT)
 
         def processed_query(self):
             new = []


Test patch: