Repository: ipython/ipython

Issue:
config option for disabling history store
I am using ipython within the django runserver app, which has a feature to auto-reload when it detects changes to project files.  Every time this happens, however, it throws a number of exceptions seemingly stemming from ipython trying to save history to an sqlite database from multiple threads.

An excerpt of the error is as follows:

```
Error in atexit._run_exitfuncs:
Traceback (most recent call last):
  File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/atexit.py", line 24, in _run_exitfuncs
    func(*targs, **kargs)
  File "/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2986, in atexit_operations
    self.history_manager.end_session()
  File "/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.py", line 432, in end_session
    self.writeout_cache()
  File "<string>", line 2, in writeout_cache
  File "/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.py", line 60, in needs_sqlite
    return f(*a,**kw)
  File "/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.py", line 607, in writeout_cache
    self._writeout_input_cache(conn)
  File "/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.py", line 591, in _writeout_input_cache
    (self.session_number,)+line)
ProgrammingError: SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 4365488128 and this is thread id 140735200143712
Error in sys.exitfunc:
Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in <module 'threading' from '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in <module 'threading' from '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in <module 'threading' from '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in <module 'threading' from '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in <module 'threading' from '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
Exception AttributeError: AttributeError("'_DummyThread' object has no attribute '_Thread__block'",) in <module 'threading' from '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc'> ignored
---------------------------------------------------------------------------
ProgrammingError                          Traceback (most recent call last)
/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/atexit.pyc in _run_exitfuncs()
     22         func, targs, kargs = _exithandlers.pop()
     23         try:
---> 24             func(*targs, **kargs)
     25         except SystemExit:
     26             exc_info = sys.exc_info()

/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc in atexit_operations(self)
   2984         # this must be *before* the tempfile cleanup, in case of temporary
   2985         # history db
-> 2986         self.history_manager.end_session()
   2987 
   2988         # Cleanup all tempfiles left around

/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.pyc in end_session(self)
    430     def end_session(self):
    431         """Close the database session, filling in the end time and line count."""
--> 432         self.writeout_cache()
    433         with self.db:
    434             self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE

/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.pyc in writeout_cache(self, conn)

/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.pyc in needs_sqlite(f, *a, **kw)
     58         return []
     59     else:
---> 60         return f(*a,**kw)
     61 
     62 

/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.pyc in writeout_cache(self, conn)
    605         with self.db_input_cache_lock:
    606             try:
--> 607                 self._writeout_input_cache(conn)
    608             except sqlite3.IntegrityError:
    609                 self.new_session(conn)

/Users/shabble/work/caproj/env/lib/python2.7/site-packages/IPython/core/history.pyc in _writeout_input_cache(self, conn)
    589             for line in self.db_input_cache:
    590                 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
--> 591                                 (self.session_number,)+line)
    592 
    593     def _writeout_output_cache(self, conn):

ProgrammingError: SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 4365488128 and this is thread id 140735200143712
```

My understanding is that this is caused by the sqlite history store used by ipython is getting upset due to the threading model of the django server code.  Oddly, the same error occurs even when using a single threaded runserver (`./manage.py runserver --nothreading`)

On the suggestion of @minrk  in #ipython freenode irc channel, I tried:

`c.HistoryManager.hist_file = ':memory:'`, but this does not fix the problem.  They suggested I write this up as a ticket with the probable solution being to have an option of disabling the history store altogether.  This is probably similar to https://github.com/ipython/ipython/pull/909 which handles the case of sqlite being entirely unavailable.

Thanks.



Discussion:

Resolving patch:
diff --git a/IPython/core/history.py b/IPython/core/history.py
--- a/IPython/core/history.py
+++ b/IPython/core/history.py
@@ -27,7 +27,9 @@
 from IPython.config.configurable import Configurable
 from IPython.external.decorator import decorator
 from IPython.utils.path import locate_profile
-from IPython.utils.traitlets import Bool, Dict, Instance, Integer, List, Unicode
+from IPython.utils.traitlets import (
+    Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
+)
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------
@@ -52,12 +54,12 @@ def __exit__(self, *args, **kwargs):
 
 
 @decorator
-def needs_sqlite(f,*a,**kw):
+def needs_sqlite(f, self, *a, **kw):
     """return an empty list in the absence of sqlite"""
-    if sqlite3 is None:
+    if sqlite3 is None or not self.enabled:
         return []
     else:
-        return f(*a,**kw)
+        return f(self, *a, **kw)
 
 
 class HistoryAccessor(Configurable):
@@ -81,12 +83,33 @@ class HistoryAccessor(Configurable):
             ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
         
         """)
+    
+    enabled = Bool(True, config=True,
+        help="""enable the SQLite history
+        
+        set enabled=False to disable the SQLite history,
+        in which case there will be no stored history, no SQLite connection,
+        and no background saving thread.  This may be necessary in some
+        threaded environments where IPython is embedded.
+        """
+    )
+    
+    connection_options = Dict(config=True,
+        help="""Options for configuring the SQLite connection
+        
+        These options are passed as keyword args to sqlite3.connect
+        when establishing database conenctions.
+        """
+    )
 
     # The SQLite database
-    if sqlite3:
-        db = Instance(sqlite3.Connection)
-    else:
-        db = Instance(DummyDB)
+    db = Any()
+    def _db_changed(self, name, old, new):
+        """validate the db, since it can be an Instance of two different types"""
+        if not isinstance(new, (sqlite3.Connection, DummyDB)):
+            msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
+                    (self.__class__.__name__, new)
+            raise TraitError(msg)
     
     def __init__(self, profile='default', hist_file=u'', config=None, **traits):
         """Create a new history accessor.
@@ -113,14 +136,18 @@ def __init__(self, profile='default', hist_file=u'', config=None, **traits):
             # No one has set the hist_file, yet.
             self.hist_file = self._get_hist_file_name(profile)
 
-        if sqlite3 is None:
+        if sqlite3 is None and self.enabled:
             warn("IPython History requires SQLite, your history will not be saved\n")
-            self.db = DummyDB()
-            return
+            self.enabled = False
+        
+        if sqlite3 is not None:
+            DatabaseError = sqlite3.DatabaseError
+        else:
+            DatabaseError = Exception
         
         try:
             self.init_db()
-        except sqlite3.DatabaseError:
+        except DatabaseError:
             if os.path.isfile(self.hist_file):
                 # Try to move the file out of the way
                 base,ext = os.path.splitext(self.hist_file)
@@ -148,9 +175,14 @@ def _get_hist_file_name(self, profile='default'):
     
     def init_db(self):
         """Connect to the database, and create tables if necessary."""
+        if not self.enabled:
+            self.db = DummyDB()
+            return
+        
         # use detect_types so that timestamps return datetime objects
-        self.db = sqlite3.connect(self.hist_file,
-                    detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
+        kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
+        kwargs.update(self.connection_options)
+        self.db = sqlite3.connect(self.hist_file, **kwargs)
         self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
                         primary key autoincrement, start timestamp,
                         end timestamp, num_cmds integer, remark text)""")
@@ -402,7 +434,7 @@ def __init__(self, shell=None, config=None, **traits):
         self.save_flag = threading.Event()
         self.db_input_cache_lock = threading.Lock()
         self.db_output_cache_lock = threading.Lock()
-        if self.hist_file != ':memory:':
+        if self.enabled and self.hist_file != ':memory:':
             self.save_thread = HistorySavingThread(self)
             self.save_thread.start()
 
@@ -638,16 +670,20 @@ class HistorySavingThread(threading.Thread):
     the cache size reaches a defined threshold."""
     daemon = True
     stop_now = False
+    enabled = True
     def __init__(self, history_manager):
         super(HistorySavingThread, self).__init__()
         self.history_manager = history_manager
+        self.enabled = history_manager.enabled
         atexit.register(self.stop)
 
     @needs_sqlite
     def run(self):
         # We need a separate db connection per thread:
         try:
-            self.db = sqlite3.connect(self.history_manager.hist_file)
+            self.db = sqlite3.connect(self.history_manager.hist_file,
+                            **self.history_manager.connection_options
+            )
             while True:
                 self.history_manager.save_flag.wait()
                 if self.stop_now:


Test patch: