Repository: peter-wangxu/persist-queue

Issue:
FIFOSQLiteQueue: the get() method returns None instead of blocking
and if I specify get(block=True) it raises the empty exception

Discussion:

Patch:
diff --git a/persistqueue/__init__.py b/persistqueue/__init__.py
index c066e10..ef321ef 100644
--- a/persistqueue/__init__.py
+++ b/persistqueue/__init__.py
@@ -1,7 +1,7 @@
 # coding=utf-8
 __author__ = 'Peter Wang'
 __license__ = 'BSD License'
-__version__ = '0.3.0'
+__version__ = '0.3.1'
 
 from .exceptions import Empty, Full  # noqa
 from .pdict import PDict  # noqa
diff --git a/persistqueue/sqlbase.py b/persistqueue/sqlbase.py
index 22bdfe6..48955f7 100644
--- a/persistqueue/sqlbase.py
+++ b/persistqueue/sqlbase.py
@@ -27,15 +27,17 @@ def commit_ignore_error(conn):
     """Ignore the error of no transaction is active.
 
     The transaction may be already committed by user's task_done call.
-    It's safe to to ignore all errors of this kind.
+    It's safe to ignore all errors of this kind.
     """
     try:
         conn.commit()
     except sqlite3.OperationalError as ex:
         if 'no transaction is active' in str(ex):
-            log.warning(
+            log.debug(
                 'Not able to commit the transaction, '
                 'may already be committed.')
+        else:
+            raise
 
 
 class SQLiteBase(object):
@@ -50,17 +52,23 @@ class SQLiteBase(object):
     _MEMORY = ':memory:'  # flag indicating store DB in memory
 
     def __init__(self, path, name='default', multithreading=False,
-                 timeout=10.0, auto_commit=False):
+                 timeout=10.0, auto_commit=True):
         """Initiate a queue in sqlite3 or memory.
 
         :param path: path for storing DB file.
+        :param name: the suffix for the table name,
+                     table name would be ${_TABLE_NAME}_${name}
         :param multithreading: if set to True, two db connections will be,
                                one for **put** and one for **get**.
         :param timeout: timeout in second waiting for the database lock.
         :param auto_commit: Set to True, if commit is required on every
-                            INSERT/UPDATE action.
+                            INSERT/UPDATE action, otherwise False, whereas
+                            a **task_done** is required to persist changes
+                            after **put**.
+
 
         """
+        self.memory_sql = False
         self.path = path
         self.name = name
         self.timeout = timeout
@@ -71,19 +79,27 @@ class SQLiteBase(object):
 
     def _init(self):
         """Initialize the tables in DB."""
-        if not os.path.exists(self.path):
+        if self.path == self._MEMORY:
+            self.memory_sql = True
+            log.debug("Initializing Sqlite3 Queue in memory.")
+        elif not os.path.exists(self.path):
             os.makedirs(self.path)
-        log.debug('Initializing Sqlite3 Queue with path {}'.format(self.path))
+            log.debug(
+                'Initializing Sqlite3 Queue with path {}'.format(self.path))
 
         self._conn = self._new_db_connection(
             self.path, self.multithreading, self.timeout)
         self._getter = self._conn
         self._putter = self._conn
-        if self.multithreading:
-            self._putter = self._new_db_connection(
-                self.path, self.multithreading, self.timeout)
+
         self._conn.execute(self._sql_create)
         self._conn.commit()
+        # Setup another session only for disk-based queue.
+        if self.multithreading:
+            if not self.memory_sql:
+                self._putter = self._new_db_connection(
+                    self.path, self.multithreading, self.timeout)
+
         # SQLite3 transaction lock
         self.tran_lock = threading.Lock()
         self.put_event = threading.Event()


Test patch:
diff --git a/tests/test_sqlqueue.py b/tests/test_sqlqueue.py
index 61a31db..1e63431 100644
--- a/tests/test_sqlqueue.py
+++ b/tests/test_sqlqueue.py
@@ -150,23 +150,32 @@ class SQLite3QueueTest(unittest.TestCase):
                 queue.put('var%d' % x)
                 task_done_if_required(queue)
 
-        def consumer():
-            for _ in range(100):
+        counter = []
+        # Set all to 0
+        for _ in range(1000):
+            counter.append(0)
+
+        def consumer(index):
+            for i in range(200):
                 data = queue.get(block=True)
                 self.assertTrue('var' in data)
+                counter[index * 200 + i] = data
 
         p = Thread(target=producer)
         p.start()
         consumers = []
-        for _ in range(10):
-            t = Thread(target=consumer)
+        for index in range(5):
+            t = Thread(target=consumer, args=(index,))
             t.start()
             consumers.append(t)
 
+        p.join()
         for t in consumers:
             t.join()
 
         self.assertEqual(0, queue.qsize())
+        for x in range(1000):
+            self.assertNotEqual(0, counter[x], "0 for counter's index %s" % x)
 
 
 class SQLite3QueueAutoCommitTest(SQLite3QueueTest):
@@ -175,6 +184,24 @@ class SQLite3QueueAutoCommitTest(SQLite3QueueTest):
         self.auto_commit = True
 
 
+class SQLite3QueueInMemory(SQLite3QueueTest):
+    def setUp(self):
+        self.path = ":memory:"
+        self.auto_commit = False
+
+    def test_open_close_1000(self):
+        self.skipTest('Memory based sqlite is not persistent.')
+
+    def test_open_close_single(self):
+        self.skipTest('Memory based sqlite is not persistent.')
+
+    def test_multiple_consumers(self):
+        # TODO(peter) when the shared-cache feature is available in default
+        # Python of most Linux distros, this should be easy:).
+        self.skipTest('In-memory based sqlite needs the support '
+                      'of shared-cache')
+
+
 class FILOSQLite3QueueTest(unittest.TestCase):
     def setUp(self):
         self.path = tempfile.mkdtemp(suffix='filo_sqlqueue')