Repository: deepset-ai/haystack

Issue:
SQLite OperationalError in SQLDocumentStore
In the following lines, I get errors:
```# Delete existing documents in documents store
document_store.delete_all_documents()

# Write documents to document store
document_store.write_documents(documents)

# Add documents embeddings to index
document_store.update_embeddings(retriever=retriever)
```

The error message:
```
01/21/2021 22:03:34 - INFO - haystack.document_store.faiss -   Updating embeddings for 75 docs...
  0%|          | 0/75 [00:00<?, ?it/s]
---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1276                     self.dialect.do_execute(
-> 1277                         cursor, statement, parameters, context
   1278                     )

17 frames
/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    608     def do_execute(self, cursor, statement, parameters, context=None):
--> 609         cursor.execute(statement, parameters)
    610 

OperationalError: near "(": syntax error

The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
<ipython-input-8-0ae6e12f6b98> in <module>()
      6 
      7 # Add documents embeddings to index
----> 8 document_store.update_embeddings(retriever=retriever)

/usr/local/lib/python3.6/dist-packages/haystack/document_store/faiss.py in update_embeddings(self, retriever, index, batch_size)
    188         batched_documents = get_batches_from_generator(result, batch_size)
    189         with tqdm(total=document_count) as progress_bar:
--> 190             for document_batch in batched_documents:
    191                 embeddings = retriever.embed_passages(document_batch)  # type: ignore
    192                 assert len(document_batch) == len(embeddings)

/usr/local/lib/python3.6/dist-packages/haystack/utils.py in get_batches_from_generator(iterable, n)
    122     """
    123     it = iter(iterable)
--> 124     x = tuple(islice(it, n))
    125     while x:
    126         yield x

/usr/local/lib/python3.6/dist-packages/haystack/document_store/faiss.py in get_all_documents_generator(self, index, filters, return_embedding, batch_size)
    241             return_embedding = self.return_embedding
    242 
--> 243         for doc in documents:
    244             if return_embedding:
    245                 if doc.meta and doc.meta.get("vector_id") is not None:

/usr/local/lib/python3.6/dist-packages/haystack/document_store/sql.py in get_all_documents_generator(self, index, filters, return_embedding, batch_size)
    183 
    184         documents_map = {}
--> 185         for i, row in enumerate(self._windowed_query(documents_query, DocumentORM.id, batch_size), start=1):
    186             documents_map[row.id] = Document(
    187                 id=row.id,

/usr/local/lib/python3.6/dist-packages/haystack/document_store/sql.py in _windowed_query(self, q, column, windowsize)
    464         for whereclause in self._column_windows(
    465                 q.session,
--> 466                 column, windowsize):
    467             for row in q.filter(whereclause).order_by(column):
    468                 yield row

/usr/local/lib/python3.6/dist-packages/haystack/document_store/sql.py in _column_windows(self, session, column, windowsize)
    449             q = q.filter(text("rownum %% %d=1" % windowsize))
    450 
--> 451         intervals = [id for id, in q]
    452 
    453         while intervals:

/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/query.py in __iter__(self)
   3533         if self._autoflush and not self._populate_existing:
   3534             self.session._autoflush()
-> 3535         return self._execute_and_instances(context)
   3536 
   3537     def __str__(self):

/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/query.py in _execute_and_instances(self, querycontext)
   3558         )
   3559 
-> 3560         result = conn.execute(querycontext.statement, self._params)
   3561         return loading.instances(querycontext.query, result, querycontext)
   3562 

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in execute(self, object_, *multiparams, **params)
   1009             )
   1010         else:
-> 1011             return meth(self, multiparams, params)
   1012 
   1013     def _execute_function(self, func, multiparams, params):

/usr/local/lib/python3.6/dist-packages/sqlalchemy/sql/elements.py in _execute_on_connection(self, connection, multiparams, params)
    296     def _execute_on_connection(self, connection, multiparams, params):
    297         if self.supports_execution:
--> 298             return connection._execute_clauseelement(self, multiparams, params)
    299         else:
    300             raise exc.ObjectNotExecutableError(self)

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_clauseelement(self, elem, multiparams, params)
   1128             distilled_params,
   1129             compiled_sql,
-> 1130             distilled_params,
   1131         )
   1132         if self._has_events or self.engine._has_events:

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1315         except BaseException as e:
   1316             self._handle_dbapi_exception(
-> 1317                 e, statement, parameters, cursor, context
   1318             )
   1319 

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1509             elif should_wrap:
   1510                 util.raise_(
-> 1511                     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   1512                 )
   1513             else:

/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py in raise_(***failed resolving arguments***)
    180 
    181         try:
--> 182             raise exception
    183         finally:
    184             # credit to

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1275                 if not evt_handled:
   1276                     self.dialect.do_execute(
-> 1277                         cursor, statement, parameters, context
   1278                     )
   1279 

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    607 
    608     def do_execute(self, cursor, statement, parameters, context=None):
--> 609         cursor.execute(statement, parameters)
    610 
    611     def do_execute_no_params(self, cursor, statement, context=None):

OperationalError: (sqlite3.OperationalError) near "(": syntax error
[SQL: SELECT anon_1.document_id AS anon_1_document_id 
FROM (SELECT document.id AS document_id, row_number() OVER (ORDER BY document.id) AS rownum 
FROM document) AS anon_1 
WHERE rownum % 10000=1]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
```




Discussion:
Can you please check the SQLite version?

Error is coming from windowing query and it's support was introduced since [`3.25` version](https://sqlite.org/windowfunctions.html#history). So I am suspecting your colab have lower version of SQLite install. Try upgrading to latest version and try again.

@tanaysoni I think we might need to disable windowing queries by default.
SQLite3 version (3, 22, 0)

it seems updating sqlite3 on colab is not as easy


Try this https://stackoverflow.com/questions/59427642/upgrading-sqlite-in-colab

Resolving patch:
diff --git a/haystack/document_store/sql.py b/haystack/document_store/sql.py
--- a/haystack/document_store/sql.py
+++ b/haystack/document_store/sql.py
@@ -96,6 +96,11 @@ def __init__(
         self.update_existing_documents = update_existing_documents
         if getattr(self, "similarity", None) is None:
             self.similarity = None
+        self.use_windowed_query = True
+        if "sqlite" in url:
+            import sqlite3
+            if sqlite3.sqlite_version < "3.25":
+                self.use_windowed_query = False
 
     def get_document_by_id(self, id: str, index: Optional[str] = None) -> Optional[Document]:
         """Fetch a document by specifying its text id string"""
@@ -182,7 +187,11 @@ def get_all_documents_generator(
                 )
 
         documents_map = {}
-        for i, row in enumerate(self._windowed_query(documents_query, DocumentORM.id, batch_size), start=1):
+
+        if self.use_windowed_query:
+            documents_query = self._windowed_query(documents_query, DocumentORM.id, batch_size)
+
+        for i, row in enumerate(documents_query, start=1):
             documents_map[row.id] = Document(
                 id=row.id,
                 text=row.text,


Test patch:
diff --git a/test/test_document_store.py b/test/test_document_store.py
--- a/test/test_document_store.py
+++ b/test/test_document_store.py
@@ -51,6 +51,19 @@ def test_get_all_documents_with_correct_filters(document_store_with_docs):
     assert {d.meta["meta_field"] for d in documents} == {"test1", "test3"}
 
 
+@pytest.mark.parametrize("document_store_with_docs", ["sql"], indirect=True)
+def test_get_all_documents_with_correct_filters_legacy_sqlite(document_store_with_docs):
+    document_store_with_docs.use_windowed_query = False
+    documents = document_store_with_docs.get_all_documents(filters={"meta_field": ["test2"]})
+    assert len(documents) == 1
+    assert documents[0].meta["name"] == "filename2"
+
+    documents = document_store_with_docs.get_all_documents(filters={"meta_field": ["test1", "test3"]})
+    assert len(documents) == 2
+    assert {d.meta["name"] for d in documents} == {"filename1", "filename3"}
+    assert {d.meta["meta_field"] for d in documents} == {"test1", "test3"}
+
+
 @pytest.mark.elasticsearch
 def test_get_all_documents_with_incorrect_filter_name(document_store_with_docs):
     documents = document_store_with_docs.get_all_documents(filters={"incorrect_meta_field": ["test2"]})