# Buggy code
def get_all_documents_generator(
    self,
    index: Optional[str] = None,
    filters: Optional[Dict[str, List[str]]] = None,
    return_embedding: Optional[bool] = None,
    batch_size: int = 10_000,
) -> Generator[Document, None, None]:
    """
    Get documents from the document store. Under-the-hood, documents are fetched in batches from the
    document store and yielded as individual documents. This method can be used to iteratively process
    a large number of documents without having to load all documents in memory.

    :param index: Name of the index to get the documents from. If None, the
                  DocumentStore's default index (self.index) will be used.
    :param filters: Optional filters to narrow down the documents to return.
                    Example: {"name": ["some", "more"], "category": ["only_one"]}
    :param return_embedding: Whether to return the document embeddings.
    """

    index = index or self.index
    # Generally ORM objects kept in memory cause performance issue
    # Hence using directly column name improve memory and performance.
    # Refer https://stackoverflow.com/questions/23185319/why-is-loading-sqlalchemy-objects-via-the-orm-5-8x-slower-than-rows-via-a-raw-my
    documents_query = self.session.query(
        DocumentORM.id, DocumentORM.text, DocumentORM.vector_id
    ).filter_by(index=index)

    if filters:
        documents_query = documents_query.join(MetaORM)
        for key, values in filters.items():
            documents_query = documents_query.filter(
                MetaORM.name == key,
                MetaORM.value.in_(values),
                DocumentORM.id == MetaORM.document_id,
            )

    documents_map = {}
    for i, row in enumerate(
        self._windowed_query(documents_query, DocumentORM.id, batch_size), start=1
    ):
        documents_map[row.id] = Document(
            id=row.id,
            text=row.text,
            meta=None if row.vector_id is None else {"vector_id": row.vector_id},  # type: ignore
        )
        if i % batch_size == 0:
            documents_map = self._get_documents_meta(documents_map)
            yield from documents_map.values()
            documents_map = {}
    if documents_map:
        documents_map = self._get_documents_meta(documents_map)
        yield from documents_map.values()


# Traceback (OperationalError)
01/21/2021 22:03:34 - INFO - haystack.document_store.faiss -   Updating embeddings for 75 docs...
0%|          | 0/75 [00:00<?, ?it/s]
---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
1276                     self.dialect.do_execute(
-> 1277                         cursor, statement, parameters, context
1278                     )

17 frames
/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
608     def do_execute(self, cursor, statement, parameters, context=None):
--> 609         cursor.execute(statement, parameters)
610

OperationalError: near "(": syntax error

The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
<ipython-input-8-0ae6e12f6b98> in <module>()
6
7 # Add documents embeddings to index
----> 8 document_store.update_embeddings(retriever=retriever)

/usr/local/lib/python3.6/dist-packages/haystack/document_store/faiss.py in update_embeddings(self, retriever, index, batch_size)
188         batched_documents = get_batches_from_generator(result, batch_size)
189         with tqdm(total=document_count) as progress_bar:
--> 190             for document_batch in batched_documents:
191                 embeddings = retriever.embed_passages(document_batch)  # type: ignore
192                 assert len(document_batch) == len(embeddings)

/usr/local/lib/python3.6/dist-packages/haystack/utils.py in get_batches_from_generator(iterable, n)
122     """
123     it = iter(iterable)
--> 124     x = tuple(islice(it, n))
125     while x:
126         yield x

/usr/local/lib/python3.6/dist-packages/haystack/document_store/faiss.py in get_all_documents_generator(self, index, filters, return_embedding, batch_size)
241             return_embedding = self.return_embedding
242
--> 243         for doc in documents:
244             if return_embedding:
245                 if doc.meta and doc.meta.get("vector_id") is not None:

/usr/local/lib/python3.6/dist-packages/haystack/document_store/sql.py in get_all_documents_generator(self, index, filters, return_embedding, batch_size)
183
184         documents_map = {}
--> 185         for i, row in enumerate(self._windowed_query(documents_query, DocumentORM.id, batch_size), start=1):
186             documents_map[row.id] = Document(
187                 id=row.id,

/usr/local/lib/python3.6/dist-packages/haystack/document_store/sql.py in _windowed_query(self, q, column, windowsize)
464         for whereclause in self._column_windows(
465                 q.session,
--> 466                 column, windowsize):
467             for row in q.filter(whereclause).order_by(column):
468                 yield row

/usr/local/lib/python3.6/dist-packages/haystack/document_store/sql.py in _column_windows(self, session, column, windowsize)
449             q = q.filter(text("rownum %% %d=1" % windowsize))
450
--> 451         intervals = [id for id, in q]
452
453         while intervals:

/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/query.py in __iter__(self)
3533         if self._autoflush and not self._populate_existing:
3534             self.session._autoflush()
-> 3535         return self._execute_and_instances(context)
3536
3537     def __str__(self):

/usr/local/lib/python3.6/dist-packages/sqlalchemy/orm/query.py in _execute_and_instances(self, querycontext)
3558         )
3559
-> 3560         result = conn.execute(querycontext.statement, self._params)
3561         return loading.instances(querycontext.query, result, querycontext)
3562

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in execute(self, object_, *multiparams, **params)
1009             )
1010         else:
-> 1011             return meth(self, multiparams, params)
1012
1013     def _execute_function(self, func, multiparams, params):

/usr/local/lib/python3.6/dist-packages/sqlalchemy/sql/elements.py in _execute_on_connection(self, connection, multiparams, params)
296     def _execute_on_connection(self, connection, multiparams, params):
297         if self.supports_execution:
--> 298             return connection._execute_clauseelement(self, multiparams, params)
299         else:
300             raise exc.ObjectNotExecutableError(self)

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_clauseelement(self, elem, multiparams, params)
1128             distilled_params,
1129             compiled_sql,
-> 1130             distilled_params,
1131         )
1132         if self._has_events or self.engine._has_events:

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
1315         except BaseException as e:
1316             self._handle_dbapi_exception(
-> 1317                 e, statement, parameters, cursor, context
1318             )
1319

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
1509             elif should_wrap:
1510                 util.raise_(
-> 1511                     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
1512                 )
1513             else:

/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py in raise_(***failed resolving arguments***)
180
181         try:
--> 182             raise exception
183         finally:
184             # credit to

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
1275                 if not evt_handled:
1276                     self.dialect.do_execute(
-> 1277                         cursor, statement, parameters, context
1278                     )
1279

/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
607
608     def do_execute(self, cursor, statement, parameters, context=None):
--> 609         cursor.execute(statement, parameters)
610
611     def do_execute_no_params(self, cursor, statement, context=None):

OperationalError: (sqlite3.OperationalError) near "(": syntax error
[SQL: SELECT anon_1.document_id AS anon_1_document_id
FROM (SELECT document.id AS document_id, row_number() OVER (ORDER BY document.id) AS rownum
FROM document) AS anon_1
WHERE rownum % 10000=1]
(Background on this error at: http://sqlalche.me/e/13/e3q8)

# Fixed code
def get_all_documents_generator(
    self,
    index: Optional[str] = None,
    filters: Optional[Dict[str, List[str]]] = None,
    return_embedding: Optional[bool] = None,
    batch_size: int = 10_000,
) -> Generator[Document, None, None]:
    """
    Get documents from the document store. Under-the-hood, documents are fetched in batches from the
    document store and yielded as individual documents. This method can be used to iteratively process
    a large number of documents without having to load all documents in memory.

    :param index: Name of the index to get the documents from. If None, the
                  DocumentStore's default index (self.index) will be used.
    :param filters: Optional filters to narrow down the documents to return.
                    Example: {"name": ["some", "more"], "category": ["only_one"]}
    :param return_embedding: Whether to return the document embeddings.
    """

    index = index or self.index
    # Generally ORM objects kept in memory cause performance issue
    # Hence using directly column name improve memory and performance.
    # Refer https://stackoverflow.com/questions/23185319/why-is-loading-sqlalchemy-objects-via-the-orm-5-8x-slower-than-rows-via-a-raw-my
    documents_query = self.session.query(
        DocumentORM.id, DocumentORM.text, DocumentORM.vector_id
    ).filter_by(index=index)

    if filters:
        documents_query = documents_query.join(MetaORM)
        for key, values in filters.items():
            documents_query = documents_query.filter(
                MetaORM.name == key,
                MetaORM.value.in_(values),
                DocumentORM.id == MetaORM.document_id,
            )

    documents_map = {}

    if self.use_windowed_query:
        documents_query = self._windowed_query(
            documents_query, DocumentORM.id, batch_size
        )

    for i, row in enumerate(documents_query, start=1):
        documents_map[row.id] = Document(
            id=row.id,
            text=row.text,
            meta=None if row.vector_id is None else {"vector_id": row.vector_id},  # type: ignore
        )
        if i % batch_size == 0:
            documents_map = self._get_documents_meta(documents_map)
            yield from documents_map.values()
            documents_map = {}
    if documents_map:
        documents_map = self._get_documents_meta(documents_map)
        yield from documents_map.values()