Repository: PaddlePaddle/PaddleNLP

Issue:
使用语义检索系统中的example semantic_search_example.py报错
ValueError: (InvalidArgument) Variable value (input) of OP(fluid.layers.embedding) expected >= 0 and < 513, but got 513. Please check input value.   [Hint: Expected ids[i] < row_number, but received ids[i]:513 >= row_number:513.] (at /home/Paddle/paddle/fluid/operators/lookup_table_v2_op.h:77)   [operator < lookup_table_v2 > error]。
查看代码好像是tokenizer query和paragraph没有设置max_seq_len



Discussion:
请问使用的 CPU 环境还是 GPU 环境？
> 请问使用的 CPU 环境还是 GPU 环境？

cpu
> 因为模型和数据量不小，建议基于 GPU 环境跑哈，目前仅支持 GPU 环境， CPU 环境我们会尽快支持，欢迎关注。

ok，我gpu环境上试下，不过看报错好像不是环境问题。

Resolving patch:
diff --git a/applications/experimental/pipelines/examples/question-answering/dense_qa_example.py b/applications/experimental/pipelines/examples/question-answering/dense_qa_example.py
--- a/applications/experimental/pipelines/examples/question-answering/dense_qa_example.py
+++ b/applications/experimental/pipelines/examples/question-answering/dense_qa_example.py
@@ -1,30 +1,40 @@
 ### 城市百科知识智能问答系统
-
-import paddle
+import argparse
 import logging
 import os
+
+import paddle
 from pipelines.document_stores import FAISSDocumentStore
-from pipelines.utils import clean_wiki_text, convert_files_to_dicts, fetch_archive_from_http, print_answers, launch_es
+from pipelines.utils import convert_files_to_dicts, fetch_archive_from_http, print_answers
 from pipelines.nodes import ErnieReader, ErnieRanker, DensePassageRetriever
 
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to run dense_qa system, defaults to gpu.")
+parser.add_argument("--index_name", default='faiss_index', type=str, help="The ann index name of FAISS.")
+parser.add_argument("--max_seq_len_query", default=64, type=int, help="The maximum total length of query after tokenization.")
+parser.add_argument("--max_seq_len_passage", default=256, type=int, help="The maximum total length of passage after tokenization.")
+parser.add_argument("--retriever_batch_size", default=16, type=int, help="The batch size of retriever to extract passage embedding for building ANN index.")
+args = parser.parse_args()
+# yapf: enable
 
 def dense_qa_pipeline():
-    logger = logging.getLogger(__name__)
 
-    faiss_index_path = "faiss_index"
+    use_gpu = True if args.device == 'gpu' else False
+
     faiss_document_store = "faiss_document_store.db"
-    if os.path.exists(faiss_index_path) and os.path.exists(
+    if os.path.exists(args.index_name) and os.path.exists(
             faiss_document_store):
         # connect to existed FAISS Index
-        document_store = FAISSDocumentStore.load(faiss_index_path)
+        document_store = FAISSDocumentStore.load(args.index_name)
         retriever = DensePassageRetriever(
             document_store=document_store,
             query_embedding_model="rocketqa-zh-dureader-query-encoder",
             passage_embedding_model="rocketqa-zh-dureader-query-encoder",
-            max_seq_len_query=64,
-            max_seq_len_passage=256,
-            batch_size=16,
-            use_gpu=True,
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
             embed_title=False, )
     else:
         doc_dir = "data/baike"
@@ -32,8 +42,8 @@ def dense_qa_pipeline():
         fetch_archive_from_http(url=city_data, output_dir=doc_dir)
         dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True)
 
-        if os.path.exists(faiss_index_path):
-            os.remove(faiss_index_path)
+        if os.path.exists(args.index_name):
+            os.remove(args.index_name)
         if os.path.exists(faiss_document_store):
             os.remove(faiss_document_store)
 
@@ -45,25 +55,25 @@ def dense_qa_pipeline():
             document_store=document_store,
             query_embedding_model="rocketqa-zh-dureader-query-encoder",
             passage_embedding_model="rocketqa-zh-dureader-query-encoder",
-            max_seq_len_query=64,
-            max_seq_len_passage=256,
-            batch_size=16,
-            use_gpu=True,
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
             embed_title=False, )
 
         # update Embedding
         document_store.update_embeddings(retriever)
 
         # save index
-        document_store.save(faiss_index_path)
+        document_store.save(args.index_name)
 
     ### Ranker
     ranker = ErnieRanker(
-        model_name_or_path="rocketqa-zh-dureader-cross-encoder", use_gpu=True)
+        model_name_or_path="rocketqa-zh-dureader-cross-encoder", use_gpu=use_gpu)
 
     reader = ErnieReader(
         model_name_or_path="ernie-gram-zh-finetuned-dureader-robust",
-        use_gpu=True,
+        use_gpu=use_gpu,
         num_processes=1)
 
     # ### Pipeline
@@ -100,4 +110,4 @@ def dense_qa_pipeline():
 
 
 if __name__ == "__main__":
-    dense_qa_pipeline()
+    dense_qa_pipeline()
\ No newline at end of file
diff --git a/applications/experimental/pipelines/examples/semantic-search/semantic_search_example.py b/applications/experimental/pipelines/examples/semantic-search/semantic_search_example.py
--- a/applications/experimental/pipelines/examples/semantic-search/semantic_search_example.py
+++ b/applications/experimental/pipelines/examples/semantic-search/semantic_search_example.py
@@ -1,37 +1,50 @@
-import paddle
 import os
+import argparse
 
+import paddle
 from pipelines.document_stores import FAISSDocumentStore
 from pipelines.nodes import DensePassageRetriever, ErnieRanker
-from pipelines.utils import convert_files_to_dicts, fetch_archive_from_http
+from pipelines.utils import convert_files_to_dicts, fetch_archive_from_http, print_documents
 
-# Note: Here import paddle occure core
+
+# yapf: disable
+parser = argparse.ArgumentParser()
+parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to run dense_qa system, defaults to gpu.")
+parser.add_argument("--index_name", default='faiss_index', type=str, help="The ann index name of FAISS.")
+parser.add_argument("--max_seq_len_query", default=64, type=int, help="The maximum total length of query after tokenization.")
+parser.add_argument("--max_seq_len_passage", default=256, type=int, help="The maximum total length of passage after tokenization.")
+parser.add_argument("--retriever_batch_size", default=16, type=int, help="The batch size of retriever to extract passage embedding for building ANN index.")
+args = parser.parse_args()
+# yapf: enable
 
 
 def semantic_search_tutorial():
-    faiss_index_path = "faiss_index"
+
+    use_gpu = True if args.device == 'gpu' else False
+
     faiss_document_store = "faiss_document_store.db"
-    if os.path.exists(faiss_index_path) and os.path.exists(
+    if os.path.exists(args.index_name) and os.path.exists(
             faiss_document_store):
         # connect to existed FAISS Index
-        document_store = FAISSDocumentStore.load(faiss_index_path)
+        document_store = FAISSDocumentStore.load(args.index_name)
         retriever = DensePassageRetriever(
             document_store=document_store,
             query_embedding_model="rocketqa-zh-dureader-query-encoder",
             passage_embedding_model="rocketqa-zh-dureader-query-encoder",
-            max_seq_len_query=64,
-            max_seq_len_passage=256,
-            batch_size=16,
-            use_gpu=True,
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
             embed_title=False, )
     else:
         doc_dir = "data/dureader_robust_processed"
         dureader_data = "https://paddlenlp.bj.bcebos.com/applications/dureader_robust_processed.zip"
+
         fetch_archive_from_http(url=dureader_data, output_dir=doc_dir)
         dicts = convert_files_to_dicts(dir_path=doc_dir, split_paragraphs=True)
 
-        if os.path.exists(faiss_index_path):
-            os.remove(faiss_index_path)
+        if os.path.exists(args.index_name):
+            os.remove(args.index_name)
         if os.path.exists(faiss_document_store):
             os.remove(faiss_document_store)
 
@@ -43,38 +56,26 @@ def semantic_search_tutorial():
             document_store=document_store,
             query_embedding_model="rocketqa-zh-dureader-query-encoder",
             passage_embedding_model="rocketqa-zh-dureader-query-encoder",
-            max_seq_len_query=64,
-            max_seq_len_passage=256,
-            batch_size=8,
-            use_gpu=True,
+            max_seq_len_query=args.max_seq_len_query,
+            max_seq_len_passage=args.max_seq_len_passage,
+            batch_size=args.retriever_batch_size,
+            use_gpu=use_gpu,
             embed_title=False, )
 
         # update Embedding
         document_store.update_embeddings(retriever)
 
         # save index
-        document_store.save(faiss_index_path)
-
-    ### Retriever
-    retriever = DensePassageRetriever(
-        document_store=document_store,
-        query_embedding_model="rocketqa-zh-dureader-query-encoder",
-        passage_embedding_model="rocketqa-zh-dureader-query-encoder",
-        max_seq_len_query=64,
-        max_seq_len_passage=256,
-        batch_size=16,
-        use_gpu=True,
-        embed_title=False, )
+        document_store.save(args.index_name)
 
     ### Ranker
     ranker = ErnieRanker(
-        model_name_or_path="rocketqa-zh-dureader-cross-encoder", use_gpu=True)
+        model_name_or_path="rocketqa-zh-dureader-cross-encoder", use_gpu=use_gpu)
 
     ### Pipeline
     from pipelines.pipelines import SemanticSearchPipeline
     pipe = SemanticSearchPipeline(retriever, ranker)
 
-    ## ask question.
     prediction = pipe.run(
         query="亚马逊河流的介绍",
         params={"Retriever": {
@@ -84,8 +85,7 @@ def semantic_search_tutorial():
                     "top_k": 5
                 }})
 
-    print(prediction)
-
+    print_documents(prediction)
 
 if __name__ == "__main__":
     semantic_search_tutorial()


Test patch: