NCTCMumbai
commited on
Commit
•
2765734
1
Parent(s):
eebaf09
Update backend/semantic_search.py
Browse files
backend/semantic_search.py
CHANGED
@@ -4,8 +4,16 @@ import lancedb
|
|
4 |
import os
|
5 |
from pathlib import Path
|
6 |
from sentence_transformers import SentenceTransformer
|
|
|
|
|
|
|
|
|
|
|
7 |
|
8 |
-
|
|
|
|
|
|
|
9 |
#DB_TABLE_NAME = "Huggingface_docs"
|
10 |
DB_TABLE_NAME = "doc_embed1"
|
11 |
# Setting up the logging
|
|
|
4 |
import os
|
5 |
from pathlib import Path
|
6 |
from sentence_transformers import SentenceTransformer
|
7 |
+
from FlagEmbedding import LLMEmbedder, FlagReranker # Al document present here https://github.com/FlagOpen/FlagEmbedding/tree/master
|
8 |
+
#EMB_MODEL_NAME = "thenlper/gte-base"
|
9 |
+
EMB_MODEL_NAME = 'BAAI/llm-embedder'
|
10 |
+
task = "qa" # Encode for a specific task (qa, icl, chat, lrlm, tool, convsearch)
|
11 |
+
#EMB_MODEL_NAME = LLMEmbedder('BAAI/llm-embedder', use_fp16=False) # Load model (automatically use GPUs)
|
12 |
|
13 |
+
#reranker_model = FlagReranker('BAAI/bge-reranker-base', use_fp16=True) # use_fp16 speeds up computation with a slight performance degradation
|
14 |
+
|
15 |
+
|
16 |
+
#EMB_MODEL_NAME = "thenlper/gte-base"
|
17 |
#DB_TABLE_NAME = "Huggingface_docs"
|
18 |
DB_TABLE_NAME = "doc_embed1"
|
19 |
# Setting up the logging
|