Update app.py
Browse files
app.py
CHANGED
@@ -84,6 +84,9 @@ qa_embeddings = semantic_model.encode(qa_questions, convert_to_tensor=True)
|
|
84 |
#-------------------------bm25---------------------------------
|
85 |
from rank_bm25 import BM25Okapi
|
86 |
from nltk.tokenize import word_tokenize
|
|
|
|
|
|
|
87 |
|
88 |
def rerank_with_bm25(docs, query):
|
89 |
tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
|
|
|
84 |
#-------------------------bm25---------------------------------
|
85 |
from rank_bm25 import BM25Okapi
|
86 |
from nltk.tokenize import word_tokenize
|
87 |
+
import nltk
|
88 |
+
nltk.download('punkt')
|
89 |
+
|
90 |
|
91 |
def rerank_with_bm25(docs, query):
|
92 |
tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
|