melk2025 commited on
Commit
032ce2e
·
verified ·
1 Parent(s): 8c3f7aa

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -0
app.py CHANGED
@@ -84,6 +84,9 @@ qa_embeddings = semantic_model.encode(qa_questions, convert_to_tensor=True)
84
  #-------------------------bm25---------------------------------
85
  from rank_bm25 import BM25Okapi
86
  from nltk.tokenize import word_tokenize
 
 
 
87
 
88
  def rerank_with_bm25(docs, query):
89
  tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
 
84
  #-------------------------bm25---------------------------------
85
  from rank_bm25 import BM25Okapi
86
  from nltk.tokenize import word_tokenize
87
+ import nltk
88
+ nltk.download('punkt')
89
+
90
 
91
  def rerank_with_bm25(docs, query):
92
  tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]