Update app.py
Browse files
app.py
CHANGED
@@ -16,8 +16,9 @@ import xml.etree.ElementTree as ET
|
|
16 |
from nltk.tokenize import word_tokenize
|
17 |
from langchain_community.vectorstores import FAISS
|
18 |
from youtube_transcript_api import YouTubeTranscriptApi
|
|
|
19 |
from langchain.schema import SystemMessage, HumanMessage, AIMessage
|
20 |
-
from
|
21 |
from langchain_community.embeddings import SentenceTransformerEmbeddings
|
22 |
from youtube_transcript_api._errors import NoTranscriptFound, TranscriptsDisabled, VideoUnavailable
|
23 |
nltk.download('punkt')
|
@@ -157,7 +158,7 @@ def normalize_text(text):
|
|
157 |
words = word_tokenize(text)
|
158 |
return ' '.join(words)
|
159 |
|
160 |
-
|
161 |
repo_id="HuggingFaceH4/starchat2-15b-v0.1",
|
162 |
task="text-generation",
|
163 |
max_new_tokens=4096,
|
@@ -167,7 +168,7 @@ chat_model = HuggingFaceEndpoint(
|
|
167 |
repetition_penalty=1.2,
|
168 |
do_sample=True,
|
169 |
)
|
170 |
-
|
171 |
|
172 |
model_name = "sentence-transformers/all-mpnet-base-v2"
|
173 |
embedding_llm = SentenceTransformerEmbeddings(model_name=model_name)
|
|
|
16 |
from nltk.tokenize import word_tokenize
|
17 |
from langchain_community.vectorstores import FAISS
|
18 |
from youtube_transcript_api import YouTubeTranscriptApi
|
19 |
+
from langchain_community.llms import HuggingFaceEndpoint
|
20 |
from langchain.schema import SystemMessage, HumanMessage, AIMessage
|
21 |
+
from langchain_community.chat_models.huggingface import ChatHuggingFace
|
22 |
from langchain_community.embeddings import SentenceTransformerEmbeddings
|
23 |
from youtube_transcript_api._errors import NoTranscriptFound, TranscriptsDisabled, VideoUnavailable
|
24 |
nltk.download('punkt')
|
|
|
158 |
words = word_tokenize(text)
|
159 |
return ' '.join(words)
|
160 |
|
161 |
+
llm = HuggingFaceEndpoint(
|
162 |
repo_id="HuggingFaceH4/starchat2-15b-v0.1",
|
163 |
task="text-generation",
|
164 |
max_new_tokens=4096,
|
|
|
168 |
repetition_penalty=1.2,
|
169 |
do_sample=True,
|
170 |
)
|
171 |
+
chat_model = ChatHuggingFace(llm=llm)
|
172 |
|
173 |
model_name = "sentence-transformers/all-mpnet-base-v2"
|
174 |
embedding_llm = SentenceTransformerEmbeddings(model_name=model_name)
|