Spaces:
Runtime error
Runtime error
update tests
Browse files- tests/test_chatbot.py +62 -53
tests/test_chatbot.py
CHANGED
@@ -67,22 +67,15 @@ def test_chatbot_mock_data(tmp_path, monkeypatch):
|
|
67 |
hf_transformers_cfg = BusterConfig(
|
68 |
unknown_prompt="This doesn't seem to be related to the huggingface library. I am not sure how to answer.",
|
69 |
embedding_model="text-embedding-ada-002",
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
"name": "
|
77 |
-
"text_before_prompt": (
|
78 |
-
"""You are a slack chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python.\n"""
|
79 |
-
"""Make sure to format your answers in Markdown format, including code block and snippets.\n"""
|
80 |
-
"""Do not include any links to urls or hyperlinks in your answers.\n\n"""
|
81 |
-
"""Now answer the following question:\n"""
|
82 |
-
),
|
83 |
-
"text_before_documents": "",
|
84 |
"completion_kwargs": {
|
85 |
-
"engine": "
|
86 |
"max_tokens": 200,
|
87 |
"temperature": None,
|
88 |
"top_p": None,
|
@@ -90,6 +83,16 @@ def test_chatbot_mock_data(tmp_path, monkeypatch):
|
|
90 |
"presence_penalty": 1,
|
91 |
},
|
92 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
93 |
)
|
94 |
filepath = tmp_path / "not_a_real_file.tar.gz"
|
95 |
retriever = MockRetriever(filepath)
|
@@ -103,22 +106,21 @@ def test_chatbot_real_data__chatGPT():
|
|
103 |
hf_transformers_cfg = BusterConfig(
|
104 |
unknown_prompt="I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
|
105 |
embedding_model="text-embedding-ada-002",
|
106 |
-
|
107 |
-
thresh=0.7,
|
108 |
-
max_words=3000,
|
109 |
-
response_format="slack",
|
110 |
-
completer_cfg={
|
111 |
"name": "ChatGPT",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
112 |
"text_before_prompt": (
|
113 |
"""You are a slack chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python.\n"""
|
114 |
"""Make sure to format your answers in Markdown format, including code block and snippets.\n"""
|
115 |
"""Do not include any links to urls or hyperlinks in your answers.\n\n"""
|
116 |
"""Now answer the following question:\n"""
|
117 |
),
|
118 |
-
"text_before_documents": "Only use these documents as reference:\n",
|
119 |
-
"completion_kwargs": {
|
120 |
-
"model": "gpt-3.5-turbo",
|
121 |
-
},
|
122 |
},
|
123 |
)
|
124 |
retriever = get_retriever_from_extension(DOCUMENTS_FILE)(DOCUMENTS_FILE)
|
@@ -131,11 +133,18 @@ def test_chatbot_real_data__chatGPT_OOD():
|
|
131 |
buster_cfg = BusterConfig(
|
132 |
unknown_prompt="I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
|
133 |
embedding_model="text-embedding-ada-002",
|
134 |
-
|
135 |
-
thresh=0.7,
|
136 |
-
max_words=3000,
|
137 |
-
completer_cfg={
|
138 |
"name": "ChatGPT",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
139 |
"text_before_prompt": (
|
140 |
"""You are a chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python. """
|
141 |
"""Make sure to format your answers in Markdown format, including code block and snippets. """
|
@@ -149,11 +158,7 @@ def test_chatbot_real_data__chatGPT_OOD():
|
|
149 |
"""Now answer the following question:\n"""
|
150 |
),
|
151 |
"text_before_documents": "Only use these documents as reference:\n",
|
152 |
-
"completion_kwargs": {
|
153 |
-
"model": "gpt-3.5-turbo",
|
154 |
-
},
|
155 |
},
|
156 |
-
response_format="gradio",
|
157 |
)
|
158 |
retriever = get_retriever_from_extension(DOCUMENTS_FILE)(DOCUMENTS_FILE)
|
159 |
buster = Buster(cfg=buster_cfg, retriever=retriever)
|
@@ -163,34 +168,38 @@ def test_chatbot_real_data__chatGPT_OOD():
|
|
163 |
|
164 |
|
165 |
def test_chatbot_real_data__GPT():
|
166 |
-
|
167 |
-
unknown_prompt="
|
168 |
embedding_model="text-embedding-ada-002",
|
169 |
-
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
175 |
"text_before_prompt": (
|
176 |
-
"""You are a chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python
|
177 |
-
"""Make sure to format your answers in Markdown format, including code block and snippets
|
178 |
-
"""Do not include any links to urls or hyperlinks in your answers
|
|
|
|
|
|
|
|
|
|
|
|
|
179 |
"""Now answer the following question:\n"""
|
180 |
),
|
181 |
-
"text_before_documents": "",
|
182 |
-
"completion_kwargs": {
|
183 |
-
"engine": "text-davinci-003",
|
184 |
-
"max_tokens": 200,
|
185 |
-
"temperature": None,
|
186 |
-
"top_p": None,
|
187 |
-
"frequency_penalty": 1,
|
188 |
-
"presence_penalty": 1,
|
189 |
-
},
|
190 |
},
|
191 |
)
|
192 |
retriever = get_retriever_from_extension(DOCUMENTS_FILE)(DOCUMENTS_FILE)
|
193 |
-
buster = Buster(cfg=
|
194 |
response = buster.process_input("What is a transformer?")
|
195 |
assert isinstance(response.completion.text, str)
|
196 |
assert response.is_relevant == True
|
|
|
67 |
hf_transformers_cfg = BusterConfig(
|
68 |
unknown_prompt="This doesn't seem to be related to the huggingface library. I am not sure how to answer.",
|
69 |
embedding_model="text-embedding-ada-002",
|
70 |
+
retriever_cfg={
|
71 |
+
"top_k": 3,
|
72 |
+
"thresh": 0.7,
|
73 |
+
},
|
74 |
+
document_source="fake source",
|
75 |
+
completion_cfg={
|
76 |
+
"name": "ChatGPT",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
77 |
"completion_kwargs": {
|
78 |
+
"engine": "gpt-3.5-turbo",
|
79 |
"max_tokens": 200,
|
80 |
"temperature": None,
|
81 |
"top_p": None,
|
|
|
83 |
"presence_penalty": 1,
|
84 |
},
|
85 |
},
|
86 |
+
prompt_cfg={
|
87 |
+
"max_words": 2000,
|
88 |
+
"text_before_documents": "",
|
89 |
+
"text_before_prompt": (
|
90 |
+
"""You are a slack chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python.\n"""
|
91 |
+
"""Make sure to format your answers in Markdown format, including code block and snippets.\n"""
|
92 |
+
"""Do not include any links to urls or hyperlinks in your answers.\n\n"""
|
93 |
+
"""Now answer the following question:\n"""
|
94 |
+
),
|
95 |
+
},
|
96 |
)
|
97 |
filepath = tmp_path / "not_a_real_file.tar.gz"
|
98 |
retriever = MockRetriever(filepath)
|
|
|
106 |
hf_transformers_cfg = BusterConfig(
|
107 |
unknown_prompt="I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
|
108 |
embedding_model="text-embedding-ada-002",
|
109 |
+
completion_cfg={
|
|
|
|
|
|
|
|
|
110 |
"name": "ChatGPT",
|
111 |
+
"completion_kwargs": {
|
112 |
+
"model": "gpt-3.5-turbo",
|
113 |
+
},
|
114 |
+
},
|
115 |
+
prompt_cfg={
|
116 |
+
"max_words": 2000,
|
117 |
+
"text_before_documents": "",
|
118 |
"text_before_prompt": (
|
119 |
"""You are a slack chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python.\n"""
|
120 |
"""Make sure to format your answers in Markdown format, including code block and snippets.\n"""
|
121 |
"""Do not include any links to urls or hyperlinks in your answers.\n\n"""
|
122 |
"""Now answer the following question:\n"""
|
123 |
),
|
|
|
|
|
|
|
|
|
124 |
},
|
125 |
)
|
126 |
retriever = get_retriever_from_extension(DOCUMENTS_FILE)(DOCUMENTS_FILE)
|
|
|
133 |
buster_cfg = BusterConfig(
|
134 |
unknown_prompt="I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
|
135 |
embedding_model="text-embedding-ada-002",
|
136 |
+
completion_cfg={
|
|
|
|
|
|
|
137 |
"name": "ChatGPT",
|
138 |
+
"completion_kwargs": {
|
139 |
+
"model": "gpt-3.5-turbo",
|
140 |
+
},
|
141 |
+
},
|
142 |
+
retriever_cfg={
|
143 |
+
"top_k": 3,
|
144 |
+
"thresh": 0.7,
|
145 |
+
},
|
146 |
+
prompt_cfg={
|
147 |
+
"max_words": 3000,
|
148 |
"text_before_prompt": (
|
149 |
"""You are a chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python. """
|
150 |
"""Make sure to format your answers in Markdown format, including code block and snippets. """
|
|
|
158 |
"""Now answer the following question:\n"""
|
159 |
),
|
160 |
"text_before_documents": "Only use these documents as reference:\n",
|
|
|
|
|
|
|
161 |
},
|
|
|
162 |
)
|
163 |
retriever = get_retriever_from_extension(DOCUMENTS_FILE)(DOCUMENTS_FILE)
|
164 |
buster = Buster(cfg=buster_cfg, retriever=retriever)
|
|
|
168 |
|
169 |
|
170 |
def test_chatbot_real_data__GPT():
|
171 |
+
buster_cfg = BusterConfig(
|
172 |
+
unknown_prompt="I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?",
|
173 |
embedding_model="text-embedding-ada-002",
|
174 |
+
completion_cfg={
|
175 |
+
"name": "ChatGPT",
|
176 |
+
"completion_kwargs": {
|
177 |
+
"model": "gpt-3.5-turbo",
|
178 |
+
},
|
179 |
+
},
|
180 |
+
retriever_cfg={
|
181 |
+
"top_k": 3,
|
182 |
+
"thresh": 0.7,
|
183 |
+
},
|
184 |
+
prompt_cfg={
|
185 |
+
"max_words": 3000,
|
186 |
"text_before_prompt": (
|
187 |
+
"""You are a chatbot assistant answering technical questions about huggingface transformers, a library to train transformers in python. """
|
188 |
+
"""Make sure to format your answers in Markdown format, including code block and snippets. """
|
189 |
+
"""Do not include any links to urls or hyperlinks in your answers. """
|
190 |
+
"""If you do not know the answer to a question, or if it is completely irrelevant to the library usage, let the user know you cannot answer. """
|
191 |
+
"""Use this response: """
|
192 |
+
"""'I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?'\n"""
|
193 |
+
"""For example:\n"""
|
194 |
+
"""What is the meaning of life for huggingface?\n"""
|
195 |
+
"""I'm sorry, but I am an AI language model trained to assist with questions related to the huggingface transformers library. I cannot answer that question as it is not relevant to the library or its usage. Is there anything else I can assist you with?"""
|
196 |
"""Now answer the following question:\n"""
|
197 |
),
|
198 |
+
"text_before_documents": "Only use these documents as reference:\n",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
199 |
},
|
200 |
)
|
201 |
retriever = get_retriever_from_extension(DOCUMENTS_FILE)(DOCUMENTS_FILE)
|
202 |
+
buster = Buster(cfg=buster_cfg, retriever=retriever)
|
203 |
response = buster.process_input("What is a transformer?")
|
204 |
assert isinstance(response.completion.text, str)
|
205 |
assert response.is_relevant == True
|