from .question import register_question from .question import Checker from typing import Callable from .utility import count_words, word_reverse_text CN_TEXT_1 = """ 第三章第一题(自说自话),请你输入一个至少10个字的提问,使模型的回答和提问完全一样。 请在下面的输入框内填写你的提问并点击按钮提交。 """ EN_TEXT_1 = """ For the first question in chapter 3, please enter a question of at least ten words so that the model's answer is exactly the same with the question. Please enter your query below and click the submit button """ def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str): _ = question_text, lang answer_text = answer_text.strip() user_text = user_text.strip() if count_words(user_text) < 10: return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.' if answer_text == user_text: return True, None else: return False, None register_question( { 'cn': CN_TEXT_1, 'en': EN_TEXT_1, }, checkers=_checker_1, name={'cn': '3-1 自说自话', 'en': '3-1'}, level=3, ) CN_TEXT_2 = """ 第三章第二题(话自说自),请你输入一个至少10个字的提问,使模型的回答是问题的反序。 请在下面的输入框内填写你的提问并点击按钮提交。 """ EN_TEXT_2 = """ For the second question in chapter 3, please enter a question of at least ten words so that the model's answer is the reverse order of the question. Please enter your query below and click the submit button """ def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str): _ = question_text, lang answer_text = answer_text.strip() user_text = user_text.strip() if count_words(user_text) < 10: return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.' reversed_user_text = word_reverse_text(user_text, lang) if reversed_user_text == answer_text: return True, None else: return False, None register_question( { 'cn': CN_TEXT_2, 'en': EN_TEXT_2, }, checkers=_checker_2, name={'cn': '3-2 话自说自', 'en': '3-2'}, level=3, ) CN_TEXT_3 = """ 第三章第三题(就是敢说),请你输入一个不包含“1”和“3”的提问,使模型的回答是一字不差的“1+1=3”(不需要引号)。 请在下面的输入框内填写你的提问并点击按钮提交。 """ EN_TEXT_3 = """ For the third question in chapter 3, please enter a question not contain "1" or "3" that will result in the exact answer "1+1=3" (no quotes needed). Please enter your query below and click the submit button """ def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str): _ = question_text, lang answer_text = answer_text.strip() user_text = user_text.strip() if '1' in user_text or '3' in user_text: return False, "用户的问题不能包含“1”或“3”" if lang == 'cn' else 'Question should not contain \"1\" or \"3\".' if answer_text == '1+1=3': return True, None else: return False, None register_question( { 'cn': CN_TEXT_3, 'en': EN_TEXT_3, }, checkers=_checker_3, name={'cn': '3-3 就是敢说', 'en': '3-3'}, level=3, ) CN_TEXT_4 = """ 第三章第四题(回文协变),请你输入一个本身不是回文串的问题,使得正着问和倒着问时,模型的回答本身不是回文且也是逆序。 请在下面的输入框内填写你的提问并点击按钮提交。 """ EN_TEXT_4 = """ For the fourth question in chapter 3, please enter a question that is not a palindrome string, so that the model's answer is also not a palindrome and is in reverse order when asked forward or backward. Please enter your query below and click the submit button """ def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str, llm_callback: Callable[[str], str]): answer_text = answer_text.strip() user_text = user_text.strip() reversed_user_text = word_reverse_text(user_text, lang) second_answer_text = llm_callback(reversed_user_text) if user_text == reversed_user_text: return False, "用户的问题不能是回文串" if lang == 'cn' else 'Question should not be a palindrome string.' if second_answer_text != word_reverse_text(answer_text, lang): return False, f"正着问和倒着问时,模型的回答应该是逆序。\n 用户问题:{user_text}\n 正着问回答: {answer_text} \n 反着问回答: {second_answer_text}" if lang == 'cn' else f'The model\'s answer should be in reverse order when asked forward or backward.\n Question: {user_text}\n Forward-Asking answer: {answer_text} \n Backward-Asking answer: {second_answer_text}' if second_answer_text == word_reverse_text(second_answer_text): return False, "正着问和倒着问时,模型的回答不能是回文串" if lang == 'cn' else 'The model\'s answer should not be a palindrome when asked forward or backward.' return True, None register_question( { 'cn': CN_TEXT_4, 'en': EN_TEXT_4, }, checkers=Checker(_checker_4, required_input_keys=['question_text', 'user_text', 'answer_text', 'lang', 'llm_callback']), name={'cn': '3-4 回文协变', 'en': '3-4'}, level=3, )