from toolbox import update_ui, trimmed_format_exc, gen_time_str from toolbox import CatchException, report_execption, write_results_to_file fast_debug = False class PaperFileGroup(): def __init__(self): self.file_paths = [] self.file_contents = [] self.sp_file_contents = [] self.sp_file_index = [] self.sp_file_tag = [] # count_token from request_llm.bridge_all import model_info enc = model_info["gpt-3.5-turbo"]['tokenizer'] def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) self.get_token_num = get_token_num def run_file_split(self, max_token_limit=1900): """ 将长文本分离开来 """ for index, file_content in enumerate(self.file_contents): if self.get_token_num(file_content) < max_token_limit: self.sp_file_contents.append(file_content) self.sp_file_index.append(index) self.sp_file_tag.append(self.file_paths[index]) else: from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) for j, segment in enumerate(segments): self.sp_file_contents.append(segment) self.sp_file_index.append(index) self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.md") print('Segmentation: done') def merge_result(self): self.file_result = ["" for _ in range(len(self.file_paths))] for r, k in zip(self.sp_file_result, self.sp_file_index): self.file_result[k] += r def write_result(self): manifest = [] for path, res in zip(self.file_paths, self.file_result): with open(path + f'.{gen_time_str()}.trans.md', 'w', encoding='utf8') as f: manifest.append(path + '.trans.md') f.write(res) return manifest def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en'): import time, os, re from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency # <-------- 读取Markdown文件，删除其中的所有注释 ----------> pfg = PaperFileGroup() for index, fp in enumerate(file_manifest): with open(fp, 'r', encoding='utf-8', errors='replace') as f: file_content = f.read() # 记录删除注释后的文本 pfg.file_paths.append(fp) pfg.file_contents.append(file_content) # <-------- 拆分过长的Markdown文件 ----------> pfg.run_file_split(max_token_limit=1500) n_split = len(pfg.sp_file_contents) # <-------- 多线程翻译开始 ----------> if language == 'en->zh': inputs_array = ["This is a Markdown file, translate it into Chinese, do not modify any existing Markdown commands:" + f"\n\n{frag}" for frag in pfg.sp_file_contents] inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] elif language == 'zh->en': inputs_array = [f"This is a Markdown file, translate it into English, do not modify any existing Markdown commands:" + f"\n\n{frag}" for frag in pfg.sp_file_contents] inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] else: inputs_array = [f"This is a Markdown file, translate it into {language}, do not modify any existing Markdown commands, only answer me with translated results:" + f"\n\n{frag}" for frag in pfg.sp_file_contents] inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( inputs_array=inputs_array, inputs_show_user_array=inputs_show_user_array, llm_kwargs=llm_kwargs, chatbot=chatbot, history_array=[[""] for _ in range(n_split)], sys_prompt_array=sys_prompt_array, # max_workers=5, # OpenAI所允许的最大并行过载 scroller_max_len = 80 ) try: pfg.sp_file_result = [] for i_say, gpt_say in zip(gpt_response_collection[0::2], gpt_response_collection[1::2]): pfg.sp_file_result.append(gpt_say) pfg.merge_result() pfg.write_result() except: print(trimmed_format_exc()) # <-------- 整理结果，退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) history = gpt_response_collection chatbot.append((f"{fp}完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 def get_files_from_everything(txt): import glob, os success = True if txt.startswith('http'): # 网络的远程文件 txt = txt.replace("https://github.com/", "https://raw.githubusercontent.com/") txt = txt.replace("/blob/", "/") import requests from toolbox import get_conf proxies, = get_conf('proxies') r = requests.get(txt, proxies=proxies) with open('./gpt_log/temp.md', 'wb+') as f: f.write(r.content) project_folder = './gpt_log/' file_manifest = ['./gpt_log/temp.md'] elif txt.endswith('.md'): # 直接给定文件 file_manifest = [txt] project_folder = os.path.dirname(txt) elif os.path.exists(txt): # 本地路径，递归搜索 project_folder = txt file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.md', recursive=True)] else: success = False return success, file_manifest, project_folder @CatchException def Markdown英译中(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): # 基本信息：功能、贡献者 chatbot.append([ "函数插件功能？", "对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 尝试导入依赖，如果缺少依赖，则给出安装建议 try: import tiktoken import glob, os except: report_execption(chatbot, history, a=f"解析项目: {txt}", b=f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade tiktoken```。") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return history = [] # 清空历史，以免输入溢出 success, file_manifest, project_folder = get_files_from_everything(txt) if not success: # 什么都没有 if txt == "": txt = '空空如也的输入栏' report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en->zh') @CatchException def Markdown中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): # 基本信息：功能、贡献者 chatbot.append([ "函数插件功能？", "对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 尝试导入依赖，如果缺少依赖，则给出安装建议 try: import tiktoken import glob, os except: report_execption(chatbot, history, a=f"解析项目: {txt}", b=f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade tiktoken```。") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return history = [] # 清空历史，以免输入溢出 success, file_manifest, project_folder = get_files_from_everything(txt) if not success: # 什么都没有 if txt == "": txt = '空空如也的输入栏' report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='zh->en') @CatchException def Markdown翻译指定语言(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): # 基本信息：功能、贡献者 chatbot.append([ "函数插件功能？", "对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 尝试导入依赖，如果缺少依赖，则给出安装建议 try: import tiktoken import glob, os except: report_execption(chatbot, history, a=f"解析项目: {txt}", b=f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade tiktoken```。") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return history = [] # 清空历史，以免输入溢出 success, file_manifest, project_folder = get_files_from_everything(txt) if not success: # 什么都没有 if txt == "": txt = '空空如也的输入栏' report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}") yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") language = plugin_kwargs.get("advanced_arg", 'Chinese') yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language=language)