import os, shutil import re import numpy as np PRESERVE = 0 TRANSFORM = 1 pj = os.path.join class LinkedListNode(): """ Linked List Node """ def __init__(self, string, preserve=True) -> None: self.string = string self.preserve = preserve self.next = None self.range = None # self.begin_line = 0 # self.begin_char = 0 def convert_to_linklist(text, mask): root = LinkedListNode("", preserve=True) current_node = root for c, m, i in zip(text, mask, range(len(text))): if (m==PRESERVE and current_node.preserve) \ or (m==TRANSFORM and not current_node.preserve): # add current_node.string += c else: current_node.next = LinkedListNode(c, preserve=(m==PRESERVE)) current_node = current_node.next return root def post_process(root): # 修复括号 node = root while True: string = node.string if node.preserve: node = node.next if node is None: break continue def break_check(string): str_stack = [""] # (lv, index) for i, c in enumerate(string): if c == '{': str_stack.append('{') elif c == '}': if len(str_stack) == 1: print('stack fix') return i str_stack.pop(-1) else: str_stack[-1] += c return -1 bp = break_check(string) if bp == -1: pass elif bp == 0: node.string = string[:1] q = LinkedListNode(string[1:], False) q.next = node.next node.next = q else: node.string = string[:bp] q = LinkedListNode(string[bp:], False) q.next = node.next node.next = q node = node.next if node is None: break # 屏蔽空行和太短的句子 node = root while True: if len(node.string.strip('\n').strip(''))==0: node.preserve = True if len(node.string.strip('\n').strip(''))<42: node.preserve = True node = node.next if node is None: break node = root while True: if node.next and node.preserve and node.next.preserve: node.string += node.next.string node.next = node.next.next node = node.next if node is None: break # 将前后断行符脱离 node = root prev_node = None while True: if not node.preserve: lstriped_ = node.string.lstrip().lstrip('\n') if (prev_node is not None) and (prev_node.preserve) and (len(lstriped_)!=len(node.string)): prev_node.string += node.string[:-len(lstriped_)] node.string = lstriped_ rstriped_ = node.string.rstrip().rstrip('\n') if (node.next is not None) and (node.next.preserve) and (len(rstriped_)!=len(node.string)): node.next.string = node.string[len(rstriped_):] + node.next.string node.string = rstriped_ # ===== prev_node = node node = node.next if node is None: break # 标注节点的行数范围 node = root n_line = 0 expansion = 2 while True: n_l = node.string.count('\n') node.range = [n_line-expansion, n_line+n_l+expansion] # 失败时,扭转的范围 n_line = n_line+n_l node = node.next if node is None: break return root """ =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Latex segmentation with a binary mask (PRESERVE=0, TRANSFORM=1) =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= """ def set_forbidden_text(text, mask, pattern, flags=0): """ Add a preserve text area in this paper e.g. with pattern = r"\\begin\{algorithm\}(.*?)\\end\{algorithm\}" you can mask out (mask = PRESERVE so that text become untouchable for GPT) everything between "\begin{equation}" and "\end{equation}" """ if isinstance(pattern, list): pattern = '|'.join(pattern) pattern_compile = re.compile(pattern, flags) for res in pattern_compile.finditer(text): mask[res.span()[0]:res.span()[1]] = PRESERVE return text, mask def reverse_forbidden_text(text, mask, pattern, flags=0, forbid_wrapper=True): """ Move area out of preserve area (make text editable for GPT) count the number of the braces so as to catch compelete text area. e.g. \begin{abstract} blablablablablabla. \end{abstract} """ if isinstance(pattern, list): pattern = '|'.join(pattern) pattern_compile = re.compile(pattern, flags) for res in pattern_compile.finditer(text): if not forbid_wrapper: mask[res.span()[0]:res.span()[1]] = TRANSFORM else: mask[res.regs[0][0]: res.regs[1][0]] = PRESERVE # '\\begin{abstract}' mask[res.regs[1][0]: res.regs[1][1]] = TRANSFORM # abstract mask[res.regs[1][1]: res.regs[0][1]] = PRESERVE # abstract return text, mask def set_forbidden_text_careful_brace(text, mask, pattern, flags=0): """ Add a preserve text area in this paper (text become untouchable for GPT). count the number of the braces so as to catch compelete text area. e.g. \caption{blablablablabla\texbf{blablabla}blablabla.} """ pattern_compile = re.compile(pattern, flags) for res in pattern_compile.finditer(text): brace_level = -1 p = begin = end = res.regs[0][0] for _ in range(1024*16): if text[p] == '}' and brace_level == 0: break elif text[p] == '}': brace_level -= 1 elif text[p] == '{': brace_level += 1 p += 1 end = p+1 mask[begin:end] = PRESERVE return text, mask def reverse_forbidden_text_careful_brace(text, mask, pattern, flags=0, forbid_wrapper=True): """ Move area out of preserve area (make text editable for GPT) count the number of the braces so as to catch compelete text area. e.g. \caption{blablablablabla\texbf{blablabla}blablabla.} """ pattern_compile = re.compile(pattern, flags) for res in pattern_compile.finditer(text): brace_level = 0 p = begin = end = res.regs[1][0] for _ in range(1024*16): if text[p] == '}' and brace_level == 0: break elif text[p] == '}': brace_level -= 1 elif text[p] == '{': brace_level += 1 p += 1 end = p mask[begin:end] = TRANSFORM if forbid_wrapper: mask[res.regs[0][0]:begin] = PRESERVE mask[end:res.regs[0][1]] = PRESERVE return text, mask def set_forbidden_text_begin_end(text, mask, pattern, flags=0, limit_n_lines=42): """ Find all \begin{} ... \end{} text block that with less than limit_n_lines lines. Add it to preserve area """ pattern_compile = re.compile(pattern, flags) def search_with_line_limit(text, mask): for res in pattern_compile.finditer(text): cmd = res.group(1) # begin{what} this = res.group(2) # content between begin and end this_mask = mask[res.regs[2][0]:res.regs[2][1]] white_list = ['document', 'abstract', 'lemma', 'definition', 'sproof', 'em', 'emph', 'textit', 'textbf', 'itemize', 'enumerate'] if (cmd in white_list) or this.count('\n') >= limit_n_lines: # use a magical number 42 this, this_mask = search_with_line_limit(this, this_mask) mask[res.regs[2][0]:res.regs[2][1]] = this_mask else: mask[res.regs[0][0]:res.regs[0][1]] = PRESERVE return text, mask return search_with_line_limit(text, mask) """ =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Latex Merge File =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= """ def find_main_tex_file(file_manifest, mode): """ 在多Tex文档中,寻找主文件,必须包含documentclass,返回找到的第一个。 P.S. 但愿没人把latex模板放在里面传进来 (6.25 加入判定latex模板的代码) """ canidates = [] for texf in file_manifest: if os.path.basename(texf).startswith('merge'): continue with open(texf, 'r', encoding='utf8', errors='ignore') as f: file_content = f.read() if r'\documentclass' in file_content: canidates.append(texf) else: continue if len(canidates) == 0: raise RuntimeError('无法找到一个主Tex文件(包含documentclass关键字)') elif len(canidates) == 1: return canidates[0] else: # if len(canidates) >= 2 通过一些Latex模板中常见(但通常不会出现在正文)的单词,对不同latex源文件扣分,取评分最高者返回 canidates_score = [] # 给出一些判定模板文档的词作为扣分项 unexpected_words = ['\LaTeX', 'manuscript', 'Guidelines', 'font', 'citations', 'rejected', 'blind review', 'reviewers'] expected_words = ['\input', '\ref', '\cite'] for texf in canidates: canidates_score.append(0) with open(texf, 'r', encoding='utf8', errors='ignore') as f: file_content = f.read() file_content = rm_comments(file_content) for uw in unexpected_words: if uw in file_content: canidates_score[-1] -= 1 for uw in expected_words: if uw in file_content: canidates_score[-1] += 1 select = np.argmax(canidates_score) # 取评分最高者返回 return canidates[select] def rm_comments(main_file): new_file_remove_comment_lines = [] for l in main_file.splitlines(): # 删除整行的空注释 if l.lstrip().startswith("%"): pass else: new_file_remove_comment_lines.append(l) main_file = '\n'.join(new_file_remove_comment_lines) # main_file = re.sub(r"\\include{(.*?)}", r"\\input{\1}", main_file) # 将 \include 命令转换为 \input 命令 main_file = re.sub(r'(? 0 and node_string.count('\_') > final_tex.count('\_'): # walk and replace any _ without \ final_tex = re.sub(r"(?