Spaces:
Running
Running
File size: 4,636 Bytes
bc11e8d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
import re
import html
import uuid
import subprocess
import unicodedata
from bs4 import BeautifulSoup
def normalized_html_table(text):
def process_table_html(md_i):
"""
pred_md format edit
"""
def process_table_html(html_content):
soup = BeautifulSoup(html_content, 'html.parser')
th_tags = soup.find_all('th')
for th in th_tags:
th.name = 'td'
thead_tags = soup.find_all('thead')
for thead in thead_tags:
thead.unwrap() # unwrap()会移除标签但保留其内容
math_tags = soup.find_all('math')
for math_tag in math_tags:
alttext = math_tag.get('alttext', '')
alttext = f'${alttext}$'
if alttext:
math_tag.replace_with(alttext)
span_tags = soup.find_all('span')
for span in span_tags:
span.unwrap()
return str(soup)
table_res=''
table_res_no_space=''
if '<table' in md_i.replace(" ","").replace("'",'"'):
md_i = process_table_html(md_i)
table_res = html.unescape(md_i).replace('\n', '')
table_res = unicodedata.normalize('NFKC', table_res).strip()
pattern = r'<table\b[^>]*>(.*)</table>'
tables = re.findall(pattern, table_res, re.DOTALL | re.IGNORECASE)
table_res = ''.join(tables)
# table_res = re.sub('<table.*?>','',table_res)
table_res = re.sub('( style=".*?")', "", table_res)
table_res = re.sub('( style=".*?")', "", table_res)
table_res = re.sub('( height=".*?")', "", table_res)
table_res = re.sub('( width=".*?")', "", table_res)
table_res = re.sub('( colwidth=".*?")', "", table_res)
table_res = re.sub('( colheight=".*?")', "", table_res)
table_res = re.sub('( rowwidth=".*?")', "", table_res)
table_res = re.sub('( rowheight=".*?")', "", table_res)
table_res = re.sub('( align=".*?")', "", table_res)
table_res = re.sub('( class=".*?")', "", table_res)
table_res = re.sub('( rowspan="1")', "", table_res)
table_res = re.sub('( colspan="1")', "", table_res)
table_res = re.sub('</?tbody>',"",table_res)
table_res = re.sub(r'\s+', " ", table_res)
table_res_no_space = '<html><body><table>' + table_res.replace(' ','') + '</table></body></html>'
# table_res_no_space = re.sub(' (style=".*?")',"",table_res_no_space)
# table_res_no_space = re.sub(r'[ ]', " ", table_res_no_space)
table_res_no_space = re.sub('colspan="', ' colspan="', table_res_no_space)
table_res_no_space = re.sub('rowspan="', ' rowspan="', table_res_no_space)
table_res_no_space = re.sub('border="', ' border="', table_res_no_space)
table_res = '<html><body><table>' + table_res + '</table></body></html>'
# table_flow.append(table_res)
# table_flow_no_space.append(table_res_no_space)
return table_res, table_res_no_space
def clean_table(input_str,flag=True):
if flag:
input_str = input_str.replace('<sup>', '').replace('</sup>', '')
input_str = input_str.replace('<sub>', '').replace('</sub>', '')
input_str = input_str.replace('<span>', '').replace('</span>', '')
input_str = input_str.replace('<div>', '').replace('</div>', '')
input_str = input_str.replace('<p>', '').replace('</p>', '')
input_str = input_str.replace('<spandata-span-identity="">', '')
input_str = re.sub('<colgroup>.*?</colgroup>','',input_str)
return input_str
def process_formula(input_str):
# 处理行内数学公式
inline_pattern = r'<p><span\s+data-latex="([^"]*)"\s+data-type="inline-math"></span></p>'
input_str = re.sub(inline_pattern, r'\(\1\)', input_str)
# 处理块级数学公式（如果有的话）
block_pattern = r'<p><span\s+data-latex="([^"]*)"\s+data-type="display-math"></span></p>'
input_str = re.sub(block_pattern, r'\[\1\]', input_str)
return input_str
def process_uline(input_str):
return input_str.replace('<u>', '<u>').replace('</u>', '</u>')
text = process_formula(text)
text = process_uline(text)
norm_text, _ = process_table_html(text)
norm_text = clean_table(norm_text)
return norm_text.replace('> ', '>').replace(" </td>", "</td>") |