File size: 3,752 Bytes
9fa4f9e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# MTUOC_preprocess
# Copyright (C) 2023 Antoni Oliver
# v. 23/05/2023
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import config
from MTUOC_misc import printLOG
import regex as rx
import re
import html
def remove_control_characters(cadena):
return rx.sub(r'\p{C}', '', cadena)
re_num = re.compile(r'[\d,\.]+')
def replace_NUMs(segment,code="@NUM@"):
trobatsEXPRNUM=re.finditer(re_num,segment)
for trobat in trobatsEXPRNUM:
if not trobat.group(0) in [".",","]:
segment=segment.replace(trobat.group(0),code,1)
return(segment)
def splitnumbers(segment,joiner=""):
joiner=joiner+" "
xifres = re.findall(re_num,segment)
for xifra in xifres:
xifrastr=str(xifra)
xifrasplit=xifra.split()
xifra2=joiner.join(xifra)
segment=segment.replace(xifra,xifra2)
return(segment)
def preprocess_segment(segment):
if config.escape_html_input:
segment=html.escape(segment)
if config.unescape_html_input:
segment=html.unescape(segment)
segment=segment.replace(" <tag0>"," <tag0> ")
segment=segment.replace(" </tag0>"," </tag0> ")
segment=remove_control_characters(segment)
hastags=config.tagrestorer.has_tags(segment)
originaltags=config.tagrestorer.get_tags(segment)
printLOG(3,"HAS TAGS",hastags)
printLOG(3,"TAGS",originaltags)
#leading and trailing spaces
config.leading_spaces=len(segment)-len(segment.lstrip())
config.trailing_spaces=len(segment)-len(segment.rstrip())-1
segment=segment.lstrip().rstrip()
if config.pre_replace_NUMs:
segment=replace_NUMs(segment)
if config.pre_split_NUMs:
segment=splitnumbers(segment)
if config.sentencepiece:
try:
segmentPre=" ".join(config.spSL.encode(segment))
except:
printLOG(1,"ERROR preprocess segment:",sys.exc_info())
else:
segmentPre=segment
return(segmentPre)
def postprocess_segment(segmentPre):
try:
if config.sentencepiece:
segmentPost=config.spTL.decode(segmentPre.split())
except:
printLOG(1,"ERROR preprocess segment:",sys.exc_info())
return(segmentPost)
def tokenizationSL(segment):
if config.tokenize_SL and not config.tokenizerSL==None:
if config.tokenizerSLType=="MTUOC":
tokens=config.tokenizerSL.tokenize(segment)
elif config.tokenizerSLType=="Moses":
tokens=" ".join(config.tokenizerSL(segment))
else:
tokens=segment
return(tokens)
def tokenizationTL(segment):
if config.tokenize_TL and not config.tokenizerTL==None:
tokens=config.tokenizerTL.tokenize(segment)
else:
tokens=segment
return(tokens)
def detokenizationSL(tokens):
if not config.tokenizerSL==None:
segment=config.tokenizerSL.detokenize(tokens)
else:
segment=tokens
return(tokens)
def detokenizationTL(tokens):
if not config.tokenizerTL==None:
segment=config.tokenizerL.detokenize(tokens)
else:
segment=tokens
return(tokens)
|