Upload 9 files

app.py

@@ -6,50 +6,22 @@
 
 import gradio as gr
 
-import ctranslate2
-from split import split_string
-
-translator_zh2en = ctranslate2.Translator("zh-en_model/", device="cpu")##路径
-translator2_zh2en = ctranslate2.Translator("zh2en_cmodel/", device="cpu")##路径
-translator_en2zh = ctranslate2.Translator("en-zh_model/", device="cpu")##路径
-translator2_en2zh = ctranslate2.Translator("en2zh_cmodel", device="cpu")##路径
-
-def translate(input_tokens, input_tokens2, mode):
-    
-    input_tokens = input_tokens.split()
-    input_tokens2 = input_tokens2.split()
-
-    source = split_string(input_tokens)
-    lenth = len(source)
-
-    source2 = split_string(input_tokens2)
-    lenth2 = len(source2)
-
-    results = []
-    results2 = []
-
-    if mode == "汉译英" :
-        results = translator_zh2en.translate_batch(source)##翻译的分词分句
-        results2 = translator2_zh2en.translate_batch(source2)##翻译的分词分句
-    else :
-        results = translator_en2zh.translate_batch(source)##翻译的分词分句
-        results2 = translator2_en2zh.translate_batch(source2)##翻译的分词分句
-
-    target = []
-    target2 = []
-    
-    for i in range(0, lenth, 1):
-        target = target + results[i].hypotheses[0]
-    for i in range(0, lenth2, 1):
-        target2 = target2 + results2[i].hypotheses[0]
-
-    #print(results[0].hypotheses[0])##results[0]为第0句，hypotheses[0]保持0
-    ##print(results[1].hypotheses[0])
-    #return results[0].hypotheses[0]
-    return ' '.join(target),' '.join(target2)
-
-demo = gr.Interface(fn=translate,
-                     inputs=["text", "text", gr.Dropdown(["汉译英", "英译汉"])],
-                     outputs=["text", "text"],)
+from tokenizer import tokenize, tokenize2
+from translater import translate
+from detokenizer import detokenize, detokenize2
+
+def run(source_text, mode):
+    source_tokens = tokenize(source_text, mode)
+    source_tokens2 = tokenize2(source_text, mode)
+    source_tokenized_text = ' '.join(source_tokens)
+    target_tokens, target_tokens2 = translate(source_tokens, source_tokens2, mode)
+    target_text = detokenize(target_tokens, mode)
+    target_text2 = detokenize2(target_tokens2, mode)
+    return target_text, target_text2, source_tokenized_text
+
+
+demo = gr.Interface(fn=run,
+                     inputs=["text", gr.Dropdown(["汉译英", "英译汉"])],
+                     outputs=["text", "text", "text"],)
 
 demo.launch()

detokenizer.py

@@ -0,0 +1,32 @@
+import re
+import sys
+from sacremoses import MosesDetokenizer
+
+md_en = MosesDetokenizer(lang='en')
+md_zh = MosesDetokenizer(lang='zh')
+
+def moses_detokenize(tokens, language='en'):
+    en_detokenizer = MosesDetokenizer(lang=language)
+    
+    stdout = en_detokenizer.detokenize(tokens,return_str=True)
+
+    # 返回处理后的句子
+    return stdout.strip()
+
+def detokenize(tokens, mode):
+    if mode == "汉译英" :
+        text = moses_detokenize(tokens)
+        text = re.sub(r" n't", "n't",text)
+    else :
+        text = ''.join(tokens)
+
+    return text
+
+def detokenize2(tokens, mode):
+    if mode == "汉译英" :
+        answer_en_bpe = md_en.detokenize(tokens,return_str=True)
+        text = re.sub(r"@@ ", "",answer_en_bpe)
+    else :    
+        answer_zh_bpe = md_zh.detokenize(tokens,return_str=True)
+        text = re.sub(r"@@ ", "",answer_zh_bpe) 
+    return text

requirements.txt

@@ -1 +1,6 @@
-ctranslate2==4.1.0
+ctranslate2==4.1.0
+spacy==3.7.4
+nltk==3.8.1
+jieba==0.42.1
+sacremoses==0.1.1
+subword_nmt==0.3.8

tokenizer.py

@@ -0,0 +1,78 @@
+import spacy
+from spacy.tokens import Doc
+
+# 加载英文模型
+nlp = spacy.load('en_core_web_sm')
+
+import nltk
+from nltk.tokenize import word_tokenize
+
+import jieba
+
+from sacremoses import MosesTokenizer
+from subword_nmt import apply_bpe
+import codecs
+
+jieba1 = jieba.Tokenizer()
+jieba2 = jieba.Tokenizer()
+jieba2.load_userdict('model2_data/dict.zh.txt')
+
+mt_zh = MosesTokenizer(lang='zh')
+with codecs.open('model2_data/bpecode.zh', 'r', 'utf-8') as f:
+    bpe_zh_f = apply_bpe.BPE(f)  
+
+#英文部分初始化，定义tokenize等等
+mt_en = MosesTokenizer(lang='en')
+with codecs.open('model2_data/bpecode.en', 'r', 'utf-8') as f:
+    bpe_en_f = apply_bpe.BPE(f)
+
+def spacy_tokenize(line):
+    # 使用spaCy处理文本
+    doc = nlp(line)
+    # 获取单词列表
+    words = [token.text for token in doc]
+    # 将单词连接成一个字符串，单词间用一个空格间隔
+    return ' '.join(words)
+
+
+def nltk_tokenize(line):
+    # 使用NLTK的word_tokenize进行分词
+    tokens = word_tokenize(line)
+    #print(tokens)
+    return tokens
+
+
+def jieba_tokenize(line):
+    # 使用jieba进行分词
+    tokens = list(jieba1.cut(line.strip()))  # strip用于去除可能的空白字符
+    #print(tokens)
+    return tokens
+
+def tokenize(line, mode):
+    if mode == "汉译英" :
+        return jieba_tokenize(line)
+    else :
+        return nltk_tokenize(spacy_tokenize(line))
+
+
+def jieba_tokenize2(line):
+    tokens = list(jieba2.cut(line.strip()))
+    return tokens
+
+def mt_bpe_zh(line):
+    zh_tok = mt_zh.tokenize(line)
+    bpe_zh = bpe_zh_f.segment_tokens(zh_tok)
+    print(bpe_zh)
+    return bpe_zh
+
+def mt_bpe_en(line):
+    en_tok = mt_en.tokenize(line)
+    bpe_en = bpe_en_f.segment_tokens(en_tok)
+    print(bpe_en)
+    return bpe_en
+
+def tokenize2(line, mode):
+    if mode == "汉译英" :
+        return mt_bpe_zh(' '.join(jieba_tokenize2(line)))
+    else :
+        return mt_bpe_en(line)

translater.py

@@ -0,0 +1,35 @@
+import ctranslate2
+from split import split_string
+
+translator_zh2en = ctranslate2.Translator("zh-en_model/", device="cpu")##路径
+translator2_zh2en = ctranslate2.Translator("zh2en_cmodel/", device="cpu")##路径
+translator_en2zh = ctranslate2.Translator("en-zh_model/", device="cpu")##路径
+translator2_en2zh = ctranslate2.Translator("en2zh_cmodel", device="cpu")##路径
+
+def translate(input_tokens, input_tokens2, mode):
+    
+    source = split_string(input_tokens)
+    lenth = len(source)
+
+    source2 = split_string(input_tokens2)
+    lenth2 = len(source2)
+
+    if mode == "汉译英" :
+        results = translator_zh2en.translate_batch(source)##翻译的分词分句
+        results2 = translator2_zh2en.translate_batch(source2)##翻译的分词分句
+    else :
+        results = translator_en2zh.translate_batch(source)##翻译的分词分句
+        results2 = translator2_en2zh.translate_batch(source2)##翻译的分词分句
+
+    target = []
+    target2 = []
+    
+    for i in range(0, lenth, 1):
+        target = target + results[i].hypotheses[0]
+    for i in range(0, lenth2, 1):
+        target2 = target2 + results2[i].hypotheses[0]
+
+    #print(results[0].hypotheses[0])##results[0]为第0句，hypotheses[0]保持0
+    ##print(results[1].hypotheses[0])
+    #return results[0].hypotheses[0]
+    return target,target2