File size: 7,019 Bytes
1022069
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46f04af
1022069
 
 
 
 
65226ab
 
 
1022069
 
 
 
4eacea4
1022069
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
# -*- coding: utf-8 -*-
"""ABSTRACTGEN_ES FINAL.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1XdfeMcdDbRuRmOGGiOmkiCP9Yih5JXyF

# installs
"""

import os 
os.system('pip install gpt_2_simple')
os.system('pip install os.system')
os.system('pip install gradio')
os.system('pip install huggingface_hub')
os.system('pip install easynmt')
os.system('pip install sentence-transformers')
os.system('curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash')
os.system('apt-get install git-lfs')
os.system('git lfs install')
os.system('git clone https://huggingface.co/franz96521/AbstractGeneratorES ')
#os.system('cd AbstractGeneratorES')
print(os.getcwd())
print(os.listdir())
# Commented out IPython magic to ensure Python compatibility.
# %cd '/content/AbstractGeneratorES'

"""# Init"""

import gpt_2_simple as gpt2
import os
import tensorflow as tf
import pandas as pd
import re

model_name = "124M"
if not os.path.isdir(os.path.join("models", model_name)):
	print(f"Downloading {model_name} model...")
	gpt2.download_gpt2(model_name=model_name)

path = os.getcwd()+'/AbstractGeneratorES/AbstractGenerator/'
checkpoint_dir =path+'weights/'
data_path = path+'TrainigData/'



file_name_en = 'en'
file_path_en = data_path+file_name_en

file_name_es = 'es'
file_path_es = data_path+file_name_es


prefix= '<|startoftext|>'
sufix ='<|endoftext|>'

import gradio as gr
import random
from easynmt import EasyNMT

from sentence_transformers import SentenceTransformer, util

def generateAbstract(text):
    tf.compat.v1.reset_default_graph()
    sess = gpt2.start_tf_sess()
    gpt2.load_gpt2(sess,checkpoint_dir=checkpoint_dir,run_name='run1')
    txt = gpt2.generate(sess,prefix=str(text)+"\nABSTRACT", return_as_list=True,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)[0]
    return txt
def removeAbstract(text):
    p = text.find("Introducción")
    p2 = text.find("INTRODUCCIÓN")
    print(p,p2)
    if(p != -1):        
       return (text[:p] , text[p:] )  
    if(p2 != -1):        
       return (text[:p2] , text[p2:] )  

def generated_similarity(type_of_input, cn_text): 
  if(type_of_input == "English"):
    tf.compat.v1.reset_default_graph()
    model2 = EasyNMT('opus-mt')
    cn_text = model2.translate(cn_text, target_lang='es')


    print(cn_text)
    abstract_original , body  = removeAbstract(cn_text)
    tf.compat.v1.reset_default_graph()

    generated_Abstract = generateAbstract(body)

    sentences = [abstract_original, generated_Abstract]

    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')  

    #Compute embedding for both lists
    embedding_1= model.encode(sentences[0], convert_to_tensor=True)
    embedding_2 = model.encode(sentences[1], convert_to_tensor=True)

    generated_similarity = util.pytorch_cos_sim(embedding_1, embedding_2)
    ## tensor([[0.6003]])
    return f'''TEXTO SIN ABSTRACT\n
        {body}\n
        ABSTRACT ORIGINAL\n
        {abstract_original}\n
        ABSTRACT GENERADO\n
        {generated_Abstract}\n
        SIMILARIDAD DE ABSTRACT: {float(round(generated_similarity.item()*100, 3))}%
        '''
  elif type_of_input == "Spanish":
    abstract_original , body  = removeAbstract(cn_text)
    tf.compat.v1.reset_default_graph()

    generated_Abstract = generateAbstract(body)

    sentences = [abstract_original, generated_Abstract]

    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')  

    #Compute embedding for both lists
    embedding_1= model.encode(sentences[0], convert_to_tensor=True)
    embedding_2 = model.encode(sentences[1], convert_to_tensor=True)

    generated_similarity = util.pytorch_cos_sim(embedding_1, embedding_2)
    return f'''TEXTO SIN ABSTRACT\n
    {body}\n
    ABSTRACT ORIGINAL\n
    {abstract_original}\n
    ABSTRACT GENERADO\n
    {generated_Abstract}\n
    SIMILARIDAD DE ABSTRACT: {float(round(generated_similarity.item()*100, 3))}%
    '''
def generated_abstract(type_of_input, cn_text):
  if type_of_input == "English":
    tf.compat.v1.reset_default_graph()
    model2 = EasyNMT('opus-mt')
    cn_text = model2.translate(cn_text, target_lang='es')
    generated_Abstract = generateAbstract(cn_text)
    return f'''TEXTO SIN ABSTRACT\n
    {cn_text}\n
    ABSTRACT GENERADO\n
    {generated_Abstract}\n
    '''
  elif type_of_input == "Spanish":
    tf.compat.v1.reset_default_graph()
    generated_Abstract = generateAbstract(cn_text)
    return f'''TEXTO SIN ABSTRACT\n
    {cn_text}\n
    ABSTRACT GENERADO\n
    {generated_Abstract}\n
    '''

block = gr.Blocks()

with block:
    gr.Markdown('''ABSTRACTGEN_ES''')
    gr.Markdown('''An app that can generate abstracts in Spanish based on the text that you input via document text and if you already have an abstract and need a different idea, check how similar the new abstract is to the original one.
        ''')
    gr.Markdown('''FUNCTIONING:
    - Upload your paper with abstract  (text without abstract + original abstract by itself): our app will generate an abstract by its own, and then you can compare how similar it is in content itself with the original abstract that was contained in the file
- Upload your paper without abstract (text without abstract only): our app will generate an abstract that you can use for your paper and work in order for it to be used directly or to inspire you to write a good and well written abstract in Spanish''')
    gr.Markdown(''' We used Blocks (beta), which allows you to build web-based demos in a flexible way using the gradio library. Blocks is a more low-level and flexible alternative to the core Interface class.
        The main problem with this library right now is that
        it doesn't support some functionality that Interface
        class has''')
    gr.Markdown('''To get more info about this project go to: https://sites.google.com/up.edu.mx/somos-pln-abstractgen-es/inicio''')
    with gr.Tab("Full text and text similarity"):
        gr.Markdown("Choose the language:")
        type_of_input = gr.inputs.Radio(["English", "Spanish"], label="Input Language")
        with gr.Row():
            cn_text = gr.inputs.Textbox(placeholder="Full text", lines=7)
        with gr.Row():
            cn_results1 = gr.outputs.Textbox(label="Abstract generado")
        cn_run = gr.Button("Run")
        cn_run.click(generated_similarity, inputs=[type_of_input, cn_text], outputs=[cn_results1])

    with gr.Tab("Only text with no abstract"):
        gr.Markdown("Choose the language:")
        type_of_input = gr.inputs.Radio(["English", "Spanish"], label="Input Language")
        with gr.Row():
            cn_text = gr.inputs.Textbox(placeholder="Text without abstract", lines=7)
        with gr.Row():
            cn_results1 = gr.outputs.Textbox(label="Abstract generado")
        cn_run = gr.Button("Run")
        cn_run.click(generated_abstract, inputs=[type_of_input, cn_text], outputs=cn_results1)

block.launch(debug = True)