Spaces:
Runtime error
Runtime error
| # -*- coding: utf-8 -*- | |
| """ABSTRACTGEN_ES FINAL.ipynb | |
| Automatically generated by Colaboratory. | |
| Original file is located at | |
| https://colab.research.google.com/drive/1XdfeMcdDbRuRmOGGiOmkiCP9Yih5JXyF | |
| # installs | |
| """ | |
| import os | |
| os.system('pip install gpt_2_simple') | |
| os.system('pip install os.system') | |
| os.system('pip install gradio') | |
| os.system('pip install huggingface_hub') | |
| os.system('pip install easynmt') | |
| os.system('pip install sentence-transformers') | |
| os.system('curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash') | |
| os.system('apt-get install git-lfs') | |
| os.system('git lfs install') | |
| os.system('git clone https://huggingface.co/franz96521/AbstractGeneratorES ') | |
| #os.system('cd AbstractGeneratorES') | |
| print(os.getcwd()) | |
| print(os.listdir()) | |
| # Commented out IPython magic to ensure Python compatibility. | |
| # %cd '/content/AbstractGeneratorES' | |
| """# Init""" | |
| import gpt_2_simple as gpt2 | |
| import os | |
| import tensorflow as tf | |
| import pandas as pd | |
| import re | |
| model_name = "124M" | |
| if not os.path.isdir(os.path.join("models", model_name)): | |
| print(f"Downloading {model_name} model...") | |
| gpt2.download_gpt2(model_name=model_name) | |
| path = os.getcwd()+'/AbstractGeneratorES/AbstractGenerator/' | |
| checkpoint_dir =path+'weights/' | |
| data_path = path+'TrainigData/' | |
| file_name_en = 'en' | |
| file_path_en = data_path+file_name_en | |
| file_name_es = 'es' | |
| file_path_es = data_path+file_name_es | |
| prefix= '<|startoftext|>' | |
| sufix ='<|endoftext|>' | |
| import gradio as gr | |
| import random | |
| from easynmt import EasyNMT | |
| from sentence_transformers import SentenceTransformer, util | |
| def generateAbstract(text): | |
| tf.compat.v1.reset_default_graph() | |
| sess = gpt2.start_tf_sess() | |
| gpt2.load_gpt2(sess,checkpoint_dir=checkpoint_dir,run_name='run1') | |
| txt = gpt2.generate(sess,prefix=str(text)+"\nABSTRACT", return_as_list=True,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)[0] | |
| return txt | |
| def removeAbstract(text): | |
| p = text.find("Introducción") | |
| p2 = text.find("INTRODUCCIÓN") | |
| print(p,p2) | |
| if(p != -1): | |
| return (text[:p] , text[p:] ) | |
| if(p2 != -1): | |
| return (text[:p2] , text[p2:] ) | |
| def generated_similarity(type_of_input, cn_text): | |
| if(type_of_input == "English"): | |
| tf.compat.v1.reset_default_graph() | |
| model2 = EasyNMT('opus-mt') | |
| cn_text = model2.translate(cn_text, target_lang='es') | |
| print(cn_text) | |
| abstract_original , body = removeAbstract(cn_text) | |
| tf.compat.v1.reset_default_graph() | |
| generated_Abstract = generateAbstract(body) | |
| sentences = [abstract_original, generated_Abstract] | |
| model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2') | |
| #Compute embedding for both lists | |
| embedding_1= model.encode(sentences[0], convert_to_tensor=True) | |
| embedding_2 = model.encode(sentences[1], convert_to_tensor=True) | |
| generated_similarity = util.pytorch_cos_sim(embedding_1, embedding_2) | |
| ## tensor([[0.6003]]) | |
| return f'''TEXTO SIN ABSTRACT\n | |
| {body}\n | |
| ABSTRACT ORIGINAL\n | |
| {abstract_original}\n | |
| ABSTRACT GENERADO\n | |
| {generated_Abstract}\n | |
| SIMILARIDAD DE ABSTRACT: {float(round(generated_similarity.item()*100, 3))}% | |
| ''' | |
| elif type_of_input == "Spanish": | |
| abstract_original , body = removeAbstract(cn_text) | |
| tf.compat.v1.reset_default_graph() | |
| generated_Abstract = generateAbstract(body) | |
| sentences = [abstract_original, generated_Abstract] | |
| model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2') | |
| #Compute embedding for both lists | |
| embedding_1= model.encode(sentences[0], convert_to_tensor=True) | |
| embedding_2 = model.encode(sentences[1], convert_to_tensor=True) | |
| generated_similarity = util.pytorch_cos_sim(embedding_1, embedding_2) | |
| return f'''TEXTO SIN ABSTRACT\n | |
| {body}\n | |
| ABSTRACT ORIGINAL\n | |
| {abstract_original}\n | |
| ABSTRACT GENERADO\n | |
| {generated_Abstract}\n | |
| SIMILARIDAD DE ABSTRACT: {float(round(generated_similarity.item()*100, 3))}% | |
| ''' | |
| def generated_abstract(type_of_input, cn_text): | |
| if type_of_input == "English": | |
| tf.compat.v1.reset_default_graph() | |
| model2 = EasyNMT('opus-mt') | |
| cn_text = model2.translate(cn_text, target_lang='es') | |
| generated_Abstract = generateAbstract(cn_text) | |
| return f'''TEXTO SIN ABSTRACT\n | |
| {cn_text}\n | |
| ABSTRACT GENERADO\n | |
| {generated_Abstract}\n | |
| ''' | |
| elif type_of_input == "Spanish": | |
| tf.compat.v1.reset_default_graph() | |
| generated_Abstract = generateAbstract(cn_text) | |
| return f'''TEXTO SIN ABSTRACT\n | |
| {cn_text}\n | |
| ABSTRACT GENERADO\n | |
| {generated_Abstract}\n | |
| ''' | |
| block = gr.Blocks() | |
| with block: | |
| gr.Markdown('''ABSTRACTGEN_ES''') | |
| gr.Markdown('''An app that can generate abstracts in Spanish based on the text that you input via document text and if you already have an abstract and need a different idea, check how similar the new abstract is to the original one. | |
| ''') | |
| gr.Markdown('''FUNCTIONING: | |
| - Upload your paper with abstract (text without abstract + original abstract by itself): our app will generate an abstract by its own, and then you can compare how similar it is in content itself with the original abstract that was contained in the file | |
| - Upload your paper without abstract (text without abstract only): our app will generate an abstract that you can use for your paper and work in order for it to be used directly or to inspire you to write a good and well written abstract in Spanish''') | |
| gr.Markdown(''' We used Blocks (beta), which allows you to build web-based demos in a flexible way using the gradio library. Blocks is a more low-level and flexible alternative to the core Interface class. | |
| The main problem with this library right now is that | |
| it doesn't support some functionality that Interface | |
| class has''') | |
| gr.Markdown('''To get more info about this project go to: https://sites.google.com/up.edu.mx/somos-pln-abstractgen-es/inicio''') | |
| with gr.Tab("Full text and text similarity"): | |
| gr.Markdown("Choose the language:") | |
| type_of_input = gr.inputs.Radio(["English", "Spanish"], label="Input Language") | |
| with gr.Row(): | |
| cn_text = gr.inputs.Textbox(placeholder="Full text", lines=7) | |
| with gr.Row(): | |
| cn_results1 = gr.outputs.Textbox(label="Abstract generado") | |
| cn_run = gr.Button("Run") | |
| cn_run.click(generated_similarity, inputs=[type_of_input, cn_text], outputs=[cn_results1]) | |
| with gr.Tab("Only text with no abstract"): | |
| gr.Markdown("Choose the language:") | |
| type_of_input = gr.inputs.Radio(["English", "Spanish"], label="Input Language") | |
| with gr.Row(): | |
| cn_text = gr.inputs.Textbox(placeholder="Text without abstract", lines=7) | |
| with gr.Row(): | |
| cn_results1 = gr.outputs.Textbox(label="Abstract generado") | |
| cn_run = gr.Button("Run") | |
| cn_run.click(generated_abstract, inputs=[type_of_input, cn_text], outputs=cn_results1) | |
| block.launch(debug = True) | |