|
import pandas as pd |
|
import streamlit as st |
|
from transformers import * |
|
from carga_articulos import cargar_articulos |
|
from preprocesamiento_articulos import limpieza_articulos |
|
from entrenamiento_modelo import term_document_matrix, tf_idf_score |
|
from resultados_consulta import resultados_consulta, detalles_resultados |
|
import tensorflow as tf |
|
|
|
def crear_indice(): |
|
df=cargar_articulos() |
|
vocab = limpieza_articulos(df) |
|
|
|
td_matrix=term_document_matrix(df, vocab, 'ID', 'titulo') |
|
td_idf_matrix=tf_idf_score(td_matrix, df.ID.values) |
|
|
|
td_idf_matrix.to_csv('articulos_indexados.csv') |
|
|
|
def load_qa_model(): |
|
|
|
tokenizer = AutoTokenizer.from_pretrained('mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es', use_fast="false") |
|
model = TFDistilBertForQuestionAnswering.from_pretrained("mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es", from_pt=True) |
|
return tokenizer, model |
|
|
|
|
|
def main(): |
|
|
|
|
|
|
|
|
|
|
|
st.set_page_config(page_title="Buscador de noticias periodicos dominicanos", page_icon="📰") |
|
st.image('repartidor_periodicos.jpeg', width=150) |
|
st.header('El Repartidor Dominicano') |
|
|
|
|
|
|
|
st.sidebar.header("Acerca de") |
|
st.sidebar.markdown( |
|
"[Streamlit](https://streamlit.io) is a Python library that allows the creation of interactive, data-driven web applications in Python." |
|
) |
|
|
|
st.sidebar.header("Artículos Indexados") |
|
st.sidebar.markdown( |
|
""" |
|
- [Streamlit Documentation](https://docs.streamlit.io/) |
|
- [Cheat sheet](https://docs.streamlit.io/library/cheatsheet) |
|
- [Book](https://www.amazon.com/dp/180056550X) (Getting Started with Streamlit for Data Science) |
|
- [Blog](https://blog.streamlit.io/how-to-master-streamlit-for-data-science/) (How to master Streamlit for data science) |
|
""" |
|
) |
|
|
|
st.sidebar.header("Disclaimer") |
|
st.sidebar.markdown( |
|
"You can quickly deploy Streamlit apps using [Streamlit Community Cloud](https://streamlit.io/cloud) in just a few clicks." |
|
) |
|
|
|
st.sidebar.header("¿Te gustó mi sitio? ¡Cómprame una café!") |
|
with st.sidebar: |
|
components.html( |
|
""" |
|
<div id="donate-button-container"> |
|
<div id="donate-button"></div> |
|
<script src="https://www.paypalobjects.com/donate/sdk/donate-sdk.js" charset="UTF-8"></script> |
|
<script> |
|
PayPal.Donation.Button({ |
|
env:'production', |
|
hosted_button_id:'VK5ZAB52ZYDNA', |
|
image: { |
|
src:'https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif', |
|
alt:'Donate with PayPal button', |
|
title:'PayPal - The safer, easier way to pay online!', |
|
} |
|
}).render('#donate-button'); |
|
</script> |
|
</div> |
|
""" |
|
) |
|
|
|
df=cargar_articulos() |
|
articulos_indexados = pd.read_csv('articulos_indexados.csv') |
|
articulos_indexados = articulos_indexados.set_index('Unnamed: 0') |
|
tokenizer, qa_model = load_qa_model() |
|
|
|
|
|
query = st.text_input( |
|
"Escribe tus términos de búsqueda o haz una pregunta terminando con el caracter ?:" |
|
) |
|
|
|
if query: |
|
|
|
if ('?' in query): |
|
st.write("Contestando a: ", query) |
|
text='Un texto es una composición de signos codificados en un sistema de escritura que forma una unidad de sentido.' |
|
inputs = tokenizer(query, text, return_tensors='tf') |
|
outputs = qa_model(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask']) |
|
answer_start_index = int(tf.math.argmax(outputs.start_logits, axis=-1)[0]) |
|
answer_end_index = int(tf.math.argmax(outputs.end_logits, axis=-1)[0]) |
|
predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1] |
|
answer=tokenizer.decode(predict_answer_tokens) |
|
st.info(answer) |
|
|
|
else: |
|
|
|
st.write("Buscando: ", query) |
|
result = resultados_consulta(df,articulos_indexados, query) |
|
|
|
if result.empty: |
|
st.info("No se encontraron artículos para la búsqueda solicitada") |
|
|
|
else: |
|
|
|
df_results=detalles_resultados(df,result) |
|
N_cards_per_row = 1 |
|
for n_row, row in df_results.reset_index().iterrows(): |
|
i = n_row%N_cards_per_row |
|
if i==0: |
|
st.write("---") |
|
cols = st.columns(N_cards_per_row, gap="large") |
|
|
|
with cols[n_row%N_cards_per_row]: |
|
st.caption(f"{row['feed'].strip()} - {row['seccion'].strip()} - {row['fecha'].strip()} ") |
|
st.markdown(f"**{row['titulo'].strip()}**") |
|
st.markdown(f"{row['resumen'].strip()}") |
|
st.markdown(f"{row['link']}") |
|
if __name__ == "__main__": |
|
main() |
|
|