File size: 1,891 Bytes
955df66 1c19d29 6665931 1c19d29 955df66 a6f5220 f6818d0 955df66 22b1490 955df66 6ca90ab 955df66 6ca90ab e3b62d5 955df66 22b1490 aa01ead 22b1490 6ca90ab 22b1490 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
import streamlit as st
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import PyPDFLoader
from transformers import T5Tokenizer, T5ForConditionalGeneration
from transformers import pipeline
import torch
import base64
from PIL import Image
st.image("https://huggingface.co/spaces/wiwaaw/summary/resolve/main/banner.png")
# Model and tokenizer
model_checkpoint = "MBZUAI/LaMini-Flan-T5-783M"
model_tokenizer = T5Tokenizer.from_pretrained(model_checkpoint)
model = T5ForConditionalGeneration.from_pretrained(model_checkpoint)
# File loader and preprocessing
def preprocess_pdf(file):
loader = PyPDFLoader(file)
pages = loader.load_and_split()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=170, chunk_overlap=70)
texts = text_splitter.split_documents(pages)
final_text = ""
for text in texts:
final_text = final_text + text.page_content
return final_text
@st.cache_data
# Language Model pipeline
def language_model_pipeline(filepath):
summarization_pipeline = pipeline(
'summarization',
model=model,
tokenizer=model_tokenizer,
max_length=500,
min_length=32)
input_text = preprocess_pdf(filepath)
summary_result = summarization_pipeline(input_text)
summarized_text = summary_result[0]['summary_text']
return summarized_text
title = st.title("PDF Summarization using LaMini")
uploaded_file = st.file_uploader("Upload your PDF file", type=['pdf'])
if uploaded_file is not None:
st.success("File Uploaded")
if st.button("Summarize"):
filepath = uploaded_file.name
with open(filepath, "wb") as temp_file:
temp_file.write(uploaded_file.read())
summarized_result = language_model_pipeline(filepath)
st.info("Summarization Complete")
st.success(summarized_result) |