Update pages/Captionize.py

pages/Captionize.py

@@ -5,6 +5,11 @@ import requests
 import streamlit as st
 from transformers import AutoTokenizer, ViTFeatureExtractor, VisionEncoderDecoderModel 
 
+from PIL import Image
+import requests
+from langchain.indexes import VectorstoreIndexCreator
+from langchain.document_loaders import ImageCaptionLoader
+
 
 st.set_page_config(page_title="Captionize")
 
@@ -26,30 +31,18 @@ div.stButton > button:hover {
     }
 </style>""", unsafe_allow_html=True)
 
-
-device='cpu'
-encoder_checkpoint = "nlpconnect/vit-gpt2-image-captioning"
-decoder_checkpoint = "nlpconnect/vit-gpt2-image-captioning"
-model_checkpoint = "nlpconnect/vit-gpt2-image-captioning"
-feature_extractor = ViTFeatureExtractor.from_pretrained(encoder_checkpoint)
-tokenizer = AutoTokenizer.from_pretrained(decoder_checkpoint)
-model = VisionEncoderDecoderModel.from_pretrained(model_checkpoint).to(device)
-
-def predict(image,max_length=64, num_beams=4):
-  #image = image.convert('RGB')
-  image = Image.open(requests.get(image, stream=True).raw).convert("RGB")
-  image = feature_extractor(image, return_tensors="pt").pixel_values.to(device)
-  clean_text = lambda x: x.replace('<|endoftext|>','').split('\n')[0]
-  caption_ids = model.generate(image, max_length = max_length)[0]
-  caption_text = clean_text(tokenizer.decode(caption_ids))
-  return caption_text
-
 pic = st.file_uploader(label="Please upload any Image here 😎",type=['png', 'jpeg', 'jpg'], help="Only 'png', 'jpeg' or 'jpg' formats allowed")
 
+#Image.open(requests.get(pic, stream=True).raw).convert("RGB")
+loader = ImageCaptionLoader(path_images=pic)
+list_docs = loader.load()
+index = VectorstoreIndexCreator().from_loaders([loader])
 
 button = st.button("Generate Caption")
+query = st.text_area("Enter your query 🔍")
 
 if button:
+    Image.open(requests.get(pic, stream=True).raw).convert("RGB")
     # Get Response
-    caption =  predict(pic)
+    caption =  index.query(query)
     st.write(caption)