fixing image2text

image2text.py

@@ -4,7 +4,8 @@ from utils import text_encoder, image_encoder
 from PIL import Image
 from jax import numpy as jnp
 import pandas as pd
-
+import requests
+import jax
 
 def app():
     st.title("From Image to Text")
@@ -17,23 +18,31 @@ def app():
         image classification task!
 
         🤌 Italian mode on! 🤌
+        
+        For example, try to write "cat" in the space for label1 and "dog" in the space for label2 and the run
+        "classify"!
 
         """
     )
 
-    filename = st.file_uploader(
-        "Choose an image from your computer", type=["jpg", "jpeg", "png"]
+    image_url = st.text_input(
+
+        "You can input the URL of an image",
+
+        value="https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Domestic_Cat_Face_Shot.jpg/1280px-Domestic_Cat_Face_Shot.jpg",
+
     )
 
+
     MAX_CAP = 4
 
     col1, col2 = st.beta_columns([3, 1])
 
     with col2:
         captions_count = st.selectbox(
-            "Number of labels", options=range(1, MAX_CAP + 1)
+            "Number of labels", options=range(1, MAX_CAP + 1), index=1
         )
-        compute = st.button("Compute")
+        compute = st.button("Classify")
 
     with col1:
         captions = list()
@@ -43,7 +52,7 @@ def app():
     if compute:
         captions = [c for c in captions if c != ""]
 
-        if not captions or not filename:
+        if not captions or not image_url:
             st.error("Please choose one image and at least one label")
         else:
             with st.spinner("Computing..."):
@@ -55,13 +64,14 @@ def app():
                     text_embeds.extend(text_encoder(c, model, tokenizer))
 
                 text_embeds = jnp.array(text_embeds)
+                image_raw = requests.get(image_url, stream=True).raw
 
-                image = Image.open(filename).convert("RGB")
+                image = Image.open(image_raw).convert("RGB")
                 transform = get_image_transform(model.config.vision_config.image_size)
                 image_embed = image_encoder(transform(image), model)
 
                 # we could have a softmax here
-                cos_similarities = jnp.matmul(image_embed, text_embeds.T)
+                cos_similarities = jax.nn.softmax(jnp.matmul(image_embed, text_embeds.T))
 
                 chart_data = pd.Series(cos_similarities[0], index=captions)
 

requirements.txt

@@ -5,4 +5,5 @@ torch
 torchvision
 natsort
 stqdm
-pandas
+pandas
+requests