init

.streamlit/secrets.toml

@@ -0,0 +1 @@
+OPENAI_KEY="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"

app.py

@@ -0,0 +1,52 @@
+import streamlit as st
+from PIL import Image
+from utils import get_gpt4V_response
+
+
+st.set_page_config(page_title="GPT-4V Demo", page_icon="🧠", layout="wide")
+
+with st.sidebar:
+    st.title("Parameters")
+    st.write("This is a demo of GPT-4V model. It takes a story, goal, entity and an image as input and generates a response.")
+
+    st.subheader("Sampling Temperature")
+    temperature = st.slider(label="", min_value=0.1, max_value=1.0, value=0.5, step=0.1)
+    st.write("The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.")
+
+    st.subheader("Entity?")
+    entity_opt = st.radio(label="With or Without", options=[1, 0], format_func=lambda x: ["Without", "With"][x])
+
+
+def main():
+    global temperature, entity
+    st.title('What can go wrong?')
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        story = st.text_area("Story", placeholder="Enter the story here")
+
+        goal = st.text_input("Goal", placeholder="Enter the goal here")
+    
+        entity = None
+        if entity_opt:
+            entity = st.text_input("Entity", placeholder="Enter the entity here")
+
+        image = st.file_uploader("Upload Image", type=['jpg', 'png'])
+
+        if image:
+            cols = st.columns(3)
+            with cols[1]:
+                st.image(image, caption="Uploaded Image", use_column_width=True)
+        
+
+        if st.button("Submit"):
+            if not story or not goal or (entity_opt and not entity) or not image:
+                st.error("Please fill all the fields")
+                return
+            with col2:
+                with st.status("Generating response...", expanded=True):
+                    response = get_gpt4V_response(story, goal, entity, image, temperature=temperature)
+                st.write(response)
+
+main()

utils.py

@@ -0,0 +1,119 @@
+from openai import OpenAI
+import streamlit as st
+from PIL import Image
+from io import BytesIO
+import base64
+
+# Convert Image to Base64 
+def im_2_b64(image):
+    image = Image.open(image)
+    buff = BytesIO()
+    image.save(buff, format="JPEG")
+    img_str = base64.b64encode(buff.getvalue())
+    return img_str
+
+
+RANDOM_SEED = 42
+client = OpenAI(api_key=st.secrets["OPENAI_KEY"])
+
+
+with_prompt = """
+In the following task, you will be presented with a image and a story that is, in some manner, related to that goal. You will be given a specific  goal and entity (generally, person or object), and be asked to identify condition necessary for that goal and the alternate condition that could prevent that goal.
+
+Conditions for the output:
+1. Condition: The condition is the necessary condition for the goal to be achieved. If the condition is not met, the goal cannot be achieved. The condition must be related to the entity.
+
+2. Alternate Condition: The alternate condition is a different version of the condition that would prevent the goal from being achieved. It is likely that this alternate condition will contradict information provided in the images and/or story.
+
+Output in a python dictionary where it should have the following keys: 'condition', 'alternate_condition'.
+
+Story: {story}
+
+Entity: {entity}
+
+Goal: {goal}
+"""
+
+wo_prompt = """
+In the following task, you will be presented with a image and a story that is, in some manner, related to that goal. You will be given a specific  goal, and be asked to identify an entity (person or object), condition necessary for that goal and the alternate condition that could prevent that goal.
+
+Conditions for the output:
+1. Entity: The entity is the person or object that the goal is related to. The entity should be a crucial part for achieving the goal.
+
+2. Condition: The condition is the necessary condition for the goal to be achieved. If the condition is not met, the goal cannot be achieved. The condition must be related to the entity.
+
+3. Alternate Condition: The alternate condition is a different version of the condition that would prevent the goal from being achieved. It is likely that this alternate condition will contradict information provided in the images and/or story.
+
+Output in a python dictionary where it should have the following keys: 'entity', 'condition', 'alternate_condition'.
+
+Story: {story}
+
+Goal: {goal}
+"""
+
+data = {
+    "Story id": [],
+    "Prompt": [],
+    "entity": [],
+    "agent": [],
+    "story": [],
+    "Image1": [],
+    "Image2": [],
+    "Image3": [],
+    "GPT-4 Output": [],
+}
+
+
+def get_gpt4V_response(story, goal, entity, image, temperature=0.5):
+    # Convert image to base64
+    image_b64 = im_2_b64(image)
+    image_url = f"data:image/jpeg;base64,{image_b64.decode('utf-8')}"
+    st.write("✅ Image converted")
+
+    if entity:
+        prompt = with_prompt
+        now_prompt = prompt.format(story=story, goal=goal, entity=entity)
+    else:
+        prompt = wo_prompt
+        now_prompt = prompt.format(story=story, goal=goal)
+
+    content = [
+        {"type": "text", "text": now_prompt},
+    ]
+
+    st.write("✅ Prompt created")
+
+    content.append({
+        "type": "image_url",
+        "image_url": {
+            "url": image_url,
+        },
+    })
+    
+    st.write("🚀 Getting Response from GPT4V")
+    response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        seed=RANDOM_SEED,
+        messages=[
+            {
+                "role": "user",
+                "content": content
+            }
+        ],
+        temperature=temperature,
+        max_tokens=256,
+        # top_p=1,
+        # frequency_penalty=0,
+        # presence_penalty=0,
+    )
+    print(response)
+    print("Prompt:")
+    print(now_prompt)
+    out = response.choices[0].message.content
+    print("OUTPUT:", out)
+    print("====================================")
+    print()
+
+    st.write("✅ Response generated")
+
+    return out