gpt4V_analysis.py

from secrets_key import OPENAI_KEY, RANDOM_SEED
from openai import OpenAI
import json
import pandas as pd
from pprint import pprint


client = OpenAI(api_key=OPENAI_KEY)


ti_prompt = """
In the following HIT, you will be presented with a collection of images and a story that is, in some manner, related to that goal. You are also given a specific agent and an entity (generally, person or object).
1. Identify a goal that the agent in the story is trying to achieve. 
2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the images and/or story.

Make each response minimum 5 words long and maximum 25 words long.

Story: {story}

Agent: {agent}
Entity: {entity}
"""

t_prompt = """
In the following HIT, you will be presented with a story. You are also given a specific agent and an entity (generally, person or object).
1. Identify a goal that the agent in the story is trying to achieve.
2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the story.

Story: {story}

Agent: {agent}
Entity: {entity}
"""

i_prompt = """
In the following HIT, you will be presented with a collection of images of a story. You are also given a specific agent and an entity (generally, person or object).
1. Identify a goal that the agent in the story is trying to achieve.
2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the images.

Agent: {agent}
Entity: {entity}
"""



def analysis(story, agent, entity, images, text=True, image=True):
    if text and image:
        now_prompt = ti_prompt.format(story=story, agent=agent, entity=entity)
    elif text:
        now_prompt = t_prompt.format(story=story, agent=agent, entity=entity)
    elif image:
        now_prompt = i_prompt.format(agent=agent, entity=entity)
    else:
        raise ValueError("text and image cannot both be False")

    content = [
        {"type": "text", "text": now_prompt},
    ]
    if image:
        content.extend([{"type": "image_url", "image_url": image} for image in images])
    
    response = client.chat.completions.create(
        model="gpt-4-vision-preview",
        seed=RANDOM_SEED,
        messages=[
            {
                "role": "user",
                "content": content
            }
        ],
        temperature=1,
        max_tokens=256,
        top_p=1,
        frequency_penalty=0,
        presence_penalty=0,
    )
    out = response.choices[0].message.content
    print("OUTPUT:", out)
    print()

    return out

if __name__ == '__main__':
    df = pd.read_csv('./results.csv')
    df = df.sample(frac=1, random_state=42).reset_index(drop=True)
    count = 0
    done = set()
    data = []
    for ind, row in df.iterrows():
        item_id = row['Input.item_id']
        if item_id in done:
            continue
        done.add(item_id)

        try:
            story = row['Input.story']
            agent = row['Input.agent']
            entity = row['Input.entity']
            images = [row[f'Input.image{i}'] for i in range(1,4)]

            print("HITId:", row['HITId'])
            print("Prompt:", ti_prompt.format(story=story, agent=agent, entity=entity))
            print()
            print("Image1:", images[0])
            print("Image2:", images[1])
            print("Image3:", images[2])
            print()

            gpt4v_out = ""
            for text in [True, False]:
                for image in [True, False]:
                    if text or image:
                        print("Text Visible:", text)
                        print("Image Visible:", image)
                        out = analysis(story, agent, entity, images, text=text, image=image)

                        # gpt4v_out += f"#### Text Visible: {text}\n"
                        # gpt4v_out += f"#### Image Visible: {image}\n"

                        if text and image:
                            gpt4v_out += f"### Both text and image are visible.\n"
                        elif text:
                            gpt4v_out += f"### Only text is visible.\n"
                        elif image:
                            gpt4v_out += f"### Only image is visible.\n"
                        
                        gpt4v_out += f"#### Output: \n"
                        gpt4v_out += out
                        gpt4v_out += "\n"

            obj = row.to_dict()
            obj['GPT4V.out'] = gpt4v_out

            data.append(obj)

            print("====================================")
            print()

            count += 1
        except Exception as e:
            done.remove(item_id)
            print("ERROR:", e)
            print("====================================")
            print()

        if count == 10:
            break

    df = pd.DataFrame(data)
    df.to_csv('./results_with_gpt4v.csv', index=False)