init

.gitignore

@@ -0,0 +1,179 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python
+
+.streamlit/
+secrets_key.py

app.py

@@ -0,0 +1,87 @@
+import streamlit as st
+import pandas as pd
+import random
+from auth import check_password
+
+if not check_password():
+    st.stop()
+
+st.set_page_config(
+    page_title="ARL - Pilot 2b - Agent 80 - GPT4V",
+    page_icon="🎆",
+    layout="wide",
+)
+
+df = pd.read_csv('./results_with_gpt4v.csv')
+print(df.columns)
+
+
+query = st.experimental_get_query_params()
+if not query or 'id4' not in query or not query['id4']:
+    rnd_ind = random.randint(0, len(df) - 1)
+    # id = rnd_ind
+    id = 0
+    st.experimental_set_query_params(id=id)
+    file_id = id
+else:
+    file_id = int(query['id4'][0])
+
+
+st.experimental_set_query_params(id4=file_id)
+
+data = df.iloc[file_id]
+# st.table(data)
+# print(data)
+
+def show_story():
+    st.subheader('Impacted Story')
+    number_of_story_lines = int(data['Input.number_of_lines'])
+    print(number_of_story_lines)
+    story = data['Input.story'].split('. ')
+    for i in range(number_of_story_lines):
+        row_name = f'Answer.line{i}_changed.on'
+        print(row_name, data[row_name])
+        if data[row_name]:
+            st.markdown(f'**Line {i+1}:** <mark style="background-color: white">{story[i]}</mark>', unsafe_allow_html=True)
+        else:
+            st.markdown(f'**Line {i+1}:** {story[i]}')
+
+
+def show_data():
+    col1, col2 = st.columns(2)
+
+    with col1:
+        img1, img2, img3 = st.columns(3)
+        with img1:
+            st.image(data['Input.image1'])
+        with img2:
+            st.image(data['Input.image2'])
+        with img3:
+            st.image(data['Input.image3'])
+
+        st.subheader('Entity')
+        st.write(data['Input.entity'])
+        st.subheader('Agent')
+        st.write(data['Input.agent'])
+        # st.subheader('Input Goal')
+        # st.write(data['Input.goal'])
+        st.subheader('Story')
+        st.write(data['Input.story'])
+
+    with col2:
+        st.header('GPT4V Output')
+        st.markdown(data['GPT4V.out'], unsafe_allow_html=True)
+
+    # from  api import show_likert_scale
+    # done = show_likert_scale('step_all', 'Whole', data['AssignmentId'])
+    
+    done = True
+    if st.button("Next", disabled=not done):
+        id = file_id + 1
+        st.experimental_set_query_params(id4=id)
+        st.experimental_rerun()
+
+
+
+
+show_data()

auth.py

@@ -0,0 +1,29 @@
+import streamlit as st
+
+def check_password():
+    """Returns `True` if the user had the correct password."""
+
+    def password_entered():
+        """Checks whether a password entered by the user is correct."""
+        if st.session_state["password"] in st.secrets["passwords"]:
+            st.session_state['user'] = st.session_state["password"]
+            st.session_state["password_correct"] = True
+        else:
+            st.session_state["password_correct"] = False
+
+    if "password_correct" not in st.session_state:
+        # First run, show input for password.
+        st.text_input(
+            "Password", type="password", on_change=password_entered, key="password"
+        )
+        return False
+    elif not st.session_state["password_correct"]:
+        # Password not correct, show input + error.
+        st.text_input(
+            "Password", type="password", on_change=password_entered, key="password"
+        )
+        st.error("😕 Password incorrect")
+        return False
+    else:
+        # Password correct.
+        return True

gpt4V_analysis.py

@@ -0,0 +1,153 @@
+from secrets_key import OPENAI_KEY, RANDOM_SEED
+from openai import OpenAI
+import json
+import pandas as pd
+from pprint import pprint
+
+
+client = OpenAI(api_key=OPENAI_KEY)
+
+
+ti_prompt = """
+In the following HIT, you will be presented with a collection of images and a story that is, in some manner, related to that goal. You are also given a specific agent and an entity (generally, person or object).
+1. Identify a goal that the agent in the story is trying to achieve. 
+2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
+3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the images and/or story.
+
+Make each response minimum 5 words long and maximum 25 words long.
+
+Story: {story}
+
+Agent: {agent}
+Entity: {entity}
+"""
+
+t_prompt = """
+In the following HIT, you will be presented with a story. You are also given a specific agent and an entity (generally, person or object).
+1. Identify a goal that the agent in the story is trying to achieve.
+2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
+3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the story.
+
+Story: {story}
+
+Agent: {agent}
+Entity: {entity}
+"""
+
+i_prompt = """
+In the following HIT, you will be presented with a collection of images of a story. You are also given a specific agent and an entity (generally, person or object).
+1. Identify a goal that the agent in the story is trying to achieve.
+2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
+3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the images.
+
+Agent: {agent}
+Entity: {entity}
+"""
+
+
+
+def analysis(story, agent, entity, images, text=True, image=True):
+    if text and image:
+        now_prompt = ti_prompt.format(story=story, agent=agent, entity=entity)
+    elif text:
+        now_prompt = t_prompt.format(story=story, agent=agent, entity=entity)
+    elif image:
+        now_prompt = i_prompt.format(agent=agent, entity=entity)
+    else:
+        raise ValueError("text and image cannot both be False")
+
+    content = [
+        {"type": "text", "text": now_prompt},
+    ]
+    if image:
+        content.extend([{"type": "image_url", "image_url": image} for image in images])
+    
+    response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        seed=RANDOM_SEED,
+        messages=[
+            {
+                "role": "user",
+                "content": content
+            }
+        ],
+        temperature=1,
+        max_tokens=256,
+        top_p=1,
+        frequency_penalty=0,
+        presence_penalty=0,
+    )
+    out = response.choices[0].message.content
+    print("OUTPUT:", out)
+    print()
+
+    return out
+
+if __name__ == '__main__':
+    df = pd.read_csv('./results.csv')
+    df = df.sample(frac=1, random_state=42).reset_index(drop=True)
+    count = 0
+    done = set()
+    data = []
+    for ind, row in df.iterrows():
+        item_id = row['Input.item_id']
+        if item_id in done:
+            continue
+        done.add(item_id)
+
+        try:
+            story = row['Input.story']
+            agent = row['Input.agent']
+            entity = row['Input.entity']
+            images = [row[f'Input.image{i}'] for i in range(1,4)]
+
+            print("HITId:", row['HITId'])
+            print("Prompt:", ti_prompt.format(story=story, agent=agent, entity=entity))
+            print()
+            print("Image1:", images[0])
+            print("Image2:", images[1])
+            print("Image3:", images[2])
+            print()
+
+            gpt4v_out = ""
+            for text in [True, False]:
+                for image in [True, False]:
+                    if text or image:
+                        print("Text Visible:", text)
+                        print("Image Visible:", image)
+                        out = analysis(story, agent, entity, images, text=text, image=image)
+
+                        # gpt4v_out += f"#### Text Visible: {text}\n"
+                        # gpt4v_out += f"#### Image Visible: {image}\n"
+
+                        if text and image:
+                            gpt4v_out += f"### Both text and image are visible.\n"
+                        elif text:
+                            gpt4v_out += f"### Only text is visible.\n"
+                        elif image:
+                            gpt4v_out += f"### Only image is visible.\n"
+                        
+                        gpt4v_out += f"#### Output: \n"
+                        gpt4v_out += out
+                        gpt4v_out += "\n"
+
+            obj = row.to_dict()
+            obj['GPT4V.out'] = gpt4v_out
+
+            data.append(obj)
+
+            print("====================================")
+            print()
+
+            count += 1
+        except Exception as e:
+            done.remove(item_id)
+            print("ERROR:", e)
+            print("====================================")
+            print()
+
+        if count == 1:
+            break
+
+    df = pd.DataFrame(data)
+    df.to_csv('./results_with_gpt4v.csv', index=False)

gpt4V_entity.py

@@ -0,0 +1,76 @@
+from secrets_key import OPENAI_KEY, RANDOM_SEED
+from openai import OpenAI
+import json
+import pandas as pd
+from pprint import pprint
+
+
+client = OpenAI(api_key=OPENAI_KEY)
+
+
+prompt = """
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: {story}
+"""
+
+
+def get_entity_gpt4V(row):
+    story = row['Input.story']
+    now_prompt = prompt.format(story=story)
+    content = [
+        {"type": "text", "text": now_prompt},
+    ]
+    images = []
+    for i in range(1,4):
+        image_url = row[f'Input.image{i}']
+        images.append(image_url)
+        content.append({
+            "type": "image_url",
+            "image_url": {
+                "url": image_url,
+            },
+        })
+    
+    response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        seed=RANDOM_SEED,
+        messages=[
+            {
+                "role": "user",
+                "content": content
+            }
+        ],
+        temperature=1,
+        max_tokens=256,
+        top_p=1,
+        frequency_penalty=0,
+        presence_penalty=0,
+    )
+
+    print(row['HITId'])
+    print(now_prompt)
+    pprint(images)
+    out = response.choices[0].message.content
+    print("OUTPUT:", out)
+    print("====================================")
+    print()
+
+if __name__ == '__main__':
+    df = pd.read_csv('./results.csv')
+
+    count = 0
+    done = set()
+    for ind, row in df.iterrows():
+        item_id = row['Input.item_id']
+        if item_id in done:
+            continue
+        done.add(item_id)
+        get_entity_gpt4V(row)
+        count += 1
+
+        if count == 10:
+            break

gpt4V_entity.txt

@@ -0,0 +1,208 @@
+36MUZ9VAE8MI4ZXG5C562HJOH58ED2
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: The parade is starting and people line the streets. The folks who live upstairs in the buildings get a good view. The really cool floats are coming next. It's raining harder but no one seems to mind. The people came prepared with umbrellas for this special occasion.The more the parade goes on, the better the displays seem to be. The zoo animals are very well done and everybody is standing in the rain to see them. The man's dog is a bit confused by these animals. Everyone is having a good time watching the parade and chatting with their neighbors.
+
+['https://drive.google.com/uc?export=view&id=1LhLpWAkuIIVDf5jl1E7_DppoHQwl-7P9',
+ 'https://drive.google.com/uc?export=view&id=11r_tEEa7QcEkalT_wcZOtOxnaztqtETh',
+ 'https://drive.google.com/uc?export=view&id=1NsZpx_otGbh9Jz00A9eyF3xoUfTnvNxH']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'building with a sign "Buck's 5th Avenue"'},
+    {'image_number': 2, 'person/object': 'white dog'},
+    {'image_number': 3, 'person/object': 'person in blue with umbrella'}
+]
+```
+====================================
+
+3P0I4CQYV0R3PWAWGLDSZH2NSNJWOW
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: The townsfolk are gathered for a parade. There are many families with children. This is a special day. Everyone is here for the celebration. It has been raining, so many people brought their umbrellas and rain caps. The parade is downtown, so people who live in apartments there can just open their windows and watch from home.The floats are comprised of different animals. Everyone is having a great time.This is a special day and everyone has been looking forward to viewing all the different animal floats people made for the parade.Families are gathered together, along with friends. It turns out the parade happens on a rainy day, so everyone brought their rain gear. The rain doesn't dampen the mood though, and everyone has a great time.There are people dressed up like flamingos, a rhinoceros float and a zebra float, along with many other festive floats.
+
+['https://drive.google.com/uc?export=view&id=1IzZcjV22ftn_wK9Ug7zihsCB_KKnsc8k',
+ 'https://drive.google.com/uc?export=view&id=1RDdlOJP5udN_uSlamKEF9qnEC_aWw5yI',
+ 'https://drive.google.com/uc?export=view&id=1DzEy5rzApDKpFdMbblkDJcQ1c1abU-WE']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'building'},
+    {'image_number': 2, 'person/object': 'dog'},
+    {'image_number': 3, 'person/object': 'camera'}
+]
+```
+====================================
+
+3RBI0I35XGNISCCACOBOWBPS84O3Y8
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: This was the perfect place for a meeting. Jack had made contact with his asset the night before and was ready for the next step on the plan. The large crowd paired with the rain made for the perfect cover. The go signal for the drop was when the rhino float went by the middle of the street.Scanning the crowd, Jack finally saw his asset. He was low to the ground, wearing a red raincoat. `Could you be anymore conspicuous?` Jack said to himself. Just then the rhino float went by and Jack's asset darted across the street. Jack felt the envelope leave his hand and a wet nose graze his thumb. By the time Jack looked back up his asset was across the street again, watching the zebra float go by.
+
+['https://drive.google.com/uc?export=view&id=1SeI0CQU_jMNTGvu5E5GfT-4Sh4jMOILA',
+ 'https://drive.google.com/uc?export=view&id=1JDv6-QjI-alRhd2e5yv6ZaUgs7dE-k-N',
+ 'https://drive.google.com/uc?export=view&id=1G2-19uq96Y2NB12l6QpsCelVY7KQFiJ6']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'person in a blue dress'},
+    {'image_number': 2, 'person/object': 'person with a white dog'},
+    {'image_number': 3, 'person/object': 'person in a yellow jacket'}
+]
+```
+====================================
+
+311HQEI8RU00YLN2AMGXNZAAJ0DZ7Z
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: One gloomy cool day, Jack, his dad and grandfather all decided to go on a hike on the mountain nearby their town. They started on a trail that leads up to the mountain. The trail was covered with leaves that had fallen from the surrounding trees. They were so dry and brittle the sound of them crunching under their shoes sounded like breaking bones. They had their backpacks ready with any necessities they thought would be somewhat needed up the mountain and their time there. They packed it with some first aid kits, drinks, snacks, and other things. At one point they had to go through what's now an old tunnel in the middle of the woods. That part was always Jacks favorites because they would always stop and mark their names on the walls of the old tunnel.As they continued to walk through the trail, they came upon a very old part of a bridge. Jack was the first one to arrive in front of it, his father and grandfather walked slowly behind him. They stopped and admired the old structure, walls filled with stones, the details amazed them. There was a tunnel going under the bridge which they would have to go under to continue their path and get to their destination. Once they were done admiring the old structure they continued their way that led to their final destination and went ahead and started their hike for that afternoon. The trail on the way to the mountain was always an important and admiring part of the hike, Jack could even say it was his favorite part!
+
+['https://drive.google.com/uc?export=view&id=1rERt9XDOv1HvXTgePqUq99QqXlwuQ5BL',
+ 'https://drive.google.com/uc?export=view&id=1v0VtbVKuHkkqIljVOCVd2_EtJkIw1zp9',
+ 'https://drive.google.com/uc?export=view&id=15PzKneScAIvHKukEfF_lC9NhJav49UFb']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'stone arch'},
+    {'image_number': 2, 'person/object': 'leaf-covered ground'},
+    {'image_number': 3, 'person/object': 'person in foreground'}
+]
+```
+====================================
+
+3S4TINXCC27B1TCLLUHWC3P2IM2BOW
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: A family on vacation sees a sign for Wing Cave National Park and decides to check it out. After parking their vehicle, they begin walking through the park, and see a sign which gives a full name of the national park. The first thing they see is a beautiful field with dozens of green trees in the distance.As the family continues to walk through the national park, they begin to realize there is not many animals. On top of that all of the grass and shrubs have died off. While they are walking, they recognize a distressed deer looking for grass. The family watches the deer for a few minutes, to see if they finally find grass to munch on. As far as they were able to see, the deer never found grass. The family realized how dire the situation was at that national park for the animals in it, and decided to call the park ranger before continuing their vacation.
+
+['https://drive.google.com/uc?export=view&id=1l5wkpwaM5kg3tpZE1rWOpfhKJtpzaQaR',
+ 'https://drive.google.com/uc?export=view&id=1Tj8p07jP9V-FFI1-m6FjS7NoQ7xUEmfr',
+ 'https://drive.google.com/uc?export=view&id=1l9FGUGIcpOOP87jtVgOGgSZGKB4-9qsi']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'park entrance sign'},
+    {'image_number': 2, 'person/object': 'dry grass'},
+    {'image_number': 3, 'person/object': 'distressed deer'}
+]
+```
+====================================
+
+3X2YVV51PWOIT43VAJFHNLPFZCA1WF
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: We visited a new National Park every year. It was tradition, my favorite tradition. Mom loved Yellowstone. Dad loved the Everglades. I hated it all. But since the divorce, I've been craving these trips more than anything I can remember. Together as a family, that's what these trips did for us. This year Mom and Dad agreed to come together, to be civil around each other. We went to Wind Cave National Park, who had ever heard of that? The first day on the drive in we went through the entrance and immediately saw a buffalo, all on its own.We got to our camping site and unloaded. Things were going well so far, no fighting. This could maybe be ok? First activity of the day was a day hike to Wind Cave. It's cool outside. The sun begins to go down. We see a deer on its own, grazing. This is not the same.
+
+['https://drive.google.com/uc?export=view&id=1I6Js6YB4EjVYooRni1gc-1qO79sBV8X2',
+ 'https://drive.google.com/uc?export=view&id=1OPdP6gue8TFu3V-kFhMQ1mb6RonOcfFt',
+ 'https://drive.google.com/uc?export=view&id=1JqIkEbPtsIXrhMLfkR9-R6lUQgs9l9q4']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'entrance sign'},
+    {'image_number': 2, 'person/object': 'trees'},
+    {'image_number': 3, 'person/object': 'grass'}
+]
+```
+====================================
+
+39WICJI5AVCIX2MF8AD7AIYUEYS3ZZ
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: While reading their kids books at the children's table at the local library, Jane and Timmy are being very good.  Their mother told them to sit and read while she goes to the adult section to get some books for her.  They sat quietly and read for a few minutes.  Then they got up and snuck around all of the book shelves.  They were curious as to where the mother had disappeared to. Instead of sitting and reading, Jane and Timmy decide to go on an excursion to see if they could find their mother, while exploring the rest of the library.Jane and Timmy search for a bit and found themselves having a little bit too much fun. They decide to play tag right inside the library. They run back and forth and in and out of the aisles.  Suddenly they are stopped in their tracks by a security man.  He asked them who they were, where was their mother, and why they were running around the library and being so noisy.  Jane and Timmy were scared.  They were just having fun.  The security guard them took them to the circulation desk so they could sit while he found their mother.
+
+['https://drive.google.com/uc?export=view&id=1qWrBBA2qxofeP5dOHhro1agUGgarJXk_',
+ 'https://drive.google.com/uc?export=view&id=1QlT-AWLSiMz0vB1FPper-2NncvQ8dhEu',
+ 'https://drive.google.com/uc?export=view&id=1wqGZzMN0WxEW6b769kfNdzlARg6EQPB8']
+OUTPUT: [
+    {"image_number": 1, "person/object": "children's chair"},
+    {"image_number": 2, "person/object": "patterned armchair"},
+    {"image_number": 3, "person/object": "circulation desk"}
+]
+====================================
+
+3QXFBUZ4ZM025F4TX2LJCK8OGLHUG8
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: Janice and Joseph are in the golden years of their lives.  They're on vacation in the mountains.  They rented a secluded cabin so that they could enjoy each others company without any distractions.  Today, they decided to trek into town and check out the historic steam engine.  It's well known around these parts.  The conductor has oodles of stories to share with anyone that cares enough to ask.Janice and Joseph take a walk up to the actual train depot and board a bigger train to go on a tour.  The tour takes a couple hours and weaves through the magical mountain forest.  As they're riding, they can't help but be smitten by the beautiful scenery passing them by.  Some young children are dangling their entire bodies out the window of the train, though.  This is giving Joseph anxiety and he badly wants to say something to them.  However, he abstains in the interest of not causing a scene in front of the love of his life.
+
+['https://drive.google.com/uc?export=view&id=1J3MHdFt5UzFraZ_pB8NzRkbtxoGkse3X',
+ 'https://drive.google.com/uc?export=view&id=1rO8oTfkvz_QRK-EgGgIt_940dB5Q3uti',
+ 'https://drive.google.com/uc?export=view&id=1KcjSxEzOnuuL1P7OL0GgTQfY8nQKliRG']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'green foliage'},
+    {'image_number': 2, 'person/object': 'conductor'},
+    {'image_number': 3, 'person/object': 'grass field'}
+]
+```
+====================================
+
+3H6W48L9F69LAWMXVLQ7OH7D2SFWPD
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: There is a place in the world where the most people go to in their holidays they visit the biggest desert where people gets into helicopter and sees the enormous land from above and the warm air they you feel and there are other people who just like to travel by travel by walking with your family and friends.There is a ancient history with that log old cowboys of the western side used to camp there and have fum stories with their family and friends but one day a disaster happened which caused the log get red and rotten and from all over there world tourist goes to that wonderful place.
+
+['https://drive.google.com/uc?export=view&id=1DyIOAuMsSy1flFmvB255cT5F3ScI2qfd',
+ 'https://drive.google.com/uc?export=view&id=1rn7_WOGfwMoUurHog8DQV5GEIA0kugGB',
+ 'https://drive.google.com/uc?export=view&id=112TxpwMDA34EO0Rrleh7XnBG56YN0vHt']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'dirt road'},
+    {'image_number': 2, 'person/object': 'wooden fence'},
+    {'image_number': 3, 'person/object': 'piece of wood'}
+]
+```
+====================================
+
+3R4QIDVOJRWNW5BG17GQ8Z19HH9EEV
+
+
+You are given a story and 3 images related to the story. Identify a person/object that can be visually identified in the images but not directly mentioned on the story. Use as few words as possible to describe each person/object. Also, mention the image number (1, 2 or 3) where the person/object can be found.
+Output in a python list of dictionaries. Each dictionary should have the following keys: 'image_number', 'person/object'.
+
+
+Story: The American Indian Sioux tribe ran off all white settlers in a certain region of Arizona for decades, and returned to the traditional native way of life free from interference of the white man. The Sioux headed down into the Red Valley earlier this year to look for more wild American buffalo, and began to venture further out into their territory. As they travelled further on horseback they saw something in the distance. A giant picnic table, sitting under a pavilion to the East. The tribe snuck closer to investigate, and what they saw shocked them. Dozens of white people marching like zombies toward the massive relic.The Sioux were shocked and confused. They continued to come closer, and followed the white travelers from a distance. All the travelers seemed to stop at a certain location miles before the monument. They would stop for a few minutes, looking down, then continue. The Sioux waited until it had cleared out and came up to where this happened, looked down and saw a rusty red rock in a tuft of grass. The Sioux were puzzled, scared, and concerned. They decided to go home and worry about themselves.
+
+['https://drive.google.com/uc?export=view&id=1VD1OaSBpYpbDvslubnKPyGmzl7WviMaY',
+ 'https://drive.google.com/uc?export=view&id=1-yaC9PVnIW-xJjV6STfJguK_zDs6QcWW',
+ 'https://drive.google.com/uc?export=view&id=1KN13XPAEPynxzrk60GI-R-Pgrz4sp2jb']
+OUTPUT: ```python
+[
+    {'image_number': 1, 'person/object': 'winding dirt road'},
+    {'image_number': 2, 'person/object': 'wooden fence'},
+    {'image_number': 3, 'person/object': 'petrified wood'}
+]
+```
+====================================
+

requirements.txt

@@ -0,0 +1,2 @@
+google-cloud-firestore
+path

results_with_gpt4v.csv

@@ -0,0 +1,24 @@
+HITId,HITTypeId,Title,Description,Keywords,Reward,CreationTime,MaxAssignments,RequesterAnnotation,AssignmentDurationInSeconds,AutoApprovalDelayInSeconds,Expiration,NumberOfSimilarHITs,LifetimeInSeconds,AssignmentId,WorkerId,AssignmentStatus,AcceptTime,SubmitTime,AutoApprovalTime,ApprovalTime,RejectionTime,RequesterFeedback,WorkTimeInSeconds,LifetimeApprovalRate,Last30DaysApprovalRate,Last7DaysApprovalRate,Input.item_id,Input.id,Input.image_set,Input.entity,Input.agent,Input.image1,Input.image2,Input.image3,Input.story,Input.number_of_lines,Answer.agent_makes_no_sense.on,Answer.agent_makes_no_sense_text,Answer.condition,Answer.goal,Answer.line0_changed.on,Answer.line10_changed.on,Answer.line11_changed.on,Answer.line12_changed.on,Answer.line13_changed.on,Answer.line14_changed.on,Answer.line15_changed.on,Answer.line16_changed.on,Answer.line17_changed.on,Answer.line18_changed.on,Answer.line19_changed.on,Answer.line1_changed.on,Answer.line20_changed.on,Answer.line21_changed.on,Answer.line22_changed.on,Answer.line23_changed.on,Answer.line24_changed.on,Answer.line2_changed.on,Answer.line3_changed.on,Answer.line4_changed.on,Answer.line5_changed.on,Answer.line6_changed.on,Answer.line7_changed.on,Answer.line8_changed.on,Answer.line9_changed.on,Answer.not_condition,Answer.not_condition_ep,Answer.rating.1,Answer.rating.2,Answer.rating.3,Answer.rating.4,Answer.rating.5,Answer.story_impact.no,Answer.story_impact.yes,Answer.why_no_impact,Approve,Reject,GPT4V.out
+35JDMRECC6TLGJ77726E82MLUXOEGC,38JF4YFDOM8MELZ3H6MMOC0A1QZ96M,Understanding how changing an attribute of an agent affects the visual story,Help us to generate alternate story,"story, multimodal, counterfactual",$0.45,Tue Nov 14 14:25:40 PST 2023,1,BatchId:5156379;OriginalHitTemplateId:920937340;,3600,1209600,Tue Nov 21 14:25:40 PST 2023,,,3JWH6J9I9VXW6RA3AHMLLYLPLR3BN0,A3B84NGQT7Z9WE,Submitted,Wed Nov 15 08:38:23 PST 2023,Wed Nov 15 08:48:47 PST 2023,Wed Nov 29 08:48:47 PST 2023,,,,624,100% (1401/1401),0% (0/0),0% (0/0),98,57,Flickr 10,A few scattered spectators,Surgeon,https://drive.google.com/uc?export=view&id=1dx4FK7tJ15MHcKC9vwhEmznxFHwSZSdb,https://drive.google.com/uc?export=view&id=1zkiU8D4VfGq5KrvzVOTAQbMBajnCJAMP,https://drive.google.com/uc?export=view&id=1vmL9PnIVrm37C4cRgrrS0h9OKOVZI3rG,"The three friends loved the soccer fields. Even when there wasn't a game, they would sit on the bleachers and reminisce about games gone by. Raquel couldn't play ever since his knee injury and he missed running up and down the field and being part of the team. He could imagine the games in his mind, projecting them onto the empty field.He had dreamed of making it out of the small municipal field and into the big leagues. To play in the World Cup was a dream he could taste. That was before another player plowed into him. Some say they planned it to take him out for the season. They ended up taking him out of the game for good. The surgeon wasn't able to fully repair the knee. He'd never be the same.",10,False,,"After Raquel's injury, a few of his friends helped him with his rehabilitation exercises.",To restore Raquel's knee back to its original condition.,False,,,,,,,,,,,False,,,,,,True,False,False,False,False,False,True,True,Raquel reinjures his knee during recovery which worsens the condition.,"The friends are not able to be at Raquel's side 24x7, and thus he sustains further injury when they are not around.",False,False,False,True,False,False,True,,,,"### Both text and image are visible.
+#### Output: 
+1. Goal: Successfully repair Raquel's knee injury.
+2. Necessary condition: Advanced surgical techniques and equipment.
+3. Alternate condition: Lack of appropriate medical facilities.
+### Only text is visible.
+#### Output: 
+1. The goal that the surgeon is trying to achieve is to successfully repair Raquel's knee injury so he can return to playing soccer.
+2. A condition that is necessary for goal completion is that the few scattered spectators must not interfere or disrupt the surgeon's work during the surgery.
+3. An alternate condition that will make the goal unlikely to happen is if the few scattered spectators interrupt or distract the surgeon during the surgery, causing a mistake or complication.
+### Only image is visible.
+#### Output: 
+The images you've provided depict a sports stadium with spectators in the stands, and there is no explicit connection to a surgeon or a narrative context involving surgery. However, to engage with the exercise, let's invent a hypothetical storyline.
+
+1. Identify a goal that the agent in the story is trying to achieve.
+Goal: The surgeon's goal is to provide immediate medical aid to injured players on the field.
+
+2. Identify and write a condition that is necessary for goal completion. The condition should be related to the entity.
+Condition: A necessary condition for the goal to be completed is that the few scattered spectators include off-duty medical personnel who can assist the surgeon in administering medical aid quickly.
+
+3. Write an alternate condition that will make the goal unlikely to happen. It is likely that this alternate condition will contradict information provided in the images.
+Alternate condition: The goal would be unlikely to happen if the scattered spectators are not medically trained and panic in the event of an emergency, causing chaos and impeding the surgeon's ability to reach the injured player in a timely manner.
+"