AI-ANK commited on
Commit
3e7ad82
1 Parent(s): 9d7ccd9

Upload 2 files

Browse files
Files changed (2) hide show
  1. app.py +165 -0
  2. requirements.txt +11 -0
app.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import extra_streamlit_components as stx
3
+ import requests
4
+ from PIL import Image
5
+ from transformers import AutoProcessor, AutoModelForVision2Seq
6
+ from io import BytesIO
7
+ import replicate
8
+ from llama_index.llms.palm import PaLM
9
+ from llama_index import ServiceContext, VectorStoreIndex, Document
10
+ from llama_index.memory import ChatMemoryBuffer
11
+ import os
12
+ import datetime
13
+
14
+ # Set up the title of the application
15
+ #st.title("PaLM-Kosmos-Vision")
16
+ st.set_page_config(layout="wide")
17
+ st.write("My version of ChatGPT vision. You can upload an image and start chatting with the LLM about the image")
18
+
19
+ # Sidebar
20
+ st.sidebar.markdown('## Created By')
21
+ st.sidebar.markdown("""
22
+ [Harshad Suryawanshi](https://www.linkedin.com/in/harshadsuryawanshi/)
23
+ """)
24
+
25
+ st.sidebar.markdown('## Other Projects')
26
+ st.sidebar.markdown("""
27
+ - [AI Equity Research Analyst](https://ai-eqty-rsrch-anlyst.streamlit.app/)
28
+ - [Recasting "The Office" Scene](https://blackmirroroffice.streamlit.app/)
29
+ - [Story Generator](https://appstorycombined-agaf9j4ceit.streamlit.app/)
30
+ """)
31
+
32
+ st.sidebar.markdown('## Disclaimer')
33
+ st.sidebar.markdown("""
34
+ This application is a conceptual prototype created to demonstrate the potential of Large Language Models (LLMs) in generating equity research reports. The contents generated by this application are purely illustrative and should not be construed as financial advice, endorsements, or recommendations. The author and the application do not provide any guarantee regarding the accuracy, completeness, or timeliness of the information provided.
35
+ """)
36
+
37
+ # Initialize the cookie manager
38
+ cookie_manager = stx.CookieManager()
39
+
40
+ # Function to get image caption via Kosmos2.
41
+ @st.cache_data
42
+ def get_image_caption(image_data):
43
+ input_data = {
44
+ "image": image_data,
45
+ "description_type": "Brief"
46
+ }
47
+ output = replicate.run(
48
+ "lucataco/kosmos-2:3e7b211c29c092f4bcc8853922cc986baa52efe255876b80cac2c2fbb4aff805",
49
+ input=input_data
50
+ )
51
+ # Split the output string on the newline character and take the first item
52
+ text_description = output.split('\n\n')[0]
53
+ return text_description
54
+
55
+ # Function to create the chat engine.
56
+ @st.cache_resource
57
+ def create_chat_engine(img_desc, api_key):
58
+ llm = PaLM(api_key=api_key)
59
+ service_context = ServiceContext.from_defaults(llm=llm, embed_model="local")
60
+ doc = Document(text=img_desc)
61
+ index = VectorStoreIndex.from_documents([doc], service_context=service_context)
62
+ chatmemory = ChatMemoryBuffer.from_defaults(token_limit=1500)
63
+
64
+ chat_engine = index.as_chat_engine(
65
+ chat_mode="context",
66
+ system_prompt=(
67
+ f"You are a chatbot, able to have normal interactions, as well as talk. "
68
+ "You always answer in great detail and are polite. Your responses always descriptive. "
69
+ "Your job is to talk about an image the user has uploaded. Image description: {img_desc}."
70
+ ),
71
+ verbose=True,
72
+ memory=chatmemory
73
+ )
74
+ return chat_engine
75
+
76
+ # Clear chat function
77
+ def clear_chat():
78
+ if "messages" in st.session_state:
79
+ del st.session_state.messages
80
+ if "image_file" in st.session_state:
81
+ del st.session_state.image_file
82
+
83
+ # Callback function to clear the chat when a new image is uploaded
84
+ def on_image_upload():
85
+ clear_chat()
86
+
87
+ # Retrieve the message count from cookies
88
+ message_count = cookie_manager.get(cookie='message_count')
89
+ if message_count is None:
90
+ message_count = 0
91
+ else:
92
+ message_count = int(message_count)
93
+
94
+ # If the message limit has been reached, disable the inputs
95
+ if message_count >= 20:
96
+ st.error("Notice: The maximum message limit for this demo version has been reached.")
97
+ # Disabling the uploader and input by not displaying them
98
+ image_uploader_placeholder = st.empty() # Placeholder for the uploader
99
+ chat_input_placeholder = st.empty() # Placeholder for the chat input
100
+ else:
101
+ # Add a clear chat button
102
+ if st.button("Clear Chat"):
103
+ clear_chat()
104
+
105
+ # Image upload section.
106
+ image_file = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"], key="uploaded_image", on_change=on_image_upload)
107
+ if image_file:
108
+ # Display the uploaded image at a standard width.
109
+ st.image(image_file, caption='Uploaded Image.', width=200)
110
+ # Process the uploaded image to get a caption.
111
+ image_data = BytesIO(image_file.getvalue())
112
+ img_desc = get_image_caption(image_data)
113
+ st.write("Image Uploaded Successfully. Ask me anything about it.")
114
+
115
+ # Initialize the chat engine with the image description.
116
+ chat_engine = create_chat_engine(img_desc, os.environ["GOOGLE_API_KEY"])
117
+
118
+ # Initialize session state for messages if it doesn't exist
119
+ if "messages" not in st.session_state:
120
+ st.session_state.messages = []
121
+
122
+ # Display previous messages
123
+ for message in st.session_state.messages:
124
+ with st.chat_message(message["role"]):
125
+ st.markdown(message["content"])
126
+
127
+ # Handle new user input
128
+ user_input = st.chat_input("Ask me about the image:", key="chat_input")
129
+ if user_input:
130
+ # Append user message to the session state
131
+ st.session_state.messages.append({"role": "user", "content": user_input})
132
+
133
+ # Display user message immediately
134
+ with st.chat_message("user"):
135
+ st.markdown(user_input)
136
+
137
+ # Call the chat engine to get the response if an image has been uploaded
138
+ if image_file and user_input:
139
+ try:
140
+ with st.spinner('Waiting for the chat engine to respond...'):
141
+ # Get the response from your chat engine
142
+ response = chat_engine.chat(user_input)
143
+
144
+ # Append assistant message to the session state
145
+ st.session_state.messages.append({"role": "assistant", "content": response})
146
+
147
+ # Display the assistant message
148
+ with st.chat_message("assistant"):
149
+ st.markdown(response)
150
+
151
+ except Exception as e:
152
+ st.error(f'An error occurred.')
153
+ # Optionally, you can choose to break the flow here if a critical error happens
154
+ # return
155
+
156
+ # Increment the message count and update the cookie
157
+ message_count += 1
158
+ cookie_manager.set('message_count', str(message_count), expires_at=datetime.datetime.now() + datetime.timedelta(days=30))
159
+
160
+
161
+
162
+
163
+ # Set Replicate and Google API keys
164
+ os.environ['REPLICATE_API_TOKEN'] = st.secrets['REPLICATE_API_TOKEN']
165
+ os.environ["GOOGLE_API_KEY"] = st.secrets['GOOGLE_API_KEY']
requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ streamlit
2
+ google-generativeai
3
+ llama-hub
4
+ llama-index
5
+ transformers
6
+ Pillow
7
+ requests
8
+ nest_asyncio
9
+ torch
10
+ extra-streamlit-components
11
+ replicate