Update app.py

app.py

@@ -1,65 +1,107 @@
-from PIL import Image
-from transformers import BlipProcessor, BlipForConditionalGeneration
-from langchain import HuggingFaceHub, LLMChain, PromptTemplate
-import gradio as gr
-import numpy as np
-import requests
 import os
+import streamlit as st
+import requests
+from transformers import pipeline
+import openai
+from langchain import LLMChain, PromptTemplate
+from langchain import HuggingFaceHub
+
+# Suppressing all warnings
+import warnings
+warnings.filterwarnings("ignore")
+
+api_token = os.getenv('HUGGING_FACE')
+
+# Image-to-text
+def img2txt(url):
+    print("Initializing captioning model...")
+    captioning_model = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base")
+    
+    print("Generating text from the image...")
+    text = captioning_model(url, max_new_tokens=20)[0]["generated_text"]
+    
+    print(text)
+    return text
+
+# Text-to-story
+
+model = "tiiuae/falcon-7b-instruct"
+llm = HuggingFaceHub(
+    huggingfacehub_api_token = api_token,
+    repo_id = model,
+    verbose = False,
+    model_kwargs = {"temperature":0.2, "max_new_tokens": 4000})
+
+def generate_story(scenario, llm):
+  template= """You are a story teller.
+               You get a scenario as an input text, and generates a short story out of it.
+               Context: {scenario}
+               Story:
+               """
+  prompt = PromptTemplate(template=template, input_variables=["scenario"])
+  #Let's create our LLM chain now
+  chain = LLMChain(prompt=prompt, llm=llm)
+  story = chain.predict(scenario=scenario)
+  start_index = story.find("Story:") + len("Story:")
+
+  # Extract the text after "Story:"
+  story = story[start_index:].strip()
+  return story
+
 
-# Load image captioning model
-processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
-model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
-
-def generate_caption_from_image(image_path):
-    # Process the image and generate caption
-    raw_image = Image.open(image_path).convert("RGB")
-    inputs = processor(raw_image, return_tensors="pt")
-    out = model.generate(**inputs)
-    caption = processor.decode(out[0], skip_special_tokens=True)
-    return caption
-
-def generate_story_from_caption(caption):
-    # Generate story based on caption
-    llm = HuggingFaceHub(huggingfacehub_api_token=os.getenv('HUGGING_FACE'),
-                            repo_id="tiiuae/falcon-7b-instruct",
-                            verbose=False,
-                            model_kwargs={"temperature": 0.2, "max_new_tokens": 4000})
-    template = """You are a story teller.
-    You get a scenario as an input text, and generate a short story out of it.
-    Context: {scenario}
-    Story:"""
-    prompt = PromptTemplate(template=template, input_variables=["scenario"])
-    # Let's create our LLM chain now
-    chain = LLMChain(prompt=prompt, llm=llm)
-    story = chain.run(caption)
-    start_index = story.find("Story:") + len("Story:")
-    # Extract the text after "Story:"
-    story = story[start_index:].strip()
-    return story
-
-def text_to_speech(text):
-    headers = {"Authorization": f"Bearer {os.getenv('HUGGING_FACE')}"}
-    payload = {"inputs": text}
+# Text-to-speech
+def txt2speech(text):
+    print("Initializing text-to-speech conversion...")
     API_URL = "https://api-inference.huggingface.co/models/espnet/kan-bayashi_ljspeech_vits"
-    response = requests.post(API_URL, headers=headers, json=payload)
+    headers = {"Authorization": f"Bearer {api_token }"}
+    payloads = {'inputs': text}
+
+    response = requests.post(API_URL, headers=headers, json=payloads)
     
-    if response.status_code == 200:
-        with open("output.mp3", "wb") as f:
-            f.write(response.content)
-        return "output.mp3"
-
-def generate_story_from_image(image_input):
-    input_image = Image.fromarray(image_input)
-    input_image.save("input_image.jpg")
-    image_path = 'input_image.jpg'
-    caption = generate_caption_from_image(image_path)
-    story = generate_story_from_caption(caption)
-    audio = text_to_speech(story)
-    return audio
-
-# Define the input and output components
-inputs = gr.Image(label="Image")
-outputs = gr.Audio(label="Story Audio")
-
-# Create the Gradio interface
-gr.Interface(fn=generate_story_from_image, inputs=inputs, outputs=outputs, title="Story Teller").launch(debug=True)
+    with open('audio_story.mp3', 'wb') as file:
+        file.write(response.content)
+        
+
+        
+# Streamlit web app main function
+def main():
+    st.set_page_config(page_title="🎨 Image-to-Audio Story 🎧", page_icon="🖼️")
+    st.title("Turn the Image into Audio Story")
+
+    # Allows users to upload an image file
+    uploaded_file = st.file_uploader("# 📷 Upload an image...", type=["jpg", "jpeg", "png"])
+
+    # Parameters for LLM model (in the sidebar)
+    st.sidebar.markdown("# LLM Inference Configuration Parameters")
+    top_k = st.sidebar.number_input("Top-K", min_value=1, max_value=100, value=5)
+    top_p = st.sidebar.number_input("Top-P", min_value=0.0, max_value=1.0, value=0.8)
+    temperature = st.sidebar.number_input("Temperature", min_value=0.1, max_value=2.0, value=1.5)
+
+    if uploaded_file is not None:
+        # Reads and saves uploaded image file
+        bytes_data = uploaded_file.read()
+        with open("uploaded_image.jpg", "wb") as file:
+            file.write(bytes_data)
+
+        st.image(uploaded_file, caption='🖼️ Uploaded Image', use_column_width=True)
+
+        # Initiates AI processing and story generation
+        with st.spinner("## 🤖 AI is at Work! "):
+            scenario = img2txt("uploaded_image.jpg")  # Extracts text from the image
+            story = generate_story(scenario, llm)  # Generates a story based on the image text, LLM params
+            txt2speech(story)  # Converts the story to audio
+
+            st.markdown("---")
+            st.markdown("## 📜 Image Caption")
+            st.write(scenario)
+
+            st.markdown("---")
+            st.markdown("## 📖 Story")
+            st.write(story)
+
+            st.markdown("---")
+            st.markdown("## 🎧 Audio Story")
+            st.audio("audio_story.mp3")
+
+if __name__ == '__main__':
+    main()