Spaces:
Running
Running
# Imports | |
import os | |
import streamlit as st | |
import requests | |
from transformers import pipeline | |
import openai | |
# Suppressing all warnings | |
import warnings | |
warnings.filterwarnings("ignore") | |
# Image-to-text | |
def img2txt(url): | |
print("Initializing captioning model...") | |
captioning_model = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base") | |
print("Generating text from the image...") | |
text = captioning_model(url, max_new_tokens=20)[0]["generated_text"] | |
print(text) | |
return text | |
# Text-to-story | |
def txt2story(img_text, mood, word_limit): | |
headers = {"Authorization": f"Bearer {os.environ['TOGETHER_API_KEY']}"} | |
data = { | |
"model": "togethercomputer/llama-2-70b-chat", | |
"messages": [ | |
{"role": "system", "content": f'''As an experienced short story writer, write story title and then create a meaningful story influenced by provided words. | |
Ensure stories have a {mood} mood within {word_limit} words. Remember the story must end within {word_limit} words''', "temperature": 1.5}, | |
{"role": "user", "content": f"Here is input set of words: {img_text}", "temperature": 1.5} | |
], | |
"top_k": 5, | |
"top_p": 0.8, | |
"temperature": 1.5 | |
} | |
response = requests.post("https://api.together.xyz/inference", headers=headers, json=data) | |
story = response.json()["output"]["choices"][0]["text"] | |
return story | |
# Text-to-speech | |
def txt2speech(text): | |
print("Initializing text-to-speech conversion...") | |
API_URL = "https://api-inference.huggingface.co/models/espnet/kan-bayashi_ljspeech_vits" | |
headers = {"Authorization": f"Bearer {os.environ['HUGGINGFACEHUB_API_TOKEN']}"} | |
payloads = {'inputs': text} | |
response = requests.post(API_URL, headers=headers, json=payloads) | |
with open('audio_story.mp3', 'wb') as file: | |
file.write(response.content) | |
# Streamlit web app main function | |
def main(): | |
st.set_page_config(page_title="π¨ Image-to-Audio Story π§", page_icon="πΌοΈ") | |
st.title("Image to story") | |
# Allows users to upload an image file | |
uploaded_file = st.file_uploader("# π· Upload an image...", type=["jpg", "jpeg", "png"]) | |
if uploaded_file is not None: | |
# Reads and saves uploaded image file | |
bytes_data = uploaded_file.read() | |
with open("uploaded_image.jpg", "wb") as file: | |
file.write(bytes_data) | |
st.image(uploaded_file, caption='πΌοΈ Uploaded Image', use_column_width=True) | |
mood = st.radio("Select mood of story", ["happy", "sad", "horror"], horizontal=True) | |
word_limit = st.slider("Set word limit", min_value=20, max_value=100, value=50) | |
if st.button("Analyze"): | |
# Initiates AI processing and story generation | |
with st.spinner("Loading... "): | |
scenario = img2txt("uploaded_image.jpg") # Extracts text from the image | |
story = txt2story(scenario, mood, word_limit) # Generates a story based on the image text, LLM params | |
txt2speech(story) # Converts the story to audio | |
st.divider() | |
st.markdown("## π Image Caption") | |
st.write(scenario) | |
st.divider() | |
st.markdown("## π Story") | |
st.write(story) | |
st.divider() | |
st.markdown("## π§ Audio") | |
st.audio("audio_story.mp3") | |
if __name__ == '__main__': | |
main() |