import os import streamlit as st #from dotenv import load_dotenv #load_dotenv() # for audio stuff import whisper from pydub import AudioSegment from io import BytesIO from gtts import gTTS # for langchain stuff from langchain.llms import OpenAI from langchain.agents import create_sql_agent,load_tools,initialize_agent from langchain.agents.agent_toolkits import SQLDatabaseToolkit from langchain.sql_database import SQLDatabase from langchain.agents import AgentExecutor from langchain import SQLDatabaseChain # for streaming stdout stuff from contextlib import contextmanager from io import StringIO from streamlit.runtime.scriptrunner.script_run_context import SCRIPT_RUN_CONTEXT_ATTR_NAME from threading import current_thread import sys @contextmanager def st_redirect(src, dst): placeholder = st.empty() output_func = getattr(placeholder, dst) with StringIO() as buffer: old_write = src.write def new_write(b): if getattr(current_thread(), SCRIPT_RUN_CONTEXT_ATTR_NAME, None): buffer.write(b) output_func(buffer.getvalue()) sys.stdout.write('\n') else: old_write(b) try: src.write = new_write yield finally: src.write = old_write @contextmanager def st_stdout(dst): with st_redirect(sys.stdout, dst): yield @contextmanager def st_stderr(dst): with st_redirect(sys.stderr, dst): yield # Code taken from https://github.com/prateekralhan/OpenAI_Whisper_ASR/tree/main st.set_page_config( page_title="OpeneXplore insights", page_icon="musical_note", layout="wide", initial_sidebar_state="auto", ) audio_tags = {'comments': 'Converted using pydub!'} upload_path = "uploads/" download_path = "downloads/" transcript_path = "transcripts/" # @st.cache(persist=True,allow_output_mutation=False,show_spinner=True,suppress_st_warning=True) def to_mp3(audio_file, output_audio_file, upload_path, download_path): ## Converting Different Audio Formats To MP3 ## if audio_file.name.split('.')[-1].lower()=="wav": audio_data = AudioSegment.from_wav(os.path.join(upload_path,audio_file.name)) audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="mp3": audio_data = AudioSegment.from_mp3(os.path.join(upload_path,audio_file.name)) audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="ogg": audio_data = AudioSegment.from_ogg(os.path.join(upload_path,audio_file.name)) audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="wma": audio_data = AudioSegment.from_file(os.path.join(upload_path,audio_file.name),"wma") audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="aac": audio_data = AudioSegment.from_file(os.path.join(upload_path,audio_file.name),"aac") audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="flac": audio_data = AudioSegment.from_file(os.path.join(upload_path,audio_file.name),"flac") audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="flv": audio_data = AudioSegment.from_flv(os.path.join(upload_path,audio_file.name)) audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) elif audio_file.name.split('.')[-1].lower()=="mp4": audio_data = AudioSegment.from_file(os.path.join(upload_path,audio_file.name),"mp4") audio_data.export(os.path.join(download_path,output_audio_file), format="mp3", tags=audio_tags) return output_audio_file # @st.cache(persist=True,allow_output_mutation=False,show_spinner=True,suppress_st_warning=True) def process_audio(filename, model_type): model = whisper.load_model(model_type) result = model.transcribe(filename) return result["text"] # @st.cache(persist=True,allow_output_mutation=False,show_spinner=True,suppress_st_warning=True) def save_transcript(transcript_data, txt_file): with open(os.path.join(transcript_path, txt_file),"w") as f: f.write(transcript_data) # generic search agent llm = OpenAI(temperature=0) tools = load_tools(['serpapi','llm-math'], llm=llm, verbose=True) agent = initialize_agent(tools, llm, agent='zero-shot-react-description', verbose=True) # import pandas as pd # df = pd.DataFrame({'date':pd.date_range('2023-01-01','2023-04-30'), # 'city':[np.random.choice(['Edmonton']) for i in range(120)], # 'sales':[np.random.randint(640000,1800000) for i in range(120)], # 'max_temperature':[np.random.randint(25) for i in range(120)], # 'min_temperature':[np.random.randint(-10,10) for i in range(120)], # 'weather':[np.random.choice(['sunny','partly cloudy']) for i in range(120)] # }) # df1 = pd.DataFrame({'date':pd.date_range('2023-01-01','2023-04-30'), # 'city':[np.random.choice(['Kitchener']) for i in range(120)], # 'sales':[np.random.randint(640000,1800000) for i in range(120)], # 'max_temperature':[np.random.randint(25) for i in range(120)], # 'min_temperature':[np.random.randint(-10,10) for i in range(120)], # 'weather':[np.random.choice(['sunny','partly cloudy']) for i in range(120)] # }) # df = pd.concat([df,df1]) # from sqlalchemy import create_engine # disk_engine = create_engine('sqlite:///company_database.db') # df.to_sql('daily_sales', disk_engine, if_exists='replace',index=False) # db agent db = SQLDatabase.from_uri('sqlite:///company_database.db') db_chain = SQLDatabaseChain(llm=llm,database=db,verbose=True) toolkit = SQLDatabaseToolkit(db=db, llm=llm) agent_executor = create_sql_agent( llm=OpenAI(temperature=0), toolkit=toolkit, verbose=True ) st.title("🗣💡 OracleXplore - last mile for insights ✨") # select journey - text or voice input_format = st.radio( "How would you like to ask the question?", ("Not sure","Type it","Say it"), horizontal=True ) transcript='' if input_format == "Type it": transcript = st.text_input("type your question here",placeholder="Example: using our own data what is the total revenue from Edmonton in Feb?") elif input_format == "Say it": st.info('✨ Supports all popular audio formats - WAV, MP3, MP4, OGG, WMA, AAC, FLAC, FLV') col1, col2 = st.columns([3, 2]) with col1: uploaded_file = st.file_uploader("Upload audio file", type=["wav","mp3","ogg","wma","aac","flac","mp4","flv"]) audio_file = None if uploaded_file is not None: generate_text = st.button("Generate speech to text") audio_bytes = uploaded_file.read() with open(os.path.join(upload_path,uploaded_file.name),"wb") as f: f.write((uploaded_file).getbuffer()) with st.spinner(f"Processing Audio ... 💫"): output_audio_file = uploaded_file.name.split('.')[0] + '.mp3' output_audio_file = to_mp3(uploaded_file, output_audio_file, upload_path, download_path) audio_file = open(os.path.join(download_path,output_audio_file), 'rb') audio_bytes = audio_file.read() # transcribe the text if generate_text: with st.spinner(f"Generating Transcript... 💫"): transcript = process_audio(str(os.path.abspath(os.path.join(download_path,output_audio_file))), 'base') st.write("Here's what our AI heard you say:") output_txt_file = str(output_audio_file.split('.')[0]+".txt") save_transcript(transcript, output_txt_file) output_file = open(os.path.join(transcript_path,output_txt_file),"r") output_file_data = output_file.read() with col2: if uploaded_file is not None: print("Opening ",audio_file) st.markdown("---") st.markdown("Feel free to play your uploaded audio file 🎼") st.audio(audio_bytes) else: st.markdown("---") st.warning('⚠ Please upload your audio file 😯') else: pass # process next steps if transcript!='': st.info(f"Question: {transcript}") # run agent # with st_stdout("error"): if "using our own data" in transcript.lower(): response = agent_executor.run(transcript.replace("using our own data","")) # with st.expander('Message history:'): # st.info(memory.buffer) else: response = agent.run(transcript) # convert this back into audio sound_file = BytesIO() tts = gTTS(response, lang='en') tts.write_to_fp(sound_file) st.info(f"Answer: {response}") st.audio(sound_file) st.markdown("
Made with ❤️ by Vivek with thanks to [Prateek Kralhan](https://github.com/prateekralhan/OpenAI_Whisper_ASR/tree/main) and other open source legends ✨
", unsafe_allow_html=True)