feat: additional audio files and supported formats

app.py

@@ -3,6 +3,7 @@ import numpy as np
 import streamlit as st
 import streamlit.components.v1 as components
 from io import BytesIO
+from pathlib import Path
 from time import time
 from src.authentication import authenticate
 from src.metahuman import MetaHuman
@@ -103,13 +104,16 @@ def run_helper(response: str):
     return audio_file, transcription
 
 
-def autoplay_audio(file_path: str):
+def autoplay_audio(file_path: Path):
+
+    suffix = file_path.suffix.split(".")[-1]
+
     with open(file_path, "rb") as f:
         data = f.read()
         b64 = base64.b64encode(data).decode()
         md = f"""
             <audio controls autoplay="true">
-            <source src="data:audio/wav;base64,{b64}" type="audio/wav">
+            <source src="data:audio/{suffix};base64,{b64}" type="audio/{suffix}">
             </audio>
             """
         sound = st.empty()

data/MetaHuman_Voice/metadata.csv

@@ -1,10 +1,20 @@
-id,keywords,filename,transcription,child_ids
-3,,Video_003_Welcome-to-Enginious-MetaHuma.wav,welcome to ingenious Med Human Experience I'm Lena and I can guide you through this website please say okay to continue,4
-4,,Video_004_Would-you-like-to-know-more-a.wav,would you like to know more about ingenuous please say yes to continue,5
-5,,Video_005_Which-parts-of-the-website-wo.wav,which parts of the website would you like me to explain please say one of the following Persona application Services real team career contact or DNA,"6,8"
-6,persona,Video_006_Persona-Application-is-an-inn.wav,Persona application is an Innovative automated generator a photo-realistic virtual humans based on machine learning algorithms would you like to know more,7
-7,,Video_007_The-main-goal-of-the-project.wav,the main goal of the project is to build and publish and Janice virtual human platform which will automate speed up and reduce the cost of the process of generating virtual high-quality characters thanks to a combination of 3D Graphics visualization Technologies and artificial intelligence algorithms,
-8,service,Video_008_Which-service-would-you-like.wav,which service would you like me to explain please say one of the following virtual production virtual humans creation environments and 3D assets motion-capture photogrammetry metaverse,9
-9,production,Video_009_We-have-many-projects-in-the.wav,we have many projects in the field of virtual production behind us virtual production using the mo says tracking system and Unreal Engine virtual environments disguise server was playback of video files of the highest-quality accepted by Netflix and HBO Stipe read spy tracking system with visit and Unreal Engine 4 live Studio streaming or virtual production with green screen and live in camera bfx previs,
-10,,Video_010_Would-you-like-to-know-more.wav,would you like to know more,
-99,,Video_099_Block_I-m-sorry-I-did-not-understa.wav,I am sorry I did not understand please repeat or say home to come back to main menu,
+id,filename,transcription,keywords,child_ids
+3,Video_003_Welcome-to-Enginious-MetaHuma.wav,Welcome to Enginious MetaHuman experience. I’m Lena and I can guide you through this website. Please say ok to continue.,,4
+4,Video_004_Would-you-like-to-know-more-a.wav,Would you like to know more about Enginious? Please say yes to continue.,,5
+5,Video_005_Which-parts-of-the-website-wo.wav,"Which parts of the website would you like me to explain? Please say one of the following: Persona Application, Services, Reel, Team, Career, Contact or DNA.",,"6,8,16,17,18,19,20"
+6,Video_006_Persona-Application-is-an-inn.wav,"Persona Application is an innovative, automated, generator of photo-realistic virtual humans based on machine learning algorithms. Would you like to know more?",persona,7
+7,Video_007_The-main-goal-of-the-project.wav,"The main goal of the project is to build and publish ENGINOUS Virtual Human platform which will automate, speed up and reduce the cost of the process of generating virtual, high-quality characters thanks to a combination of 3D graphics, visualization technologies and artificial intelligence algorithms.",,
+8,Video_008_Which-service-would-you-like.wav,"Which service would you like me to explain? Please say one of the following: Virtual Production, Virtual Humans Creation, Environments & 3D Assets, Motion Capture, Photogrammetry, Metaverse.",service,"9,11,12,13,14,15"
+9,Video_009_We-have-many-projects-in-the.wav,"We have many projects in the field of Virtual Production behind us. Virtual production using the Mo-Sys tracking system and Unreal Engine virtual environments, Disguise server with playback of video files of the highest quality accepted by Netflix and HBO, Stype Red Spy tracking system with Vizrt and Unreal Engine for Live Studio Streaming or Virtual production with green screen and live in camera VFX previs.",production,
+10,Video_010_Would-you-like-to-know-more.wav,Would you like to know more?,,
+11,Video011_Through-the-use-of-real-time-g.mp3,"Through the use of real-time graphics, recurrent neural networks and deep learning alongside state-of-the-art photogrammetry, we are completely redesigning the virtual human production pipeline to make it dramatically faster and highly cost-effective. Moreover, we create MetaHuman ChatBots based on AI with the use of voice and text recognition and classification algorithms. Our Avatars can recreate human interactions and help clients across different markets from education to retail.",humans,
+12,Video012_We-create-3D-computer-generate.mp3,"We create 3D computer-generated environments of photorealistic quality with attention to every detail. We have done environments and virtual backgrounds for brands such as The Witcher, Dying Light 2, Dark Stories or Kruk and Klangor. Thanks to the use of Unreal Engine and real-time rendering, each change of location or detail takes a moment. We create environments with the possibility of changing the weather or the time of day on the fly. Whether it’s assets for Virtual Production, Green Screen or VFX - we are ready to provide top quality.",environment,
+13,Video013_We-have-vast-experience-in-3D-2.mp3,We have vast experience in 3D Character Animation using our own equipment. We chose cutting-edge Xsens and Facewear Motion Capture systems. Whether you need subtle and finesse moves or fast and furious action of the CGI characters we have the equipment and know-how to achieve it with top quality and cost efficiency.,motion,
+14,Video014_Our-tech-team-uses-The-ESPER-L.mp3,Our tech team uses The ESPER LightCage for photogrammetry solutions. This is a State-of-the-Art Lighting System for Photorealistic 3D Face Scanning & Detailed Surface Texture Capture. It is the most advanced lighting system for 3D face scanning on the market. Transform the quality and efficiency of your 3D scanning with a lighting rig which enables the recovery of detailed surface texture data.,photogrammetry,
+15,Video015_We-have-gained-experience-and.mp3,"We have gained experience and knowledge in creating architecture, avatars, cgi content and gamification solutions in the Metaverse platforms. We know it is the future of marketing with so many digital events, transactions, investors and a growing cummunity. If you are looking for custom made solutions within the meta, crypto world just step in our office in Decentraland or meet us in Sandbox and we can guide you in this journey.",metaverse,
+16,Video016_Check-out-our-showreel-consist.mp3,"Check out our showreel consisting of the projects we created using virtual production, Unreal Engine and other CGI solutions.",reel,
+17,Video017_A-world-class-team-of-experts.mp3,"A world-class team of experts in VFX specializing in creating computer generated environments, characters, objects and effects. Our multidisciplinary team consists of experienced: Producers, Creatives, Directors, Geeks, 3D Artists, Concept Artiists, Unreal Engine Specialists, Motion Designers, Houdini software experts, Animators, Maya, Nuke, Cinema 4D, Unity, Interactive, VR, Metaverse, Virtual Humans and other technology enthusiasts.",team,
+18,Video018_Virtual-Production-and-AI-base.mp3,Virtual Production and AI based MetaHumans is the future. We love the future and want to be a part of it. We have the brains and the tools. So now we need You. What are You waiting for? We are currently looking for an Unreal Engine Developer and Unreal Engine Supervisor.,career,
+19,Video_19_Go-ahead-and-drop-us-a-line-us.mp3,Go ahead and DROP US A LINE using the following email address: hello@enginious.tech,contact,
+20,Video_20_We-promote-inclusivity-and-res.mp3,"We promote inclusivity and respect, while understanding that being different is an advantage. A diversity in cultural backgrounds, interests and personalities is brought together by mutual love for all gaming, technology and entertainment as a whole. Authenticity, zeitgeist, Enginious.",DNA,
+99,Video_099_Block_I-m-sorry-I-did-not-understa.wav,I’m sorry I did not understand. Please repeat or say home to come back to the main menu.,,

src/metahuman.py

@@ -32,12 +32,12 @@ class MetaChatbot():
     def __init__(self, speech_dir_path: Path, metadata_filename: Optional[str] = "metadata.csv", compare_func: Optional[callable] = keyword_compare):
 
         self.speech_dir_path = speech_dir_path
-        
+
         self.response_df = pd.read_csv(
             speech_dir_path / metadata_filename, index_col=0)
         self.response_df.child_ids = self.response_df.child_ids.apply(
             lambda x: [int(i) for i in x.split(",")] if isinstance(x, str) else x)
-        
+
         self.compare_func = compare_func
 
     def get_state_file(self):
@@ -108,8 +108,9 @@ class MetaHuman(MetaChatbot):
         return any([re.search(r"\b(yes|yea|yep|y|sure|ok|okay|okie|yeah|yup)\b", response, re.IGNORECASE)])
 
     def compare(self, children: pd.DataFrame, response: str):
-        
-        selection = children.keywords.apply(lambda x: max([self.compare_func(y, response) for y in x.split(",")]))
+
+        selection = children.keywords.apply(lambda x: max(
+            [self.compare_func(y, response) for y in x.split(",")]))
 
         # TODO: And max only occurs once
         if selection.max() > self.context_threshold:
@@ -126,8 +127,11 @@ def create_audio_csv(dir: Path, suffix=".wav", out_name="metadata"):
 
     rows = []
     for file in audio_files:
-
-        text = transcribe_file(file)
+        
+        try:
+            text = transcribe_file(file)
+        except Exception as e:
+            text = ""
 
         # Using first number as identifier
         id = int(re.search(r"\d+", file.name).group(0))
@@ -151,6 +155,15 @@ def create_audio_csv(dir: Path, suffix=".wav", out_name="metadata"):
         dict_writer.writerows(sorted_rows)
 
 
+def update_audio_csv(dir: Path, suffix=".wav", out_name="metadata"):
+
+    out_file_path = dir / f"{out_name}.csv"
+    if not out_file_path.exists():
+        raise FileNotFoundError(f"No file named {out_name}.csv found in {dir}")
+
+    raise NotImplementedError()
+
+
 if __name__ == "__main__":
 
-    create_audio_csv(helper_path)
+    create_audio_csv(helper_path, suffix=".mp3")