Spaces:
Running
Running
SnJForever
commited on
Commit
•
ad25943
1
Parent(s):
2da487d
add video
Browse files
app.py
CHANGED
@@ -40,7 +40,8 @@ from langchain.vectorstores.faiss import FAISS
|
|
40 |
from langchain.docstore.document import Document
|
41 |
from langchain.chains.question_answering import load_qa_chain
|
42 |
import azure.cognitiveservices.speech as speechsdk
|
43 |
-
|
|
|
44 |
|
45 |
news_api_key = os.environ["NEWS_API_KEY"]
|
46 |
|
@@ -56,7 +57,7 @@ MAX_TOKENS = 512
|
|
56 |
|
57 |
LOOPING_TALKING_HEAD = "videos/Michelle.mp4"
|
58 |
TALKING_HEAD_WIDTH = "192"
|
59 |
-
MAX_TALKING_HEAD_TEXT_LENGTH =
|
60 |
|
61 |
# Pertains to Express-inator functionality
|
62 |
NUM_WORDS_DEFAULT = 0
|
@@ -431,7 +432,9 @@ class ChatWrapper:
|
|
431 |
if speak_text:
|
432 |
if talking_head:
|
433 |
if len(output) <= MAX_TALKING_HEAD_TEXT_LENGTH:
|
434 |
-
html_video, temp_file = do_html_video_speak(output, translate_to)
|
|
|
|
|
435 |
else:
|
436 |
temp_file = LOOPING_TALKING_HEAD
|
437 |
html_video = create_html_video(temp_file, TALKING_HEAD_WIDTH)
|
@@ -552,6 +555,48 @@ def create_html_video(file_name, width):
|
|
552 |
html_video = f'<video width={width} height={width} autoplay muted loop><source src={temp_file_url} type="video/mp4" poster="Michelle.png"></video>'
|
553 |
return html_video
|
554 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
555 |
|
556 |
def do_html_video_speak(words_to_speak, azure_language):
|
557 |
azure_voice = AZURE_VOICE_DATA.get_voice(azure_language, "Male")
|
|
|
40 |
from langchain.docstore.document import Document
|
41 |
from langchain.chains.question_answering import load_qa_chain
|
42 |
import azure.cognitiveservices.speech as speechsdk
|
43 |
+
import base64
|
44 |
+
|
45 |
|
46 |
news_api_key = os.environ["NEWS_API_KEY"]
|
47 |
|
|
|
57 |
|
58 |
LOOPING_TALKING_HEAD = "videos/Michelle.mp4"
|
59 |
TALKING_HEAD_WIDTH = "192"
|
60 |
+
MAX_TALKING_HEAD_TEXT_LENGTH = 20
|
61 |
|
62 |
# Pertains to Express-inator functionality
|
63 |
NUM_WORDS_DEFAULT = 0
|
|
|
432 |
if speak_text:
|
433 |
if talking_head:
|
434 |
if len(output) <= MAX_TALKING_HEAD_TEXT_LENGTH:
|
435 |
+
# html_video, temp_file = do_html_video_speak(output, translate_to)
|
436 |
+
html_audio, temp_aud_file = do_html_audio_speak_azure(output, translate_to)
|
437 |
+
html_video, temp_file = do_html_video_speak_sad_talker(output, translate_to)
|
438 |
else:
|
439 |
temp_file = LOOPING_TALKING_HEAD
|
440 |
html_video = create_html_video(temp_file, TALKING_HEAD_WIDTH)
|
|
|
555 |
html_video = f'<video width={width} height={width} autoplay muted loop><source src={temp_file_url} type="video/mp4" poster="Michelle.png"></video>'
|
556 |
return html_video
|
557 |
|
558 |
+
def ToBase64(file):
|
559 |
+
with open(file, 'rb') as fileObj:
|
560 |
+
image_data = fileObj.read()
|
561 |
+
base64_data = base64.b64encode(image_data)s
|
562 |
+
return base64_data.decode()
|
563 |
+
|
564 |
+
|
565 |
+
def do_html_video_speak_sad_talker(temp_aud_file, azure_language):
|
566 |
+
|
567 |
+
GRADIO_URL=os.environ["GRADIO_URL"],
|
568 |
+
|
569 |
+
img_data = ToBase64("images/Masahiro.png")
|
570 |
+
audio_data = ToBase64(temp_aud_file)
|
571 |
+
|
572 |
+
response = requests.post(GRADIO_URL+"/run/sad_talker", json={
|
573 |
+
"data": [
|
574 |
+
"data:image/png;base64,"+img_data,
|
575 |
+
{"name":"audio.wav","data":"data:audio/wav;base64,"+audio_data},
|
576 |
+
"crop",
|
577 |
+
False,
|
578 |
+
False,
|
579 |
+
]
|
580 |
+
},timeout=3000)
|
581 |
+
print(response.text)
|
582 |
+
res = response.json()
|
583 |
+
|
584 |
+
data = res["data"]
|
585 |
+
print(data)
|
586 |
+
video_rul = GRADIO_URL+"/file=" + data[0][0]['name']
|
587 |
+
print(video_rul)
|
588 |
+
|
589 |
+
html_video = '<pre>no video</pre>'
|
590 |
+
|
591 |
+
# with open('videos/tempfile.mp4', 'wb') as f:
|
592 |
+
# f.write(response_stream.read())
|
593 |
+
# temp_file = gr.File("videos/tempfile.mp4")
|
594 |
+
# temp_file_url = "/file=" + temp_file.value['name']
|
595 |
+
temp_file_url=video_rul
|
596 |
+
html_video = f'<video width={TALKING_HEAD_WIDTH} height={TALKING_HEAD_WIDTH} autoplay><source src={temp_file_url} type="video/mp4" poster="Michelle.png"></video>'
|
597 |
+
|
598 |
+
return html_video, "videos/tempfile.mp4"
|
599 |
+
|
600 |
|
601 |
def do_html_video_speak(words_to_speak, azure_language):
|
602 |
azure_voice = AZURE_VOICE_DATA.get_voice(azure_language, "Male")
|