Upload 3 files

Dockerfile

@@ -0,0 +1,64 @@
+FROM ubuntu:latest
+
+WORKDIR /content
+
+RUN apt-get update && apt-get install -y \
+    python3 \
+    python3-pip \
+    gnupg \
+    wget \
+    htop \
+    sudo \
+    git \
+    git-lfs \
+    software-properties-common \
+    build-essential \
+    cmake \
+    curl \
+    libavcodec-dev \
+    libavformat-dev \
+    libavdevice-dev \
+    libgl1 \
+    libgtk2.0-0 \
+    jq \
+    libraw1394-dev \
+    libopenblas-base
+    
+RUN alias python=python3    
+
+RUN apt-get install -y gnupg wget htop sudo git git-lfs software-properties-common build-essential cmake curl 
+RUN apt-get install -y ffmpeg libavcodec-dev libavformat-dev libavdevice-dev libgl1 libgtk2.0-0 jq libraw1394-dev libopenblas-base
+RUN apt-add-repository -y universe
+
+RUN pip3 install pandas scipy matplotlib torch torchvision ffmpeg-python imageio[ffmpeg] tensorboardX huggingface-hub g2p_en opencv-python fairseq imageio torchaudio gradio gtts soundfile fairseq huggingface-hub g2p_en altair imageio-ffmpeg pocketsphinx dlib ffmpeg jq "numpy==1.23.1"
+
+RUN pip install cmake==3.24.1.1
+
+RUN git clone https://github.com/TencentARC/GFPGAN.git && cd GFPGAN && pip install basicsr && pip install facexlib && pip install -r requirements.txt && python3 setup.py develop && pip install realesrgan
+RUN git clone https://github.com/chi0tzp/PyVideoFramesExtractor && cd PyVideoFramesExtractor && pip install -r requirements.txt
+
+RUN git lfs install
+RUN git clone https://huggingface.co/camenduru/pocketsphinx-20.04-t4 pocketsphinx && cd pocketsphinx && cmake -S . -B build && cmake --build build --target install
+RUN git clone https://huggingface.co/camenduru/one-shot-talking-face-20.04-t4 one-shot-talking-face && cd one-shot-talking-face && pip install -r requirements.txt && chmod 755 OpenFace/FeatureExtraction
+RUN sed -i 's/.cuda()/ /' one-shot-talking-face/test_script.py
+RUN sed -i 's/.cuda()/ /' one-shot-talking-face/tools/interface.py
+RUN sed -i 's/.load(checkpoint_path)/.load(checkpoint_path,map_location=torch.device("cpu")) /' one-shot-talking-face/tools/interface.py
+RUN sed -i 's/.load(audio2pose)/.load(audio2pose,map_location=torch.device("cpu")) /' one-shot-talking-face/tools/interface.py
+RUN mkdir /content/out
+
+COPY app.py /content/app.py
+
+
+RUN adduser --disabled-password --gecos '' admin
+RUN adduser admin sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+RUN chown -R admin:admin /content
+RUN chmod -R 777 /content
+RUN chown -R admin:admin /home
+RUN chmod -R 777 /home
+USER admin
+
+EXPOSE 7860
+
+CMD ["python3", "app.py"]

app.py

@@ -0,0 +1,257 @@
+import gradio as gr
+import os, subprocess, torchaudio
+import torch
+from PIL import Image
+import gradio as gr
+import soundfile
+from gtts import gTTS
+import tempfile
+from pydub.generators import Sine
+from pydub import AudioSegment
+import dlib
+import cv2
+import imageio
+import os
+import ffmpeg
+from io import BytesIO
+import requests
+import sys
+
+python_path = sys.executable
+
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+block = gr.Blocks()
+
+def compute_aspect_preserved_bbox(bbox, increase_area, h, w):
+    left, top, right, bot = bbox
+    width = right - left
+    height = bot - top
+
+    width_increase = max(increase_area, ((1 + 2 * increase_area) * height - width) / (2 * width))
+    height_increase = max(increase_area, ((1 + 2 * increase_area) * width - height) / (2 * height))
+
+    left_t = int(left - width_increase * width)
+    top_t = int(top - height_increase * height)
+    right_t = int(right + width_increase * width)
+    bot_t = int(bot + height_increase * height)
+
+    left_oob = -min(0, left_t)
+    right_oob = right - min(right_t, w)
+    top_oob = -min(0, top_t)
+    bot_oob = bot - min(bot_t, h)
+
+    if max(left_oob, right_oob, top_oob, bot_oob) > 0:
+        max_w = max(left_oob, right_oob)
+        max_h = max(top_oob, bot_oob)
+        if max_w > max_h:
+            return left_t + max_w, top_t + max_w, right_t - max_w, bot_t - max_w
+        else:
+            return left_t + max_h, top_t + max_h, right_t - max_h, bot_t - max_h
+
+    else:
+        return (left_t, top_t, right_t, bot_t)
+
+def crop_src_image(src_img, detector=None):
+    if  detector is None:
+        detector = dlib.get_frontal_face_detector()
+    save_img='/content/image_pre.png'
+    img = cv2.imread(src_img)
+    faces = detector(img, 0)
+    h, width, _ = img.shape
+    if len(faces) > 0:
+        bbox = [faces[0].left(), faces[0].top(),faces[0].right(), faces[0].bottom()]
+        l = bbox[3]-bbox[1]
+        bbox[1]= bbox[1]-l*0.1
+        bbox[3]= bbox[3]-l*0.1
+        bbox[1] = max(0,bbox[1])
+        bbox[3] = min(h,bbox[3])
+        bbox = compute_aspect_preserved_bbox(tuple(bbox), 0.5, img.shape[0], img.shape[1])
+        img = img[bbox[1] :bbox[3] , bbox[0]:bbox[2]]
+        img = cv2.resize(img, (256, 256))
+        cv2.imwrite(save_img,img)
+    else:
+        img = cv2.resize(img,(256,256))
+        cv2.imwrite(save_img, img)
+    return save_img
+
+def pad_image(image):
+    w, h = image.size
+    if w == h:
+        return image
+    elif w > h:
+        new_image = Image.new(image.mode, (w, w), (0, 0, 0))
+        new_image.paste(image, (0, (w - h) // 2))
+        return new_image
+    else:
+        new_image = Image.new(image.mode, (h, h), (0, 0, 0))
+        new_image.paste(image, ((h - w) // 2, 0))
+        return new_image
+
+def calculate(image_in, audio_in):
+    waveform, sample_rate = torchaudio.load(audio_in)
+    waveform = torch.mean(waveform, dim=0, keepdim=True)
+    torchaudio.save("/content/audio.wav", waveform, sample_rate, encoding="PCM_S", bits_per_sample=16)
+    image_in = image_in.replace("results/", "")
+    print("****"*100)
+    print(f" *#*#*# original image => {image_in}  *#*#*# ")
+    if os.path.exists(image_in):
+        print(f"image exists => {image_in}")
+        image = Image.open(image_in)
+    else:
+        print("image not exists reading web image")
+        image_url = "http://labelme.csail.mit.edu/Release3.0/Images/users/DNguyen91/face/m_unsexy_gr.jpg"
+        response = requests.get(image_url)
+        image = Image.open(BytesIO(response.content))
+    print("****"*100)
+    image = pad_image(image)
+    # os.system(f"rm -rf /content/image.png")
+    image.save("image.png")
+
+    pocketsphinx_run = subprocess.run(['pocketsphinx', '-phone_align', 'yes', 'single', '/content/audio.wav'], check=True, capture_output=True)
+    jq_run = subprocess.run(['jq', '[.w[]|{word: (.t | ascii_upcase | sub("<S>"; "sil") | sub("<SIL>"; "sil") | sub("\\\(2\\\)"; "") | sub("\\\(3\\\)"; "") | sub("\\\(4\\\)"; "") | sub("\\\[SPEECH\\\]"; "SIL") | sub("\\\[NOISE\\\]"; "SIL")), phones: [.w[]|{ph: .t | sub("\\\+SPN\\\+"; "SIL") | sub("\\\+NSN\\\+"; "SIL"), bg: (.b*100)|floor, ed: (.b*100+.d*100)|floor}]}]'], input=pocketsphinx_run.stdout, capture_output=True)
+    with open("test.json", "w") as f:
+        f.write(jq_run.stdout.decode('utf-8').strip())
+    # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    # os.system(f"rm -rf /content/image_audio.mp4")
+    os.system(f"cd /content/one-shot-talking-face && {python_path} -B test_script.py --img_path /content/image.png --audio_path /content/audio.wav --phoneme_path /content/test.json --save_dir /content/train")
+    return "/content/train/image_audio.mp4"
+
+def merge_frames():
+
+
+  path = '/content/video_results/restored_imgs'
+    
+  if not os.path.exists(path):
+      os.makedirs(path)
+        
+  image_folder = os.fsencode(path)
+  print(image_folder)
+  filenames = []
+
+  for file in os.listdir(image_folder):
+      filename = os.fsdecode(file)
+      if filename.endswith( ('.jpg', '.png', '.gif') ):
+          filenames.append(filename)
+
+  filenames.sort() # this iteration technique has no built in order, so sort the frames
+  print(filenames)
+  images = list(map(lambda filename: imageio.imread("/content/video_results/restored_imgs/"+filename), filenames))
+  # os.system(f"rm -rf /content/video_output.mp4")
+  imageio.mimsave('/content/video_output.mp4', images, fps=25.0) # modify the frame duration as needed
+  return "/content/video_output.mp4"
+
+def audio_video():
+
+  input_video = ffmpeg.input('/content/video_output.mp4')
+
+  input_audio = ffmpeg.input('/content/audio.wav')
+  os.system(f"rm -rf /content/final_output.mp4")
+  ffmpeg.concat(input_video, input_audio, v=1, a=1).output('/content/final_output.mp4').run()
+  
+  return "/content/final_output.mp4"
+
+def one_shot_talking(image_in,audio_in):
+
+
+  # Pre-processing of image
+  crop_img=crop_src_image(image_in)
+
+  if os.path.exists("/content/results/restored_imgs/image_pre.png"):
+    os.system(f"rm -rf /content/results/restored_imgs/image_pre.png")
+  
+  if not os.path.exists( "/content/results" ):
+    os.makedirs("/content/results")
+  
+  #Improve quality of input image
+  os.system(f"{python_path} /content/GFPGAN/inference_gfpgan.py --upscale 2 -i /content/image_pre.png -o /content/results --bg_upsampler realesrgan")
+  # time.sleep(60)
+  
+  image_in_one_shot='/content/results/image_pre.png'
+  
+  #One Shot Talking Face algorithm
+  calculate(image_in_one_shot,audio_in)
+
+  #Video Quality Improvement
+  os.system(f"rm -rf /content/extracted_frames/image_audio_frames")
+  #1. Extract the frames from the video file using PyVideoFramesExtractor
+  os.system(f"{python_path} /content/PyVideoFramesExtractor/extract.py --video=/content/train/image_audio.mp4")
+
+  #2. Improve image quality using GFPGAN on each frames
+  # os.system(f"rm -rf /content/extracted_frames/image_audio_frames")
+  os.system(f"rm -rf /content/video_results/")
+  os.system(f"{python_path} /content/GFPGAN/inference_gfpgan.py --upscale 2 -i /content/extracted_frames/image_audio_frames -o /content/video_results  --bg_upsampler realesrgan")
+
+  #3. Merge all the frames to a one video using imageio
+  merge_frames()
+  return audio_video()
+
+
+def one_shot(image_in,input_text,gender):
+    if gender == "Female":
+         tts = gTTS(input_text)
+         with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
+              tts.write_to_fp(f)
+              f.seek(0)
+              sound = AudioSegment.from_file(f.name, format="mp3")
+              os.system(f"rm -rf /content/audio.wav")
+              sound.export("/content/audio.wav", format="wav")
+              audio_in="/content/audio.wav"
+         return one_shot_talking(image_in,audio_in)
+    elif gender == 'Male':
+       
+          models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
+              "Voicemod/fastspeech2-en-male1",
+              arg_overrides={"vocoder": "hifigan", "fp16": False}
+          )
+
+          model = models[0]
+          TTSHubInterface.update_cfg_with_data_cfg(cfg, task.data_cfg)
+          generator = task.build_generator([model], cfg)
+          # next(model.parameters()).device
+
+          sample = TTSHubInterface.get_model_input(task, input_text)
+          sample["net_input"]["src_tokens"] = sample["net_input"]["src_tokens"]
+          sample["net_input"]["src_lengths"] = sample["net_input"]["src_lengths"]
+          sample["speaker"] = sample["speaker"]
+
+          wav, rate = TTSHubInterface.get_prediction(task, model, generator, sample)
+          # soundfile.write("/content/audio_before.wav", wav, rate)
+          os.system(f"rm -rf /content/audio_before.wav")
+          soundfile.write("/content/audio_before.wav", wav.cpu().clone().numpy(), rate)
+          os.system(f"rm -rf /content/audio.wav")
+          cmd='ffmpeg -i /content/audio_before.wav -filter:a "atempo=0.7" -vn /content/audio.wav'
+          os.system(cmd)
+          audio_in="/content/audio.wav"
+          
+          return one_shot_talking(image_in,audio_in)
+           
+                        
+def run():
+  with gr.Blocks(css=".gradio-container {background-color: lightgray} #radio_div {background-color: #FFD8B4; font-size: 40px;}") as demo:
+    gr.Markdown("<h1 style='text-align: center;'>"+ "One Shot Talking Face from Text" + "</h1><br/><br/>")
+    with gr.Group():
+      with gr.Box():
+        with gr.Row().style(equal_height=True):
+          image_in = gr.Image(show_label=True, type="filepath",label="Input Image")
+          input_text = gr.Textbox(show_label=True,label="Input Text")
+          gender = gr.Radio(["Female","Male"],value="Female",label="Gender")
+          video_out = gr.Video(show_label=True,label="Output")
+        with gr.Row().style(equal_height=True):
+          btn = gr.Button("Generate")   
+    # gr.Markdown(
+    #         """
+    #         <p style='text-align: center;'>Feel free to give us your thoughts on this demo and please contact us at 
+    #                 <a href="mailto:letstalk@pragnakalp.com" target="_blank">letstalk@pragnakalp.com</a> 
+    #                 <p style='text-align: center;'>Developed by: <a href="https://www.pragnakalp.com" target="_blank">Pragnakalp Techlabs</a></p>
+
+    #         """)
+
+    btn.click(one_shot, inputs=[image_in,input_text,gender], outputs=[video_out])
+    demo.queue()
+    demo.launch(server_name="0.0.0.0", server_port=7860)
+
+
+if __name__ == "__main__":
+    run()

requirements.txt

@@ -0,0 +1,5 @@
+gtts
+soundfile
+fairseq
+huggingface-hub
+g2p_en