import os import datetime import shutil import subprocess import cv2 from PIL import Image from moviepy.editor import * from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip from gradio_client import Client import gradio as gr from share_btn import community_icon_html, loading_icon_html, share_js matte_client = Client("https://fffiloni-video-matting-anything.hf.space/") # execute a CLI command def execute_command(command: str) -> None: subprocess.run(command, check=True) def infer(video_frames, masks_frames, project_name): timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") # Create the directory if it doesn't exist my_video_directory = f"{project_name}" if not os.path.exists(my_video_directory): os.makedirs(my_video_directory) else: # If the directory already exists, add a timestamp to the new directory name my_video_directory = f"{project_name}_{timestamp}" os.makedirs(my_video_directory) # Assuming 'images' is a list of image file paths for idx, image in enumerate(video_frames): # Get the base file name (without path) from the original location image_name = os.path.basename(image.name) # Construct the destination path in the working directory destination_path = os.path.join(my_video_directory, image_name) # Copy the image from the original location to the working directory shutil.copy(image.name, destination_path) # Print the image name and its corresponding save path print(f"Image {idx + 1}: {image_name} copied to {destination_path}") # Create the directory if it doesn't exist my_masks_directory = f"{project_name}_masks" if not os.path.exists(my_masks_directory): os.makedirs(my_masks_directory) else: # If the directory already exists, add a timestamp to the new directory name my_masks_directory = f"{project_name}_masks_{timestamp}" os.makedirs(my_masks_directory) # Assuming 'images' is a list of image file paths for idx, image in enumerate(masks_frames): # Get the base file name (without path) from the original location image_name = os.path.basename(image.name) # Construct the destination path in the working directory destination_path = os.path.join(my_masks_directory, image_name) # Copy the image from the original location to the working directory shutil.copy(image.name, destination_path) # Print the image name and its corresponding save path print(f"Image {idx + 1}: {image_name} copied to {destination_path}") #video_frames_folder = "inputs/object_removal/bmx-trees" #masks_folder = "inputs/object_removal/bmx-trees_mask" video_frames_folder = f"{my_video_directory}" masks_folder = f"{my_masks_directory}" # Create the "results" folder if it doesn't exist output_folder = f"results_{timestamp}" if not os.path.exists(output_folder): os.makedirs(output_folder) #bmx_trees_folder = os.path.join(output_folder, "bmx-trees") command = [ f"python", f"inference_propainter.py", f"--video={video_frames_folder}", f"--mask={masks_folder}", f"--output={output_folder}" ] execute_command(command) # Get the list of files in the "results" folder result_files = os.listdir(output_folder) # Print the content of the "results" folder print(f"Contents of the {output_folder} folder:") for item in result_files: print(item) # List the content of the "bmx-trees" folder within "results" results_folder = os.path.join(output_folder, my_video_directory) results_folder_content = [os.path.join(results_folder, item) for item in os.listdir(results_folder)] print(f"Contents of the {results_folder} folder:") for item in results_folder_content: print(item) return results_folder_content[0], results_folder_content[1], gr.Group.update(visible=True) def get_frames(video_in, img_type): frames = [] #resize the video clip = VideoFileClip(video_in) #check fps if clip.fps > 30: print("vide rate is over 30, resetting to 30") clip_resized = clip.resize(height=512) clip_resized.write_videofile(f"{img_type}_video_resized.mp4", fps=30) else: print("video rate is OK") clip_resized = clip.resize(height=512) clip_resized.write_videofile(f"{img_type}_video_resized.mp4", fps=clip.fps) print("video resized to 512 height") # Opens the Video file with CV2 cap= cv2.VideoCapture(f"{img_type}_video_resized.mp4") fps = cap.get(cv2.CAP_PROP_FPS) print("video fps: " + str(fps)) i=0 while(cap.isOpened()): ret, frame = cap.read() if ret == False: break if img_type == "source": filename = f'{i:05d}.jpg' cv2.imwrite(filename, frame) frames.append(filename) elif img_type == "mask": filename = f'{i:05d}.png' cv2.imwrite(filename, frame) frames.append(filename) i+=1 cap.release() cv2.destroyAllWindows() print("broke the video into frames") return frames, fps def get_matte(video_in, subject_to_remove): print("Trying to call video matting") result = matte_client.predict( f"{video_in}", # str (filepath on your computer (or URL) of file) in 'parameter_4' Video component 10, # int | float (numeric value between 0 and 10) in 'Cut video at (s)' Slider component f"{subject_to_remove}", # str in 'Text prompt' Textbox component "", # str in 'Background prompt' Textbox component api_name="/go_matte" ) print(result) return result[2] def infer_auto(project_name, video_in, subject_to_remove): timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") print(video_in) # Load the video clip video_clip = VideoFileClip(video_in) # Check the original FPS of the video original_fps = video_clip.fps # Set the target FPS to 24 target_fps = 24 # Check if the original FPS is greater than 24 if original_fps > target_fps: # Convert the video to the target FPS video_clip = video_clip.set_duration(video_clip.duration * (target_fps / original_fps)) video_clip = video_clip.set_fps(target_fps) # Write the new video to an output file video_clip.write_videofile(f"retimed_clip_in_{timestamp}.mp4", codec='libx264') video_in = f"retimed_clip_in_{timestamp}.mp4" matte_video = get_matte(video_in, subject_to_remove) # Cut the video to the first 3 seconds #video_cut = f"video_cut.mp4" #ffmpeg_extract_subclip(video_in, t1=0, t2=3, targetname=video_cut) video_frames = get_frames(video_in, "source") frames_list = video_frames[0] print(video_frames[0]) masks_frames = get_frames(matte_video, "mask") masks_list = masks_frames[0] print(masks_frames[0]) # Check the lengths of the two lists frames_length = len(frames_list) masks_length = len(masks_list) # Make the lists the same length if they are different if frames_length > masks_length: frames_list = frames_list[:masks_length] elif masks_length > frames_length: masks_list = masks_list[:frames_length] # Now, both lists have the same length # Create the directory if it doesn't exist my_video_directory = f"{project_name}" if not os.path.exists(my_video_directory): os.makedirs(my_video_directory) else: # If the directory already exists, add a timestamp to the new directory name my_video_directory = f"{project_name}_{timestamp}" os.makedirs(my_video_directory) print(f"Created the dir: {my_video_directory}") # Assuming 'images' is a list of image file paths for idx, image in enumerate(frames_list): # Get the base file name (without path) from the original location image_name = os.path.basename(image) # Construct the destination path in the working directory destination_path = os.path.join(my_video_directory, image_name) # Copy the image from the original location to the working directory shutil.copy(image, destination_path) # Print the image name and its corresponding save path print(f"Image {idx + 1}: {image_name} copied to {destination_path}") # Create the directory if it doesn't exist my_masks_directory = f"{project_name}_masks" if not os.path.exists(my_masks_directory): os.makedirs(my_masks_directory) else: # If the directory already exists, add a timestamp to the new directory name my_masks_directory = f"{project_name}_masks_{timestamp}" os.makedirs(my_masks_directory) print(f"Created the dir: {my_masks_directory}") # Assuming 'images' is a list of image file paths for idx, image in enumerate(masks_list): # Get the base file name (without path) from the original location image_name = os.path.basename(image) # Construct the destination path in the working directory destination_path = os.path.join(my_masks_directory, image_name) # Copy the image from the original location to the working directory shutil.copy(image, destination_path) # Print the image name and its corresponding save path print(f"Image {idx + 1}: {image_name} copied to {destination_path}") #video_frames_folder = "inputs/object_removal/bmx-trees" #masks_folder = "inputs/object_removal/bmx-trees_mask" video_frames_folder = f"{my_video_directory}" masks_folder = f"{my_masks_directory}" # Create the "results" folder if it doesn't exist output_folder = f"results_{timestamp}" if not os.path.exists(output_folder): os.makedirs(output_folder) #bmx_trees_folder = os.path.join(output_folder, "bmx-trees") # Convert the float fps to an integer needed_fps = int(video_frames[1]) command_auto= [ f"python", f"inference_propainter.py", f"--video={video_frames_folder}", f"--mask={masks_folder}", f"--output={output_folder}", f"--save_fps={int(needed_fps)}", #f"--fp16" ] execute_command(command_auto) # Get the list of files in the "results" folder result_files = os.listdir(output_folder) # Print the content of the "results" folder print(f"Contents of the {output_folder} folder:") for item in result_files: print(item) # List the content of the "bmx-trees" folder within "results" results_folder = os.path.join(output_folder, my_video_directory) results_folder_content = [os.path.join(results_folder, item) for item in os.listdir(results_folder)] print(f"Contents of the {results_folder} folder:") for item in results_folder_content: print(item) return results_folder_content[0], results_folder_content[1], gr.Group.update(visible=True) css=""" #col-container{ margin: 0 auto; max-width: 840px; text-align: left; } .animate-spin { animation: spin 1s linear infinite; } @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } } #share-btn-container { display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; max-width: 15rem; height: 32px; } div#share-btn-container > div { flex-direction: row; background: black; align-items: center; } #share-btn-container:hover { background-color: #060606; } #share-btn { all: initial; color: #ffffff; font-weight: 600; font-size: 1em; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.5rem !important; padding-bottom: 0.5rem !important; right:0; } #share-btn * { all: unset; } #share-btn-container div:nth-child(-n+2){ width: auto !important; min-height: 0px !important; } #share-btn-container .wrap { display: none !important; } #share-btn-container.hidden { display: none!important; } img[src*='#center'] { display: block; margin: unset; margin-top: 6px; } div#component-25 { align-items: center; } """ with gr.Blocks(css=css) as demo: with gr.Column(elem_id="col-container"): gr.HTML("""

ProPainter

[ICCV 2023] ProPainter: Improving Propagation and Transformer for Video Inpainting
code | project page

""") with gr.Row(): with gr.Tab("Manual"): with gr.Column(): project_name = gr.Textbox(label="Name your project", info="no spaces nor special characters", value="my-new-project") video_frames = gr.File(label="Video frames", file_types=["image"], file_count="multiple") masks_frames = gr.File(label="Masks frames", file_types=["image"], file_count="multiple") submit_btn = gr.Button("Submit") gr.Examples( examples = [ [ [ "hf-examples/manual/00000.jpg", "hf-examples/manual/00001.jpg", "hf-examples/manual/00002.jpg" ], [ "hf-examples/manual/00000.png", "hf-examples/manual/00001.png", "hf-examples/manual/00002.png" ], "manual_example" ] ], fn = infer, inputs=[video_frames, masks_frames, project_name], #outputs=[res_masked, res_files] ) with gr.Tab("Auto"): with gr.Column(): project_name_2 = gr.Textbox(label="Name your project", info="no spaces nor special characters", value="my-new-project") video_in = gr.Video(label="Source video", source="upload", format="mp4") subject_to_remove = gr.Textbox(label="Subject to remove") submit_auto_btn = gr.Button("Submit") gr.Examples( examples = [ [ "example_1", "hf-examples/knight.mp4", "knight" ], [ "example_2", "hf-examples/knight.mp4", "horse" ], [ "example_3", "hf-examples/knight.mp4", "tail" ] ], fn = infer_auto, inputs=[project_name_2, video_in, subject_to_remove], #outputs=[res_masked, res_files] ) with gr.Column(): gr.Markdown(""" ### Results """) res_masked = gr.Video(label="Masked video", elem_id="res-masked") res_files = gr.Video(label="Final result", elem_id="res_cleaned") with gr.Row(): with gr.Group(elem_id="share-btn-container", visible=False) as share_group: community_icon = gr.HTML(community_icon_html) loading_icon = gr.HTML(loading_icon_html) share_button = gr.Button("Share with Community", elem_id="share-btn") gr.Markdown(""" [![Duplicate this Space](https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-lg.svg#center)](https://huggingface.co/spaces/fffiloni/ProPainter?duplicate=true) """) submit_btn.click(fn=infer, inputs=[video_frames, masks_frames, project_name], outputs=[res_masked, res_files, share_group]) submit_auto_btn.click(fn=infer_auto, inputs=[project_name_2, video_in, subject_to_remove], outputs=[res_masked, res_files, share_group]) share_button.click(None, [], [], _js=share_js) demo.queue(max_size=12).launch()