Spaces:
Running
Running
| import gradio as gr | |
| from gradio_client import Client | |
| import numpy as np | |
| import os | |
| import random | |
| hf_token = os.environ.get("HF_TKN") | |
| # global variable | |
| MAX_SEED = np.iinfo(np.int32).max | |
| from style_template import styles | |
| STYLE_NAMES = list(styles.keys()) | |
| DEFAULT_STYLE_NAME = "(No style)" | |
| def randomize_seed_fn(seed: int, randomize_seed: bool) -> int: | |
| if randomize_seed: | |
| seed = random.randint(0, MAX_SEED) | |
| return seed | |
| def get_instantID(portrait_in, condition_pose, controlnet, prompt, style): | |
| negative_prompt = "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, (frame:1.2), deformed, ugly, deformed eyes, blur, out of focus, blurry, deformed cat, deformed, photo, anthropomorphic cat, monochrome, pet collar, gun, weapon, blue, 3d, drones, drone, buildings in background, green" | |
| seed = 42 | |
| seed = random.randint(0, MAX_SEED) | |
| client = Client("https://instantx-instantid.hf.space/") | |
| result = client.predict( | |
| portrait_in, # filepath in 'Upload a photo of your face' Image component | |
| condition_pose, # filepath in 'Upload a reference pose image (Optional)' Image component | |
| prompt, # str in 'Prompt' Textbox component | |
| negative_prompt, # str in 'Negative Prompt' Textbox component | |
| style, # Literal['(No style)', 'Spring Festival', 'Watercolor', 'Film Noir', 'Neon', 'Jungle', 'Mars', 'Vibrant Color', 'Snow', 'Line art'] in 'Style template' Dropdown component | |
| 5, # float (numeric value between 1 and 100) in 'Number of sample steps' Slider component | |
| 0.8, # float (numeric value between 0 and 1.5) in 'IdentityNet strength (for fidelity)' Slider component | |
| 0.8, # float (numeric value between 0 and 1.5) in 'Image adapter strength (for detail)' Slider component | |
| 0.4, # float (numeric value between 0 and 1.5) in 'Pose strength' Slider component | |
| 0.4, # float (numeric value between 0 and 1.5) in 'Canny strength' Slider component | |
| 0.4, # float (numeric value between 0 and 1.5) in 'Depth strength' Slider component | |
| controlnet, # List[Literal['pose', 'canny', 'depth']] in 'Controlnet' Checkboxgroup component | |
| 1.5, # float (numeric value between 0.1 and 20.0) in 'Guidance scale' Slider component | |
| seed, # float (numeric value between 0 and 2147483647) in 'Seed' Slider component | |
| "EulerDiscreteScheduler", # Literal['DEISMultistepScheduler', 'HeunDiscreteScheduler', 'EulerDiscreteScheduler', 'DPMSolverMultistepScheduler', 'DPMSolverMultistepScheduler-Karras', 'DPMSolverMultistepScheduler-Karras-SDE'] in 'Schedulers' Dropdown component | |
| True, # bool in 'Enable Fast Inference with LCM' Checkbox component | |
| True, # bool in 'Enhance non-face region' Checkbox component | |
| api_name="/generate_image" | |
| ) | |
| print(result) | |
| return result[0] | |
| def get_video_i2vgen(image_in, prompt): | |
| client = Client("https://modelscope-i2vgen-xl.hf.space/") | |
| result = client.predict( | |
| image_in, | |
| prompt, | |
| fn_index=1 | |
| #api_name="/image_to_video" | |
| ) | |
| print(result) | |
| return result | |
| def get_video_svd(image_in): | |
| from gradio_client import Client | |
| client = Client("https://multimodalart-stable-video-diffusion.hf.space/") | |
| result = client.predict( | |
| image_in, # filepath in 'Upload your image' Image component | |
| 0, # float (numeric value between 0 and 9223372036854775807) in 'Seed' Slider component | |
| True, # bool in 'Randomize seed' Checkbox component | |
| 127, # float (numeric value between 1 and 255) in 'Motion bucket id' Slider component | |
| 6, # float (numeric value between 5 and 30) in 'Frames per second' Slider component | |
| api_name="/video" | |
| ) | |
| print(result) | |
| return result[0]["video"] | |
| def load_sample_shot(camera_shot): | |
| if camera_shot == "close-up": | |
| conditional_pose = "camera_shots/close_up_shot.jpeg" | |
| elif camera_shot == "medium close-up": | |
| conditional_pose = "camera_shots/medium_close_up.jpeg" | |
| elif camera_shot == "medium shot": | |
| conditional_pose = "camera_shots/medium_shot.png" | |
| elif camera_shot == "cowboy shot": | |
| conditional_pose = "camera_shots/cowboy_shot.jpeg" | |
| elif camera_shot == "medium full shot": | |
| conditional_pose = "camera_shots/medium_full_shot.png" | |
| elif camera_shot == "full shot": | |
| conditional_pose = "camera_shots/full_shot.jpeg" | |
| elif camera_shot == "custom": | |
| conditional_pose = None | |
| return conditional_pose | |
| def use_custom_cond(): | |
| return "custom" | |
| def get_short_caption(image_in): | |
| client = Client("https://vikhyatk-moondream1.hf.space/") | |
| result = client.predict( | |
| image_in, # filepath in 'image' Image component | |
| "Describe what is happening in one sentence", # str in 'Question' Textbox component | |
| api_name="/answer_question" | |
| ) | |
| print(result) | |
| return result | |
| def infer(image_in, camera_shot, conditional_pose, controlnet_selection, prompt, style, chosen_model): | |
| if camera_shot == "custom": | |
| if conditional_pose != None: | |
| conditional_pose = conditional_pose | |
| else : | |
| raise gr.Error("No custom conditional shot found !") | |
| iid_img = get_instantID(image_in, conditional_pose, controlnet_selection, prompt, style) | |
| #short_cap = get_short_caption(iid_img) | |
| if chosen_model == "i2vgen-xl" : | |
| video_res = get_video_i2vgen(iid_img, prompt) | |
| elif chosen_model == "stable-video" : | |
| video_res = get_video_svd(image_in) | |
| print(video_res) | |
| return video_res | |
| css = """ | |
| #col-container{ | |
| margin: 0 auto; | |
| max-width: 1080px; | |
| } | |
| """ | |
| with gr.Blocks(css=css) as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.HTML(""" | |
| <h2 style="text-align: center;"> | |
| InstantID-2V | |
| </h2> | |
| <p style="text-align: center;"> | |
| Generate alive camera shot from input face | |
| </p> | |
| """) | |
| with gr.Row(): | |
| with gr.Column(): | |
| face_in = gr.Image(type="filepath", label="Face to copy", value="monalisa.png") | |
| with gr.Column(): | |
| with gr.Group(): | |
| with gr.Row(): | |
| camera_shot = gr.Dropdown( | |
| label = "Camera Shot", | |
| info = "Use standard camera shots vocabulary, or drop your custom shot as conditional pose (1280*720 ratio is recommended)", | |
| choices = [ | |
| "custom", "close-up", "medium close-up", "medium shot", "cowboy shot", "medium full shot", "full shot" | |
| ], | |
| value = "custom" | |
| ) | |
| style = gr.Dropdown(label="Style template", info="InstantID legacy templates", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME) | |
| condition_shot = gr.Image(type="filepath", label="Custom conditional shot (Important) [1280*720 recommended]") | |
| controlnet_selection = gr.CheckboxGroup( | |
| ["pose", "canny", "depth"], label="Controlnet", value=["pose"], | |
| info="Use pose for skeleton inference, canny for edge detection, and depth for depth map estimation. You can try all three to control the generation process" | |
| ) | |
| prompt = gr.Textbox(label="Short Prompt (keeping it short is better)") | |
| chosen_model = gr.Radio(label="Choose a model", choices=["i2vgen-xl", "stable-video"], value="i2vgen-xl", interactive=False, visible=False) | |
| with gr.Column(): | |
| submit_btn = gr.Button("Submit") | |
| video_out = gr.Video() | |
| camera_shot.change( | |
| fn = load_sample_shot, | |
| inputs = camera_shot, | |
| outputs = condition_shot, | |
| queue=False | |
| ) | |
| condition_shot.clear( | |
| fn = use_custom_cond, | |
| inputs = None, | |
| outputs = camera_shot, | |
| queue=False, | |
| ) | |
| submit_btn.click( | |
| fn = infer, | |
| inputs = [ | |
| face_in, | |
| camera_shot, | |
| condition_shot, | |
| controlnet_selection, | |
| prompt, | |
| style, | |
| chosen_model | |
| ], | |
| outputs = [ | |
| video_out | |
| ] | |
| ) | |
| demo.queue(max_size=3).launch(share=False, show_error=True, show_api=False) |