Align face ID app with RoPE reference conditioning

app.py

@@ -16,7 +16,6 @@ import torch
 from huggingface_hub import hf_hub_download, snapshot_download
 
 from ltx_pipelines.distilled import DistilledPipeline
-from ltx_pipelines.utils.args import ImageConditioningInput
 from ltx_pipelines.utils.media_io import encode_video
 from ltx_pipelines.utils.types import OffloadMode
 from ltx_core.loader import LoraPathStrengthAndSDOps, LTXV_LORA_COMFY_RENAMING_MAP
@@ -144,9 +143,6 @@ def preload_models_for_zerogpu() -> None:
     print("Models preloaded: weights registered for ZeroGPU streaming.")
 
 
-MODE_KEEP_PHOTO = "Animate my photo"
-MODE_NEW_SCENE = "Put my face in a new scene"
-
 # Aspect ratios offered in "new scene" mode -> exact (width, height), each a
 # multiple of 64 (so stage-1 at half resolution stays a multiple of 32 for the
 # patchifier), ~0.44-0.59 MP.
@@ -181,7 +177,6 @@ def generate_video(
     face_image,
     prompt: str,
     duration: float,
-    mode: str,
     aspect_ratio: str,
     seed: int,
     randomize_seed: bool,
@@ -202,15 +197,10 @@ def generate_video(
             present tense. Including identity attributes (hair, skin tone, eyes,
             facial hair, glasses, face shape) noticeably improves fidelity.
         duration (float): Video length in seconds, from 1.0 to 8.0.
-        mode (str): One of "Animate my photo (start from this image)" — the video
-            starts from and animates the uploaded photo — or "Put my face in a new
-            scene (text only)" — the whole scene is generated from the prompt with
-            the person's face preserved.
         seed (int): Random seed, 0 to 2147483647. Ignored when randomize_seed=True.
         randomize_seed (bool): When true, a fresh random seed is used each run.
-        aspect_ratio (str): Output aspect ratio for "new scene" mode — one of
-            "1:1", "4:3", "3:4", "16:9", "9:16". Ignored in "animate my photo"
-            mode, where the output matches the uploaded photo's aspect ratio.
+        aspect_ratio (str): Output aspect ratio — one of "1:1", "4:3", "3:4",
+            "16:9", "9:16".
 
     Returns:
         tuple[str, int]: Filepath of the generated .mp4 video, and the seed used.
@@ -220,7 +210,6 @@ def generate_video(
         if face_image is None:
             raise ValueError("Please provide a reference face image.")
 
-        use_image_conditioning = mode != MODE_NEW_SCENE
         current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
 
         frame_rate = 24.0
@@ -238,31 +227,16 @@ def generate_video(
         else:
             temp_image_path = Path(face_image)
 
-        # Output dimensions:
-        #  - "animate my photo": match the uploaded photo's aspect ratio.
-        #  - "new scene": use the selected aspect ratio (default 4:3).
-        if use_image_conditioning:
-            from PIL import Image as _PILImage
-
-            _iw, _ih = (
-                face_image.size if hasattr(face_image, "size") else _PILImage.open(temp_image_path).size
-            )
-            width, height = _dims_for_ratio(_iw / _ih)
-        else:
-            width, height = ASPECT_DIMS.get(aspect_ratio, ASPECT_DIMS[DEFAULT_ASPECT])
+        width, height = ASPECT_DIMS.get(aspect_ratio, ASPECT_DIMS[DEFAULT_ASPECT])
 
-        # First-frame (I2V) conditioning only in "animate my photo" mode. Identity
-        # itself is carried by the overlap reference (reference_image_path) in BOTH
-        # modes — that's the BFS source-phase mechanism the LoRA was trained with.
-        images = (
-            [ImageConditioningInput(path=str(temp_image_path), frame_idx=0, strength=1.0)]
-            if use_image_conditioning
-            else []
-        )
+        # Best-Face-ID was not trained as I2V. The face image must not replace
+        # frame 0; it is injected below as source-phase-tagged overlap tokens in
+        # a separate RoPE source segment.
+        images = []
 
         # `ref_t2v:` is the LoRA's trained caption trigger.
         identity_prompt = prompt if prompt.strip().startswith("ref_t2v:") else f"ref_t2v: {prompt}"
-        print(f"Generating (mode={mode!r}, {int(width)}x{int(height)}, {num_frames}f): {identity_prompt}")
+        print(f"Generating (reference-overlap, {int(width)}x{int(height)}, {num_frames}f): {identity_prompt}")
 
         # --- ZeroGPU diagnostics (safe counters only; mem_get_info() asserts on MIG) ---
         try:
@@ -302,7 +276,8 @@ def generate_video(
                 frame_rate=frame_rate,
                 images=images,
                 tiling_config=tiling_config,
-                enhance_prompt=False,  # multimodal enhancer disabled (device bug)
+                enhance_prompt=True,
+                enhance_prompt_image_path=str(temp_image_path),
                 reference_image_path=str(temp_image_path),
                 reference_source_id=2.0,
                 reference_phase_scale=1.0,
@@ -342,12 +317,6 @@ with gr.Blocks(title="LTX-2.3 Face ID 🎥🧑") as demo:
         with gr.Column():
             face_image = gr.Image(label="Reference Face Image", type="pil")
 
-            mode = gr.Radio(
-                label="Generation mode",
-                choices=[MODE_KEEP_PHOTO, MODE_NEW_SCENE],
-                value=MODE_KEEP_PHOTO,
-            )
-
             prompt = gr.Textbox(
                 label="Prompt",
                 value=DEFAULT_PROMPT,
@@ -364,7 +333,6 @@ with gr.Blocks(title="LTX-2.3 Face ID 🎥🧑") as demo:
                     label="Aspect ratio",
                     choices=list(ASPECT_DIMS.keys()),
                     value=DEFAULT_ASPECT,
-                    info="For 'Put my face in a new scene'. In 'Animate my photo' the output matches your photo.",
                 )
                 seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, value=DEFAULT_SEED, step=1)
                 randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
@@ -374,7 +342,7 @@ with gr.Blocks(title="LTX-2.3 Face ID 🎥🧑") as demo:
 
     generate_btn.click(
         fn=generate_video,
-        inputs=[face_image, prompt, duration, mode, aspect_ratio, seed, randomize_seed],
+        inputs=[face_image, prompt, duration, aspect_ratio, seed, randomize_seed],
         outputs=[output_video, seed],
     )
 
@@ -382,25 +350,31 @@ with gr.Blocks(title="LTX-2.3 Face ID 🎥🧑") as demo:
         examples=[
             [
                 str(current_dir / "face_ex1.jpg"),
-                "A young man with dark curly hair and a short beard sits in a sunlit cafe, sipping coffee and smiling warmly at the camera. Soft window light, shallow depth of field, cinematic quality, realistic skin texture.",
+                "A man sits in a sunlit cafe, sipping coffee and smiling warmly at the camera. Soft window light, shallow depth of field, cinematic quality, realistic skin texture.",
                 3.0,
-                MODE_NEW_SCENE,
+                DEFAULT_ASPECT,
+                DEFAULT_SEED,
+                True,
             ],
             [
                 str(current_dir / "face_ex2.jpg"),
-                "A young woman with long dark hair and glasses walks through a vibrant open-air market, glancing at the camera with a gentle smile as stalls of colorful fabrics pass behind her. Natural daylight, shallow depth of field, cinematic.",
+                "A woman walks through a vibrant open-air market, glancing at the camera with a gentle smile as stalls of colorful fabrics pass behind her. Natural daylight, shallow depth of field, cinematic.",
                 3.0,
-                MODE_NEW_SCENE,
+                DEFAULT_ASPECT,
+                DEFAULT_SEED,
+                True,
             ],
             [
                 str(current_dir / "face_ex3.jpg"),
-                "A young woman with shoulder-length wavy black hair stands on a rooftop at golden hour, the city skyline behind her, turning to look at the camera with a calm expression. Warm cinematic lighting.",
+                "A woman stands on a rooftop at golden hour, the city skyline behind her, turning to look at the camera with a calm expression. Warm cinematic lighting.",
                 3.0,
-                MODE_NEW_SCENE,
+                DEFAULT_ASPECT,
+                DEFAULT_SEED,
+                True,
             ],
         ],
         fn=generate_video,
-        inputs=[face_image, prompt, duration, mode],
+        inputs=[face_image, prompt, duration, aspect_ratio, seed, randomize_seed],
         outputs=[output_video, seed],
         label="Example",
         cache_examples=False,

packages/ltx-pipelines/src/ltx_pipelines/distilled.py

@@ -39,6 +39,158 @@ from ltx_pipelines.utils.media_io import encode_video, load_image_and_preprocess
 from ltx_pipelines.utils.types import ModalitySpec, OffloadMode
 
 
+IDENTITY_PROMPT_ENHANCER_SYSTEM_PROMPT = """You will receive two inputs:
+
+1. A reference face image showing the person whose appearance must be preserved.
+2. An original `ref_t2v` video caption describing the scene, action, environment, camera framing, movement, lighting, mood, clothing, and props.
+
+Your task is to analyze the reference face image, extract the person's visible appearance characteristics, and merge those characteristics naturally into the original video caption.
+
+STRICT EDITING RULE:
+
+Treat the original video caption as locked text.
+
+You are allowed to make only these edits:
+
+1. Expand the first subject phrase:
+   - Replace only `A person`, `A man`, or `A woman` with the same original subject noun plus visible appearance details.
+   - The original subject noun must remain present.
+   - Insert the appearance details immediately around the original subject noun.
+   - Do not delete or replace any words that appear after the original subject phrase.
+
+2. Optionally replace later subject pronouns:
+   - `They`, `He`, or `She` may be replaced with a short appearance-based reference.
+   - Preserve every verb, object, action, and remaining word from the original sentence.
+   - Do not alter the order of the words or actions.
+
+Everything else in the original caption is immutable.
+
+You must preserve verbatim:
+
+- clothing and costumes;
+- actions;
+- action order;
+- environments;
+- objects and props;
+- camera framing;
+- camera movement;
+- lighting;
+- mood;
+- adjectives;
+- temporal transitions;
+- all principal nouns and verbs.
+
+CRITICAL PRESERVATION RULES:
+
+- Never remove clothing or costume descriptions.
+- Never remove phrases between the original subject and the main action.
+- Never replace original words with synonyms.
+- Never summarize or simplify the caption.
+- Never improve or creatively rewrite the caption.
+- Never add new actions, clothing, props, people, or environments.
+- Never mention or infer the person's identity or name.
+- Never repeat labels such as `REFERENCE FACE IMAGE` or `ORIGINAL VIDEO CAPTION`.
+- Never write introductory text such as:
+  - `Okay`
+  - `I understand`
+  - `Let's proceed`
+  - `Here is the merged caption`
+  - `The result is`
+
+FACE IMAGE ANALYSIS:
+
+Extract only clearly visible, non-sensitive appearance details, such as:
+
+- apparent age group;
+- skin tone;
+- hair color;
+- hair length;
+- hair texture;
+- hairstyle;
+- face shape;
+- visible facial hair;
+- glasses;
+- clearly visible eye color;
+- other distinctive visible facial features.
+
+Do not:
+
+- identify the person;
+- repeat a name supplied in image metadata or surrounding text;
+- infer ethnicity, nationality, occupation, personality, religion, health, or background;
+- invent unclear characteristics;
+- infer clothing from the face image.
+
+MINIMUM-EDIT PROCEDURE:
+
+Step 1:
+Copy the original caption exactly.
+
+Step 2:
+Locate only the first subject phrase:
+- `A person`
+- `A man`
+- `A woman`
+
+Step 3:
+Expand that phrase with the extracted appearance while preserving the original subject noun.
+
+Example:
+
+Original:
+`A person dressed in a pirate costume walking along a harbor promenade`
+
+Allowed:
+`A fair-skinned adult person with short, neatly combed blond hair dressed in a pirate costume walking along a harbor promenade`
+
+Forbidden:
+`A fair-skinned adult man with short blond hair walking along a harbor promenade`
+
+The forbidden version removes:
+`dressed in a pirate costume`
+
+Step 4:
+For later pronouns, either preserve the original pronoun or replace only the pronoun with a concise appearance-based subject reference.
+
+Example:
+
+Original:
+`They adjust the hat, hold a prop telescope, and continue strolling near the water.`
+
+Allowed:
+`The short-haired person adjusts the hat, holds a prop telescope, and continues strolling near the water.`
+
+Do not modify:
+`adjusts the hat, holds a prop telescope, and continues strolling near the water.`
+
+FINAL VALIDATION:
+
+Before answering, silently compare the output with the original caption and verify:
+
+- every original costume and clothing phrase remains;
+- every original action remains;
+- every original prop remains;
+- every original environment remains;
+- every original framing term remains;
+- every original lighting and mood phrase remains;
+- the action order is unchanged;
+- no original principal word was removed;
+- no synonym replaced an original word;
+- only subject appearance information was added;
+- no introductory or explanatory text appears.
+
+OUTPUT REQUIREMENT:
+
+Output exactly one line containing only the final `ref_t2v` caption.
+
+Do not output analysis, headings, labels, explanations, acknowledgements, or quotation marks.
+
+The line must begin with exactly:
+
+ref_t2v:
+"""
+
+
 class DistilledPipeline:
     """
     Two-stage distilled video generation pipeline.
@@ -142,6 +294,7 @@ class DistilledPipeline:
         images: list[ImageConditioningInput],
         tiling_config: TilingConfig | None = None,
         enhance_prompt: bool = False,
+        enhance_prompt_image_path: str | None = None,
         stage_1_sigmas: torch.Tensor = DISTILLED_SIGMAS,
         stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
         reference_image_path: str | None = None,
@@ -157,7 +310,9 @@ class DistilledPipeline:
         (ctx_p,) = self.prompt_encoder(
             [prompt],
             enhance_first_prompt=enhance_prompt,
-            enhance_prompt_image=images[0][0] if len(images) > 0 else None,
+            enhance_prompt_image=enhance_prompt_image_path,
+            enhance_prompt_seed=seed,
+            enhance_prompt_system_prompt=IDENTITY_PROMPT_ENHANCER_SYSTEM_PROMPT if enhance_prompt_image_path else None,
         )
         video_context, audio_context = ctx_p.video_encoding, ctx_p.audio_encoding
 

packages/ltx-pipelines/src/ltx_pipelines/utils/blocks.py

@@ -563,6 +563,7 @@ class PromptEncoder:
         enhance_first_prompt: bool = False,
         enhance_prompt_image: str | None = None,
         enhance_prompt_seed: int = 42,
+        enhance_prompt_system_prompt: str | None = None,
     ) -> list[EmbeddingsProcessorOutput]:
         """Encode *prompts* through Gemma -> embeddings processor, freeing each model after use."""
         logger.info("Building text encoder from %s", self._gemma_root)
@@ -570,7 +571,11 @@ class PromptEncoder:
             if enhance_first_prompt:
                 prompts = list(prompts)
                 prompts[0] = generate_enhanced_prompt(
-                    text_encoder, prompts[0], enhance_prompt_image, seed=enhance_prompt_seed
+                    text_encoder,
+                    prompts[0],
+                    enhance_prompt_image,
+                    seed=enhance_prompt_seed,
+                    system_prompt=enhance_prompt_system_prompt,
                 )
             raw_outputs = text_encoder.encode(prompts)
         logger.info("Text encoder done, building embeddings processor from %s", self._checkpoint_path)

packages/ltx-pipelines/src/ltx_pipelines/utils/denoisers.py

@@ -51,6 +51,8 @@ def _repeat_state(state: LatentState, n: int) -> LatentState:
         positions=_repeat(state.positions),
         clean_latent=_repeat(state.clean_latent),
         attention_mask=_repeat(state.attention_mask) if state.attention_mask is not None else None,
+        ref_overlap_len=state.ref_overlap_len,
+        ref_seg_value=state.ref_seg_value,
     )
 
 

packages/ltx-pipelines/src/ltx_pipelines/utils/helpers.py

@@ -315,6 +315,7 @@ def generate_enhanced_prompt(
     image_path: str | None = None,
     image_long_side: int = 896,
     seed: int = 42,
+    system_prompt: str | None = None,
 ) -> str:
     """Generate an enhanced prompt from a text encoder and a prompt."""
     image = None
@@ -322,9 +323,9 @@ def generate_enhanced_prompt(
         image = decode_image(image_path=image_path)
         image = torch.tensor(image)
         image = resize_aspect_ratio_preserving(image, image_long_side).to(torch.uint8)
-        prompt = text_encoder.enhance_i2v(prompt, image, seed=seed)
+        prompt = text_encoder.enhance_i2v(prompt, image, seed=seed, system_prompt=system_prompt)
     else:
-        prompt = text_encoder.enhance_t2v(prompt, seed=seed)
+        prompt = text_encoder.enhance_t2v(prompt, seed=seed, system_prompt=system_prompt)
     logging.info(f"Enhanced prompt: {prompt}")
     return clean_response(prompt)