first commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+.ipynb_checkpoints/
+__pycache__/
+*.pyc

app.py

@@ -0,0 +1,375 @@
+import os
+import re
+import time
+import math
+from dataclasses import dataclass
+from glob import iglob
+import argparse
+from einops import rearrange
+from PIL import ExifTags, Image
+import torch
+import gradio as gr
+import numpy as np
+import spaces
+from huggingface_hub import login
+login(token=os.getenv('Token'))
+from flux.sampling_lore import denoise, get_schedule, prepare, unpack, get_v_mask, add_masked_noise_to_z,get_mask_one_tensor, denoise_with_noise_optim,prepare_tokens
+from flux.util_lore import (configs, embed_watermark, load_ae, load_clip,
+                       load_flow_model, load_t5)
+
+def encode(init_image, torch_device, ae):
+    init_image = torch.from_numpy(init_image).permute(2, 0, 1).float() / 127.5 - 1
+    init_image = init_image.unsqueeze(0) 
+    init_image = init_image.to(torch_device)
+    init_image = ae.encode(init_image.to()).to(torch.bfloat16)
+    return init_image
+from torchvision import transforms
+transform = transforms.ToTensor()
+
+class FluxEditor_lore_demo:
+    def __init__(self, model_name):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.offload = False
+
+        self.name = model_name
+        self.is_schnell = model_name == "flux-schnell"
+        self.resize_longside = 800
+        self.save = False
+
+        self.output_dir = 'outputs_gradio'
+
+        self.t5 = load_t5(self.device, max_length=256 if self.name == "flux-schnell" else 512)
+        self.clip = load_clip(self.device)
+        self.model = load_flow_model(model_name, device=self.device)
+        self.ae = load_ae(self.name, device="cpu" if self.offload else self.device)
+
+        self.t5.eval()
+        self.clip.eval()
+        self.ae.eval()
+        self.info = {}
+        if self.offload:
+            self.model.cpu()
+            torch.cuda.empty_cache()
+            self.ae.encoder.to(self.device)
+        for param in self.model.parameters():
+            param.requires_grad = False  # freeze the model
+        for param in self.t5.parameters():
+            param.requires_grad = False  # freeze the model
+        for param in self.clip.parameters():
+            param.requires_grad = False  # freeze the model
+        for param in self.ae.parameters():
+            param.requires_grad = False  # freeze the model
+        
+    def resize_image(self,image):
+        pil_image = Image.fromarray(image)
+        h, w = pil_image.size[1], pil_image.size[0]
+        if h <= self.resize_longside and w <= self.resize_longside:
+            return image
+    
+        if h >= w:
+            new_h = self.resize_longside 
+            new_w = int(w * self.resize_longside  / h)
+        else:
+            new_w = self.resize_longside 
+            new_h = int(h * self.resize_longside  / w)
+    
+        resized_image = pil_image.resize((new_w, new_h), Image.LANCZOS)
+        return np.array(resized_image)
+        
+    def resize_mask(self,mask,height,width):
+        pil_mask = Image.fromarray(mask.astype(np.uint8))  # ensure it's 8-bit for PIL
+        resized_pil = pil_mask.resize((width, height), Image.NEAREST)  # width first!
+        return np.array(resized_pil)
+
+    @spaces.GPU(duration=240)
+    def inverse(self, brush_canvas,src_prompt, 
+                inversion_num_steps, injection_num_steps, 
+                inversion_guidance,
+             ):
+        print(f"Inversing {src_prompt}, guidance {inversion_guidance}, inje/step {injection_num_steps}/{inversion_num_steps}")
+        self.z0 = None
+        self.zt = None
+        torch.cuda.empty_cache()
+        if self.info:
+            del self.info        
+        self.info = {'src_p':src_prompt}
+        
+        rgba_init_image = brush_canvas["background"]
+        init_image = rgba_init_image[:,:,:3]
+        
+            
+        if self.resize_longside != -1:
+            init_image = self.resize_image(init_image)
+        shape = init_image.shape
+
+        new_h = shape[0] if shape[0] % 16 == 0 else shape[0] - shape[0] % 16
+        new_w = shape[1] if shape[1] % 16 == 0 else shape[1] - shape[1] % 16
+    
+        init_image = init_image[:new_h, :new_w, :]
+        width, height = init_image.shape[0], init_image.shape[1]
+        self.init_image = encode(init_image, self.device, self.ae)
+
+        if self.save:
+            ori_output_path = os.path.join(self.output_dir,f'{src_prompt[:20]}_ori.png')
+            Image.fromarray(init_image,'RGB').save(ori_output_path)
+        
+        t0 = time.perf_counter()
+
+        self.info['feature'] = {}
+        self.info['inject_step'] = injection_num_steps
+        self.info['wh'] = (width, height)
+        
+        torch.cuda.empty_cache()
+
+        inp = prepare(self.t5, self.clip, self.init_image, prompt=src_prompt)
+        timesteps = get_schedule(inversion_num_steps, inp["img"].shape[1], shift=True)
+        self.info['x_ori'] = inp["img"].clone()
+    
+        # inversion initial noise
+        torch.set_grad_enabled(False)
+        z, info, _, _ = denoise(self.model, **inp, timesteps=timesteps, guidance=inversion_guidance, inverse=True, info=self.info)
+        self.z0 = z
+        self.info = info
+            
+        t1 = time.perf_counter()
+        print(f"inversion Done in {t1 - t0:.1f}s.")
+        return init_image
+        
+    @spaces.GPU(duration=240)
+    def edit(self, brush_canvas, source_prompt, inversion_guidance,
+                target_prompt, target_object,target_object_index,
+                inversion_num_steps, injection_num_steps, 
+                training_epochs, 
+                denoise_guidance,noise_scale,seed,
+             ):
+        
+        torch.cuda.empty_cache()
+        if 'src_p' not in self.info or self.info['src_p'] != source_prompt:
+            print('src prompt changed. inverse again')
+            self.inverse(brush_canvas,source_prompt, 
+                inversion_num_steps, injection_num_steps, 
+                inversion_guidance)
+        
+        rgba_init_image = brush_canvas["background"]
+        rgba_mask = brush_canvas["layers"][0]
+        init_image = rgba_init_image[:,:,:3]
+        if self.resize_longside != -1:
+            init_image = self.resize_image(init_image)
+        width, height = self.info['wh']
+        init_image = init_image[:width, :height, :]
+        #rgba_init_image = rgba_init_image[:height, :width, :]
+        
+        if self.resize_longside != -1:
+            mask = self.resize_mask(rgba_mask[:,:,3],height,width)
+        else:            
+            mask = rgba_mask[:width, :height, 3]
+        mask = mask.astype(int)
+        
+        rgba_mask[:,:,3] = rgba_mask[:,:,3]//2
+        masked_image = Image.alpha_composite(Image.fromarray(rgba_init_image, 'RGBA'), Image.fromarray(rgba_mask, 'RGBA'))
+        masked_image = masked_image.resize((height, width), Image.LANCZOS)
+            
+
+        # prepare source mask and vmask
+        inp_optim = prepare(self.t5, self.clip, self.init_image, prompt=target_prompt)
+        inp_target = prepare(self.t5, self.clip, self.init_image, prompt=target_prompt)
+        v_mask,source_mask = self.get_v_src_masks(mask,width,height,self.device)
+        self.info['change_v'] = 2 # v_mask
+        self.info['v_mask'] = v_mask
+        self.info['source_mask'] = source_mask
+        self.info['inject_step'] = injection_num_steps
+        timesteps = get_schedule(inversion_num_steps, inp_optim["img"].shape[1], shift=True)
+        seed = int(seed)
+        if seed == -1:
+            seed = torch.randint(0, 2**32, (1,)).item()
+
+        # prepare token_ids
+        token_ids=[]
+        replacements = [[None,target_object,-1,int(target_object_index)]]
+        src_dif_ids,tgt_dif_ids = prepare_tokens(self.t5, source_prompt, target_prompt, replacements,True)
+        for t_ids in tgt_dif_ids:
+            token_ids.append([t_ids,True,1])
+        print('token_ids',token_ids)
+        # do latent optim
+        
+        t0 = time.perf_counter() 
+        print(f'optimizing & editing noise, {target_prompt} with seed {seed}, noise_scale {noise_scale}, training_epochs {training_epochs}')
+        if training_epochs != 0:
+            torch.set_grad_enabled(True)
+            inp_optim["img"] = self.z0
+            _, info, _, _, trainable_noise_list = denoise_with_noise_optim(self.model,**inp_optim,token_ids=token_ids,source_mask=source_mask,training_steps=1,training_epochs=training_epochs,learning_rate=0.01,seed=seed,noise_scale=noise_scale,timesteps=timesteps,info=self.info,guidance=denoise_guidance)
+            z_optim = trainable_noise_list[0]
+            self.info = info
+        else:
+            z_optim = add_masked_noise_to_z(self.z0,source_mask,width,height,seed=seed,noise_scale=noise_scale)
+            trainable_noise_list = None
+
+        # denoise (editing)
+        inp_target["img"] = z_optim
+        timesteps = get_schedule(inversion_num_steps, inp_target["img"].shape[1], shift=True)
+        self.model.eval()
+        torch.set_grad_enabled(False)
+        x, _, _, _ = denoise(self.model, **inp_target, timesteps=timesteps, guidance=denoise_guidance, inverse=False, info=self.info, trainable_noise_list = trainable_noise_list)
+        
+         # decode latents to pixel space
+        batch_x = unpack(x.float(), width,height)
+    
+        for x in batch_x:
+            x = x.unsqueeze(0)
+            
+    
+            with torch.autocast(device_type=self.device.type, dtype=torch.bfloat16):
+                x = self.ae.decode(x)
+    
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+            # bring into PIL format and save
+            x = x.clamp(-1, 1)
+            x = embed_watermark(x.float())
+            x = rearrange(x[0], "c h w -> h w c")
+    
+            img = Image.fromarray((127.5 * (x + 1.0)).cpu().byte().numpy())
+            exif_data = Image.Exif()
+            exif_data[ExifTags.Base.Software] = "AI generated;txt2img;flux"
+            exif_data[ExifTags.Base.Make] = "Black Forest Labs"
+            if self.save:
+                output_path = os.path.join(self.output_dir,f'{target_object}_{injection_num_steps:02d}_{inversion_num_steps}_seed_{seed}_epoch_{training_epochs:03d}_scale_{noise_scale:.2f}.png')
+                img.save(output_path, exif=exif_data, quality=95, subsampling=0)
+                masked_image.save(output_path.replace(target_object,f'{target_object}_masked'))
+                binary_mask = np.where(mask != 0, 255, 0).astype(np.uint8)
+                Image.fromarray(binary_mask, mode="L").save(output_path.replace(target_object,f'{target_object}_mask'))
+            t1 = time.perf_counter()  
+            print(f"Done in {t1 - t0:.1f}s.", f'Saving {output_path} .' if self.save else 'No saving files.')
+        
+        return img
+
+    def encode(self,init_image, torch_device):
+        init_image = torch.from_numpy(init_image).permute(2, 0, 1).float() / 127.5 - 1
+        init_image = init_image.unsqueeze(0) 
+        init_image = init_image.to(torch_device)
+        self.ae.encoder.to(torch_device)
+        
+        init_image = self.ae.encode(init_image).to(torch.bfloat16)
+        return init_image
+
+    def get_v_src_masks(self,mask,width,height,device,txt_length=512):
+        # resize mask to token size
+        mask = (mask > 127).astype(np.uint8)
+        mask = mask * 255
+        pil_mask = Image.fromarray(mask)
+        pil_mask = pil_mask.resize((math.ceil(height/16), math.ceil(width/16)), Image.Resampling.LANCZOS)
+
+        mask = transform(pil_mask)
+        mask = mask.flatten().to(device)
+
+        s_mask = mask.view(1, 1, -1, 1)
+        s_mask = s_mask.to(torch.bfloat16)
+        v_mask = torch.cat([torch.ones(txt_length).to(device),mask])
+        v_mask = v_mask.view(1, 1, -1, 1)
+        v_mask = v_mask.to(torch.bfloat16)
+        return v_mask,s_mask
+
+def create_demo(model_name: str):
+    editor = FluxEditor_lore_demo(model_name)
+    is_schnell = model_name == "flux-schnell"
+    
+    title = r"""
+        <h1 align="center">🎨 LORE Image Editing </h1>
+        """
+        
+    description = r"""
+        <b>Official 🤗 Gradio demo</b> <br>
+        <b>LORE: Latent Optimization for Precise Semantic Control in Rectified Flow-based Image Editing.</b><br>
+        <b>Here are editing steps:</b> <br>
+        1️⃣ Upload your source image. <br>
+        2️⃣ Fill in your source prompt and click the "Inverse" button to perform image inversion. <br>
+        3️⃣ Use the brush tool to draw your mask. (on layer 1) <br>
+        4️⃣ Fill in your target prompt, then adjust the hyperparameters. <br>
+        5️⃣ Click the "Edit" button to generate your edited image! <br>
+        6️⃣ If source image and prompt are not changed, you can click 'Edit' for next generation. <br>
+        
+        🔔 [<b>Note</b>] Due to limited resources, we will resize image to <=800 longside. <br>
+        """
+    article = r"""
+    https://github.com/oyly16/LORE 
+    """
+    
+    with gr.Blocks() as demo:
+        gr.HTML(title)
+        gr.Markdown(description)
+        
+        with gr.Row():
+            with gr.Column():
+                src_prompt = gr.Textbox(label="Source Prompt", value='' )
+                inversion_num_steps = gr.Slider(1, 50, 15, step=1, label="Number of inversion/denoise steps")
+                injection_num_steps = gr.Slider(1, 50, 12, step=1, label="Number of masked value injection steps")
+                target_prompt = gr.Textbox(label="Target Prompt", value='' )
+                target_object = gr.Textbox(label="Target Object", value='' )
+                target_object_index = gr.Textbox(label="Target Object Index (start index from 0 in target prompt)", value='' )
+                brush_canvas = gr.ImageEditor(label="Brush Canvas",
+                                                sources=('upload'), 
+                                                brush=gr.Brush(colors=["#ff0000"],color_mode='fixed',default_color="#ff0000"),
+                                                interactive=True,
+                                                transforms=[],
+                                                container=True,
+                                                format='png',scale=1)
+                
+                inv_btn = gr.Button("inverse")
+                edit_btn = gr.Button("edit")
+                
+                
+            with gr.Column():
+                with gr.Accordion("Advanced Options", open=True):
+
+                    training_epochs = gr.Slider(0, 30, 10, step=1, label="Number of LORE training epochs")
+                    inversion_guidance = gr.Slider(1.0, 10.0, 1.0, step=0.1, label="inversion Guidance", interactive=not is_schnell)
+                    denoise_guidance = gr.Slider(1.0, 10.0, 2.0, step=0.1, label="denoise Guidance", interactive=not is_schnell)
+                    noise_scale = gr.Slider(0.0, 1.0, 0.9, step=0.1, label="renoise scale")
+                    seed = gr.Textbox('0', label="Seed (-1 for random)", visible=True)
+
+                
+                output_image = gr.Image(label="Generated Image")
+                gr.Markdown(article)
+        inv_btn.click(
+            fn=editor.inverse,
+            inputs=[brush_canvas,src_prompt, 
+                    inversion_num_steps, injection_num_steps, 
+                    inversion_guidance,
+                    ],
+            outputs=[output_image]
+        )
+        edit_btn.click(
+            fn=editor.edit,
+            inputs=[brush_canvas,src_prompt,inversion_guidance,
+                    target_prompt, target_object,target_object_index,
+                    inversion_num_steps, injection_num_steps, 
+                    training_epochs, 
+                    denoise_guidance,noise_scale,seed,
+                    ],
+            outputs=[output_image]
+        )
+        gr.Examples(
+            examples=[
+                ["examples/woman.png", "a young woman", 15, 12, "a young woman with a necklace", "necklace", "5", 10, 0.9, "3"],
+                ["examples/car.png", "a taxi in a neon-lit street", 30, 24, "a race car in a neon-lit street", "race car", "1", 5, 0.1, "2388791121"],
+                ["examples/cup.png", "a cup on a wooden table", 10, 8, "a wooden table", "table", "2", 2, 0, "0"],
+            ],
+            inputs=[
+                brush_canvas,
+                src_prompt,
+                inversion_num_steps,
+                injection_num_steps,
+                target_prompt,
+                target_object,
+                target_object_index,
+                training_epochs,
+                noise_scale,
+                seed,
+            ],
+            label="Examples (Click to load)"
+        )
+
+    return demo
+
+demo = create_demo("flux-dev")    
+demo.launch()

flux/__init__.py

@@ -0,0 +1,11 @@
+try:
+    from ._version import version as __version__  # type: ignore
+    from ._version import version_tuple
+except ImportError:
+    __version__ = "unknown (no version information available)"
+    version_tuple = (0, 0, "unknown", "noinfo")
+
+from pathlib import Path
+
+PACKAGE = __package__.replace("_", "-")
+PACKAGE_ROOT = Path(__file__).parent

flux/__main__.py

@@ -0,0 +1,4 @@
+from .cli import app
+
+if __name__ == "__main__":
+    app()

flux/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.0.post61+g0274301.d20250318'
+__version_tuple__ = version_tuple = (0, 0, 'g0274301.d20250318')

flux/api.py

@@ -0,0 +1,194 @@
+import io
+import os
+import time
+from pathlib import Path
+
+import requests
+from PIL import Image
+
+API_ENDPOINT = "https://api.bfl.ml"
+
+
+class ApiException(Exception):
+    def __init__(self, status_code: int, detail: str | list[dict] | None = None):
+        super().__init__()
+        self.detail = detail
+        self.status_code = status_code
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+    def __repr__(self) -> str:
+        if self.detail is None:
+            message = None
+        elif isinstance(self.detail, str):
+            message = self.detail
+        else:
+            message = "[" + ",".join(d["msg"] for d in self.detail) + "]"
+        return f"ApiException({self.status_code=}, {message=}, detail={self.detail})"
+
+
+class ImageRequest:
+    def __init__(
+        self,
+        prompt: str,
+        width: int = 1024,
+        height: int = 1024,
+        name: str = "flux.1-pro",
+        num_steps: int = 50,
+        prompt_upsampling: bool = False,
+        seed: int | None = None,
+        validate: bool = True,
+        launch: bool = True,
+        api_key: str | None = None,
+    ):
+        """
+        Manages an image generation request to the API.
+
+        Args:
+            prompt: Prompt to sample
+            width: Width of the image in pixel
+            height: Height of the image in pixel
+            name: Name of the model
+            num_steps: Number of network evaluations
+            prompt_upsampling: Use prompt upsampling
+            seed: Fix the generation seed
+            validate: Run input validation
+            launch: Directly launches request
+            api_key: Your API key if not provided by the environment
+
+        Raises:
+            ValueError: For invalid input
+            ApiException: For errors raised from the API
+        """
+        if validate:
+            if name not in ["flux.1-pro"]:
+                raise ValueError(f"Invalid model {name}")
+            elif width % 32 != 0:
+                raise ValueError(f"width must be divisible by 32, got {width}")
+            elif not (256 <= width <= 1440):
+                raise ValueError(f"width must be between 256 and 1440, got {width}")
+            elif height % 32 != 0:
+                raise ValueError(f"height must be divisible by 32, got {height}")
+            elif not (256 <= height <= 1440):
+                raise ValueError(f"height must be between 256 and 1440, got {height}")
+            elif not (1 <= num_steps <= 50):
+                raise ValueError(f"steps must be between 1 and 50, got {num_steps}")
+
+        self.request_json = {
+            "prompt": prompt,
+            "width": width,
+            "height": height,
+            "variant": name,
+            "steps": num_steps,
+            "prompt_upsampling": prompt_upsampling,
+        }
+        if seed is not None:
+            self.request_json["seed"] = seed
+
+        self.request_id: str | None = None
+        self.result: dict | None = None
+        self._image_bytes: bytes | None = None
+        self._url: str | None = None
+        if api_key is None:
+            self.api_key = os.environ.get("BFL_API_KEY")
+        else:
+            self.api_key = api_key
+
+        if launch:
+            self.request()
+
+    def request(self):
+        """
+        Request to generate the image.
+        """
+        if self.request_id is not None:
+            return
+        response = requests.post(
+            f"{API_ENDPOINT}/v1/image",
+            headers={
+                "accept": "application/json",
+                "x-key": self.api_key,
+                "Content-Type": "application/json",
+            },
+            json=self.request_json,
+        )
+        result = response.json()
+        if response.status_code != 200:
+            raise ApiException(status_code=response.status_code, detail=result.get("detail"))
+        self.request_id = response.json()["id"]
+
+    def retrieve(self) -> dict:
+        """
+        Wait for the generation to finish and retrieve response.
+        """
+        if self.request_id is None:
+            self.request()
+        while self.result is None:
+            response = requests.get(
+                f"{API_ENDPOINT}/v1/get_result",
+                headers={
+                    "accept": "application/json",
+                    "x-key": self.api_key,
+                },
+                params={
+                    "id": self.request_id,
+                },
+            )
+            result = response.json()
+            if "status" not in result:
+                raise ApiException(status_code=response.status_code, detail=result.get("detail"))
+            elif result["status"] == "Ready":
+                self.result = result["result"]
+            elif result["status"] == "Pending":
+                time.sleep(0.5)
+            else:
+                raise ApiException(status_code=200, detail=f"API returned status '{result['status']}'")
+        return self.result
+
+    @property
+    def bytes(self) -> bytes:
+        """
+        Generated image as bytes.
+        """
+        if self._image_bytes is None:
+            response = requests.get(self.url)
+            if response.status_code == 200:
+                self._image_bytes = response.content
+            else:
+                raise ApiException(status_code=response.status_code)
+        return self._image_bytes
+
+    @property
+    def url(self) -> str:
+        """
+        Public url to retrieve the image from
+        """
+        if self._url is None:
+            result = self.retrieve()
+            self._url = result["sample"]
+        return self._url
+
+    @property
+    def image(self) -> Image.Image:
+        """
+        Load the image as a PIL Image
+        """
+        return Image.open(io.BytesIO(self.bytes))
+
+    def save(self, path: str):
+        """
+        Save the generated image to a local path
+        """
+        suffix = Path(self.url).suffix
+        if not path.endswith(suffix):
+            path = path + suffix
+        Path(path).resolve().parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "wb") as file:
+            file.write(self.bytes)
+
+
+if __name__ == "__main__":
+    from fire import Fire
+
+    Fire(ImageRequest)

flux/math.py

@@ -0,0 +1,57 @@
+import torch
+from einops import rearrange
+from torch import Tensor
+
+
+def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor) -> Tensor:
+    q, k = apply_rope(q, k, pe)
+
+    x = torch.nn.functional.scaled_dot_product_attention(q, k, v)
+    x = rearrange(x, "B H L D -> B L (H D)")
+
+    return x
+
+def attention_with_attnmap(q: Tensor, k: Tensor, v: Tensor, pe: Tensor) -> Tensor:
+    q, k = apply_rope(q, k, pe)
+
+    x= torch.nn.functional.scaled_dot_product_attention(q, k, v)
+    x = rearrange(x, "B H L D -> B L (H D)")
+
+    # get attn map
+    d_k = q.shape[-1]  # head_dim (D)
+    attn_map = torch.matmul(q, k.transpose(-2, -1)) / (d_k ** 0.5)  # [B, H, L, L]
+    return x, attn_map
+
+def attention_with_attnmap_injection(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, attnmap_idxs, old_attnmaps) -> Tensor:
+    q, k = apply_rope(q, k, pe)
+
+    # original attn
+    # x= torch.nn.functional.scaled_dot_product_attention(q, k, v)
+    # x = rearrange(x, "B H L D -> B L (H D)")
+
+    # get attn map
+    d_k = q.shape[-1]  # head_dim (D)
+    attn_map = torch.matmul(q, k.transpose(-2, -1)) / (d_k ** 0.5)  # [B, H, L, L]
+    attn_map = torch.softmax(attn_map, dim=-1)
+    # inject attn map
+    for idx,old_attnmap in zip(attnmap_idxs,old_attnmaps):
+        attn_map[:,:,512:,idx] = old_attnmap
+    x = attn_map @ v
+    return x, attn_map
+
+def rope(pos: Tensor, dim: int, theta: int) -> Tensor:
+    assert dim % 2 == 0
+    scale = torch.arange(0, dim, 2, dtype=torch.float64, device=pos.device) / dim
+    omega = 1.0 / (theta**scale)
+    out = torch.einsum("...n,d->...nd", pos, omega)
+    out = torch.stack([torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1)
+    out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2)
+    return out.float()
+
+
+def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor) -> tuple[Tensor, Tensor]:
+    xq_ = xq.float().reshape(*xq.shape[:-1], -1, 1, 2)
+    xk_ = xk.float().reshape(*xk.shape[:-1], -1, 1, 2)
+    xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1]
+    xk_out = freqs_cis[..., 0] * xk_[..., 0] + freqs_cis[..., 1] * xk_[..., 1]
+    return xq_out.reshape(*xq.shape).type_as(xq), xk_out.reshape(*xk.shape).type_as(xk)

flux/model_lore.py

@@ -0,0 +1,124 @@
+from dataclasses import dataclass
+
+import torch
+from torch import Tensor, nn
+
+from flux.modules.layers_lore import (DoubleStreamBlock, EmbedND, LastLayer,
+                                 MLPEmbedder, SingleStreamBlock,
+                                 timestep_embedding)
+
+
+@dataclass
+class FluxParams:
+    in_channels: int
+    vec_in_dim: int
+    context_in_dim: int
+    hidden_size: int
+    mlp_ratio: float
+    num_heads: int
+    depth: int
+    depth_single_blocks: int
+    axes_dim: list[int]
+    theta: int
+    qkv_bias: bool
+    guidance_embed: bool
+
+
+class Flux(nn.Module):
+    """
+    Transformer model for flow matching on sequences.
+    """
+
+    def __init__(self, params: FluxParams):
+        super().__init__()
+
+        self.params = params
+        self.in_channels = params.in_channels
+        self.out_channels = self.in_channels
+        if params.hidden_size % params.num_heads != 0:
+            raise ValueError(
+                f"Hidden size {params.hidden_size} must be divisible by num_heads {params.num_heads}"
+            )
+        pe_dim = params.hidden_size // params.num_heads
+        if sum(params.axes_dim) != pe_dim:
+            raise ValueError(f"Got {params.axes_dim} but expected positional dim {pe_dim}")
+        self.hidden_size = params.hidden_size
+        self.num_heads = params.num_heads
+        self.pe_embedder = EmbedND(dim=pe_dim, theta=params.theta, axes_dim=params.axes_dim)
+        self.img_in = nn.Linear(self.in_channels, self.hidden_size, bias=True)
+        self.time_in = MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size)
+        self.vector_in = MLPEmbedder(params.vec_in_dim, self.hidden_size)
+        self.guidance_in = (
+            MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size) if params.guidance_embed else nn.Identity()
+        )
+        self.txt_in = nn.Linear(params.context_in_dim, self.hidden_size)
+
+        self.double_blocks = nn.ModuleList(
+            [
+                DoubleStreamBlock(
+                    self.hidden_size,
+                    self.num_heads,
+                    mlp_ratio=params.mlp_ratio,
+                    qkv_bias=params.qkv_bias,
+                )
+                for _ in range(params.depth)
+            ]
+        )
+
+        self.single_blocks = nn.ModuleList(
+            [
+                SingleStreamBlock(self.hidden_size, self.num_heads, mlp_ratio=params.mlp_ratio)
+                for _ in range(params.depth_single_blocks)
+            ]
+        )
+
+        self.final_layer = LastLayer(self.hidden_size, 1, self.out_channels)
+
+    def forward(
+        self,
+        img: Tensor,
+        img_ids: Tensor,
+        txt: Tensor,
+        txt_ids: Tensor,
+        timesteps: Tensor,
+        y: Tensor,
+        guidance: Tensor | None = None,
+        info = None,
+    ) -> Tensor:
+        if img.ndim != 3 or txt.ndim != 3:
+            raise ValueError("Input img and txt tensors must have 3 dimensions.")
+
+        # running on sequences img
+        img = self.img_in(img)
+        vec = self.time_in(timestep_embedding(timesteps, 256))
+        if self.params.guidance_embed:
+            if guidance is None:
+                raise ValueError("Didn't get guidance strength for guidance distilled model.")
+            vec = vec + self.guidance_in(timestep_embedding(guidance, 256))
+        vec = vec + self.vector_in(y)
+        txt = self.txt_in(txt)
+        
+
+        ids = torch.cat((txt_ids, img_ids), dim=1)
+        pe = self.pe_embedder(ids)
+
+        attn_maps = []
+
+        for block in self.double_blocks:
+            img, txt, attn_map = block(img=img, txt=txt, vec=vec, pe=pe, info=info)
+            attn_maps.append(attn_map)
+        
+
+        cnt = 0
+        img = torch.cat((txt, img), 1) 
+        info['type'] = 'single'
+        for block in self.single_blocks:
+            info['id'] = cnt
+            img, info, attn_map = block(img, vec=vec, pe=pe, info=info)
+            attn_maps.append(attn_map)
+            cnt += 1
+        attn_maps = torch.stack(attn_maps)
+        img = img[:, txt.shape[1] :, ...]
+
+        img = self.final_layer(img, vec)  # (N, T, patch_size ** 2 * out_channels) # 1, N, 64
+        return img, info, attn_maps

flux/modules/autoencoder.py

@@ -0,0 +1,313 @@
+from dataclasses import dataclass
+
+import torch
+from einops import rearrange
+from torch import Tensor, nn
+
+
+@dataclass
+class AutoEncoderParams:
+    resolution: int
+    in_channels: int
+    ch: int
+    out_ch: int
+    ch_mult: list[int]
+    num_res_blocks: int
+    z_channels: int
+    scale_factor: float
+    shift_factor: float
+
+
+def swish(x: Tensor) -> Tensor:
+    return x * torch.sigmoid(x)
+
+
+class AttnBlock(nn.Module):
+    def __init__(self, in_channels: int):
+        super().__init__()
+        self.in_channels = in_channels
+
+        self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
+
+        self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+        self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+        self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+        self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+
+    def attention(self, h_: Tensor) -> Tensor:
+        h_ = self.norm(h_)
+        q = self.q(h_)
+        k = self.k(h_)
+        v = self.v(h_)
+
+        b, c, h, w = q.shape
+        q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous()
+        k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous()
+        v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous()
+        h_ = nn.functional.scaled_dot_product_attention(q, k, v)
+
+        return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x + self.proj_out(self.attention(x))
+
+
+class ResnetBlock(nn.Module):
+    def __init__(self, in_channels: int, out_channels: int):
+        super().__init__()
+        self.in_channels = in_channels
+        out_channels = in_channels if out_channels is None else out_channels
+        self.out_channels = out_channels
+
+        self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+        self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True)
+        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+        if self.in_channels != self.out_channels:
+            self.nin_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+    def forward(self, x):
+        h = x
+        h = self.norm1(h)
+        h = swish(h)
+        h = self.conv1(h)
+
+        h = self.norm2(h)
+        h = swish(h)
+        h = self.conv2(h)
+
+        if self.in_channels != self.out_channels:
+            x = self.nin_shortcut(x)
+
+        return x + h
+
+
+class Downsample(nn.Module):
+    def __init__(self, in_channels: int):
+        super().__init__()
+        # no asymmetric padding in torch conv, must do it ourselves
+        self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+    def forward(self, x: Tensor):
+        pad = (0, 1, 0, 1)
+        x = nn.functional.pad(x, pad, mode="constant", value=0)
+        x = self.conv(x)
+        return x
+
+
+class Upsample(nn.Module):
+    def __init__(self, in_channels: int):
+        super().__init__()
+        self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+    def forward(self, x: Tensor):
+        x = nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
+        x = self.conv(x)
+        return x
+
+
+class Encoder(nn.Module):
+    def __init__(
+        self,
+        resolution: int,
+        in_channels: int,
+        ch: int,
+        ch_mult: list[int],
+        num_res_blocks: int,
+        z_channels: int,
+    ):
+        super().__init__()
+        self.ch = ch
+        self.num_resolutions = len(ch_mult)
+        self.num_res_blocks = num_res_blocks
+        self.resolution = resolution
+        self.in_channels = in_channels
+        # downsampling
+        self.conv_in = nn.Conv2d(in_channels, self.ch, kernel_size=3, stride=1, padding=1)
+
+        curr_res = resolution
+        in_ch_mult = (1,) + tuple(ch_mult)
+        self.in_ch_mult = in_ch_mult
+        self.down = nn.ModuleList()
+        block_in = self.ch
+        for i_level in range(self.num_resolutions):
+            block = nn.ModuleList()
+            attn = nn.ModuleList()
+            block_in = ch * in_ch_mult[i_level]
+            block_out = ch * ch_mult[i_level]
+            for _ in range(self.num_res_blocks):
+                block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
+                block_in = block_out
+            down = nn.Module()
+            down.block = block
+            down.attn = attn
+            if i_level != self.num_resolutions - 1:
+                down.downsample = Downsample(block_in)
+                curr_res = curr_res // 2
+            self.down.append(down)
+
+        # middle
+        self.mid = nn.Module()
+        self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+        self.mid.attn_1 = AttnBlock(block_in)
+        self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+
+        # end
+        self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
+        self.conv_out = nn.Conv2d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1)
+
+    def forward(self, x: Tensor) -> Tensor:
+        # downsampling
+        hs = [self.conv_in(x)]
+        for i_level in range(self.num_resolutions):
+            for i_block in range(self.num_res_blocks):
+                h = self.down[i_level].block[i_block](hs[-1])
+                if len(self.down[i_level].attn) > 0:
+                    h = self.down[i_level].attn[i_block](h)
+                hs.append(h)
+            if i_level != self.num_resolutions - 1:
+                hs.append(self.down[i_level].downsample(hs[-1]))
+
+        # middle
+        h = hs[-1]
+        h = self.mid.block_1(h)
+        h = self.mid.attn_1(h)
+        h = self.mid.block_2(h)
+        # end
+        h = self.norm_out(h)
+        h = swish(h)
+        h = self.conv_out(h)
+        return h
+
+
+class Decoder(nn.Module):
+    def __init__(
+        self,
+        ch: int,
+        out_ch: int,
+        ch_mult: list[int],
+        num_res_blocks: int,
+        in_channels: int,
+        resolution: int,
+        z_channels: int,
+    ):
+        super().__init__()
+        self.ch = ch
+        self.num_resolutions = len(ch_mult)
+        self.num_res_blocks = num_res_blocks
+        self.resolution = resolution
+        self.in_channels = in_channels
+        self.ffactor = 2 ** (self.num_resolutions - 1)
+
+        # compute in_ch_mult, block_in and curr_res at lowest res
+        block_in = ch * ch_mult[self.num_resolutions - 1]
+        curr_res = resolution // 2 ** (self.num_resolutions - 1)
+        self.z_shape = (1, z_channels, curr_res, curr_res)
+
+        # z to block_in
+        self.conv_in = nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1)
+
+        # middle
+        self.mid = nn.Module()
+        self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+        self.mid.attn_1 = AttnBlock(block_in)
+        self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+
+        # upsampling
+        self.up = nn.ModuleList()
+        for i_level in reversed(range(self.num_resolutions)):
+            block = nn.ModuleList()
+            attn = nn.ModuleList()
+            block_out = ch * ch_mult[i_level]
+            for _ in range(self.num_res_blocks + 1):
+                block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
+                block_in = block_out
+            up = nn.Module()
+            up.block = block
+            up.attn = attn
+            if i_level != 0:
+                up.upsample = Upsample(block_in)
+                curr_res = curr_res * 2
+            self.up.insert(0, up)  # prepend to get consistent order
+
+        # end
+        self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
+        self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1)
+
+    def forward(self, z: Tensor) -> Tensor:
+        # z to block_in
+        h = self.conv_in(z)
+
+        # middle
+        h = self.mid.block_1(h)
+        h = self.mid.attn_1(h)
+        h = self.mid.block_2(h)
+
+        # upsampling
+        for i_level in reversed(range(self.num_resolutions)):
+            for i_block in range(self.num_res_blocks + 1):
+                h = self.up[i_level].block[i_block](h)
+                if len(self.up[i_level].attn) > 0:
+                    h = self.up[i_level].attn[i_block](h)
+            if i_level != 0:
+                h = self.up[i_level].upsample(h)
+
+        # end
+        h = self.norm_out(h)
+        h = swish(h)
+        h = self.conv_out(h)
+        return h
+
+
+class DiagonalGaussian(nn.Module):
+    def __init__(self, sample: bool = True, chunk_dim: int = 1):
+        super().__init__()
+        self.sample = sample
+        self.chunk_dim = chunk_dim
+
+    def forward(self, z: Tensor) -> Tensor:
+        mean, logvar = torch.chunk(z, 2, dim=self.chunk_dim)
+        # import pdb;pdb.set_trace()
+        if self.sample:
+            std = torch.exp(0.5 * logvar)
+            return mean #+ std * torch.randn_like(mean)
+        else:
+            return mean
+
+
+class AutoEncoder(nn.Module):
+    def __init__(self, params: AutoEncoderParams):
+        super().__init__()
+        self.encoder = Encoder(
+            resolution=params.resolution,
+            in_channels=params.in_channels,
+            ch=params.ch,
+            ch_mult=params.ch_mult,
+            num_res_blocks=params.num_res_blocks,
+            z_channels=params.z_channels,
+        )
+        self.decoder = Decoder(
+            resolution=params.resolution,
+            in_channels=params.in_channels,
+            ch=params.ch,
+            out_ch=params.out_ch,
+            ch_mult=params.ch_mult,
+            num_res_blocks=params.num_res_blocks,
+            z_channels=params.z_channels,
+        )
+        self.reg = DiagonalGaussian()
+
+        self.scale_factor = params.scale_factor
+        self.shift_factor = params.shift_factor
+
+    def encode(self, x: Tensor) -> Tensor:
+        z = self.reg(self.encoder(x))
+        z = self.scale_factor * (z - self.shift_factor)
+        return z
+
+    def decode(self, z: Tensor) -> Tensor:
+        z = z / self.scale_factor + self.shift_factor
+        return self.decoder(z)
+
+    def forward(self, x: Tensor) -> Tensor:
+        return self.decode(self.encode(x))

flux/modules/conditioner_lore.py

@@ -0,0 +1,155 @@
+from torch import Tensor, nn
+from transformers import (CLIPTextModel, CLIPTokenizer, T5EncoderModel,
+                          T5Tokenizer)
+
+
+class HFEmbedder(nn.Module):
+    def __init__(self, version: str, max_length: int, is_clip, **hf_kwargs):
+        super().__init__()
+        self.is_clip = is_clip
+        self.max_length = max_length
+        self.output_key = "pooler_output" if self.is_clip else "last_hidden_state"
+
+        if self.is_clip:
+            self.tokenizer: CLIPTokenizer = CLIPTokenizer.from_pretrained(version, max_length=max_length)
+            self.hf_module: CLIPTextModel = CLIPTextModel.from_pretrained(version, **hf_kwargs)
+        else:
+            self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(version, max_length=max_length)
+            self.hf_module: T5EncoderModel = T5EncoderModel.from_pretrained(version, **hf_kwargs)
+
+        self.hf_module = self.hf_module.eval().requires_grad_(False)
+
+    def forward(self, text: list[str]) -> Tensor:
+        batch_encoding = self.tokenizer(
+            text,
+            truncation=True,
+            max_length=self.max_length,
+            return_length=False,
+            return_overflowing_tokens=False,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        if not self.is_clip:
+            pass
+
+        outputs = self.hf_module(
+            input_ids=batch_encoding["input_ids"].to(self.hf_module.device),
+            attention_mask=None,
+            output_hidden_states=False,
+        )
+        return outputs[self.output_key]
+
+    def forward_length(self, text: list[str]) -> Tensor:
+        batch_encoding = self.tokenizer(
+            text,
+            truncation=True,
+            max_length=self.max_length,
+            return_length=True,
+            return_overflowing_tokens=False,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        if not self.is_clip:
+            pass
+
+        outputs = self.hf_module(
+            input_ids=batch_encoding["input_ids"].to(self.hf_module.device),
+            attention_mask=None,
+            output_hidden_states=False,
+        )
+        # -1 to delete the end token
+        return outputs[self.output_key],batch_encoding['length']-1
+
+    def get_word_embed(self, text: list[str]) -> Tensor:
+        batch_encoding = self.tokenizer(
+            text,
+            truncation=True,
+            max_length=16,
+            return_length=True,
+            return_overflowing_tokens=False,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        
+        input_ids = batch_encoding["input_ids"].to(self.hf_module.device)
+        attention_mask = batch_encoding["attention_mask"].to(self.hf_module.device)
+        
+        outputs = self.hf_module(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            output_hidden_states=False,
+        )
+        
+        token_embeddings = outputs[self.output_key]  # [B, T, D]
+        mask = attention_mask.unsqueeze(-1).float()  # [B, T, 1]
+        summed = (token_embeddings * mask).sum(dim=1)           # [B, D]
+        counts = mask.sum(dim=1).clamp(min=1e-6)                 
+        mean_pooled = summed / counts                            # [B, D]
+        
+        return mean_pooled
+
+
+    def get_text_embeddings_with_diff(self, src_text: str, tgt_text: str, replacements: list[tuple[str, str, int, int]], show_tokens=False, return_embeds=False):
+        batch_encoding = self.tokenizer(
+            [src_text, tgt_text],
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors="pt",
+            padding="max_length",
+        )
+        
+        src_ids, tgt_ids = batch_encoding["input_ids"]
+        
+        src_tokens = self.tokenizer.tokenize(src_text)
+        tgt_tokens = self.tokenizer.tokenize(tgt_text)
+        if show_tokens:
+            print("src tokens", src_tokens)
+            print("tgt tokens", tgt_tokens)
+        
+        src_dif_ids = []
+        tgt_dif_ids = []
+        def find_mappings(tokens,words,start_idx):
+            if (words is None) or start_idx<0: # some samples do not need this
+                return [-1]
+            res = []
+            flag = 0
+            for i in range(start_idx,len(tokens)):
+                this_token = tokens[i].strip('▁')
+                if this_token == "":
+                    continue
+                if words.startswith(this_token):
+                    res.append(i)
+                    flag = 1
+                    if words.endswith(this_token):
+                        break
+                    else:
+                        continue
+                if flag and words.endswith(this_token):
+                    res.append(i)
+                    break
+                if flag:
+                    res.append(i)
+            return res
+        
+        for src_words, tgt_words, src_index, tgt_index in replacements:
+            if src_words:
+                src_dif_ids.append(find_mappings(src_tokens,src_words,src_index))
+            else:
+                src_dif_ids.append([-1])
+            if tgt_words:
+                tgt_dif_ids.append(find_mappings(tgt_tokens,tgt_words,tgt_index))
+            else:
+                tgt_dif_ids.append([-1])
+            
+        if return_embeds:
+            outputs = self.hf_module(
+                input_ids=batch_encoding["input_ids"].to(self.hf_module.device),
+                attention_mask=None,
+                output_hidden_states=False,
+            )
+            embeddings = outputs[self.output_key]
+        else:
+            embeddings = (None,None)
+        return embeddings[0], embeddings[1], src_dif_ids, tgt_dif_ids
+
+

flux/modules/layers_lore.py

@@ -0,0 +1,298 @@
+import math
+from dataclasses import dataclass
+
+import torch
+from einops import rearrange
+from torch import Tensor, nn
+
+from flux.math import attention, rope, attention_with_attnmap
+
+import os
+
+class EmbedND(nn.Module):
+    def __init__(self, dim: int, theta: int, axes_dim: list[int]):
+        super().__init__()
+        self.dim = dim
+        self.theta = theta
+        self.axes_dim = axes_dim
+
+    def forward(self, ids: Tensor) -> Tensor:
+        n_axes = ids.shape[-1]
+        emb = torch.cat(
+            [rope(ids[..., i], self.axes_dim[i], self.theta) for i in range(n_axes)],
+            dim=-3,
+        )
+
+        return emb.unsqueeze(1)
+
+
+def timestep_embedding(t: Tensor, dim, max_period=10000, time_factor: float = 1000.0):
+    """
+    Create sinusoidal timestep embeddings.
+    :param t: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param dim: the dimension of the output.
+    :param max_period: controls the minimum frequency of the embeddings.
+    :return: an (N, D) Tensor of positional embeddings.
+    """
+    t = time_factor * t
+    half = dim // 2
+    freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(
+        t.device
+    )
+
+    args = t[:, None].float() * freqs[None]
+    embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+    if dim % 2:
+        embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+    if torch.is_floating_point(t):
+        embedding = embedding.to(t)
+    return embedding
+
+
+class MLPEmbedder(nn.Module):
+    def __init__(self, in_dim: int, hidden_dim: int):
+        super().__init__()
+        self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True)
+        self.silu = nn.SiLU()
+        self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True)
+
+    def forward(self, x: Tensor) -> Tensor:
+        return self.out_layer(self.silu(self.in_layer(x)))
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int):
+        super().__init__()
+        self.scale = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x: Tensor):
+        x_dtype = x.dtype
+        x = x.float()
+        rrms = torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + 1e-6)
+        return (x * rrms).to(dtype=x_dtype) * self.scale
+
+
+class QKNorm(torch.nn.Module):
+    def __init__(self, dim: int):
+        super().__init__()
+        self.query_norm = RMSNorm(dim)
+        self.key_norm = RMSNorm(dim)
+
+    def forward(self, q: Tensor, k: Tensor, v: Tensor) -> tuple[Tensor, Tensor]:
+        q = self.query_norm(q)
+        k = self.key_norm(k)
+        return q.to(v), k.to(v)
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.norm = QKNorm(head_dim)
+        self.proj = nn.Linear(dim, dim)
+
+    def forward(self, x: Tensor, pe: Tensor) -> Tensor:
+        qkv = self.qkv(x)
+        q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads)
+        q, k = self.norm(q, k, v)
+        x = attention(q, k, v, pe=pe)
+        x = self.proj(x)
+        return x
+
+
+@dataclass
+class ModulationOut:
+    shift: Tensor
+    scale: Tensor
+    gate: Tensor
+
+
+class Modulation(nn.Module):
+    def __init__(self, dim: int, double: bool):
+        super().__init__()
+        self.is_double = double
+        self.multiplier = 6 if double else 3
+        self.lin = nn.Linear(dim, self.multiplier * dim, bias=True)
+
+    def forward(self, vec: Tensor) -> tuple[ModulationOut, ModulationOut | None]:
+        out = self.lin(nn.functional.silu(vec))[:, None, :].chunk(self.multiplier, dim=-1)
+
+        return (
+            ModulationOut(*out[:3]),
+            ModulationOut(*out[3:]) if self.is_double else None,
+        )
+
+
+class DoubleStreamBlock(nn.Module):
+    def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False):
+        super().__init__()
+
+        mlp_hidden_dim = int(hidden_size * mlp_ratio)
+        self.num_heads = num_heads
+        self.hidden_size = hidden_size
+        self.img_mod = Modulation(hidden_size, double=True)
+        self.img_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.img_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias)
+
+        self.img_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.img_mlp = nn.Sequential(
+            nn.Linear(hidden_size, mlp_hidden_dim, bias=True),
+            nn.GELU(approximate="tanh"),
+            nn.Linear(mlp_hidden_dim, hidden_size, bias=True),
+        )
+
+        self.txt_mod = Modulation(hidden_size, double=True)
+        self.txt_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.txt_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias)
+
+        self.txt_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.txt_mlp = nn.Sequential(
+            nn.Linear(hidden_size, mlp_hidden_dim, bias=True),
+            nn.GELU(approximate="tanh"),
+            nn.Linear(mlp_hidden_dim, hidden_size, bias=True),
+        )
+
+    def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor, info) -> tuple[Tensor, Tensor]:
+        img_mod1, img_mod2 = self.img_mod(vec)
+        txt_mod1, txt_mod2 = self.txt_mod(vec)
+
+        # prepare image for attention
+        img_modulated = self.img_norm1(img)
+        img_modulated = (1 + img_mod1.scale) * img_modulated + img_mod1.shift
+        img_qkv = self.img_attn.qkv(img_modulated)
+        img_q, img_k, img_v = rearrange(img_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads)
+
+        img_q, img_k = self.img_attn.norm(img_q, img_k, img_v)
+
+        # prepare txt for attention
+        txt_modulated = self.txt_norm1(txt)
+        txt_modulated = (1 + txt_mod1.scale) * txt_modulated + txt_mod1.shift
+        txt_qkv = self.txt_attn.qkv(txt_modulated)
+        txt_q, txt_k, txt_v = rearrange(txt_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads)
+        txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v)
+
+        # run actual attention
+        q = torch.cat((txt_q, img_q), dim=2) #[8, 24, 512, 128] + [8, 24, 900, 128] -> [8, 24, 1412, 128]
+        k = torch.cat((txt_k, img_k), dim=2)
+        v = torch.cat((txt_v, img_v), dim=2)
+        attn,attn_map = attention_with_attnmap(q, k, v, pe=pe)
+        attn_map = attn_map[:, :, txt.shape[1]:, :txt.shape[1]]  # text to image attn map
+        if 'txt_token_l' in info:
+            # drop all paddings
+            attn_map = attn_map[:,:,:,:info['txt_token_l']]
+        attn_map = torch.nn.functional.softmax(attn_map, dim=-1) # softmax
+        attn_map = attn_map.mean(dim=1) # avg all head(24 head)
+ 
+        txt_attn, img_attn = attn[:, : txt.shape[1]], attn[:, txt.shape[1] :]
+
+        # calculate the img bloks
+        img = img + img_mod1.gate * self.img_attn.proj(img_attn)
+        img = img + img_mod2.gate * self.img_mlp((1 + img_mod2.scale) * self.img_norm2(img) + img_mod2.shift)
+
+        # calculate the txt bloks
+        txt = txt + txt_mod1.gate * self.txt_attn.proj(txt_attn)
+        txt = txt + txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift)
+        return img, txt, attn_map
+
+
+class SingleStreamBlock(nn.Module):
+    """
+    A DiT block with parallel linear layers as described in
+    https://arxiv.org/abs/2302.05442 and adapted modulation interface.
+    """
+
+    def __init__(
+        self,
+        hidden_size: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qk_scale: float | None = None,
+    ):
+        super().__init__()
+        self.hidden_dim = hidden_size
+        self.num_heads = num_heads
+        head_dim = hidden_size // num_heads
+        self.scale = qk_scale or head_dim**-0.5
+
+        self.mlp_hidden_dim = int(hidden_size * mlp_ratio)
+        # qkv and mlp_in
+        self.linear1 = nn.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim)
+        # proj and mlp_out
+        self.linear2 = nn.Linear(hidden_size + self.mlp_hidden_dim, hidden_size)
+
+        self.norm = QKNorm(head_dim)
+
+        self.hidden_size = hidden_size
+        self.pre_norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+
+        self.mlp_act = nn.GELU(approximate="tanh")
+        self.modulation = Modulation(hidden_size, double=False)
+
+    def forward(self, x: Tensor, vec: Tensor, pe: Tensor, info) -> Tensor:
+        mod, _ = self.modulation(vec)
+        x_mod = (1 + mod.scale) * self.pre_norm(x) + mod.shift
+        qkv, mlp = torch.split(self.linear1(x_mod), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1)
+
+        q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads)
+        q, k = self.norm(q, k, v)
+
+        # Note: If the memory of your device is not enough, you may consider uncomment the following code.
+        # if info['inject'] and info['id'] > 19:
+        #     store_path = os.path.join(info['feature_path'], str(info['t']) + '_' + str(info['second_order']) + '_' + str(info['id']) + '_' + info['type'] + '_' + 'V' + '.pth')
+        #     if info['inverse']:
+        #         torch.save(v, store_path)
+        #     if not info['inverse']:
+        #         v = torch.load(store_path, weights_only=True)
+
+        # Save the features in the memory # ori: 19
+        if info['inject'] and info['id'] > 19:
+            if 'ref' not in info:
+                info['ref'] = False
+            feature_name = str(info['t']) + '_' + str(info['second_order']) + '_' + str(info['id']) + '_' + info['type'] + '_' + str(info['ref']) + '_' + 'V'
+            if info['inverse']:
+                info['feature'][feature_name] = v.cpu()
+            else:
+                # v injection with mask
+                # 0: original RF-Edit
+                # 1: new_v_text + old_v_image 
+                # 2: new_v*mask + old_v*(1-mask)
+                if info['change_v'] == 0:
+                    v = info['feature'][feature_name].cuda()
+                elif info['change_v'] == 1:
+                    old_v = info['feature'][feature_name].cuda()
+                    v = torch.cat([v[:, :, :512, :], old_v[:, :, 512:, :]], dim=2)
+                elif info['change_v'] == 2:
+                    old_v = info['feature'][feature_name].cuda()
+                    v = v * info['v_mask'] + old_v * (1 - info['v_mask'])
+                    
+                    
+
+        # compute attention
+        attn,attn_map = attention_with_attnmap(q, k, v, pe=pe)
+        attn_map = attn_map[:, :, 512:, :512]  # text to image attn map
+        if 'txt_token_l' in info:
+            # drop all paddings
+            attn_map = attn_map[:,:,:,:info['txt_token_l']]
+        attn_map = torch.nn.functional.softmax(attn_map, dim=-1) # softmax
+        attn_map = attn_map.mean(dim=1) # avg all head(24 head)
+        # compute activation in mlp stream, cat again and run second linear layer
+        output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2))
+        return x + mod.gate * output, info, attn_map
+
+
+class LastLayer(nn.Module):
+    def __init__(self, hidden_size: int, patch_size: int, out_channels: int):
+        super().__init__()
+        self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+        self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
+        self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))
+
+    def forward(self, x: Tensor, vec: Tensor) -> Tensor:
+        shift, scale = self.adaLN_modulation(vec).chunk(2, dim=1)
+        x = (1 + scale[:, None, :]) * self.norm_final(x) + shift[:, None, :]
+        x = self.linear(x)
+        return x

flux/sampling_lore.py

@@ -0,0 +1,372 @@
+import math
+import random
+import copy
+from typing import Callable
+
+import torch
+import numpy as np
+from einops import rearrange, repeat
+from torch import Tensor
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+from PIL import Image
+from torchvision import transforms
+
+from .model_lore import Flux
+from .modules.conditioner_lore import HFEmbedder
+
+def prepare_tokens(t5, source_prompt, target_prompt, replacements,show_tokens=False):
+    _, _, src_dif_ids, tgt_dif_ids=t5.get_text_embeddings_with_diff(source_prompt,target_prompt,replacements,show_tokens=show_tokens)
+    return src_dif_ids,tgt_dif_ids
+
+transform = transforms.ToTensor()
+
+def get_mask_one_tensor(mask_dirs,width,height,device):
+    res = []
+    for mask_dir in mask_dirs:
+        mask_image = Image.open(mask_dir).convert('L')
+        # resize
+        mask_image = mask_image.resize((math.ceil(height/16), math.ceil(width/16)), Image.Resampling.LANCZOS)
+        mask_tensor = transform(mask_image)
+        mask_tensor = mask_tensor.squeeze(0)
+        # to one dim
+        mask_tensor = mask_tensor.flatten()
+        mask_tensor = mask_tensor.to(device)
+        res.append(mask_tensor)
+    res = sum(res)
+    res = res.view(1, 1, -1, 1)
+    res = res.to(torch.bfloat16)
+    return res
+
+def get_v_mask(mask_dirs,width,height,device,txt_length=512):
+    res = []
+    for mask_dir in mask_dirs:
+        mask_image = Image.open(mask_dir).convert('L')
+        # resize
+        mask_image = mask_image.resize((math.ceil(height/16), math.ceil(width/16)), Image.Resampling.LANCZOS)
+        mask_tensor = transform(mask_image)
+        mask_tensor = mask_tensor.squeeze(0)
+        # to one dim
+        mask_tensor = mask_tensor.flatten()
+        mask_tensor = mask_tensor.to(device)
+        res.append(mask_tensor)
+    res = sum(res)
+    res = torch.cat([torch.ones(txt_length).to(device),res])
+    res = res.view(1, 1, -1, 1)
+    res = res.to(torch.bfloat16)
+    return res
+
+def add_masked_noise_to_z(z,mask,width,height,seed=42,noise_scale=0.1):
+    if noise_scale == 0:
+        return z
+    noise = torch.randn(z.shape,device=z.device,dtype=z.dtype,generator=torch.Generator(device=z.device).manual_seed(seed))
+    if noise_scale > 10:
+        return noise
+    # how to change z?
+    z = z*(1-mask[0])+noise_scale*noise*mask[0]+(1-noise_scale)*z*mask[0]
+    return z
+
+def prepare(t5: HFEmbedder, clip: HFEmbedder, img: Tensor, prompt: str | list[str]) -> dict[str, Tensor]:
+    bs, c, h, w = img.shape
+    if bs == 1 and not isinstance(prompt, str):
+        bs = len(prompt)
+
+    img = rearrange(img, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=2, pw=2)
+    if img.shape[0] == 1 and bs > 1:
+        img = repeat(img, "1 ... -> bs ...", bs=bs)
+
+    img_ids = torch.zeros(h // 2, w // 2, 3)
+    img_ids[..., 1] = img_ids[..., 1] + torch.arange(h // 2)[:, None]
+    img_ids[..., 2] = img_ids[..., 2] + torch.arange(w // 2)[None, :]
+    img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs)
+
+    if isinstance(prompt, str):
+        prompt = [prompt]
+    txt = t5(prompt)
+    if txt.shape[0] == 1 and bs > 1:
+        txt = repeat(txt, "1 ... -> bs ...", bs=bs)
+    txt_ids = torch.zeros(bs, txt.shape[1], 3)
+
+    vec = clip(prompt)
+    if vec.shape[0] == 1 and bs > 1:
+        vec = repeat(vec, "1 ... -> bs ...", bs=bs)
+
+    return {
+        "img": img,
+        "img_ids": img_ids.to(img.device),
+        "txt": txt.to(img.device),
+        "txt_ids": txt_ids.to(img.device),
+        "vec": vec.to(img.device),
+    }
+
+
+def time_shift(mu: float, sigma: float, t: Tensor):
+    return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma)
+
+
+def get_lin_function(
+    x1: float = 256, y1: float = 0.5, x2: float = 4096, y2: float = 1.15
+) -> Callable[[float], float]:
+    m = (y2 - y1) / (x2 - x1)
+    b = y1 - m * x1
+    return lambda x: m * x + b
+
+
+def get_schedule(
+    num_steps: int,
+    image_seq_len: int,
+    base_shift: float = 0.5,
+    max_shift: float = 1.15,
+    shift: bool = True,
+) -> list[float]:
+    # extra step for zero
+    timesteps = torch.linspace(1, 0, num_steps + 1)
+
+    # shifting the schedule to favor high timesteps for higher signal images
+    if shift:
+        # estimate mu based on linear estimation between two points
+        mu = get_lin_function(y1=base_shift, y2=max_shift)(image_seq_len)
+        timesteps = time_shift(mu, 1.0, timesteps)
+
+    return timesteps.tolist()
+
+def denoise(
+    model: Flux,
+    # model input
+    img: Tensor,
+    img_ids: Tensor,
+    txt: Tensor,
+    txt_ids: Tensor,
+    vec: Tensor,
+    # sampling parameters
+    timesteps: list[float],
+    inverse,
+    info, 
+    guidance: float = 4.0,
+    trainable_noise_list=None,
+):
+    # this is ignored for schnell
+    inject_list = [True] * info['inject_step'] + [False] * (len(timesteps[:-1]) - info['inject_step'])
+    
+
+    if inverse:
+        timesteps = timesteps[::-1]
+        inject_list = inject_list[::-1]
+    guidance_vec = torch.full((img.shape[0],), guidance, device=img.device, dtype=img.dtype)
+    
+    step_list = []
+    attn_map_list = []
+    for i, (t_curr, t_prev) in enumerate(zip(timesteps[:-1], timesteps[1:])):
+        t_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device)
+        info['t'] = t_prev if inverse else t_curr
+        info['inverse'] = inverse
+        info['second_order'] = False
+        info['inject'] = inject_list[i]
+        # when editing add optim latent for several steps
+        if trainable_noise_list and i != 0 and i<len(trainable_noise_list):
+            # smask = info['source_mask'].squeeze(0)
+            # img = trainable_noise_list[i]*smask+img*(1-smask)
+            img = trainable_noise_list[i]
+
+        pred, info, attn_maps_mid = model(
+            img=img,
+            img_ids=img_ids,
+            txt=txt,
+            txt_ids=txt_ids,
+            y=vec,
+            timesteps=t_vec,
+            guidance=guidance_vec,
+            info=info
+        )
+
+        img_mid = img + (t_prev - t_curr) / 2 * pred
+
+        t_vec_mid = torch.full((img.shape[0],), (t_curr + (t_prev - t_curr) / 2), dtype=img.dtype, device=img.device)
+        info['second_order'] = True
+        pred_mid, info, attn_maps = model(
+            img=img_mid,
+            img_ids=img_ids,
+            txt=txt,
+            txt_ids=txt_ids,
+            y=vec,
+            timesteps=t_vec_mid,
+            guidance=guidance_vec,
+            info=info
+        )
+
+        first_order = (pred_mid - pred) / ((t_prev - t_curr) / 2)
+        img = img + (t_prev - t_curr) * pred + 0.5 * (t_prev - t_curr) ** 2 * first_order
+
+        # return attnmaps L,1,512,N
+        step_list.append(t_curr)
+        attn_map_list.append((attn_maps_mid+attn_maps)/2)
+
+    attn_map_list = torch.stack(attn_map_list)
+    return img, info, step_list, attn_map_list
+
+selected_layers = range(8,44)
+
+def gaussian_smooth(attnmap,wh,kernel_size=3,sigma=0.5):
+    # to 2d
+    attnmap = rearrange(
+                        attnmap,
+                        "b (w h) -> b (w) (h)",
+                        w=math.ceil(wh[0]/16),
+                        h=math.ceil(wh[1]/16),
+                    )
+    attnmap = attnmap.unsqueeze(1)
+    # prepare kernel
+    ax = torch.arange(-kernel_size // 2 + 1., kernel_size // 2 + 1., device=attnmap.device)
+    xx, yy = torch.meshgrid(ax, ax, indexing='ij')
+    kernel = torch.exp(-(xx**2 + yy**2) / (2. * sigma**2))
+    kernel = kernel / kernel.sum()
+    kernel = kernel.view(1, 1, kernel_size, kernel_size)
+    kernel = kernel.to(dtype=attnmap.dtype)
+    # gaussian smooth
+    attnmap_smoothed = F.conv2d(attnmap, kernel, padding=kernel_size // 2)
+    return attnmap_smoothed.view(attnmap_smoothed.shape[0], -1)
+
+def compute_attn_max_loss(attnmaps,source_mask,wh):
+    # attnmaps L,1,N,k
+    attnmaps = attnmaps[selected_layers,0,:,:]
+    attnmaps = attnmaps.mean(dim=-1)
+    src_mask = source_mask.view(-1).unsqueeze(0)
+    p = attnmaps*src_mask
+    p = gaussian_smooth(p, wh, kernel_size=3, sigma=0.5)
+    p = p.max(dim=1).values
+    loss = (1 - p).mean()
+    return loss
+
+def compute_attn_min_loss(attnmaps,source_mask,wh):
+    # attnmaps L,1,N,k
+    attnmaps = attnmaps[selected_layers,0,:,:]
+    attnmaps = attnmaps.mean(dim=-1)
+    src_mask = source_mask.view(-1).unsqueeze(0)
+    p = attnmaps*src_mask
+    p = gaussian_smooth(p, wh, kernel_size=3, sigma=0.5)
+    p = p.max(dim=1).values 
+    loss = p.mean()
+    return loss
+
+def denoise_with_noise_optim(
+    model: Flux,
+    # model input
+    img: Tensor,
+    img_ids: Tensor,
+    txt: Tensor,
+    txt_ids: Tensor,
+    vec: Tensor,
+    # loss cal
+    token_ids: list[list[int]],
+    source_mask: Tensor,
+    training_steps: int,
+    training_epochs: int,
+    learning_rate: float,
+    seed: int,
+    noise_scale: float,
+    # sampling parameters
+    timesteps: list[float],
+    info, 
+    guidance: float = 4.0
+):
+    # this is ignored for schnell
+    #print(f'training the noise in last {training_steps} steps and {training_epochs} epochs')
+    #timesteps = timesteps[::-1]
+    guidance_vec = torch.full((img.shape[0],), guidance, device=img.device, dtype=img.dtype)
+
+    step_list = []
+    attn_map_list = []
+    trainable_noise_list = []
+    for i, (t_curr, t_prev) in enumerate(zip(timesteps[:-1], timesteps[1:])):
+        if i >= training_steps:
+            break
+        # prepare ori parameters
+        ori_txt = txt.clone()
+        ori_img = img.clone()
+        ori_vec = vec.clone()
+        
+        # prepare trainable noise
+        if i == 0:
+            if noise_scale == 0:
+                trainable_noise = torch.nn.Parameter(img.clone().detach(), requires_grad=True)
+            else:
+                noise = torch.randn(img.shape,device=img.device,dtype=img.dtype,generator=torch.Generator(device=img.device).manual_seed(seed))
+                noise = img*(1-source_mask[0])+ noise_scale*noise*source_mask[0] + (1-noise_scale)*img*source_mask[0]
+                trainable_noise = torch.nn.Parameter(noise.clone().detach(), requires_grad=True)
+        else:
+            trainable_noise = torch.nn.Parameter(img.clone().detach(), requires_grad=True)
+        optimizer = optim.Adam([trainable_noise], lr=learning_rate)
+
+        # run one training step
+        for j in range(training_epochs):
+            optimizer.zero_grad()
+            txt = ori_txt.clone().detach()
+            vec = ori_vec.clone().detach()
+            t_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device)
+            info['t'] = t_prev
+            info['inverse'] = False
+            info['second_order'] = False
+            info['inject'] = False # tried True, seems not necessary
+            pred, info, attn_maps_mid = model(
+                img=trainable_noise,
+                img_ids=img_ids,
+                txt=txt,
+                txt_ids=txt_ids,
+                y=vec,
+                timesteps=t_vec,
+                guidance=guidance_vec,
+                info=info
+            )
+            
+    
+            img_mid = trainable_noise + (t_prev - t_curr) / 2 * pred
+    
+            t_vec_mid = torch.full((img.shape[0],), (t_curr + (t_prev - t_curr) / 2), dtype=img.dtype, device=img.device)
+            info['second_order'] = True
+            pred_mid, info, attn_maps = model(
+                img=img_mid,
+                img_ids=img_ids,
+                txt=txt,
+                txt_ids=txt_ids,
+                y=vec,
+                timesteps=t_vec_mid,
+                guidance=guidance_vec,
+                info=info
+            )
+    
+            first_order = (pred_mid - pred) / ((t_prev - t_curr) / 2)
+            img = img + (t_prev - t_curr) * pred + 0.5 * (t_prev - t_curr) ** 2 * first_order
+    
+            # attnmaps L,1,N,512 for cal loss
+            attn_maps=(attn_maps_mid+attn_maps)/2
+            total_loss = 0.0
+            for indices,change,ratio in token_ids:
+                if change:
+                    total_loss += compute_attn_max_loss(attn_maps[:,:,:,indices], source_mask, info['wh'])
+                else:
+                    if ratio != 0:
+                        total_loss += ratio*compute_attn_min_loss(attn_maps[:,:,:,indices], source_mask, info['wh'])
+            total_loss.backward()
+            with torch.no_grad():
+                trainable_noise.grad *= source_mask[0]
+            optimizer.step()
+            print(f"Time {t_curr:.4f} Step {j+1}/{training_epochs}, Loss: {total_loss.item():.6f}")
+            
+        attn_map_list.append(attn_maps.detach())
+        step_list.append(t_curr)
+        trainable_noise = trainable_noise.detach()
+        trainable_noise_list.append(trainable_noise.clone())
+
+    attn_map_list = torch.stack(attn_map_list)
+    return img, info, step_list, attn_map_list, trainable_noise_list
+
+def unpack(x: Tensor, height: int, width: int) -> Tensor:
+    return rearrange(
+        x,
+        "b (h w) (c ph pw) -> b c (h ph) (w pw)",
+        h=math.ceil(height / 16),
+        w=math.ceil(width / 16),
+        ph=2,
+        pw=2,
+    )

flux/util_lore.py

@@ -0,0 +1,208 @@
+import os
+from dataclasses import dataclass
+
+import torch
+from einops import rearrange
+from huggingface_hub import hf_hub_download
+from imwatermark import WatermarkEncoder
+from safetensors.torch import load_file as load_sft
+
+from flux.model_lore import Flux, FluxParams
+from flux.modules.autoencoder import AutoEncoder, AutoEncoderParams
+from flux.modules.conditioner_lore import HFEmbedder
+
+
+@dataclass
+class ModelSpec:
+    params: FluxParams
+    ae_params: AutoEncoderParams
+    ckpt_path: str | None
+    ae_path: str | None
+    repo_id: str | None
+    repo_flow: str | None
+    repo_ae: str | None
+
+# download model from hf
+flux_path = "black-forest-labs/FLUX.1-dev"
+flux_ckpt_path = os.getenv("FLUX_DEV")
+flux_ae_path = os.getenv("AE")
+t5_path = "google/t5-v1_1-xxl"
+clip_path = "openai/clip-vit-large-patch14"
+
+configs = {
+    "flux-dev": ModelSpec(
+        repo_id=flux_path,
+        repo_flow="flux1-dev.safetensors",
+        repo_ae="ae.safetensors",
+        ckpt_path=flux_ckpt_path,
+        params=FluxParams(
+            in_channels=64,
+            vec_in_dim=768,
+            context_in_dim=4096,
+            hidden_size=3072,
+            mlp_ratio=4.0,
+            num_heads=24,
+            depth=19,
+            depth_single_blocks=38,
+            axes_dim=[16, 56, 56],
+            theta=10_000,
+            qkv_bias=True,
+            guidance_embed=True,
+        ),
+        ae_path=flux_ae_path,
+        ae_params=AutoEncoderParams(
+            resolution=256,
+            in_channels=3,
+            ch=128,
+            out_ch=3,
+            ch_mult=[1, 2, 4, 4],
+            num_res_blocks=2,
+            z_channels=16,
+            scale_factor=0.3611,
+            shift_factor=0.1159,
+        ),
+    ),
+    "flux-schnell": ModelSpec(
+        repo_id="black-forest-labs/FLUX.1-schnell",
+        repo_flow="flux1-schnell.safetensors",
+        repo_ae="ae.safetensors",
+        ckpt_path=os.getenv("FLUX_SCHNELL"),
+        params=FluxParams(
+            in_channels=64,
+            vec_in_dim=768,
+            context_in_dim=4096,
+            hidden_size=3072,
+            mlp_ratio=4.0,
+            num_heads=24,
+            depth=19,
+            depth_single_blocks=38,
+            axes_dim=[16, 56, 56],
+            theta=10_000,
+            qkv_bias=True,
+            guidance_embed=False,
+        ),
+        ae_path=os.getenv("AE"),
+        ae_params=AutoEncoderParams(
+            resolution=256,
+            in_channels=3,
+            ch=128,
+            out_ch=3,
+            ch_mult=[1, 2, 4, 4],
+            num_res_blocks=2,
+            z_channels=16,
+            scale_factor=0.3611,
+            shift_factor=0.1159,
+        ),
+    ),
+}
+
+
+def print_load_warning(missing: list[str], unexpected: list[str]) -> None:
+    if len(missing) > 0 and len(unexpected) > 0:
+        print(f"Got {len(missing)} missing keys:\n\t" + "\n\t".join(missing))
+        print("\n" + "-" * 79 + "\n")
+        print(f"Got {len(unexpected)} unexpected keys:\n\t" + "\n\t".join(unexpected))
+    elif len(missing) > 0:
+        print(f"Got {len(missing)} missing keys:\n\t" + "\n\t".join(missing))
+    elif len(unexpected) > 0:
+        print(f"Got {len(unexpected)} unexpected keys:\n\t" + "\n\t".join(unexpected))
+
+
+def load_flow_model(name: str, device: str | torch.device = "cuda", hf_download: bool = True):
+    # Loading Flux
+    print("Init model")
+    
+    ckpt_path = configs[name].ckpt_path
+    if (
+        ckpt_path is None
+        and configs[name].repo_id is not None
+        and configs[name].repo_flow is not None
+        and hf_download
+    ):
+        ckpt_path = hf_hub_download(configs[name].repo_id, configs[name].repo_flow)
+
+    with torch.device("meta" if ckpt_path is not None else device):
+        model = Flux(configs[name].params).to(torch.bfloat16)
+
+    if ckpt_path is not None:
+        print("Loading checkpoint on", device, ckpt_path)
+        # load_sft doesn't support torch.device
+        sd = load_sft(ckpt_path, device=str(device))
+        missing, unexpected = model.load_state_dict(sd, strict=False, assign=True)
+        print_load_warning(missing, unexpected)
+    return model
+
+
+def load_t5(device: str | torch.device = "cuda", max_length: int = 512) -> HFEmbedder:
+    # max length 64, 128, 256 and 512 should work (if your sequence is short enough)
+    return HFEmbedder(t5_path, max_length=max_length, is_clip=False, torch_dtype=torch.bfloat16).to(device)
+
+
+def load_clip(device: str | torch.device = "cuda") -> HFEmbedder:
+    return HFEmbedder(clip_path, max_length=77, is_clip=True, torch_dtype=torch.bfloat16).to(device)
+
+
+def load_ae(name: str, device: str | torch.device = "cuda", hf_download: bool = True) -> AutoEncoder:
+    ckpt_path = configs[name].ae_path
+    if (
+        ckpt_path is None
+        and configs[name].repo_id is not None
+        and configs[name].repo_ae is not None
+        and hf_download
+    ):
+        ckpt_path = hf_hub_download(configs[name].repo_id, configs[name].repo_ae)
+
+    # Loading the autoencoder
+    print("Init AE")
+    with torch.device("meta" if ckpt_path is not None else device):
+        ae = AutoEncoder(configs[name].ae_params)
+
+    if ckpt_path is not None:
+        sd = load_sft(ckpt_path, device=str(device))
+        missing, unexpected = ae.load_state_dict(sd, strict=False, assign=True)
+        print_load_warning(missing, unexpected)
+    return ae
+
+
+class WatermarkEmbedder:
+    def __init__(self, watermark):
+        self.watermark = watermark
+        self.num_bits = len(WATERMARK_BITS)
+        self.encoder = WatermarkEncoder()
+        self.encoder.set_watermark("bits", self.watermark)
+
+    def __call__(self, image: torch.Tensor) -> torch.Tensor:
+        """
+        Adds a predefined watermark to the input image
+
+        Args:
+            image: ([N,] B, RGB, H, W) in range [-1, 1]
+
+        Returns:
+            same as input but watermarked
+        """
+        image = 0.5 * image + 0.5
+        squeeze = len(image.shape) == 4
+        if squeeze:
+            image = image[None, ...]
+        n = image.shape[0]
+        image_np = rearrange((255 * image).detach().cpu(), "n b c h w -> (n b) h w c").numpy()[:, :, :, ::-1]
+        # torch (b, c, h, w) in [0, 1] -> numpy (b, h, w, c) [0, 255]
+        # watermarking libary expects input as cv2 BGR format
+        for k in range(image_np.shape[0]):
+            image_np[k] = self.encoder.encode(image_np[k], "dwtDct")
+        image = torch.from_numpy(rearrange(image_np[:, :, :, ::-1], "(n b) h w c -> n b c h w", n=n)).to(
+            image.device
+        )
+        image = torch.clamp(image / 255, min=0.0, max=1.0)
+        if squeeze:
+            image = image[0]
+        image = 2 * image - 1
+        return image
+
+
+# A fixed 48-bit message that was chosen at random
+WATERMARK_MESSAGE = 0b001010101111111010000111100111001111010100101110
+# bin(x)[2:] gives bits of x as str, use int to convert them to 0/1
+WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]]
+embed_watermark = WatermarkEmbedder(WATERMARK_BITS)

requirements.txt

@@ -0,0 +1,19 @@
+pydantic==2.10.6
+torch
+einops
+accelerate==0.34.2
+einops==0.8.0
+transformers==4.41.2
+huggingface-hub==0.24.6
+datasets
+omegaconf
+diffusers
+sentencepiece
+opencv-python
+matplotlib
+onnxruntime
+torchvision
+timm
+invisible-watermark
+fire
+tqdm