0515_first_upload

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,5 +1,5 @@
 ---
-title: CharacterFactory
+title: 'CharacterFactory'
 emoji: 🖼
 colorFrom: purple
 colorTo: red
@@ -7,7 +7,6 @@ sdk: gradio
 sdk_version: 4.26.0
 app_file: app.py
 pinned: false
-license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,146 +1,395 @@
+import os
+import io
+import IPython.display
+from PIL import Image
+import base64
+import io
+from PIL import Image
 import gradio as gr
-import numpy as np
+import requests
+import time
 import random
-from diffusers import DiffusionPipeline
+import numpy as np
 import torch
+import os
+from transformers import ViTModel, ViTImageProcessor
+from utils import text_encoder_forward
+from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
+from utils import latents_to_images, downsampling, merge_and_save_images
+from omegaconf import OmegaConf
+from accelerate.utils import set_seed
+from tqdm import tqdm
+from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
+from PIL import Image
+from models.celeb_embeddings import embedding_forward
+import models.embedding_manager
+import importlib
+import time
 
-device = "cuda" if torch.cuda.is_available() else "cpu"
+import os
+os.environ['GRADIO_TEMP_DIR'] = 'qinghewang/tmp'
 
-if torch.cuda.is_available():
-    torch.cuda.max_memory_allocated(device=device)
-    pipe = DiffusionPipeline.from_pretrained("stabilityai/sdxl-turbo", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
-    pipe.enable_xformers_memory_efficient_attention()
-    pipe = pipe.to(device)
-else: 
-    pipe = DiffusionPipeline.from_pretrained("stabilityai/sdxl-turbo", use_safetensors=True)
-    pipe = pipe.to(device)
+title = r"""
+<h1 align="center">CharacterFactory: Sampling Consistent Characters with GANs for Diffusion Models</h1>
+"""
 
-MAX_SEED = np.iinfo(np.int32).max
-MAX_IMAGE_SIZE = 1024
+description = r"""
+<b>Official Gradio demo</b> for <a href='https://qinghew.github.io/CharacterFactory/' target='_blank'><b>CharacterFactory: Sampling Consistent Characters with GANs for Diffusion Models</b></a>.<br>
 
-def infer(prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps):
+How to use:<br>
+1. Enter prompts (the character placeholder is "a person"), where each line will generate an image.
+2. You can choose to create a new character or continue to use the current one. We have provided some examples, click on the examples below to use.
+3. You can choose to use the Normal version (the gender is random), the Man version, and the Woman version.
+4. Click the <b>Generate</b> button to begin (Images are generated one by one).
+5. Our method can be applied to illustrating books and stories, creating brand ambassadors, developing presentations, art design, identity-consistent data construction and more. Looking forward to your explorations!😊
+6. If CharacterFactory is helpful, please help to ⭐ the <a href='https://github.com/qinghew/CharacterFactory' target='_blank'>Github Repo</a>. Thanks! 
+"""
 
-    if randomize_seed:
-        seed = random.randint(0, MAX_SEED)
-        
-    generator = torch.Generator().manual_seed(seed)
-    
-    image = pipe(
-        prompt = prompt, 
-        negative_prompt = negative_prompt,
-        guidance_scale = guidance_scale, 
-        num_inference_steps = num_inference_steps, 
-        width = width, 
-        height = height,
-        generator = generator
-    ).images[0] 
-    
-    return image
-
-examples = [
-    "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k",
-    "An astronaut riding a green horse",
-    "A delicious ceviche cheesecake slice",
-]
-
-css="""
-#col-container {
-    margin: 0 auto;
-    max-width: 520px;
+article = r"""
+---
+📝 **Citation**
+<br>
+If our work is helpful for your research or applications, please cite us via:
+```bibtex
+@article{wang2024characterfactory,
+title={CharacterFactory: Sampling Consistent Characters with GANs for Diffusion Models},
+author={Wang, Qinghe and Li, Baolu and Li, Xiaomin and Cao, Bing and Ma, Liqian and Lu, Huchuan and Jia, Xu},
+journal={arXiv preprint arXiv:2404.15677},
+year={2024}
 }
+```
+📧 **Contact**
+<br>
+If you have any questions, please feel free to open an issue or directly reach us out at <b>qinghewang@mail.dlut.edu.cn</b>.
 """
 
-if torch.cuda.is_available():
-    power_device = "GPU"
-else:
-    power_device = "CPU"
+css = '''
+#color-bg{display:flex;justify-content: center;align-items: center;}
+.color-bg-item{width: 100%; height: 32px}
+#main_button{width:100%}
+<style>
+'''
+
+model_id = "stabilityai/stable-diffusion-2-1-base"
+# model_path = "/home/qinghewang/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6"
+pipe = StableDiffusionPipeline.from_pretrained(model_id)   # , torch_dtype=torch.float16
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+pipe = pipe.to("cuda")
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+vae = pipe.vae
+unet = pipe.unet
+text_encoder = pipe.text_encoder
+tokenizer = pipe.tokenizer
+scheduler = pipe.scheduler
+
+input_dim = 64
+
+original_forward = text_encoder.text_model.embeddings.forward
+text_encoder.text_model.embeddings.forward = embedding_forward.__get__(text_encoder.text_model.embeddings)
+embedding_manager_config = OmegaConf.load("datasets_face/identity_space.yaml")
+
+normal_Embedding_Manager = models.embedding_manager.EmbeddingManagerId_adain(  
+        tokenizer,
+        text_encoder,
+        device = device,
+        training = True,
+        experiment_name = "normal_GAN", 
+        num_embeds_per_token = embedding_manager_config.model.personalization_config.params.num_embeds_per_token,            
+        token_dim = embedding_manager_config.model.personalization_config.params.token_dim,
+        mlp_depth = embedding_manager_config.model.personalization_config.params.mlp_depth,
+        loss_type = embedding_manager_config.model.personalization_config.params.loss_type,
+        vit_out_dim = input_dim,
+)
+
+man_Embedding_Manager = models.embedding_manager.EmbeddingManagerId_adain(  
+        tokenizer,
+        text_encoder,
+        device = device,
+        training = True,
+        experiment_name = "man_GAN", 
+        num_embeds_per_token = embedding_manager_config.model.personalization_config.params.num_embeds_per_token,            
+        token_dim = embedding_manager_config.model.personalization_config.params.token_dim,
+        mlp_depth = embedding_manager_config.model.personalization_config.params.mlp_depth,
+        loss_type = embedding_manager_config.model.personalization_config.params.loss_type,
+        vit_out_dim = input_dim,
+)
+
+woman_Embedding_Manager = models.embedding_manager.EmbeddingManagerId_adain(  
+        tokenizer,
+        text_encoder,
+        device = device,
+        training = True,
+        experiment_name = "woman_GAN", 
+        num_embeds_per_token = embedding_manager_config.model.personalization_config.params.num_embeds_per_token,            
+        token_dim = embedding_manager_config.model.personalization_config.params.token_dim,
+        mlp_depth = embedding_manager_config.model.personalization_config.params.mlp_depth,
+        loss_type = embedding_manager_config.model.personalization_config.params.loss_type,
+        vit_out_dim = input_dim,
+)
+
 
-with gr.Blocks(css=css) as demo:
+DEFAULT_STYLE_NAME = "Watercolor"
+MAX_SEED = np.iinfo(np.int32).max
+
+def remove_tips():
+    return gr.update(visible=False)
+
+def response(choice, gender_GAN):
+    c = ""
+    e = ""
+    if choice == "Create a new character":
+        c = "create"
+    elif choice == "Still use this character":
+        c = "continue"
     
-    with gr.Column(elem_id="col-container"):
-        gr.Markdown(f"""
-        # Text-to-Image Gradio Template
-        Currently running on {power_device}.
-        """)
-        
-        with gr.Row():
-            
-            prompt = gr.Text(
-                label="Prompt",
-                show_label=False,
-                max_lines=1,
-                placeholder="Enter your prompt",
-                container=False,
-            )
-            
-            run_button = gr.Button("Run", scale=0)
+    if gender_GAN == "Normal":
+        e = "normal_GAN"
+    elif gender_GAN == "Man":
+        e = "man_GAN"
+    elif gender_GAN == "Woman":
+        e = "woman_GAN"
+    
+    return c, e
+
+def replace_phrases(prompt):
+    replacements = {
+        "a person": "v1* v2*",
+        "a man": "v1* v2*",
+        "a woman": "v1* v2*",
+        "a boy": "v1* v2*",
+        "a girl": "v1* v2*"
+    }
+    for phrase, replacement in replacements.items():
+        prompt = prompt.replace(phrase, replacement)
+    return prompt
+
+
+def handle_prompts(prompts_array):
+    prompts = prompts_array.splitlines()
+    prompts = [prompt + ', facing to camera, best quality, ultra high res'  for prompt in prompts]
+    prompts = [replace_phrases(prompt) for prompt in prompts]
+    return prompts
+    
+
+
+def generate_image(experiment_name, label, prompts_array, chose_emb):
+    prompts = handle_prompts(prompts_array)
+    
+    print("experiment_name:",experiment_name)
+
+    if experiment_name == "normal_GAN":
+        steps = 10000
+        Embedding_Manager = normal_Embedding_Manager
+    elif experiment_name == "man_GAN":
+        steps = 7000
+        Embedding_Manager = man_Embedding_Manager
+    elif experiment_name == "woman_GAN":
+        steps = 6000
+        Embedding_Manager = woman_Embedding_Manager
+    else:
+        print("Hello, please notice this ^_^")
+        assert 0
+    
+    embedding_path = os.path.join("training_weight", experiment_name, "embeddings_manager-{}.pt".format(str(steps)))
+    Embedding_Manager.load(embedding_path)
+    print("embedding_path:",embedding_path)
+    print("label:",label)
+    
+    index = "0"
+    save_dir = os.path.join("test_results/" + experiment_name, index)
+    os.makedirs(save_dir, exist_ok=True)
+    ran_emb_path = os.path.join(save_dir, "ran_embeddings.pt")
+    test_emb_path = os.path.join(save_dir, "id_embeddings.pt")
+    
+    if label == "create":
+        print("new")
+        random_embedding = torch.randn(1, 1, input_dim).to(device)
+        torch.save(random_embedding, ran_emb_path)
+        _, emb_dict = Embedding_Manager(tokenized_text=None, embedded_text=None, name_batch=None, random_embeddings = random_embedding, timesteps = None,)
+        text_encoder.text_model.embeddings.forward = original_forward
+        test_emb = emb_dict["adained_total_embedding"].to(device)
+        torch.save(test_emb, test_emb_path)
+    elif label == "continue":
+        print("old")
+        test_emb = torch.load(chose_emb).cuda()
+        text_encoder.text_model.embeddings.forward = original_forward
+    
+    v1_emb = test_emb[:, 0]
+    v2_emb = test_emb[:, 1]
+    embeddings = [v1_emb, v2_emb]
+    
+    tokens = ["v1*", "v2*"]
+    tokenizer.add_tokens(tokens)
+    token_ids = tokenizer.convert_tokens_to_ids(tokens)
+    
+    text_encoder.resize_token_embeddings(len(tokenizer), pad_to_multiple_of = 8)
+    for token_id, embedding in zip(token_ids, embeddings):
+        text_encoder.get_input_embeddings().weight.data[token_id] = embedding
+    
+    total_results = []
+    for prompt in prompts:
+        image = pipe(prompt, guidance_scale = 8.5).images
+        total_results = image + total_results
+        yield total_results, test_emb_path
+
+def get_example():
+    case = [
+        [
+            'demo_embeddings/example_1.pt',
+            "Normal",
+            "Still use this character",
+            "a photo of a person\na person as a small child\na person as a 20 years old person\na person as a 80 years old person\na person reading a book\na person in the sunset\n",
+        ],
+        [
+            'demo_embeddings/example_2.pt',
+            "Man",
+            "Still use this character",
+            "a photo of a person\na person with a mustache and a hat\na person wearing headphoneswith red hair\na person with his dog\n",
+        ],
+        [
+            'demo_embeddings/example_3.pt',
+            "Woman",
+            "Still use this character",
+            "a photo of a person\na person at a beach\na person as a police officer\na person wearing a birthday hat\n",
+        ],
+        [
+            'demo_embeddings/example_4.pt',
+            "Man",
+            "Still use this character",
+            "a photo of a person\na person holding a bunch of flowers\na person in a lab coat\na person speaking at a podium\n",
+        ],
+        [
+            'demo_embeddings/example_5.pt',
+            "Woman",
+            "Still use this character",
+            "a photo of a person\na person wearing a kimono\na person in Van Gogh style\nEthereal fantasy concept art of a person\n",
+        ],
+        [
+            'demo_embeddings/example_6.pt',
+            "Man",
+            "Still use this character",
+            "a photo of a person\na person in the rain\na person meditating\na pencil sketch of a person\n",
+        ],
+    ]
+    return case
+
+def run_for_examples(example_emb, gender_GAN, choice, prompts_array):
+    prompts = handle_prompts(prompts_array)
+    label, experiment_name = response(choice, gender_GAN)
+    if experiment_name == "normal_GAN":
+        steps = 10000
+        Embedding_Manager = normal_Embedding_Manager
+    elif experiment_name == "man_GAN":
+        steps = 7000
+        Embedding_Manager = man_Embedding_Manager
+    elif experiment_name == "woman_GAN":
+        steps = 6000
+        Embedding_Manager = woman_Embedding_Manager
+    else:
+        print("Hello, please notice this ^_^")
+        assert 0
         
-        result = gr.Image(label="Result", show_label=False)
+    embedding_path = os.path.join("training_weight", experiment_name, "embeddings_manager-{}.pt".format(str(steps)))
+    Embedding_Manager.load(embedding_path)
+    print("embedding_path:",embedding_path)
+    print("label:",label)
+    
+    test_emb = torch.load(example_emb).cuda()
+    text_encoder.text_model.embeddings.forward = original_forward
+    v1_emb = test_emb[:, 0]
+    v2_emb = test_emb[:, 1]
+    embeddings = [v1_emb, v2_emb]
+    
+    tokens = ["v1*", "v2*"]
+    tokenizer.add_tokens(tokens)
+    token_ids = tokenizer.convert_tokens_to_ids(tokens)
+    
+    text_encoder.resize_token_embeddings(len(tokenizer), pad_to_multiple_of = 8)
+    for token_id, embedding in zip(token_ids, embeddings):
+        text_encoder.get_input_embeddings().weight.data[token_id] = embedding
+    
+    total_results = []
+    i = 0
+    for prompt in prompts:
+        image = pipe(prompt, guidance_scale = 8.5).images
+        total_results = image + total_results
+        i+=1
+        if i < len(prompts):
+            yield total_results, gr.update(visible=True, value="<h3>(Not Finished) Generating ···</h3>")
+        else:
+            yield total_results, gr.update(visible=True, value="<h3>Generation Finished</h3>")
 
-        with gr.Accordion("Advanced Settings", open=False):
-            
-            negative_prompt = gr.Text(
-                label="Negative prompt",
-                max_lines=1,
-                placeholder="Enter a negative prompt",
-                visible=False,
-            )
-            
-            seed = gr.Slider(
-                label="Seed",
-                minimum=0,
-                maximum=MAX_SEED,
-                step=1,
-                value=0,
-            )
+
+def set_text_unfinished():
+    return gr.update(visible=True, value="<h3>(Not Finished) Generating ···</h3>")
+ 
+def set_text_finished():
+    return gr.update(visible=True, value="<h3>Generation Finished</h3>")
+
+
+
+
+with gr.Blocks(css=css) as demo:    # css=css
+    # binary_matrixes = gr.State([])
+    # color_layout = gr.State([])
+    
+    # gr.Markdown(logo)
+    gr.Markdown(title)
+    gr.Markdown(description)
+    
+    with gr.Row():
+        with gr.Column():         
+            prompts_array = gr.Textbox(lines = 3, 
+                    label="Prompts (each line corresponds to a frame).", 
+                    info="Give simple prompt is enough to achieve good face fidelity",
+                    # placeholder="A photo of a person",
+                    value="a photo of a person\na person in front of the Great Wall\na person reading a book\na person wearing a Christmas hat\n",
+                    interactive=True)
+            choice = gr.Radio(choices=["Create a new character", "Still use this character"], label="Choose your action")
+
+            gender_GAN = gr.Radio(choices=["Normal", "Man", "Woman"], label="Choose your model version")
             
-            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+            label = gr.Text(label="Select the action you want to take", visible=False)
+            experiment_name = gr.Text(label="Select the GAN you want to take", visible=False)
+            chose_emb = gr.File(label="Uploaded files", type="filepath", visible=False)
+            example_emb = gr.File(label="Uploaded files", type="filepath", visible=False)
             
-            with gr.Row():
-                
-                width = gr.Slider(
-                    label="Width",
-                    minimum=256,
-                    maximum=MAX_IMAGE_SIZE,
-                    step=32,
-                    value=512,
-                )
-                
-                height = gr.Slider(
-                    label="Height",
-                    minimum=256,
-                    maximum=MAX_IMAGE_SIZE,
-                    step=32,
-                    value=512,
-                )
+            generate = gr.Button("Generate!😊", variant="primary")
+
+        with gr.Column():
+            gallery = gr.Gallery(label="Generated Images", columns=2, height='auto')
+            generated_information = gr.Markdown(label="Generation Details", value="",visible=False)
             
-            with gr.Row():
-                
-                guidance_scale = gr.Slider(
-                    label="Guidance scale",
-                    minimum=0.0,
-                    maximum=10.0,
-                    step=0.1,
-                    value=0.0,
-                )
-                
-                num_inference_steps = gr.Slider(
-                    label="Number of inference steps",
-                    minimum=1,
-                    maximum=12,
-                    step=1,
-                    value=2,
-                )
-        
-        gr.Examples(
-            examples = examples,
-            inputs = [prompt]
+        generate.click(
+            fn=set_text_unfinished, 
+            outputs=generated_information
+        ).then(
+            fn=response,
+            inputs=[choice, gender_GAN],
+            outputs=[label, experiment_name],    
+        ).then(
+            fn=generate_image,
+            inputs=[experiment_name, label, prompts_array, chose_emb],
+            outputs=[gallery, chose_emb] 
+        ).then(
+            fn=set_text_finished,
+            outputs=generated_information
         )
+        
 
-    run_button.click(
-        fn = infer,
-        inputs = [prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps],
-        outputs = [result]
+    gr.Examples(
+        examples=get_example(),
+        inputs=[example_emb, gender_GAN, choice, prompts_array],
+        run_on_click=True,
+        fn=run_for_examples,
+        outputs=[gallery, generated_information],
     )
+    
+    gr.Markdown(article)
+# demo.launch(server_name="0.0.0.0", share = False)
+# share_link = demo.launch(share=True)
+# print("Share this link: ", share_link)
 
-demo.queue().launch()
+demo.launch()   # share=True

datasets_face/face_id.py

@@ -0,0 +1,1007 @@
+import numpy as np
+import torch
+from torch.utils.data import Dataset
+from torchvision.transforms import transforms
+import torch.nn.functional as F
+import random
+
+imagenet_templates_small = [
+    'a photo of {}',
+    '{} is sitting at a desk, writing in a notebook',
+    '{} is standing in a kitchen, cooking a meal',
+    'In a garden, there are flowers and trees. {} is walking around',
+    '{} is playing a piano in a music room',
+    'In a garden, {} is watering plants',
+    'In a gym, there are many machines. {} is lifting weights',
+    '{} is reading a book in a cozy armchair',
+    '{} is painting on a canvas in a studio',
+    'In a museum, there are many paintings. {} is admiring them',
+    '{} is jogging on a trail in the woods',
+    'In an office, {} is working on a computer',
+    '{} is playing with a dog in a backyard',
+    '{} is taking a photograph in a city street',
+    'In a concert, there are many people. {} is enjoying the music',
+    '{} is playing chess in a park',
+    'In a library, {} is browsing through books',
+    '{} is riding a bicycle on a city street',
+    '{} is watching a movie in a living room',
+    'In a café, {} is drinking coffee and using a laptop',
+    '{} is hiking in the mountains',
+    '{} is playing a violin in a concert hall',
+    'In a gym, a {} is lifting weights',
+    '{} is gardening in his backyard',
+    '{} is swimming in a pool',
+    '{} is shopping in a grocery stor',
+    'In a museum, {} is admiring a painting',
+    'In a studio, {} is recording a podcast',
+    '{} is doing yoga in a peaceful room',
+    '{} is cooking barbecue in a park',
+    'In a laboratory, {} is conducting an experiment',
+    'In an airport, {} is waiting for a flight',
+    '{} is sitting on a bench, smiling at the camera',
+    '{} is holding a book in his hands',
+    'In a room, {}n is sitting on a chair',
+    '{} is standing in front of a window',
+    'In a kitchen, {} is cooking food',
+    'In a living room, {} is watching TV',
+    'In a bedroom, {} is sleeping',
+    '{} is holding a cup of coffee',
+    'In a classroom, {} is writing on a whiteboard',
+    'In a gym, {} is lifting weights',
+    '{} is holding a microphone',
+    'In a restaurant, {} is eating food',
+    '{} is holding a pen and writing on a paper',
+    'In a store, {} is shopping for clothes',
+    'In a museum, {} is looking at an exhibit',
+    '{} is holding a camera and taking a photo',
+    '{} is holding a baby in his arms',
+    'In a laboratory, {} is conducting an experiment',
+    '{} is holding a guitar',
+    'In a swimming pool, {} is swimming',
+    'In a cafe, {} is drinking tea',
+    'In a garden, {} is watering plants',
+    '{} is sitting on a bench in a park',
+    'In a classroom, {} is writing on a whiteboard',
+    '{} is holding a pen and writing on a paper',
+    '{} is standing in front of a building',
+    'In a museum, {} is looking at an exhibit',
+    'In a theater, {} is watching a movie',
+    '{} is standing in front of a car',
+    '{} is standing in front of a tree',
+    'In a meeting room, {} is giving a presentation',
+    'In a stadium, {} is watching a game',
+    'In a garage, {} is fixing a car',
+    '{} is holding a paintbrush and painting a picture',
+    'In a classroom, {} is listening to a lecture',
+    '{} is standing in front of a mountain',
+    'In a park, {} is flying a kite',
+    'a rendering of a {}',
+    'a cropped photo of the {}',
+    'the photo of a {}',
+    'a photo of a clean {}',
+    'a photo of a dirty {}',
+    'a dark photo of the {}',
+    'a photo of my {}',
+    'a photo of the cool {}',
+    'a close-up photo of a {}',
+    'a bright photo of the {}',
+    'a cropped photo of a {}',
+    'a photo of the {}',
+    'a good photo of the {}',
+    'a photo of one {}',
+    'a close-up photo of the {}',
+    'a rendition of the {}',
+    'a photo of the clean {}',
+    'a rendition of a {}',
+    'a photo of a nice {}',
+    'a good photo of a {}',
+    'a photo of the nice {}',
+    'a photo of the small {}',
+    'a photo of the weird {}',
+    'a photo of the large {}',
+    'a photo of a cool {}',
+    'a photo of a small {}',
+    'an illustration of a {}',
+    'a rendering of a {}',
+    'a cropped photo of the {}',
+    'the photo of a {}',
+    'an illustration of a clean {}',
+    'an illustration of a dirty {}',
+    'a dark photo of the {}',
+    'an illustration of my {}',
+    'an illustration of the cool {}',
+    'a close-up photo of a {}',
+    'a bright photo of the {}',
+    'a cropped photo of a {}',
+    'an illustration of the {}',
+    'a good photo of the {}',
+    'an illustration of one {}',
+    'a close-up photo of the {}',
+    'a rendition of the {}',
+    'an illustration of the clean {}',
+    'a rendition of a {}',
+    'an illustration of a nice {}',
+    'a good photo of a {}',
+    'an illustration of the nice {}',
+    'an illustration of the small {}',
+    'an illustration of the weird {}',
+    'an illustration of the large {}',
+    'an illustration of a cool {}',
+    'an illustration of a small {}',
+    'a depiction of a {}',
+    'a rendering of a {}',
+    'a cropped photo of the {}',
+    'the photo of a {}',
+    'a depiction of a clean {}',
+    'a depiction of a dirty {}',
+    'a dark photo of the {}',
+    'a depiction of my {}',
+    'a depiction of the cool {}',
+    'a close-up photo of a {}',
+    'a bright photo of the {}',
+    'a cropped photo of a {}',
+    'a depiction of the {}',
+    'a good photo of the {}',
+    'a depiction of one {}',
+    'a close-up photo of the {}',
+    'a rendition of the {}',
+    'a depiction of the clean {}',
+    'a rendition of a {}',
+    'a depiction of a nice {}',
+    'a good photo of a {}',
+    'a depiction of the nice {}',
+    'a depiction of the small {}',
+    'a depiction of the weird {}',
+    'a depiction of the large {}',
+    'a depiction of a cool {}',
+    'a depiction of a small {}',
+    '{} reads a newspaper in a cozy coffee shop.',
+    '{} jogs along a winding trail at sunrise.',
+    '{} takes a photograph of the bustling cityscape.',
+    '{} bakes cookies in a warm, inviting kitchen.',
+    '{} paints a mural on a large outdoor wall.',
+    '{} plants a tree in a sunny backyard.',
+    '{} repairs an old bicycle in the garage.',
+    '{} sketches a portrait with charcoal.',
+    '{} dances freely at a lively festival.',
+    '{} sows seeds in a flourishing vegetable garden.',
+    '{} plays a violin in a quiet room.',
+    '{} writes a poem under the shade of an oak tree.',
+    '{} feeds ducks at a peaceful lake.',
+    '{} practices yoga on a tranquil beach at dawn.',
+    '{} repairs a watch with skilled hands.',
+    '{} constructs a model airplane with precision.',
+    '{} decorates a cake with elaborate icing designs.',
+    '{} climbs a rock wall with determination.',
+    '{} meditates in a serene temple garden.',
+    '{} knits a colorful scarf by the fireside.',
+    '{} assembles a puzzle on a rainy afternoon.',
+    '{} examines artifacts at a history museum.',
+    '{} tends to a beehive in protective gear.',
+    '{} composes a new song on a keyboard.',
+    '{} stretches before starting a marathon.',
+    '{} recites lines for an upcoming play.',
+    '{} harvests apples in an orchard.',
+    '{} leads a tour group through ancient ruins.',
+    '{} creates a scrapbook filled with memories.',
+    '{} tutors a student in mathematics.',
+    '{} tries a new recipe from a gourmet cookbook.',
+    '{} rides a horse through an open field.',
+    '{} collects samples on a nature walk.',
+    '{} solves a complex mathematical equation.',
+    '{} fills the room with the sound of saxophone music.',
+    '{} arranges flowers in a beautiful bouquet.',
+    '{} hosts a podcast interview.',
+    '{} dives into a crystal-clear swimming pool.',
+    '{} studies a map before an expedition.',
+    '{} makes ceramic pottery on a spinning wheel.',
+    '{} trains a puppy to sit and stay.',
+    '{} rehearses for a ballet performance.',
+    '{} sails a boat across a calm lake.',
+    '{} browses through a second-hand bookstore.',
+    '{} explores a cave with a flashlight.',
+    '{} restores an old car in their home workshop.',
+    '{} conducts an orchestra with passion.',
+    '{} volunteers at a community food bank.',
+    '{} compiles a report in their office.',
+    '{} designs a website on their computer.',
+    '{} teaches a child to ride a bike.',
+    '{} performs a magic trick at a party.',
+    '{} packs a suitcase for a journey.',
+    '{} prunes roses in a sunlit garden.',
+    '{} crafts handmade jewelry from silver and gems.',
+    '{} inspects products for quality in a factory.',
+    '{} sculpts a figure from a block of marble.',
+    '{} organizes a community cleanup day.',
+    '{} swings a golf club on a green fairway.',
+    '{} develops photos in a darkroom.',
+    '{} directs a small indie film.',
+    '{} carves a wooden figure with intricate detail.',
+    '{} birdwatches with binoculars in the forest.',
+    '{} pilots a hot air balloon at dawn.',
+    '{} tutors peers in a university library.',
+    '{} rides a skateboard down a city street.',
+    '{} decorates a storefront for the holidays.',
+    '{} mixes cocktails at a busy bar.',
+    '{} cuts hair in a stylish salon.',
+    '{} researches genealogy to fill out a family tree.',
+    '{} writes calligraphy with elegant strokes.',
+    '{} edits a manuscript for publication.',
+    '{} lectures on environmental science.',
+    '{} designs a new board game.',
+    '{} hosts a charity auction.',
+    '{} laces up skates for an ice-skating lesson.',
+    '{} coordinates a wedding at a picturesque venue.',
+    '{} builds a sandcastle on the beach.',
+    '{} programs a robot for a competition.',
+    '{} captures wildlife photography in the jungle.',
+    '{} sets up a tent under the stars.',
+    '{} debuts a fashion collection on the runway.',
+    '{} curates pieces for an art exhibition.',
+    '{} conducts a science experiment in the laboratory.',
+    '{} provides a walking tour of a historic city.',
+    '{} partakes in a coffee cupping session.',
+    '{} negotiates a deal in a boardroom.',
+    '{} operates a forklift in a warehouse.',
+    '{} leads a yoga retreat in a mountain setting.',
+    '{} analyzes data on multiple computer screens.',
+    '{} paints a picket fence on a sunny day.',
+    '{} trains for gymnastics at the gym.',
+    '{} teaches a pottery class, guiding students.',
+    '{} cares for animals at a wildlife sanctuary.',
+    '{} crafts origami creations from colorful paper.',
+    '{} deejays a lively dance party.',
+    '{} writes code for a new software application.',
+    '{} grows an array of herbs in a window garden.',
+    '{} instructs a spin class with high energy.',
+    '{} navigates rapids in a whitewater raft.',
+    'Quietly, {} sets the table for dinner.',
+    'Suddenly, {} stops to pick up a fallen object.',
+    'Calmly, {} navigates through the crowd.',
+    'Gently, {} soothes a crying child.',
+    'Quickly, {} dashes out in the rain.',
+    'Joyfully, {} embraces a long-lost friend.',
+    'Firmly, {} stands their ground in debate.',
+    'Loudly, {} cheers on their favorite team.',
+    'Patiently, {} waits for their turn.',
+    'Anxiously, {} fidgets during an interview.',
+    'Easily, {} solves a complex puzzle.',
+    'Sadly, {} waves farewell at the departure gates.',
+    'Meticulously, {} organizes their collection.',
+    'Slyly, {} sneaks a cookie from the jar.',
+    'Defiantly, {} marches for change.',
+    'Warmly, {} greets newcomers.',
+    'Hungrily, {} eyes the banquet table.',
+    'Enthusiastically, {} starts their first day of work.',
+    'Stealthily, {} moves in the game of hide and seek.',
+    'Expertly, {} navigates the rapid waters.',
+    'Seamlessly, {} transitions between tasks.',
+    'Vigorously, {} cleans the cluttered garage.',
+    'Devotedly, {} tends to their garden.',
+    'Silently, {} admires the sunrise.',
+    'Righteously, {} advocates for justice.',
+    'Keenly, {} observes the wildlife.',
+    'Desperately, {} searches for their lost item.',
+    'Reverently, {} visits a historic monument.',
+    'Wistfully, {} looks back on fond memories.',
+    'Ambitiously, {} sets their career goals.',
+    'Rapidly, {} types up an urgent report.',
+    'Generously, {} shares their lunch.',
+    'Skillfully, {} crafts a beautiful piece of pottery.',
+    'Cautiously, {} approaches the unfamiliar dog.',
+    'Inquisitively, {} examines the ancient artifact.',
+    'Effortlessly, {} completes the challenging workout.',
+    'Frantically, {} looks for the exit.',
+    'Discreetly, {} passes a note during class.',
+    'Pensively, {} contemplates their next move.',
+    'Optimistically, {} plans for the future.',
+    'Sorrowfully, {} attends a memorial service.',
+    'Methodically, {} assembles the model airplane.',
+    'Lazily, {} lounges on the hammock.',
+    'Unsuccessfully, {} tries to hail a taxi.',
+    'Faithfully, {} follows the recipe.',
+    'Dramatically, {} reacts to the plot twist.',
+    'Adventurously, {} explores the dense forest.',
+    'Gracefully, {} accepts the award.',
+    'Theatrically, {} recites lines on stage.',
+    'Ardently, {} defends their thesis.',
+    'Abstractedly, {} doodles during the meeting.',
+    'Vivaciously, {} engages in the lively party.',
+    'Stoically, {} endures the challenging ordeal.',
+    'Resolutely, {} decides to change their life.',
+    'Triumphantly, {} crosses the finish line.',
+    'Suspiciously, {} glances over their shoulder.',
+    'Fervently, {} prays for good news.',
+    'Ruefully, {} acknowledges their mistake.',
+    'Industriously, {} works on the project till dusk.',
+    'Compassionately, {} comforts a stranger.',
+    'Sheepishly, {} admits they forgot the appointment.',
+    'Irately, {} disputes the incorrect charge.',
+    'Protectively, {} shields the puppy from the rain.',
+    'Serenely, {} meditates in the morning light.',
+    'Comically, {} slips on the banana peel.',
+    'Impressively, {} juggles multiple objects with ease.',
+    'Apprehensively, {} approaches the spooky house.',
+    'Unwaveringly, {} supports their friend.',
+    'Blissfully, {} soaks in the hot spring.',
+    'Compulsively, {} checks their notifications.',
+    'Tactfully, {} navigates the awkward situation.',
+    'Convincingly, {} sells their innovative idea.',
+    'Dutifully, {} fulfills their obligations.',
+    'Ingeniously, {} solves the critical problem.',
+    'Haphazardly, {} packs their suitcase.',
+    'Deftly, {} maneuvers the playing pieces.',
+    'Intriguedly, {} listens to the mysterious tale.',
+    'Ceremoniously, {} unveils the new sculpture.',
+    'Sterily, {} organizes the lab equipment.',
+    'Unintentionally, {} overhears a private conversation.',
+    'Forever, {} holds dear the cherished memories.',
+    'Nostalgically, {} revisits their old neighborhood.',
+    'Predictably, {} always laughs at the same joke.',
+    'Politely, {} inquires about the meeting agenda.',
+    'Securely, {} fastens their seatbelt before takeoff.',
+    'Casually, {} strolls through the park.',
+    'Spontaneously, {} decides to go on a road trip.',
+    'Clearly, {} expresses their feelings.',
+    'Merrily, {} decorates for the festive season.',
+    'Valiantly, {} stands up against injustice.',
+    'Diligently, {} studies for the upcoming exam.',
+    'Nonchalantly, {} brushes off the slight mishap.',
+    'Intensely, {} focuses on the target.',
+    'Subtly, {} hints at the surprise party.',
+    'Mysteriously, {} vanishes into the foggy night.',
+    'Decisively, {} makes their final choice.',
+    'Lovingly, {} prepares a home-cooked meal.',
+    'Immaculately, {} arranges the storefront display.',
+    'Vibrantly, {} adds color to the canvas.',
+    'The silhouette of {} casts a long shadow.',
+    'Through the fog, {} emerges slowly.',
+    'Over the hill, {} rides a bicycle.',
+    'After the storm, {} surveys the damage.',
+    'Around the bend, {} sails a boat.',
+    'Under the tree, {} reads a book.',
+    'Beside the fire, {} warms their hands.',
+    'Below the surface, {} discovers coral reefs.',
+    'Beyond the fence, {} tends to horses.',
+    'Above the crowd, {} waves a flag.',
+    'Among the flowers, {} finds peace.',
+    'Across the field, {} flies a kite.',
+    'Near the water, {} sketches the view.',
+    'By the road, {} waits patiently.',
+    'With careful precision, {} repairs a clock.',
+    'In the spotlight, {} performs a solo.',
+    'To the beat, {} dances joyfully.',
+    'On the stage, {} delivers a monologue.',
+    'Underneath the stars, {} makes a wish.',
+    'Beside the window, {} sips morning tea.',
+    'At the corner, {} hails a cab.',
+    'Against the odds, {} triumphs victoriously.',
+    'Beneath the waves, {} finds tranquility.',
+    'Before the race, {} stretches carefully.',
+    'Through the lens, {} captures the moment.',
+    'From the bridge, {} observes the river.',
+    'Since the dawn, {} has been fishing.',
+    'Like a statue, {} stands immovable.',
+    'Inside the house, {} feels safe.',
+    'The smile of {} brightens the room.',
+    'Upon the mountaintop, {} feels awe.',
+    'Without a doubt, {} commits to the goal.',
+    'Reflecting on life, {} contemplates deeply.',
+    'Amidst the chaos, {} remains calm.',
+    'Throughout the day, {} maintains focus.',
+    'During the performance, {} takes the stage.',
+    'Considering all options, {} makes a choice.',
+    'Alongside the path, {} picks wildflowers.',
+    'Toward the horizon, {} gazes expectantly.',
+    'Wrapped in thought, {} ponders life’s mysteries.',
+    'Accompanied by music, {} feels uplifted.',
+    'Surrounded by books, {} indulges in knowledge.',
+    'Guided by intuition, {} chooses a path.',
+    'Entertaining guests, {} tells a tale.',
+    'Admiring the artwork, {} gains inspiration.',
+    'Standing at the crossroads, {} hesitates slightly.',
+    'Lost in music, {} enjoys the concert.',
+    'Besieged by deadlines, {} works diligently.',
+    'Empowered by support, {} achieves greatness.',
+    'Gazing into space, {} dreams of stars.',
+    'Facing the challenge, {} exudes confidence.',
+    'Approaching the podium, {} clears their throat.',
+    'Enclosed in glass, {} admires the terrarium.',
+    'The reflection of {} shimmers on water.',
+    'Clutching the ticket, {} rushes to the gate.',
+    'Heeding the warning, {} takes precaution.',
+    'Observing the traditions, {} learns respect.',
+    'At the museum, {} admires ancient artifacts.',
+    'Following the recipe, {} bakes a cake.',
+    'Adjusting the telescope, {} explores the heavens.',
+    'In the garden, {} relaxes with nature.',
+    'Clinging to hope, {} perseveres through trials.',
+    'The laughter of {} fills the room.',
+    'During the lecture, {} takes diligent notes.',
+    'Sitting by the piano, {} composes a melody.',
+    'The hands of {} shape the clay.',
+    'The courage of {} inspires many others.',
+    'Laid on the canvas, {} begins to paint.',
+    'Carried by wind, {}’s kite ascends higher.',
+    'In the workshop, {} builds a dream.',
+    'Mingled with others, {} shares a story.',
+    'Learning the ropes, {} adapts quickly.',
+    'Fuelled by passion, {} pursues their dreams.',
+    'In the office, {} meets a deadline.',
+    'With each stride, {} closes the distance.',
+    'Mastering the craft, {} excels in their art.',
+    'The vision of {} leads to success.',
+    'Striving for wellness, {} embraces a change.',
+    'Buffeted by wind, {} adjusts their hat.',
+    'Engulfed in aroma, {} enjoys the spices.',
+    'Surrounded by laughter, {} feels joy.',
+    'Avoiding the puddle, {} steps carefully.',
+    'Reacting quickly, {} catches the falling vase.',
+    'Marked by time, {}’s diary tells tales.',
+    'Supported by friends, {} overcomes fear.',
+    'Puzzled by clues, {} solves the riddle.',
+    'Driving through night, {} reaches their destination.',
+    'Splashed by waves, {} laughs heartily.',
+    'Confronted with choices, {} deliberates wisely.',
+    'Hidden by shadows, {} watches the scene.',
+    'Inspired by nature, {} writes poetry.',
+    'Guarded by mystery, {}’s past intrigues.',
+    'Detouring the path, {} discovers new sights.',
+    'Greeted by dawn, {} feels renewed.',
+    'Warmed by sunlight, {} enjoys the afternoon.',
+    'Answering the call, {} takes action.',
+    'Sheltered by canopy, {} escapes the rain.',
+    'Bound by duty, {} fulfills their role.',
+    'Pulled by curiosity, {} enters the store.',
+    'Motivated by change, {} advocates for causes.',
+    'In silence, {} stares into space.',
+    'Lost in thought, {} stands still.',
+    'With excitement, {} opens a gift.',
+    'Amid laughter, {} shares a joke.',
+    'Surrounded by nature, {} takes a deep breath.',
+    'Under the sun, {} stretches out.',
+    'Against a backdrop of mountains, {} gazes afar.',
+    'Among friends, {} enjoys a conversation.',
+    'Before dinner, {} sets the table.',
+    'Behind the counter, {} makes coffee.',
+    'Below the surface, {} snorkels in clear water.',
+    'Beneath the stars, {} lights a campfire.',
+    'Beside a bicycle, {} takes a break.',
+    'By the seaside, {} collects seashells.',
+    'Near the horizon, {} sketches the view.',
+    'On the bridge, {} watches the water flow.',
+    'Through the window, {} waves goodbye.',
+    'To the music, {} taps their feet.',
+    'With a book, {} finds escape.',
+    'Without a care, {} listens to music.',
+    'Around the table, {} shares a story.',
+    'Outside the house, {} does some gardening.',
+    'From the stage, {} delivers a speech.',
+    'After the rain, {} jumps in puddles.',
+    'During the party, {} blows up balloons.',
+    'Following the path, {} takes a stroll.',
+    'Along the river, {} is fishing.',
+    'Inside the room, {} practices yoga.',
+    'Throughout the day, {} takes photos.',
+    'Across the field, {} flies a kite.',
+    'Between the lines, {} reads quietly.',
+    'Behind the lens, {} captures the moment.',
+    'Along the alley, {} walks their dog.',
+    'Before the sunrise, {} enjoys the calm.',
+    'Over the fence, {} talks to a neighbor.',
+    'Under the tree, {} has a picnic.',
+    'Beyond the gate, {} starts their journey.',
+    'Around the fire, {} tells ghost stories.',
+    'Above the clouds, {} skydives.',
+    'Among the crowd, {} cheers loudly.',
+    'Near the pond, {} feeds the ducks.',
+    'On the couch, {} takes a nap.',
+    'Before the show, {} checks their ticket.',
+    'Under the sky, {} flies a drone.',
+    'Behind the wheel, {} sings loudly.',
+    'Above the waves, {} surfs with skill.',
+    'Within the walls, {} paints their dream.',
+    'Beyond the road, {} hikes up the hill.',
+    'Beneath the quilt, {} reads at night.',
+    'Against the odds, {} tries a new trick.',
+    'During the trip, {} savors local cuisine.',
+    'Amid the shelves, {} finds an old book.',
+    'Across the room, {} waves to a friend.',
+    'By the pool, {} basks in the sun.',
+    'Beneath the lights, {} takes center stage.',
+    'Above the city, {} marvels at the view.',
+    'Behind the scenes, {} prepares diligently.',
+    'Over the moon, {} celebrates good news.',
+    'Under the arch, {} takes memorable photos.',
+    'Before the dawn, {} prepares for the day.',
+    'Throughout the match, {} cheers enthusiastically.',
+    'Between workouts, {} hydrates and rests.',
+    'Around the campfire, {} roasts marshmallows.',
+    'By the window, {} enjoys the morning light.',
+    'After the lecture, {} asks thoughtful questions.',
+    'Within the garden, {} admires the flowers.',
+    'Beneath the blanket, {} watches a movie.',
+    'Beyond the wall, {} hears echoes of laughter.',
+    'Behind the book, {} hides a surprise gift.',
+    'Under the bridge, {} sketches the river scene.',
+    'During the concert, {} loses themselves in the music.',
+    'On the terrace, {} sips on iced tea.',
+    'Before the alarm, {} wakes up naturally.',
+    'Above the rooftops, {} spots a passing balloon.',
+    'Across the street, {} helps an elderly neighbor.',
+    'Beside the lamp, {} finishes their novel.',
+    'With the crowd, {} dances to the festival music.',
+    'By the lakeside, {} sets up a fishing rod.',
+    'Before the exercise, {} stretches thoroughly.',
+    'Near the finish line, {} sprints with determination.',
+    'On the balcony, {} tends to potted plants.',
+    'After the storm, {} clears the fallen branches.',
+    'Under the covers, {} snoozes the alarm clock.',
+    'Between the curtains, {} peeks at the sunrise.',
+    'Around the corner, {} discovers a quaint café.',
+    'By the artwork, {} contemplates the message of the painter.',
+    'After the game, {} congratulates the players.',
+    'Within the studio, {} edits a documentary film.',
+    'Beneath the hat, {} grins at a private joke.',
+    'Beyond the dunes, {} takes in the beach view.',
+    'Behind the microphone, {} records a podcast.',
+    'Under the eaves, {} shelters from the rain.',
+    'During the hike, {} spots a rare bird.',
+    'On the platform, {} awaits the next train.',
+    'Before the meal, {} gives thanks.',
+    'Above the fray, {} keeps a level head.',
+    'Across the canvas, {} strokes colors with a brush.',
+    'Beside the hearth, {} warms their hands.',
+    'With affection, {} pets their sleepy cat.',
+    'By the harbor, {} watches boats come and go.',
+    'In a room, {} reads quietly.',
+    'Near the shore, {} fishes calmly.',
+    'Behind the counter, {} smiles warmly.',
+    'Among the trees, {} jogs daily.',
+    'On the bench, {} sits silently.',
+    'With a pen, {} writes diligently.',
+    'At dawn, {} stretches readily.',
+    'Under the stars, {} dreams peacefully.',
+    'With the dog, {} walks leisurely.',
+    'Against the backdrop, {} stands proudly.',
+    'On stage, {} speaks clearly.',
+    'In the garden, {} works happily.',
+    'At the table, {} eats slowly.',
+    'Beside the window, {} gazes thoughtfully.',
+    'Within the crowd, {} laughs loudly.',
+    'By the painting, {} ponders intently.',
+    'On the bridge, {} pauses reflectively.',
+    'Under the umbrella, {} waits patiently.',
+    'Before the game, {} practices routinely.',
+    'Behind the lens, {} captures moments.',
+    'In the cafe, {} sips coffee.',
+    'With a map, {} explores curiously.',
+    'On the couch, {} naps briefly.',
+    'At the wheel, {} drives safely.',
+    'Beside the fire, {} warms up.',
+    'During the concert, {} claps excitedly.',
+    'By the bookshelf, {} selects a novel.',
+    'On the path, {} bikes steadily.',
+    'Under the quilt, {} snoozes comfortably.',
+    'Before the screen, {} types consistently.',
+    'Within the room, {} dances joyfully.',
+    'At the market, {} shops carefully.',
+    'Beside the pool, {} sunbathes lazily.',
+    'On the road, {} hitches northward.',
+    'Against the clock, {} races swiftly.',
+    'By the door, {} knocks promptly.',
+    'In the silence, {} meditates profoundly.',
+    'With a brush, {} paints a canvas.',
+    'On a horse, {} rides boldly.',
+    'At the concert, {} listens attentively.',
+    'Beside the lamp, {} reads a letter.',
+    'On the field, {} throws a ball.',
+    'Under the sun, {} basks leisurely.',
+    'Before the microphone, {} sings softly.',
+    'Within the frame, {} looks stern.',
+    'In the studio, {} records a podcast.',
+    'By the seaside, {} collects shells.',
+    'On the mattress, {} lies awake.',
+    'Behind the bar, {} mixes drinks.',
+    'During the meeting, {} takes notes.',
+    'At the podium, {} delivers a speech.',
+    'Beside the pond, {} feeds ducks.',
+    'On the swing, {} rocks gently.',
+    'Under the sky, {} dreams freely.',
+    'Before the class, {} sets up.',
+    'Within the pages, {} finds adventure.',
+    'At the corner, {} waves hello.',
+    'By the stove, {} cooks breakfast.',
+    'On the terrace, {} breathes deeply.',
+    'Against the wall, {} rests momentarily.',
+    'In the lineup, {} waits calmly.',
+    'With a joystick, {} plays a game.',
+    'On the floor, {} stretches out.',
+    'At the crossroads, {} chooses a path.',
+    'Beside the bag, {} finds keys.',
+    'On the track, {} runs laps.',
+    'Under the tree, {} enjoys shade.',
+    'Before the journey, {} packs essentials.',
+    'Within the box, {} discovers treasures.',
+    'In the mirror, {} sees reflection.',
+    'By the lake, {} skips stones.',
+    'On the steps, {} sits waiting.',
+    'Against the flow, {} stands firm.',
+    'Before the event, {} feels nervous.',
+    'Within the heart, {} holds love.',
+    'At the keyboard, {} composes music.',
+    'By the fence, {} watches sunset.',
+    'On the ledge, {} takes in views.',
+    'Under the moon, {} makes wishes.',
+    'Before the crowd, {} shows courage.',
+    'Within the house, {} calls family.',
+    'At the desk, {} solves puzzles.',
+    'Beside the car, {} checks tires.',
+    'On the peak, {} celebrates triumph.',
+    'Against the odds, {} perseveres always.',
+    'In the foyer, {} welcomes guests.',
+    'With the team, {} collaborates effectively.',
+    'On the grass, {} rolls playfully.',
+    'At the junction, {} signals left.',
+    'Beside the easel, {} studies the painting.',
+    'On the quilt, {} patches holes.',
+    'Under the coat, {} hides a gift.',
+    'Before the dawn, {} dreams of success.',
+    'Within the shadows, {} moves silently.',
+    'At the beach, {} builds castles.',
+    'By the gate, {} waits anxiously.',
+    'On the island, {} finds peace.',
+    'Against the breeze, {} flies a kite.',
+    'Before the altar, {} takes a vow.',
+    'Within the orchestra, {} tunes their instrument.',
+    'An exciting magic trick is being performed by {}.',
+    'A quiet library is being enjoyed by {}.',
+    'A delicious meal is being cooked in the kitchen by {}.',
+    'A challenging rock wall is being climbed by {}.',
+    'A fast-paced basketball game is being played by {}.',
+    'A beautiful melody is being played on a violin by {}.',
+    'A serene lake is being fished by {}.',
+    'An intense workout is being completed in the gym by {}.',
+    'A mysterious book is being read under the tree by {}.',
+    'A spirited dance is being performed on stage by {}.',
+    'A serene afternoon picnic is being enjoyed by {}.',
+    'A thrilling skateboarding trick is being attempted by {}.',
+    'An intricate jigsaw puzzle is being solved by {}.',
+    'A high note is being sung in a rehearsal room by {}.',
+    'A new recipe is being tried out in the kitchen by {}.',
+    'A bookshelf is being organized in the study by {}.',
+    'A large canvas is being painted with bold colors by {}.',
+    'An ancient ruin is being carefully explored by {}.',
+    'A lengthy novel is being written at the desk by {}.',
+    'A pottery wheel is being used to shape clay by {}.',
+    'A soft melody is being played on a guitar by {}.',
+    'A new language is being learned with enthusiasm by {}.',
+    'An early morning jog is being taken along the beach by {}.',
+    'A handmade quilt is being stitched with care by {}.',
+    'A tropical fruit stand is being set up at the market by {}.',
+    'A hot beverage is being brewed in a cozy cafe by {}.',
+    'A winter bonfire is being lit to warm up the night by {}.',
+    'A peaceful kayak trip is being embarked upon by {}.',
+    'Bold graffiti is being sprayed on an urban wall by {}.',
+    'A lively story is being told around the campfire by {}.',
+    'A crafty sculpture is being created from recycled materials by {}.',
+    'A vibrant mural is being painted on a downtown alley by {}.',
+    'A dusty trail is being hiked at dawn by {}.',
+    'A tricky crossword puzzle is being filled out by {}.',
+    'A homemade pie is being baked for a special occasion by {}.',
+    'An elaborate garden is being tended to by {}.',
+    'A suspenseful movie is being watched with excitement by {}.',
+    'A difficult yoga pose is being mastered in the studio by {}.',
+    'A new skateboard is being ridden down a hill by {}.',
+    'A savory soup is being stirred in a pot by {}.',
+    'Cheerful holiday decorations are being hung around the house by {}.',
+    'A thrilling novel is being devoured on a rainy afternoon by {}.',
+    'A chess game is being thoughtfully played in the park by {}.',
+    'A burst of laughter is being shared with friends by {}.',
+    'Bright city lights are being admired from a rooftop by {}.',
+    'An old family recipe is being followed in the kitchen by {}.',
+    'A marshmallow is being roasted over a campfire by {}.',
+    'Careful brush strokes are being applied to a model figurine by {}.',
+    'A challenging video game is being played with focus by {}.',
+    'An evening class is being attended with interest by {}.',
+    'A delicate pastry is being decorated with icing by {}.',
+    'An excited puppy is being trained in the backyard by {}.',
+    'A basketball is being shot into a hoop by {}.',
+    'A lively drumbeat is being played at a concert by {}.',
+    'Colorful fall leaves are being photographed in the woods by {}.',
+    'A new song is being composed on the piano by {}.',
+    'A long-lost friend is being hugged in a warm embrace by {}.',
+    'Bright fireworks are being watched in awe by {}.',
+    'A favorite TV show is being binge-watched by {}.',
+    'A new trail is being biked through the forest by {}.',
+    'Freshly baked cookies are being taken out of the oven by {}.',
+    'A difficult problem is being solved with satisfaction by {}.',
+    'Colorful balloons are being blown up for a party by {}.',
+    'A joyful tune is being whistled while walking by {}.',
+    'An old film camera is being loaded with film by {}.',
+    'An empty canvas is being gazed upon before painting by {}.',
+    'An exciting soccer match is being watched with friends by {}.',
+    'A warm cup of tea is being sipped quietly by {}.',
+    'A good book is being enjoyed in a comfy armchair by {}.',
+    'A gentle horse is being groomed in the stable by {}.',
+    'A tense board game is being strategized over by {}.',
+    'Fresh laundry is being folded neatly by {}.',
+    'A thrilling roller coaster ride is being braved by {}.',
+    'A favorite song is being sung in the shower by {}.',
+    'A rainy day is being spent baking cookies by {}.',
+    'Classic tunes are being listened to on vinyl by {}.',
+    'An interesting documentary is being watched intently by {}.',
+    'A busy day is being relaxed from with a bubble bath by {}.',
+    'A sunflower field is being walked through by {}.',
+    'A new plant is being potted with care by {}.',
+    'A sunny terrace is being enjoyed with a cold drink by {}.',
+    'Morning birds are being listened to at dawn by {}.',
+    'A quiet museum hall is being wandered through by {}.',
+    'An experimental recipe is being tested in the kitchen by {}.',
+    'A homemade kite is being flown on a breezy day by {}.',
+    'A colorful aquarium is being cleaned by {}.',
+    'A new blog post is being composed on a laptop by {}.',
+    'A wild trail is being trekked with enthusiasm by {}.',
+    'An ice cream cone is being savored on a warm day by {}.',
+    'A peaceful sunrise is being watched from a hilltop by {}.',
+    'Freshly ground coffee is being brewed in the morning by {}.',
+    'A comfortable hammock is being swayed in gently by {}.',
+    'A nostalgic video game is being revisited with joy by {}.',
+    'A challenging Sudoku puzzle is being completed by {}.',
+    'A dusty attic is being explored for treasures by {}.',
+    'A hefty stack of pancakes is being devoured for breakfast by {}.',
+    'Delicate origami is being folded by {}.',
+    'A peaceful moment is being cherished on a quiet porch by {}.',
+    'On a quiet street, {} is jogging.',
+    'With a gentle smile, {} offers help.',
+    'Behind the old bookstore, {} reads quietly.',
+    'Near a calm lake, {} sketches the scenery.',
+    'By the bright window, {} sips coffee.',
+    'Under the warm sun, {} relaxes.',
+    'Around the bustling square, {} dances.',
+    'Beside the campfire, {} tells stories.',
+    'Above the city noise, {} daydreams.',
+    'Through the crowded fair, {} navigates.',
+    'Against the evening sky, {} takes photos.',
+    'Among the tall trees, {} hikes.',
+    'Before the morning rush, {} stretches.',
+    'Amid the garden blooms, {} seeks peace.',
+    'Across the open field, {} runs freely.',
+    'During the lively party, {} laughs.',
+    'Following the winding path, {} explores.',
+    'Outside the cozy cottage, {} gazes at stars.',
+    'Within the silent walls, {} contemplates.',
+    'Beneath the ancient arch, {} pauses reflectively.',
+    'Along the riverbank, {} fishes.',
+    'Beside a bubbling brook, {} writes poetry.',
+    'Underneath the vibrant mural, {} admires art.',
+    'Beyond the bustling streets, {} finds quiet.',
+    'Behind the heavy curtain, {} rehearses lines.',
+    'Upon the windswept hill, {} flies a kite.',
+    'Throughout the sunny day, {} tends the shop.',
+    'Despite the hectic pace, {} stays calm.',
+    'Behind the lens of a camera, {} captures moments.',
+    'Inside the warm bakery, {} savors aromas.',
+    'Beneath the star-filled sky, {} makes a wish.',
+    'Beyond the garden gate, {} enters serenity.',
+    'Between the bookshelves, {} finds adventure.',
+    'Across the dance floor, {} moves gracefully.',
+    'Around the festive decorations, {} feels joy.',
+    'Amidst the quiet sanctuary, {} prays.',
+    'Near the bustling café, {} watches the world.',
+    'Under the shade of a tree, {} enjoys a picnic.',
+    'By the glow of the fireplace, {} reads.',
+    'After the long journey, {} rests.',
+    'Outside the lively market, {} samples flavors.',
+    'Upon the old wooden bench, {} sits.',
+    'Around the warm campfire, {} sings.',
+    'Through the busy terminal, {} travels.',
+    'Within the walls of home, {} feels safe.',
+    'Beside the flowing river, {} reflects.',
+    'Against the cool breeze, {} wraps up warm.',
+    'Across the silent library, {} seeks knowledge.',
+    'Beneath the towering cliff, {} gazes up.',
+    'Beyond the colorful horizon, {} dreams.',
+    'Between the office cubicles, {} takes a breath.',
+    'Behind the vibrant easel, {} paints.',
+    'Upon the peaceful shore, {} collects shells.',
+    'Throughout the old village, {} discovers history.',
+    'Despite the falling rain, {} smiles.',
+    'Inside the bustling diner, {} enjoys breakfast.',
+    'By the edge of the fountain, {} tosses a coin.',
+    'Outside the charming bookstore, {} chooses a novel.',
+    'Upon the rooftop terrace, {} views the skyline.',
+    'Through the frosty window, {} longs for spring.',
+    'Within the hushed auditorium, {} listens intently.',
+    'Beside the crackling bonfire, {} cozies up.',
+    'Against the morning chill, {} jogs.',
+    'Across the golden meadow, {} strolls.',
+    'Amidst the echo of laughter, {} joins in.',
+    'Beyond the realm of the city, {} seeks nature.',
+    'Between the lush vines, {} harvests grapes.',
+    'Behind the frosted glass, {} sips tea.',
+    'Upon the creaky floorboards, {} tip-toes.',
+    'Throughout the silent movie, {} is mesmerized.',
+    'Despite the room’s clutter, {} finds order.',
+    'Beneath the bright marquee, {} awaits the opening.',
+    'By the light of the lanterns, {} feels warmth.',
+    'After the rain has passed, {} splashes in puddles.',
+    'Outside the local theater, {} buys a ticket.',
+    'Upon the green expanse, {} practices yoga.',
+    'Through the historic district, {} admires architecture.',
+    'Within the quiet of dawn, {} takes a moment.',
+    'Beside the ice-covered pond, {} feeds the ducks.',
+    'Against the setting sun, {} cherishes the moment.',
+    'Across the crowded room, {} finds a friend.',
+    'Amidst the morning calm, {} sows seeds.',
+    'Beneath the overcast sky, {} contemplates change.',
+    'Beyond the busy crosswalk, {} finds solitude.',
+    'Between two towering pines, {} hangs a hammock.',
+    'Behind the cool shade, {} enjoys an ice cream.',
+    'Upon the deserted path, {} embraces stillness.',
+    'Throughout the lively tune, {} taps their foot.',
+    'Despite the distance apart, {} feels connected.',
+    'Inside the crowded bus, {} daydreams.',
+    'Beneath the vast universe, {} feels wonder.',
+    'By the vibrant mural, {} appreciates art.',
+    'After the final curtain call, {} feels inspired.',
+    'Outside the quaint café, {} inhales the fresh morning air.',
+    'Sitting calmly with a book in their lap is {}.',
+    'Holding the reins of a horse stands {}.',
+    'Laughing at a joke just heard is {}.',
+    'Taking a deep breath of fresh air on a hike is {}.',
+    'Reaching for an apple on a tree is {}.',
+    'Playing a violin with focused attention is {}.',
+    'Taking a photo of the sunset is {}.',
+    'Lying on the grass and looking at the clouds is {}.',
+    'Standing with an umbrella in the rain is {}.',
+    'Throwing a frisbee in the park is {}.',
+    'Riding a skateboard down the sidewalk is {}.',
+    'Juggling three balls skillfully is {}.',
+    'Swinging on a swing with a smile is {}.',
+    'Pulling a suitcase in an airport is {}.',
+    'Dipping a paintbrush into paint before a canvas is {}.',
+    'Stretching before a run along the track is {}.',
+    'Pouring a cup of coffee in the morning is {}.',
+    'Bouncing a basketball on the court is {}.',
+    'Holding an ice cream cone upside down is {}.',
+    'Standing at the podium about to speak is {}.',
+    'Waiting for a train at the station is {}.',
+    'Typing rapidly on a keyboard is {}.',
+    'Riding a bicycle along the river path is {}.',
+    'Blowing out candles on a birthday cake is {}.',
+    'Feeding ducks by the pond is {}.',
+    'Hiking with a backpack up a mountain trail is {}.',
+    'Lifting weights in the gym is {}.',
+    'Contemplating a piece of art in a gallery is {}.',
+    'Sipping a milkshake through a straw is {}.',
+    'Planting seedlings in a garden bed is {}.',
+    'Wading through a stream with a fishing pole is {}.',
+    'Assembling a model airplane with focus is {}.',
+    'Whipping up a smoothie in a blender is {}.',
+    'Rolling out dough for a pie is {}.',
+    'Peering through a telescope at night is {}.',
+    'Flying a kite in the open field is {}.',
+    'Playing chess and contemplating the next move is {}.',
+    'Brushing a horse in the stable is {}.',
+    'Sitting on the pier with feet dangling over water is {}.',
+    'Tuning a guitar before a performance is {}.',
+    'Practicing yoga in a peaceful room is {}.',
+    'Sculpting clay on a pottery wheel is {}.',
+    'Skimming a stone across a lake is {}.',
+    'Building a sandcastle at the beach is {}.',
+    'Fishing at the crack of dawn on a boat is {}.',
+    'Roasting marshmallows over a campfire is {}.',
+    'Watching the horizon from the deck of a ship is {}.',
+    'Admiring the view from the top of a ferris wheel is {}.',
+    'Reading a map under the streetlight is {}.',
+    'Twirling a pen thoughtfully while studying is {}.',
+    'Writing in a journal quietly is {}.',
+    'Inspecting a gadget with curiosity is {}.',
+    'Balancing on a slackline between two trees is {}.',
+    'Mixing ingredients for a recipe is {}.',
+    'Waiting patiently for the crosswalk signal is {}.',
+    'Riding an escalator up to the next floor is {}.',
+    'Sitting on a bench feeding pigeons is {}.',
+    'Standing at the edge of the diving board is {}.',
+    'Looking at merchandise in a shop window is {}.',
+    'Sitting on the floor wrapping gifts is {}.',
+    'Climbing up a ladder to reach a high shelf is {}.',
+    'Waiting for the bus at the bus stop is {}.',
+    'Sipping tea while gazing out the window is {}.',
+    'Swinging a tennis racquet on the court is {}.',
+    'Watching a movie with 3D glasses on is {}.',
+    'Carving a piece of wood into a sculpture is {}.',
+    'Hula hooping in the backyard is {}.',
+    'Rowing a boat down the river is {}.',
+    'Bending down to tie a shoelace is {}.',
+    'Playing the drums with enthusiasm is {}.',
+    'Waiting in line at the grocery store checkout is {}.',
+    'Blowing bubbles with gum is {}.',
+    'Sketching a landscape on a notepad is {}.',
+    'Jumping into a pile of autumn leaves is {}.',
+    'Standing with hands on hips after a workout is {}.',
+    'Conducting an orchestra with intensity is {}.',
+    'Leaning against a fence watching the sunrise is {}.',
+    'Tossing a salad in a bowl for dinner is {}.',
+    'Crossing a footbridge over a stream is {}.',
+    'Bobbing their head to music on headphones is {}.',
+    'Attaching a lock to a bridge railing as a symbol of love is {}.',
+    'Pumping air into a bicycle tire is {}.',
+    'Repairing a computer with various tools is {}.',
+    'Doodling in a notebook during a lecture is {}.',
+    'Lining up a shot with a camera is {}.',
+    'Kneading dough on a floured surface is {}.',
+    'Waving goodbye at the train station is {}.',
+    'Lying on the beach soaking up the sun is {}.',
+    'Reading street signs in an unfamiliar city is {}.',
+    'Casting a fishing line from the shore is {}.',
+    'Blowing on a dandelion with seeds dispersing is {}.',
+    'Dancing alone in the living room is {}.',
+    'Watching the stars with a blanket wrapped around is {}.',
+    'Peeling an orange in one long spiral is {}.',
+    'Picking flowers from a field is {}.',
+    'Studying a museum exhibit with interest is {}.',
+    'Hanging laundry out to dry on a sunny day is {}.',
+    'Cuddling a pet cat on the couch is {}.',
+    'Arranging books on a shelf by color is {}.',
+    'Standing silent in a moment of gratitude is {}.'
+]
+
+
+random.shuffle(imagenet_templates_small)
+
+per_img_token_list = ['*']
+
+
+class FaceIdDataset(Dataset):
+    def __init__(self, experiment_name, **kwargs):
+        super(FaceIdDataset, self).__init__()
+        
+        self.experiment_name = experiment_name
+        if self.experiment_name == "normal_GAN":
+            name_path = "datasets_face/good_names.txt"
+        elif self.experiment_name == "man_GAN":
+            name_path = "datasets_face/good_names_man.txt" 
+        elif self.experiment_name == "woman_GAN":            
+            name_path = "datasets_face/good_names_woman.txt"
+        else:
+            print("Hello, please notice this ^_^")
+            assert 0
+        print("now experiment_name:", self.experiment_name)
+        
+        with open(name_path, "r") as f:
+            self.names = f.read().splitlines()
+            
+        if self.experiment_name == "normal_GAN":
+            with open("datasets_face/good_names_man.txt", "r") as f_man, open("datasets_face/good_names_woman.txt", "r") as f_woman:
+                self.man_names = f_man.read().splitlines()
+                self.woman_names = f_woman.read().splitlines()  
+                 
+        self._length = len(self.names)
+
+
+
+    def __len__(self):
+        return self._length
+
+    def __getitem__(self, i):   
+        example = {}
+        
+        name = self.names[i]
+        
+        # if normal_GAN, this trick will be used for gender balance.
+        if self.experiment_name == "normal_GAN":
+            if random.random() < 0.5:
+                name = random.choice(self.man_names)
+            else:
+                name = random.choice(self.woman_names)      
+
+        ''' text '''
+        placeholder_string = per_img_token_list[0]
+        text = random.choice(imagenet_templates_small).format('%s person' % placeholder_string)
+
+        example["caption"] = text
+        example["name"] = name
+        
+        return example

datasets_face/good_names.txt

@@ -0,0 +1,326 @@
+Adam Savage
+Adam Scott
+Alan Alda
+Alan Hale
+Albert Brooks
+Alec Baldwin
+Alec Guinness
+Alice Cooper
+Alicia Alonso
+Amy Adams
+Amy Schumer
+Anderson Cooper
+Andrea Martin
+Andy Richter
+Angelina Jolie
+Ann Curry
+Ann Miller
+Anne Hathaway
+Anne Murray
+Aubrey Plaza
+Audrey Hepburn
+Aziz Ansari
+BD Wong
+Barbara Walters
+Ben Affleck
+Ben Kingsley
+Ben Miller
+Ben Schwartz
+Benedict Cumberbatch
+Bill Burr
+Bill Cosby
+Bill Irwin
+Bill Maher
+Bill Murray
+Bill Nye
+Billy Chow
+Billy Connolly
+Billy Crystal
+Billy Joel
+Billy Porter
+Billy Wilder
+Bob Hope
+Bob Marley
+Bonnie Hunt
+Brad Pitt
+Brandon Lee
+Brian Cox
+Brian Tee
+Britney Spears
+Bron James
+Bruce Springsteen
+Bruce Willis
+Bryan Cranston
+Buck Henry
+Burt Lancaster
+Burt Reynolds
+Cameron Diaz
+Carol Burnett
+Carol Channing
+Carol Kane
+Carrie Underwood
+Cary Grant
+Cate Blanchett
+Celia Cruz
+Celine Dion
+Charlie Sheen
+Cheryl Hines
+Chris Pratt
+Christina Hendricks
+Christopher Guest
+Cindy Williams
+Claire Danes
+Craig Ferguson
+Craig Robinson
+Cristiano Ronaldo
+Crystal Gayle
+Dan Harmon
+Dan Levy
+Dan Rather
+Dana Gould
+Daniel Radcliffe
+Danny Thomas
+Daryl Hall
+Dave Bautista
+Dave Matthews
+David Beckham
+David Bowie
+David Butler
+David Spade
+Dax Shepard
+Dean Martin
+Debra Messing
+Dennis Chan
+Dennis Franz
+Dennis Hopper
+Dennis Quaid
+Dev Patel
+Devon Aoki
+Diana Ross
+Diane Sawyer
+Dizzy Gillespie
+Donald Crisp
+Donald Glover
+Donna Reed
+Donnie Yen
+Doris Roberts
+Drew Barrymore
+Drew Carey
+Dudley Moore
+Dwayne Johnson
+Ed Sheeran
+Eddie Murphy
+Edgar Wright
+Edward Norton
+Elaine May
+Eleanor Powell
+Eli Roth
+Elizabeth Banks
+Ellen Pompeo
+Elon Musk
+Elton John
+Emma Thompson
+Eric Idle
+Ernie Reyes
+Floyd Mayweather
+Forest Whitaker
+Fred Savage
+Garry Marshall
+Gene Lockhart
+George Benson
+George Burns
+George Clooney
+George Lopez
+George Lucas
+George Marshall
+George Miller
+Gillian Murphy
+Ginger Rogers
+Gregory Hines
+Gregory Peck
+Halle Berry
+Harold Lloyd
+Harrison Ford
+Harry Carey
+Helen Mirren
+Helen Reddy
+Howard Stern
+Hugh Jackman
+Hugh Laurie
+Ira Glass
+Isabel Sanford
+Jack Conway
+Jack Nicholson
+Jackie Chan
+Jackie Mason
+James Burrows
+James Cameron
+James Franco
+James Patterson
+Jamie Foxx
+Jane Lynch
+Janet Jackson
+Jason Alexander
+Jason Bateman
+Jason Biggs
+Jason Nash
+Jay Leno
+Jay Pharoah
+Jeff Gordon
+Jennifer Aniston
+Jennifer Garner
+Jennifer Hudson
+Jennifer Lopez
+Jennifer Saunders
+Jenny Slate
+Jerome Robbins
+Jerry Lewis
+Jerry Seinfeld
+Jim Parsons
+Jodie Foster
+Joe Cornish
+John Cho
+John Legend
+John Ritter
+Johnny Depp
+Jon Hamm
+Joseph Gordon
+Josh Gad
+Julia Roberts
+Julie Bowen
+Julie Kent
+Julie Walters
+Justin Bieber
+Kanye West
+Katy Perry
+Kay Cannon
+Keanu Reeves
+Kelly Clarkson
+Kelly Hu
+Ken Dodd
+Ken Jeong
+Kenny Ortega
+Kerry Washington
+Kevin Dillon
+Kevin Hart
+Kevin James
+Kevin Kline
+Kevin Spacey
+Kiefer Sutherland
+Kim Coles
+Kim Kardashian
+Kobe Bryant
+Kristen Bell
+Kylie Jenner
+Lady Gaga
+Larry King
+LeBron James
+Lee Daniels
+Lena Dunham
+Leonardo DiCaprio
+Leslie Mann
+Leslie Nielsen
+Lillian Hurst
+Lilly Singh
+Lily Tomlin
+Lionel Messi
+Loretta Lynn
+Lucy Liu
+Mackenzie Crook
+Madeline Kahn
+Marcia Wallace
+Margaret Cho
+Mariah Carey
+Mark Wahlberg
+Martin Scorsese
+Mel Brooks
+Mel Gibson
+Michael Cera
+Michael Jackson
+Michael Jordan
+Michael Landon
+Michael Palin
+Mike Myers
+Molly Shannon
+Morgan Freeman
+Naomi Watts
+Natalie Morales
+Natalie Portman
+Nathan Fielder
+Nathan Lane
+Nick Park
+Nicolas Cage
+Nicole Kidman
+Norman Lear
+Patrick Stewart
+Paul McCartney
+Paul Rudd
+Paula Abdul
+Penny Marshall
+Pete Holmes
+Peter Jackson
+Phil McGraw
+Piers Morgan
+Quentin Tarantino
+Randy Jackson
+Randy Travis
+Ray Romano
+Rich Sommer
+Richard Attenborough
+Ricky Gervais
+Ridley Scott
+Rita Moreno
+Rob Lowe
+Robert Downey
+Robin Williams
+Roger Federer
+Roger Moore
+Ron Howard
+Rose Marie
+Russell Brand
+Ryan Murphy
+Ryan Reynolds
+Sally Field
+Sandra Bullock
+Sarah Shahi
+Seth Rogen
+Shirley Jones
+Sidney Franklin
+Simon Cowell
+Snoop Dogg
+Spike Lee
+Stan Lee
+Stephen Curry
+Stephen Fry
+Stephen King
+Stephen Merchant
+Steven Spielberg
+Sung Kang
+Susan Egan
+Taylor Swift
+Terrence Howard
+Terry Bradshaw
+Terry Jones
+Tim Conway
+Tim Robbins
+Tina Fey
+Tom Cruise
+Tom Hanks
+Tom Hiddleston
+Tom Jones
+Tommy Chong
+Tony Bennett
+Tracy Morgan
+Trey Parker
+Tyler Perry
+Valerie Harper
+Vanessa Bayer
+Vanessa Williams
+Viola Davis
+Walt Disney
+Wanda Sykes
+Wayne Brady
+Wendy Whelan
+Will Ferrell
+Will Smith
+Zachary Levi

datasets_face/good_names_man.txt

@@ -0,0 +1,226 @@
+Adam Savage
+Adam Scott
+Alan Alda
+Alan Hale
+Albert Brooks
+Alec Baldwin
+Alec Guinness
+Alice Cooper
+Amy Adams
+Anderson Cooper
+Andy Richter
+Aziz Ansari
+BD Wong
+Ben Affleck
+Ben Kingsley
+Ben Miller
+Ben Schwartz
+Benedict Cumberbatch
+Bill Burr
+Bill Cosby
+Bill Irwin
+Bill Maher
+Bill Murray
+Bill Nye
+Billy Chow
+Billy Connolly
+Billy Crystal
+Billy Joel
+Billy Porter
+Billy Wilder
+Bob Hope
+Bob Marley
+Brad Pitt
+Brandon Lee
+Brian Cox
+Brian Tee
+Britney Spears
+Bron James
+Bruce Springsteen
+Bruce Willis
+Bryan Cranston
+Buck Henry
+Burt Lancaster
+Burt Reynolds
+Cary Grant
+Charlie Sheen
+Chris Pratt
+Christopher Guest
+Craig Ferguson
+Craig Robinson
+Cristiano Ronaldo
+Dan Harmon
+Dan Levy
+Dan Rather
+Dana Gould
+Daniel Radcliffe
+Danny Thomas
+Daryl Hall
+Dave Bautista
+Dave Matthews
+David Beckham
+David Bowie
+David Butler
+David Spade
+Dax Shepard
+Dean Martin
+Dennis Chan
+Dennis Franz
+Dennis Hopper
+Dennis Quaid
+Dev Patel
+Dizzy Gillespie
+Donald Crisp
+Donald Glover
+Donnie Yen
+Drew Carey
+Dudley Moore
+Dwayne Johnson
+Ed Sheeran
+Eddie Murphy
+Edgar Wright
+Edward Norton
+Eli Roth
+Elon Musk
+Elton John
+Eric Idle
+Ernie Reyes
+Floyd Mayweather
+Forest Whitaker
+Fred Savage
+Garry Marshall
+Gene Lockhart
+George Benson
+George Burns
+George Clooney
+George Lopez
+George Lucas
+George Marshall
+George Miller
+Gregory Hines
+Gregory Peck
+Harold Lloyd
+Harrison Ford
+Harry Carey
+Howard Stern
+Hugh Jackman
+Hugh Laurie
+Ira Glass
+Jack Conway
+Jack Nicholson
+Jackie Chan
+Jackie Mason
+James Burrows
+James Cameron
+James Franco
+James Patterson
+Jamie Foxx
+Jason Alexander
+Jason Bateman
+Jason Biggs
+Jason Nash
+Jay Leno
+Jay Pharoah
+Jeff Gordon
+Jerome Robbins
+Jerry Lewis
+Jerry Seinfeld
+Jim Parsons
+Joe Cornish
+John Cho
+John Legend
+John Ritter
+Johnny Depp
+Jon Hamm
+Joseph Gordon
+Josh Gad
+Justin Bieber
+Kanye West
+Keanu Reeves
+Ken Dodd
+Ken Jeong
+Kenny Ortega
+Kevin Dillon
+Kevin Hart
+Kevin James
+Kevin Kline
+Kevin Spacey
+Kiefer Sutherland
+Kobe Bryant
+Larry King
+LeBron James
+Lee Daniels
+Leonardo DiCaprio
+Lionel Messi
+Mackenzie Crook
+Mark Wahlberg
+Martin Scorsese
+Mel Brooks
+Mel Gibson
+Michael Cera
+Michael Jackson
+Michael Jordan
+Michael Landon
+Michael Palin
+Mike Myers
+Morgan Freeman
+Nathan Fielder
+Nathan Lane
+Nick Park
+Nicolas Cage
+Norman Lear
+Patrick Stewart
+Paul McCartney
+Paul Rudd
+Pete Holmes
+Peter Jackson
+Phil McGraw
+Piers Morgan
+Quentin Tarantino
+Randy Jackson
+Randy Travis
+Ray Romano
+Rich Sommer
+Richard Attenborough
+Ricky Gervais
+Ridley Scott
+Rob Lowe
+Robert Downey
+Robin Williams
+Roger Federer
+Roger Moore
+Ron Howard
+Russell Brand
+Ryan Murphy
+Ryan Reynolds
+Seth Rogen
+Sidney Franklin
+Simon Cowell
+Snoop Dogg
+Spike Lee
+Stan Lee
+Stephen Curry
+Stephen Fry
+Stephen King
+Stephen Merchant
+Steven Spielberg
+Sung Kang
+Terrence Howard
+Terry Bradshaw
+Terry Jones
+Tim Conway
+Tim Robbins
+Tom Cruise
+Tom Hanks
+Tom Hiddleston
+Tom Jones
+Tommy Chong
+Tony Bennett
+Tracy Morgan
+Trey Parker
+Tyler Perry
+Walt Disney
+Wayne Brady
+Will Ferrell
+Will Smith
+Zachary Levi

datasets_face/good_names_woman.txt

@@ -0,0 +1,100 @@
+Alicia Alonso
+Amy Schumer
+Andrea Martin
+Angelina Jolie
+Ann Curry
+Ann Miller
+Anne Hathaway
+Anne Murray
+Aubrey Plaza
+Audrey Hepburn
+Barbara Walters
+Bonnie Hunt
+Cameron Diaz
+Carol Burnett
+Carol Channing
+Carol Kane
+Carrie Underwood
+Cate Blanchett
+Celia Cruz
+Celine Dion
+Cheryl Hines
+Christina Hendricks
+Cindy Williams
+Claire Danes
+Crystal Gayle
+Debra Messing
+Devon Aoki
+Diana Ross
+Diane Sawyer
+Donna Reed
+Doris Roberts
+Drew Barrymore
+Elaine May
+Eleanor Powell
+Elizabeth Banks
+Ellen Pompeo
+Emma Thompson
+Gillian Murphy
+Ginger Rogers
+Halle Berry
+Helen Mirren
+Helen Reddy
+Isabel Sanford
+Jane Lynch
+Janet Jackson
+Jennifer Aniston
+Jennifer Garner
+Jennifer Hudson
+Jennifer Lopez
+Jennifer Saunders
+Jenny Slate
+Jodie Foster
+Julia Roberts
+Julie Bowen
+Julie Kent
+Julie Walters
+Katy Perry
+Kay Cannon
+Kelly Clarkson
+Kelly Hu
+Kerry Washington
+Kim Coles
+Kim Kardashian
+Kristen Bell
+Kylie Jenner
+Lady Gaga
+Lena Dunham
+Leslie Mann
+Leslie Nielsen
+Lillian Hurst
+Lilly Singh
+Lily Tomlin
+Loretta Lynn
+Lucy Liu
+Madeline Kahn
+Marcia Wallace
+Margaret Cho
+Mariah Carey
+Molly Shannon
+Naomi Watts
+Natalie Morales
+Natalie Portman
+Nicole Kidman
+Paula Abdul
+Penny Marshall
+Rita Moreno
+Rose Marie
+Sally Field
+Sandra Bullock
+Sarah Shahi
+Shirley Jones
+Susan Egan
+Taylor Swift
+Tina Fey
+Valerie Harper
+Vanessa Bayer
+Vanessa Williams
+Viola Davis
+Wanda Sykes
+Wendy Whelan

datasets_face/identity_space.yaml

@@ -0,0 +1,38 @@
+model:
+  use_celeb: True
+  use_svd: True
+  rm_repeats: True
+  n_components: 512  # consistent with meta_inner_dim, should be <= n_samples-1
+  use_sample_reduce: False
+  n_samples: 513
+  use_flatten: False
+  num_embeds_per_token: 2  # consistent with personalization_config
+  target: models.embedding_manager.EmbeddingManagerId
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: image
+    cond_stage_key: caption
+    image_size: 64
+    channels: 4
+    cond_stage_trainable: true   # Note: different from the one we trained before
+    conditioning_key: crossattn
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: False
+    embedding_reg_weight: 0.0
+    unfreeze_model: False
+    model_lr: 0.0
+
+  personalization_config:
+    params:
+      num_embeds_per_token: 2  # consistent with cond_stage_config
+      mlp_depth: 2
+      input_dim: 64
+      token_dim: 1024
+      loss_type: 'none'
+
+

demo_embeddings/example_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d77090f8d1c6cb049c491dd0ffc74a05c1df9e272d9a5788f358b4073f63b75
+size 9288

demo_embeddings/example_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:523e07d8ea4af7f74a4bc4e40ea3bd246562389b6f52aa81ad79ba900eeef040
+size 9288

demo_embeddings/example_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a09e71305dd6a1a6ad2f8e387012c64f7f38728f33667a776e40456435e4d5a
+size 9288

demo_embeddings/example_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03deeeb1438237f2e69467a8afc0841c644c336c74f7703b39c78cbf211983f7
+size 9288

demo_embeddings/example_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b3a8e990d1b374748d514a031e617e5230215e9619d97441855bf734b05439
+size 9288

demo_embeddings/example_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5888c106b5b574a0f015e9d294f86cc62c77f2d3afb197cdc3f71fbabbceec5d
+size 9283

models/celeb_embeddings.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+from functools import partial
+import clip
+from einops import rearrange, repeat
+from transformers import CLIPTokenizer, CLIPTextModel
+import kornia
+import numpy as np
+import os
+
+def embedding_forward(
+        self,
+        input_ids = None,    
+        position_ids = None,
+        name_batch = None,   
+        inputs_embeds = None,
+        embedding_manager = None,
+        only_embedding=True,
+        random_embeddings = None,
+        timesteps = None,
+    ) -> torch.Tensor:
+
+        seq_length = input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2]
+
+        if inputs_embeds is None:
+            inputs_embeds = self.token_embedding(input_ids)
+            if only_embedding:      
+                return inputs_embeds
+            
+        if embedding_manager is not None:
+            inputs_embeds, other_return_dict = embedding_manager(input_ids, inputs_embeds, name_batch, random_embeddings, timesteps)
+        
+        if position_ids is None:
+            position_ids = self.position_ids[:, :seq_length]   
+
+        position_embeddings = self.position_embedding(position_ids)
+        embeddings = inputs_embeds + position_embeddings
+        
+        return embeddings, other_return_dict
+
+
+@torch.no_grad()
+def _get_celeb_embeddings_basis(tokenizer, text_encoder, good_names_txt):
+    
+    device = text_encoder.device
+    max_length = 77  
+    
+    with open(good_names_txt, "r") as f:
+        celeb_names = f.read().splitlines()
+
+    ''' get tokens and embeddings '''
+    all_embeddings = []
+    for name in celeb_names:
+        batch_encoding = tokenizer(name, truncation=True, return_tensors="pt")
+        tokens = batch_encoding["input_ids"].to(device)[:, 1:3] 
+        embeddings = text_encoder.text_model.embeddings(input_ids=tokens, only_embedding=True)  
+        all_embeddings.append(embeddings)
+
+    all_embeddings: torch.Tensor = torch.cat(all_embeddings, dim=0)
+    
+    
+    print('[all_embeddings loaded] shape =', all_embeddings.shape,
+            'max:', all_embeddings.max(),
+            'min={}', all_embeddings.min())  
+    
+    name_emb_mean = all_embeddings.mean(0)
+    name_emb_std = all_embeddings.std(0)    
+    
+    print('[name_emb_mean loaded] shape =', name_emb_mean.shape,
+            'max:', name_emb_mean.max(),
+            'min={}', name_emb_mean.min())  
+    
+    return name_emb_mean, name_emb_std
+

models/embedding_manager.py

@@ -0,0 +1,217 @@
+import torch
+from torch import nn
+from einops import rearrange
+import numpy as np
+from typing import List
+from models.id_embedding.helpers import get_rep_pos, shift_tensor_dim0
+from models.id_embedding.meta_net import StyleVectorizer
+from models.celeb_embeddings import _get_celeb_embeddings_basis
+
+from functools import partial
+import torch.nn.functional as F
+import torch.nn as nn
+import torch.nn.init as init
+
+
+DEFAULT_PLACEHOLDER_TOKEN = ["*"]
+
+PROGRESSIVE_SCALE = 2000
+
+def get_clip_token_for_string(tokenizer, string):
+    batch_encoding = tokenizer(string, return_length=True, padding=True, truncation=True, return_overflowing_tokens=False, return_tensors="pt")
+    tokens = batch_encoding["input_ids"]
+    
+    return tokens 
+
+
+def get_embedding_for_clip_token(embedder, token):
+    return embedder(token.unsqueeze(0))
+
+
+class EmbeddingManagerId_adain(nn.Module):
+    def __init__(
+            self,
+            tokenizer,
+            text_encoder,
+            device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu"),  
+            experiment_name = "normal_GAN",                      
+            num_embeds_per_token: int = 2,  
+            loss_type: str = None,
+            mlp_depth: int = 2,    
+            token_dim: int = 1024,   
+            input_dim: int = 1024, 
+            **kwargs
+    ):
+        super().__init__()
+        self.device = device
+        self.num_es = num_embeds_per_token
+
+        self.get_token_for_string = partial(get_clip_token_for_string, tokenizer)        
+        self.get_embedding_for_tkn = partial(get_embedding_for_clip_token, text_encoder.text_model.embeddings)  
+        
+
+        self.token_dim = token_dim
+
+        ''' 1. Placeholder mapping dicts '''
+        self.placeholder_token = self.get_token_for_string("*")[0][1]    
+        
+        if experiment_name == "normal_GAN":
+            self.celeb_embeddings_mean, self.celeb_embeddings_std = _get_celeb_embeddings_basis(tokenizer, text_encoder, "datasets_face/good_names.txt")
+        elif experiment_name == "man_GAN":
+            self.celeb_embeddings_mean, self.celeb_embeddings_std = _get_celeb_embeddings_basis(tokenizer, text_encoder, "datasets_face/good_names_man.txt")
+        elif experiment_name == "woman_GAN":            
+            self.celeb_embeddings_mean, self.celeb_embeddings_std = _get_celeb_embeddings_basis(tokenizer, text_encoder, "datasets_face/good_names_woman.txt")
+        else:
+            print("Hello, please notice this ^_^")
+            assert 0
+        print("now experiment_name:", experiment_name)
+        
+        self.celeb_embeddings_mean = self.celeb_embeddings_mean.to(device)   
+        self.celeb_embeddings_std = self.celeb_embeddings_std.to(device)  
+
+        self.name_projection_layer = StyleVectorizer(input_dim, self.token_dim * self.num_es, depth=mlp_depth, lr_mul=0.1) 
+        self.embedding_discriminator = Embedding_discriminator(self.token_dim * self.num_es, dropout_rate = 0.2)
+
+        self.adain_mode = 0
+        
+    def forward(
+            self,
+            tokenized_text, 
+            embedded_text, 
+            name_batch,
+            random_embeddings = None,
+            timesteps = None,
+    ):
+        
+        if tokenized_text is not None:
+            batch_size, n, device = *tokenized_text.shape, tokenized_text.device
+        other_return_dict = {}
+        
+        if random_embeddings is not None:
+            mlp_output_embedding = self.name_projection_layer(random_embeddings)   
+            total_embedding = mlp_output_embedding.view(mlp_output_embedding.shape[0], 2, 1024)   
+
+            if self.adain_mode == 0:          
+                adained_total_embedding = total_embedding * self.celeb_embeddings_std + self.celeb_embeddings_mean
+            else:
+                adained_total_embedding = total_embedding
+                
+            other_return_dict["total_embedding"] = total_embedding
+            other_return_dict["adained_total_embedding"] = adained_total_embedding
+
+        if name_batch is not None:
+            if isinstance(name_batch, list): 
+                name_tokens = self.get_token_for_string(name_batch)[:, 1:3]
+                name_embeddings = self.get_embedding_for_tkn(name_tokens.to(random_embeddings.device))[0] 
+                
+                other_return_dict["name_embeddings"] = name_embeddings
+            else:
+                assert 0
+
+        if tokenized_text is not None:
+            placeholder_pos = get_rep_pos(tokenized_text,
+                                        [self.placeholder_token])
+            placeholder_pos = np.array(placeholder_pos)
+            if len(placeholder_pos) != 0:
+                batch_size = adained_total_embedding.shape[0]  
+                end_index = min(batch_size, placeholder_pos.shape[0]) 
+                embedded_text[placeholder_pos[:, 0], placeholder_pos[:, 1]] = adained_total_embedding[:end_index,0,:]
+                embedded_text[placeholder_pos[:, 0], placeholder_pos[:, 1] + 1] = adained_total_embedding[:end_index,1,:]
+
+        return embedded_text, other_return_dict
+
+
+
+    def load(self, ckpt_path):
+        ckpt = torch.load(ckpt_path, map_location='cuda')
+        if ckpt.get("name_projection_layer") is not None:
+            self.name_projection_layer = ckpt.get("name_projection_layer").float()
+
+        print('[Embedding Manager] weights loaded.')
+
+
+
+    def save(self, ckpt_path):
+        save_dict = {}
+        save_dict["name_projection_layer"] = self.name_projection_layer
+        
+        torch.save(save_dict, ckpt_path)
+
+
+    def trainable_projection_parameters(self):  
+        trainable_list = []
+        trainable_list.extend(list(self.name_projection_layer.parameters())) 
+
+        return trainable_list
+   
+
+
+class Embedding_discriminator(nn.Module):
+    def __init__(self, input_size, dropout_rate):
+        super(Embedding_discriminator, self).__init__()
+        self.input_size = input_size
+
+        self.fc1 = nn.Linear(input_size, 512)
+        self.fc2 = nn.Linear(512, 256)
+        self.fc3 = nn.Linear(256, 1)
+
+        self.LayerNorm1 = nn.LayerNorm(512)
+        self.LayerNorm2 = nn.LayerNorm(256)
+
+        self.leaky_relu = nn.LeakyReLU(0.2)
+
+        self.dropout_rate = dropout_rate
+        if self.dropout_rate > 0:
+            self.dropout1 = nn.Dropout(dropout_rate)
+            self.dropout2 = nn.Dropout(dropout_rate)
+
+    def forward(self, input):
+        x = input.view(-1, self.input_size)
+
+        if self.dropout_rate > 0:
+            x = self.leaky_relu(self.dropout1(self.fc1(x)))
+        else:
+            x = self.leaky_relu(self.fc1(x))
+        
+        if self.dropout_rate > 0:
+            x = self.leaky_relu(self.dropout2(self.fc2(x)))
+        else:
+            x = self.leaky_relu(self.fc2(x))
+
+        x = self.fc3(x)
+
+        return x
+    
+    
+    def save(self, ckpt_path):
+        save_dict = {}
+ 
+        save_dict["fc1"] = self.fc1
+        save_dict["fc2"] = self.fc2
+        save_dict["fc3"] = self.fc3
+        save_dict["LayerNorm1"] = self.LayerNorm1
+        save_dict["LayerNorm2"] = self.LayerNorm2
+        save_dict["leaky_relu"] = self.leaky_relu
+        save_dict["dropout1"] = self.dropout1
+        save_dict["dropout2"] = self.dropout2
+        
+        torch.save(save_dict, ckpt_path)
+    
+    def load(self, ckpt_path):
+        ckpt = torch.load(ckpt_path, map_location='cuda')
+        
+        if ckpt.get("first_name_proj_layer") is not None:
+            self.fc1 = ckpt.get("fc1").float()
+            self.fc2 = ckpt.get("fc2").float()
+            self.fc3 = ckpt.get("fc3").float()
+            self.LayerNorm1 = ckpt.get("LayerNorm1").float()
+            self.LayerNorm2 = ckpt.get("LayerNorm2").float()
+            self.leaky_relu = ckpt.get("leaky_relu").float()
+            self.dropout1 = ckpt.get("dropout1").float()
+            self.dropout2 = ckpt.get("dropout2").float()
+            
+        print('[Embedding D] weights loaded.')
+
+
+
+    

models/id_embedding/helpers.py

@@ -0,0 +1,63 @@
+# -*-coding:utf-8-*-
+import torch
+import numpy as np
+from typing import List
+
+
+def get_rep_pos(tokenized: torch.Tensor, rep_tokens: list):
+    pos_list = []
+    for token in rep_tokens:
+        pos_list = torch.stack(torch.where(tokenized == token)).T.tolist()
+    return pos_list
+
+
+def shift_tensor_dim0(ori: torch.Tensor, r_pos: List[np.ndarray], reps: int):
+    assert reps >= 1  
+    device = ori.device
+    d = ori.shape[0]
+    offset = np.zeros(d, dtype=np.int64) 
+    r_pos_cat = np.concatenate(r_pos)
+    for p in r_pos_cat:
+        offset[p + 1:] += (reps - 1)
+
+    r_cnt = r_pos_cat.shape[0] 
+    target_pos = (np.arange(d) + offset)[:d - r_cnt * (reps - 1)] 
+    ori[target_pos] = ori[np.arange(target_pos.shape[0])]
+
+    rep_final_pos: np.ndarray = target_pos[r_pos_cat].repeat(reps) + np.tile(np.arange(reps), r_cnt)
+    ori[rep_final_pos] = ori[target_pos[r_pos_cat].repeat(reps)]
+
+    rep_final_pos_list = []
+    lo = 0
+    for i in range(len(r_pos)):
+        r_one_times = r_pos[i].shape[0]
+        r_one_nums = r_one_times * reps 
+        rep_final_pos_list.append(rep_final_pos[lo: lo + r_one_nums].reshape(r_one_times, reps))
+        lo += r_one_nums
+    return ori, rep_final_pos_list
+
+
+def _test_get_rep_pos():
+    tokenized = torch.LongTensor([0, 1, 2, 2, 3, 4, 5, 6, 7, 99] + [99] * 20)
+    print('[from]:', tokenized)
+    rep_tokens = [2, 6]
+    rep_times = 2
+
+    rep_pos = get_rep_pos(tokenized, rep_tokens)
+    print('[rep_pos]:', rep_pos)
+    res, rep_pos_final = shift_tensor_dim0(tokenized, rep_pos, rep_times)
+    print('[to]:', res)
+    print('[final pos]:', rep_pos_final)
+
+
+def _test_shift_tensor_dim0():
+    embedded = torch.arange(20)
+    print(embedded)
+    pos = np.array([3, 6, 8])
+    times = 1
+    output = shift_tensor_dim0(embedded, pos, times)
+    print(output)
+
+
+if __name__ == "__main__":
+    _test_get_rep_pos()

models/id_embedding/meta_net.py

@@ -0,0 +1,67 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import kornia
+from einops import rearrange
+import torch.nn.init as init
+
+def leaky_relu(p=0.2):
+    return nn.LeakyReLU(p, inplace=True)
+
+class Residual(nn.Module):
+    def __init__(self,
+                 fn):
+        super().__init__()
+        self.fn = fn
+
+    def forward(self, x, **kwargs):
+        return x + self.fn(x, **kwargs)
+
+
+class EqualLinear(nn.Module):
+    def __init__(self, in_dim, out_dim, lr_mul=1, bias=True, pre_norm=False, activate = False):
+        super().__init__()
+        self.weight = nn.Parameter(torch.randn(out_dim, in_dim))
+        if bias:
+            self.bias = nn.Parameter(torch.zeros(out_dim))
+
+        self.lr_mul = lr_mul
+
+        self.pre_norm = pre_norm
+        if pre_norm:
+            self.norm = nn.LayerNorm(in_dim, eps=1e-5)
+        self.activate = activate
+        if self.activate == True:
+            self.non_linear = leaky_relu()
+
+    def forward(self, input):
+        if hasattr(self, 'pre_norm') and self.pre_norm:
+            out = self.norm(input)
+            out = F.linear(out, self.weight * self.lr_mul, bias=self.bias * self.lr_mul)
+        else:
+            out = F.linear(input, self.weight * self.lr_mul, bias=self.bias * self.lr_mul)
+        
+        if self.activate == True:
+            out = self.non_linear(out)
+        return out
+
+
+class StyleVectorizer(nn.Module):
+    def __init__(self, dim_in, dim_out, depth, lr_mul = 0.1):
+        super().__init__()
+
+        layers = []
+        for i in range(depth):
+            if i == 0: 
+                layers.extend([EqualLinear(dim_in, dim_out, lr_mul, pre_norm=False, activate = True)]) 
+            elif i == depth - 1:
+                layers.extend([EqualLinear(dim_out, dim_out, lr_mul, pre_norm=True, activate = False)]) 
+            else:
+                layers.extend([Residual(EqualLinear(dim_out, dim_out, lr_mul, pre_norm=True, activate = True))])
+
+        self.net = nn.Sequential(*layers)
+        self.norm = nn.LayerNorm(dim_out, eps=1e-5)
+        
+    def forward(self, x):
+        return self.norm(self.net(x))
+

requirements.txt

@@ -1,6 +1,15 @@
-accelerate
-diffusers
-invisible_watermark
-torch
-transformers
-xformers
+torch==2.0.1
+torchvision==0.15.2
+diffusers==0.23.0
+transformers==4.33.2
+xformers==0.0.20
+accelerate==0.23.0
+omegaconf
+clip==0.2.0
+einops
+kornia==0.6.12
+opencv-python
+opencv-contrib-python
+gradio
+huggingface_hub==0.22.2
+IPython

test.ipynb

@@ -0,0 +1,243 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import os\n",
+    "from transformers import ViTModel, ViTImageProcessor\n",
+    "from utils import text_encoder_forward\n",
+    "from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler\n",
+    "from utils import latents_to_images, downsampling, merge_and_save_images\n",
+    "from omegaconf import OmegaConf\n",
+    "from accelerate.utils import set_seed\n",
+    "from tqdm import tqdm\n",
+    "from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput\n",
+    "from PIL import Image\n",
+    "from models.celeb_embeddings import embedding_forward\n",
+    "import models.embedding_manager\n",
+    "import importlib\n",
+    "\n",
+    "# seed = 42\n",
+    "# set_seed(seed)  \n",
+    "# torch.cuda.set_device(0)\n",
+    "\n",
+    "# set your sd2.1 path\n",
+    "model_path = \"/home/user/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6\"\n",
+    "pipe = StableDiffusionPipeline.from_pretrained(model_path)   \n",
+    "pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)\n",
+    "pipe = pipe.to(\"cuda\")\n",
+    "\n",
+    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
+    "\n",
+    "vae = pipe.vae\n",
+    "unet = pipe.unet\n",
+    "text_encoder = pipe.text_encoder\n",
+    "tokenizer = pipe.tokenizer\n",
+    "scheduler = pipe.scheduler\n",
+    "\n",
+    "input_dim = 64\n",
+    "\n",
+    "experiment_name = \"normal_GAN\"   # \"normal_GAN\", \"man_GAN\", \"woman_GAN\" , \n",
+    "if experiment_name == \"normal_GAN\":\n",
+    "    steps = 10000\n",
+    "elif experiment_name == \"man_GAN\":\n",
+    "    steps = 7000\n",
+    "elif experiment_name == \"woman_GAN\":\n",
+    "    steps = 6000\n",
+    "else:\n",
+    "    print(\"Hello, please notice this ^_^\")\n",
+    "    assert 0\n",
+    "\n",
+    "\n",
+    "original_forward = text_encoder.text_model.embeddings.forward\n",
+    "text_encoder.text_model.embeddings.forward = embedding_forward.__get__(text_encoder.text_model.embeddings)\n",
+    "embedding_manager_config = OmegaConf.load(\"datasets_face/identity_space.yaml\")\n",
+    "Embedding_Manager = models.embedding_manager.EmbeddingManagerId_adain(  \n",
+    "        tokenizer,\n",
+    "        text_encoder,\n",
+    "        device = device,\n",
+    "        training = True,\n",
+    "        experiment_name = experiment_name, \n",
+    "        num_embeds_per_token = embedding_manager_config.model.personalization_config.params.num_embeds_per_token,            \n",
+    "        token_dim = embedding_manager_config.model.personalization_config.params.token_dim,\n",
+    "        mlp_depth = embedding_manager_config.model.personalization_config.params.mlp_depth,\n",
+    "        loss_type = embedding_manager_config.model.personalization_config.params.loss_type,\n",
+    "        vit_out_dim = input_dim,\n",
+    ")\n",
+    "embedding_path = os.path.join(\"training_weight\", experiment_name, \"embeddings_manager-{}.pt\".format(str(steps)))\n",
+    "Embedding_Manager.load(embedding_path)\n",
+    "text_encoder.text_model.embeddings.forward = original_forward\n",
+    "\n",
+    "print(\"finish init\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "1. create a new character and test with prompts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# sample a z\n",
+    "random_embedding = torch.randn(1, 1, input_dim).to(device)\n",
+    "\n",
+    "# map z to pseudo identity embeddings\n",
+    "_, emb_dict = Embedding_Manager(tokenized_text=None, embedded_text=None, name_batch=None, random_embeddings = random_embedding, timesteps = None,)\n",
+    "\n",
+    "test_emb = emb_dict[\"adained_total_embedding\"].to(device)\n",
+    "\n",
+    "v1_emb = test_emb[:, 0]\n",
+    "v2_emb = test_emb[:, 1]\n",
+    "embeddings = [v1_emb, v2_emb]\n",
+    "\n",
+    "index = \"0000\"\n",
+    "save_dir = os.path.join(\"test_results/\" + experiment_name, index)\n",
+    "os.makedirs(save_dir, exist_ok=True)\n",
+    "test_emb_path = os.path.join(save_dir, \"id_embeddings.pt\")\n",
+    "torch.save(test_emb, test_emb_path)\n",
+    "\n",
+    "'''insert into tokenizer & embedding layer'''\n",
+    "tokens = [\"v1*\", \"v2*\"]\n",
+    "embeddings = [v1_emb, v2_emb]\n",
+    "# add tokens and get ids\n",
+    "tokenizer.add_tokens(tokens)\n",
+    "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
+    "\n",
+    "# resize token embeddings and set new embeddings\n",
+    "text_encoder.resize_token_embeddings(len(tokenizer), pad_to_multiple_of = 8)\n",
+    "for token_id, embedding in zip(token_ids, embeddings):\n",
+    "    text_encoder.get_input_embeddings().weight.data[token_id] = embedding\n",
+    "\n",
+    "prompts_list = [\"a photo of v1* v2*, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a Superman outfit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a spacesuit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a red sweater, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a blue hoodie, facing to camera, best quality, ultra high res\",\n",
+    "]\n",
+    "\n",
+    "for prompt in prompts_list:\n",
+    "    image = pipe(prompt, guidance_scale = 8.5).images[0]\n",
+    "    save_img_path = os.path.join(save_dir, prompt.replace(\"v1* v2*\", \"a person\") + '.png')\n",
+    "    image.save(save_img_path)\n",
+    "    print(save_img_path)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "2. directly use a chosen generated pseudo identity embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the path of your generated embeddings\n",
+    "test_emb_path = \"demo_embeddings/856.pt\"  # \"test_results/normal_GAN/0000/id_embeddings.pt\"\n",
+    "test_emb = torch.load(test_emb_path).cuda()\n",
+    "v1_emb = test_emb[:, 0]\n",
+    "v2_emb = test_emb[:, 1]\n",
+    "\n",
+    "\n",
+    "index = \"chosen_index\"\n",
+    "save_dir = os.path.join(\"test_results/\" + experiment_name, index)\n",
+    "os.makedirs(save_dir, exist_ok=True)\n",
+    "\n",
+    "\n",
+    "'''insert into tokenizer & embedding layer'''\n",
+    "tokens = [\"v1*\", \"v2*\"]\n",
+    "embeddings = [v1_emb, v2_emb]\n",
+    "# add tokens and get ids\n",
+    "tokenizer.add_tokens(tokens)\n",
+    "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
+    "\n",
+    "# resize token embeddings and set new embeddings\n",
+    "text_encoder.resize_token_embeddings(len(tokenizer), pad_to_multiple_of = 8)\n",
+    "for token_id, embedding in zip(token_ids, embeddings):\n",
+    "    text_encoder.get_input_embeddings().weight.data[token_id] = embedding\n",
+    "\n",
+    "prompts_list = [\"a photo of v1* v2*, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a Superman outfit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a spacesuit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a red sweater, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a purple wizard outfit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a blue hoodie, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing headphones, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* with red hair, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing headphones with red hair, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a Christmas hat, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses and necklace, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a blue cap, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a doctoral cap, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* with white hair, wearing glasses, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* in a helmet and vest riding a motorcycle, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* holding a bottle of red wine, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* driving a bus in the desert, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* playing basketball, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* playing the violin, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* piloting a spaceship, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* riding a horse, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* coding in front of a computer, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* laughing on the lawn, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* frowning at the camera, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* happily smiling, looking at the camera, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* crying disappointedly, with tears flowing, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* playing the guitar in the view of left side, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* holding a bottle of red wine, upper body, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses and necklace, close-up, in the view of right side, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* riding a horse, in the view of the top, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a doctoral cap, upper body, with the left side of the face facing the camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* crying disappointedly, with tears flowing, with left side of the face facing the camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* sitting in front of the camera, with a beautiful purple sunset at the beach in the background, best quality, ultra high res\",\n",
+    "    \"v1* v2* swimming in the pool, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* climbing a mountain, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* skiing on the snowy mountain, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* in the snow, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* in space wearing a spacesuit, facing to camera, best quality, ultra high res\",\n",
+    "]\n",
+    "\n",
+    "for prompt in prompts_list:\n",
+    "    image = pipe(prompt, guidance_scale = 8.5).images[0]\n",
+    "    save_img_path = os.path.join(save_dir, prompt.replace(\"v1* v2*\", \"a person\") + '.png')\n",
+    "    image.save(save_img_path)\n",
+    "    print(save_img_path)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "lbl",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

test_create_many_characters.ipynb

@@ -0,0 +1,255 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import os\n",
+    "from transformers import ViTModel, ViTImageProcessor\n",
+    "from utils import text_encoder_forward\n",
+    "from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler\n",
+    "from utils import latents_to_images, downsampling, merge_and_save_images\n",
+    "from omegaconf import OmegaConf\n",
+    "from accelerate.utils import set_seed\n",
+    "from tqdm import tqdm\n",
+    "from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput\n",
+    "from PIL import Image\n",
+    "from models.celeb_embeddings import embedding_forward\n",
+    "import models.embedding_manager\n",
+    "import importlib\n",
+    "\n",
+    "# seed = 42\n",
+    "# set_seed(seed)  \n",
+    "# torch.cuda.set_device(0)\n",
+    "\n",
+    "# set your sd2.1 path\n",
+    "model_path = \"/home/user/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6\"\n",
+    "pipe = StableDiffusionPipeline.from_pretrained(model_path)   \n",
+    "pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)\n",
+    "pipe = pipe.to(\"cuda\")\n",
+    "\n",
+    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
+    "\n",
+    "vae = pipe.vae\n",
+    "unet = pipe.unet\n",
+    "text_encoder = pipe.text_encoder\n",
+    "tokenizer = pipe.tokenizer\n",
+    "scheduler = pipe.scheduler\n",
+    "\n",
+    "input_dim = 64\n",
+    "\n",
+    "experiment_name = \"normal_GAN\"   # \"normal_GAN\", \"man_GAN\", \"woman_GAN\" , \n",
+    "if experiment_name == \"normal_GAN\":\n",
+    "    steps = 10000\n",
+    "elif experiment_name == \"man_GAN\":\n",
+    "    steps = 7000\n",
+    "elif experiment_name == \"woman_GAN\":\n",
+    "    steps = 6000\n",
+    "else:\n",
+    "    print(\"Hello, please notice this ^_^\")\n",
+    "    assert 0\n",
+    "\n",
+    "\n",
+    "original_forward = text_encoder.text_model.embeddings.forward\n",
+    "text_encoder.text_model.embeddings.forward = embedding_forward.__get__(text_encoder.text_model.embeddings)\n",
+    "embedding_manager_config = OmegaConf.load(\"datasets_face/identity_space.yaml\")\n",
+    "Embedding_Manager = models.embedding_manager.EmbeddingManagerId_adain(  \n",
+    "        tokenizer,\n",
+    "        text_encoder,\n",
+    "        device = device,\n",
+    "        training = True,\n",
+    "        experiment_name = experiment_name, \n",
+    "        num_embeds_per_token = embedding_manager_config.model.personalization_config.params.num_embeds_per_token,            \n",
+    "        token_dim = embedding_manager_config.model.personalization_config.params.token_dim,\n",
+    "        mlp_depth = embedding_manager_config.model.personalization_config.params.mlp_depth,\n",
+    "        loss_type = embedding_manager_config.model.personalization_config.params.loss_type,\n",
+    "        vit_out_dim = input_dim,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "embedding_path = os.path.join(\"training_weight\", experiment_name, \"embeddings_manager-{}.pt\".format(str(steps)))\n",
+    "Embedding_Manager.load(embedding_path)\n",
+    "text_encoder.text_model.embeddings.forward = original_forward\n",
+    "\n",
+    "print(\"finish init\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "1. create a new character and test with prompts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# sample a z\n",
+    "for index in range(100):\n",
+    "\n",
+    "    random_embedding = torch.randn(1, 1, input_dim).to(device)\n",
+    "\n",
+    "    # map z to pseudo identity embeddings\n",
+    "    _, emb_dict = Embedding_Manager(tokenized_text=None, embedded_text=None, name_batch=None, random_embeddings = random_embedding, timesteps = None,)\n",
+    "\n",
+    "    test_emb = emb_dict[\"adained_total_embedding\"].to(device)\n",
+    "\n",
+    "    v1_emb = test_emb[:, 0]\n",
+    "    v2_emb = test_emb[:, 1]\n",
+    "    embeddings = [v1_emb, v2_emb]\n",
+    "\n",
+    "    save_dir = os.path.join(\"test_results/\" + experiment_name, str(index))\n",
+    "    os.makedirs(save_dir, exist_ok=True)    \n",
+    "    test_emb_path = os.path.join(save_dir, \"id_embeddings.pt\")\n",
+    "    torch.save(test_emb, test_emb_path)\n",
+    "\n",
+    "\n",
+    "\n",
+    "    '''insert into tokenizer & embedding layer'''\n",
+    "    tokens = [\"v1*\", \"v2*\"]\n",
+    "    embeddings = [v1_emb, v2_emb]\n",
+    "    # add tokens and get ids\n",
+    "    tokenizer.add_tokens(tokens)\n",
+    "    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
+    "\n",
+    "    # resize token embeddings and set new embeddings\n",
+    "    text_encoder.resize_token_embeddings(len(tokenizer), pad_to_multiple_of = 8)\n",
+    "    for token_id, embedding in zip(token_ids, embeddings):\n",
+    "        text_encoder.get_input_embeddings().weight.data[token_id] = embedding\n",
+    "\n",
+    "    prompts_list = [\"a photo of v1* v2*, facing to camera, best quality, ultra high res\",\n",
+    "        \"v1* v2* wearing a Superman outfit, facing to camera, best quality, ultra high res\",\n",
+    "        \"v1* v2* wearing a spacesuit, facing to camera, best quality, ultra high res\",\n",
+    "        \"v1* v2* wearing a red sweater, facing to camera, best quality, ultra high res\",\n",
+    "        \"v1* v2* wearing a blue hoodie, facing to camera, best quality, ultra high res\",\n",
+    "    ]\n",
+    "\n",
+    "    for prompt in prompts_list:\n",
+    "        image = pipe(prompt, guidance_scale = 8.5).images[0]\n",
+    "        save_img_path = os.path.join(save_dir, prompt.replace(\"v1* v2*\", \"a person\") + '.png')\n",
+    "        image.save(save_img_path)\n",
+    "        print(save_img_path)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "2. directly use a chosen generated pseudo identity embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the path of your generated embeddings\n",
+    "test_emb_path = \"test_results/normal_GAN/0000/id_embeddings.pt\"\n",
+    "test_emb = torch.load(test_emb_path).cuda()\n",
+    "v1_emb = test_emb[:, 0]\n",
+    "v2_emb = test_emb[:, 1]\n",
+    "\n",
+    "\n",
+    "index = \"chosen_index\"\n",
+    "save_dir = os.path.join(\"test_results/\" + experiment_name, index)\n",
+    "os.makedirs(save_dir, exist_ok=True)\n",
+    "\n",
+    "\n",
+    "'''insert into tokenizer & embedding layer'''\n",
+    "tokens = [\"v1*\", \"v2*\"]\n",
+    "embeddings = [v1_emb, v2_emb]\n",
+    "# add tokens and get ids\n",
+    "tokenizer.add_tokens(tokens)\n",
+    "token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
+    "\n",
+    "# resize token embeddings and set new embeddings\n",
+    "text_encoder.resize_token_embeddings(len(tokenizer), pad_to_multiple_of = 8)\n",
+    "for token_id, embedding in zip(token_ids, embeddings):\n",
+    "    text_encoder.get_input_embeddings().weight.data[token_id] = embedding\n",
+    "\n",
+    "prompts_list = [\"a photo of v1* v2*, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a Superman outfit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a spacesuit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a red sweater, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a purple wizard outfit, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a blue hoodie, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing headphones, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* with red hair, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing headphones with red hair, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a Christmas hat, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses and necklace, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a blue cap, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a doctoral cap, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* with white hair, wearing glasses, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* in a helmet and vest riding a motorcycle, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* holding a bottle of red wine, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* driving a bus in the desert, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* playing basketball, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* playing the violin, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* piloting a spaceship, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* riding a horse, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* coding in front of a computer, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* laughing on the lawn, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* frowning at the camera, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* happily smiling, looking at the camera, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* crying disappointedly, with tears flowing, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* playing the guitar in the view of left side, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* holding a bottle of red wine, upper body, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing sunglasses and necklace, close-up, in the view of right side, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* riding a horse, in the view of the top, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* wearing a doctoral cap, upper body, with the left side of the face facing the camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* crying disappointedly, with tears flowing, with left side of the face facing the camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* sitting in front of the camera, with a beautiful purple sunset at the beach in the background, best quality, ultra high res\",\n",
+    "    \"v1* v2* swimming in the pool, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* climbing a mountain, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* skiing on the snowy mountain, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* in the snow, facing to camera, best quality, ultra high res\",\n",
+    "    \"v1* v2* in space wearing a spacesuit, facing to camera, best quality, ultra high res\",\n",
+    "]\n",
+    "\n",
+    "for prompt in prompts_list:\n",
+    "    image = pipe(prompt, guidance_scale = 8.5).images[0]\n",
+    "    save_img_path = os.path.join(save_dir, prompt.replace(\"v1* v2*\", \"a person\") + '.png')\n",
+    "    image.save(save_img_path)\n",
+    "    print(save_img_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "lbl",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

train.py

@@ -0,0 +1,767 @@
+import argparse
+import itertools
+import logging
+import math
+import os
+from pathlib import Path
+import accelerate
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+import transformers
+from accelerate import Accelerator
+from accelerate.logging import get_logger
+from accelerate.utils import ProjectConfiguration, set_seed
+from packaging import version
+from PIL import Image
+from torch.utils.data import Dataset
+from torchvision import transforms
+from tqdm.auto import tqdm
+from transformers import AutoTokenizer, PretrainedConfig
+import diffusers
+from diffusers import (
+    AutoencoderKL,
+    DDPMScheduler,
+    DiffusionPipeline,
+    UNet2DConditionModel,
+    StableDiffusionPipeline,
+    DPMSolverMultistepScheduler,
+)
+from diffusers.optimization import get_scheduler
+from diffusers.utils.import_utils import is_xformers_available
+import numpy as np
+from omegaconf import OmegaConf
+import random
+from transformers import ViTModel, ViTImageProcessor
+from models.celeb_embeddings import embedding_forward  
+from models.embedding_manager import EmbeddingManagerId_adain, Embedding_discriminator
+from datasets_face.face_id import FaceIdDataset
+from utils import text_encoder_forward, set_requires_grad, add_noise_return_paras, latents_to_images, discriminator_r1_loss, discriminator_r1_loss_accelerator, downsampling, GANLoss
+import types
+import torch.nn as nn
+from tqdm import tqdm
+from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
+import importlib
+
+logger = get_logger(__name__)
+
+def import_model_class_from_model_name_or_path(pretrained_model_name_or_path: str, revision: str):
+    text_encoder_config = PretrainedConfig.from_pretrained(
+        pretrained_model_name_or_path,
+        subfolder="text_encoder",
+        revision=revision,
+    )
+    model_class = text_encoder_config.architectures[0]
+
+    if model_class == "CLIPTextModel":
+        from transformers import CLIPTextModel
+
+        return CLIPTextModel
+    elif model_class == "RobertaSeriesModelWithTransformation":
+        from diffusers.pipelines.alt_diffusion.modeling_roberta_series import RobertaSeriesModelWithTransformation
+
+        return RobertaSeriesModelWithTransformation
+    elif model_class == "T5EncoderModel":
+        from transformers import T5EncoderModel
+
+        return T5EncoderModel
+    else:
+        raise ValueError(f"{model_class} is not supported.")
+
+def parse_args(input_args=None):
+    parser = argparse.ArgumentParser(description="Simple example of a script for training Cones 2.")          
+    parser.add_argument(
+        "--embedding_manager_config", 
+        type=str,
+        default="datasets_face/identity_space.yaml",
+        help=('config to load the train model and dataset'),
+    )
+    parser.add_argument(
+        "--d_reg_every",  
+        type=int, 
+        default=16,
+        help="interval for applying r1 regularization"
+    )    
+    parser.add_argument(
+        "--r1", 
+        type=float, 
+        default=1, 
+        help="weight of the r1 regularization"
+    )    
+    parser.add_argument(
+        "--l_gan_lambda",  
+        type=float,
+        default=1,
+        help="Initial learning rate (after the potential warmup period) to use.",
+    )  
+    parser.add_argument(
+        "--l_consis_lambda",  
+        type=float,
+        default=8,
+        help="Initial learning rate (after the potential warmup period) to use.",
+    )                             
+    parser.add_argument(
+        "--pretrained_model_name_or_path", 
+        type=str,
+        default="/home/user/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6",
+        help="Path to pretrained model or model identifier from huggingface.co/models.",
+    )
+    parser.add_argument(
+        "--pretrained_embedding_manager_path",  
+        type=str,
+        default=None,          
+        help="pretrained_embedding_manager_path",
+    )      
+    parser.add_argument(
+        "--pretrained_embedding_manager_epoch",  
+        type=str,
+        default=800,
+        help="pretrained_embedding_manager_epoch",
+    )                
+    parser.add_argument(
+        "--revision",
+        type=str,
+        default=None,
+        required=False,
+        help=(
+            "Revision of pretrained model identifier from huggingface.co/models. Trainable model components should be"
+            " float32 precision."
+        ),
+    )
+    parser.add_argument(
+        "--tokenizer_name",
+        type=str,
+        default=None,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )      
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="training_weight/normal_GAN",   # training_weight/woman_GAN  training_weight/man_GAN
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
+    parser.add_argument("--seed", type=int, default= None, help="A seed for reproducible training.")
+    parser.add_argument(
+        "--resolution",
+        type=int,
+        default=512,
+        help=(
+            "The resolution for input images, all the images in the train/validation dataset will be resized to this"
+            " resolution"
+        ),
+    )
+    parser.add_argument(
+        "--center_crop",
+        default=False,
+        action="store_true",
+        help=(
+            "Whether to center crop the input images to the resolution. If not set, the images will be randomly"
+            " cropped. The images will be resized to the resolution first before cropping."
+        ),
+    )
+    parser.add_argument(
+        "--train_batch_size", 
+        type=int, default=8, 
+        help="Batch size (per device) for the training dataloader."
+    )
+    parser.add_argument(
+        "--num_train_epochs", 
+        type=int, 
+        default=None
+    )
+    parser.add_argument(
+        "--max_train_steps",
+        type=int,
+        # default=None,
+        default=10001,
+        help="Total number of training steps to perform.  If provided, overrides num_train_epochs.",
+    )
+    parser.add_argument(
+        "--checkpointing_steps",
+        type=int,
+        default=1000,
+        help=(
+            "Save a checkpoint of the training state every X updates. Checkpoints can be used for resuming training via"
+            " `--resume_from_checkpoint`. In the case that the checkpoint is better than the final trained model, the"
+            " checkpoint can also be used for inference. Using a checkpoint for inference requires separate loading of"
+            " the original pipeline and the individual checkpointed model components."
+        ),
+    )
+    parser.add_argument(
+        "--resume_from_checkpoint",
+        type=str,
+        default=None,
+        help=(
+            "Whether training should be resumed from a previous checkpoint. Use a path saved by"
+            ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.'
+        ),
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument(
+        "--gradient_checkpointing",
+        action="store_true",
+        help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.",
+    )
+    parser.add_argument(
+        "--learning_rate",
+        type=float,
+        default=5e-5,          
+        help="Initial learning rate (after the potential warmup period) to use.",
+    )
+    parser.add_argument(
+        "--scale_lr",
+        action="store_true",
+        default=False,
+        help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.",
+    )
+    parser.add_argument(
+        "--lr_scheduler",
+        type=str,
+        default="constant",
+        help=(
+            'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",'
+            ' "constant", "constant_with_warmup"]'
+        ),
+    )
+    parser.add_argument(
+        "--lr_warmup_steps", type=int, default=0, help="Number of steps for the warmup in the lr scheduler."
+    )
+    parser.add_argument(
+        "--lr_num_cycles",
+        type=int,
+        default=1,
+        help="Number of hard resets of the lr in cosine_with_restarts scheduler.",
+    )
+    parser.add_argument("--lr_power", type=float, default=1.0, help="Power factor of the polynomial scheduler.")
+    parser.add_argument(
+        "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes."
+    )
+    parser.add_argument(
+        "--dataloader_num_workers",
+        type=int,
+        default=2,
+        help=(
+            "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process."
+        ),
+    )
+    parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.")
+    parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.")
+    parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.")
+    parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
+    parser.add_argument(
+        "--logging_dir",
+        type=str,
+        default="logs",
+        help=(
+            "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to"
+            " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***."
+        ),
+    )
+    parser.add_argument(
+        "--allow_tf32",
+        action="store_true",
+        help=(
+            "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see"
+            " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices"
+        ),
+    )
+    parser.add_argument(
+        "--report_to",
+        type=str,
+        default="tensorboard",
+        help=(
+            'The integration to report the results and logs to. Supported platforms are `"tensorboard"`'
+            ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.'
+        ),
+    )
+    parser.add_argument(
+        "--mixed_precision",
+        type=str,
+        default=None,
+        choices=["no", "fp16", "bf16"],
+        help=(
+            "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >="
+            " 1.10.and an Nvidia Ampere GPU.  Default to the value of accelerate config of the current system or the"
+            " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config."
+        ),
+    )
+    parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
+    parser.add_argument(
+        "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers."
+    )
+    parser.add_argument(
+        "--set_grads_to_none",
+        action="store_true",
+        help=(
+            "Save more memory by using setting grads to None instead of zero. Be aware, that this changes certain"
+            " behaviors, so disable this argument if it causes any problems. More info:"
+            " https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html"
+        ),
+    )
+    parser.add_argument(
+        "--input_dim",   
+        type=int, 
+        default=64,
+        help="randomly sampled vectors and dimensions of MLP input"
+    )
+    parser.add_argument(
+        "--experiment_name",   
+        type=str, 
+        default="normal_GAN",  # "man_GAN"  "woman_GAN"
+        help="randomly sampled vectors and dimensions of MLP input"
+    )
+    
+    
+    if input_args is not None:  
+        args = parser.parse_args(input_args)
+    else:
+        args = parser.parse_args()
+
+    env_local_rank = int(os.environ.get("LOCAL_RANK", -1))
+    if env_local_rank != -1 and env_local_rank != args.local_rank:
+        args.local_rank = env_local_rank
+
+    return args
+
+def encode_prompt(prompt_batch, name_batch, text_encoder, tokenizer, embedding_manager, is_train=True,
+                  random_embeddings = None, timesteps = None):
+    captions = []
+    proportion_empty_prompts = 0     
+    
+    for caption in prompt_batch:
+        if random.random() < proportion_empty_prompts:
+            captions.append("")  
+        elif isinstance(caption, str):
+            captions.append(caption)
+        elif isinstance(caption, (list, np.ndarray)):
+            captions.append(random.choice(caption) if is_train else caption[0])
+
+    text_inputs = tokenizer(
+        captions,
+        padding="max_length",
+        max_length=tokenizer.model_max_length,
+        truncation=True,
+        return_tensors="pt",
+    )
+    text_input_ids = text_inputs.input_ids.to(text_encoder.device) 
+    
+    positions_list = []
+    for prompt_ids in text_input_ids:
+        position = int(torch.where(prompt_ids == 265)[0][0])
+        positions_list.append(position)
+
+    prompt_embeds, other_return_dict = text_encoder_forward(
+                                            text_encoder = text_encoder, 
+                                            input_ids = text_input_ids,
+                                            name_batch = name_batch,
+                                            output_hidden_states=True,
+                                            embedding_manager = embedding_manager,
+                                            random_embeddings = random_embeddings,
+                                            timesteps = timesteps)
+
+    return prompt_embeds, other_return_dict, positions_list
+
+
+def weights_init_normal(m):
+    classname = m.__class__.__name__
+    if classname.find("Linear") != -1:  
+        torch.nn.init.normal_(m.weight.data, 0.0, 0.02)  
+        torch.nn.init.constant_(m.bias.data, 0.0)   
+
+
+def main(args):
+    args.output_dir = os.path.join(args.output_dir, args.experiment_name)
+    print("output_dir", args.output_dir)    
+    logging_dir = Path(args.output_dir, args.logging_dir) 
+    accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir)
+    accelerator = Accelerator(
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        mixed_precision=args.mixed_precision,
+        log_with=args.report_to,
+        project_config=accelerator_project_config,
+    )
+
+    # Currently, it's not possible to do gradient accumulation when training two models with accelerate.accumulate
+    # This will be enabled soon in accelerate. For now, we don't allow gradient accumulation when training two models.
+    if args.gradient_accumulation_steps > 1 and accelerator.num_processes > 1:
+        raise ValueError(
+            "Gradient accumulation is not supported when training the text encoder in distributed training. "
+            "Please set gradient_accumulation_steps to 1. This feature will be supported in the future."
+        )
+
+    # Make one log on every process with the configuration for debugging.
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,
+    )
+    logger.info(accelerator.state, main_process_only=False)
+    if accelerator.is_local_main_process:
+        transformers.utils.logging.set_verbosity_warning()
+        diffusers.utils.logging.set_verbosity_info()
+    else:
+        transformers.utils.logging.set_verbosity_error()
+        diffusers.utils.logging.set_verbosity_error()
+
+    if args.seed is not None:
+        set_seed(args.seed)
+
+    if accelerator.is_main_process:
+        if args.output_dir is not None:
+            os.makedirs(args.output_dir, exist_ok=True)
+
+    # Load the tokenizer  
+    if args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, revision=args.revision, use_fast=False)
+    elif args.pretrained_model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(
+            args.pretrained_model_name_or_path,
+            subfolder="tokenizer",
+            revision=args.revision,
+            use_fast=False,
+        )
+    # import correct text encoder class
+    text_encoder_cls = import_model_class_from_model_name_or_path(args.pretrained_model_name_or_path, args.revision)
+
+    # Load scheduler and models  
+    noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler")
+    noise_scheduler.add_noise = types.MethodType(add_noise_return_paras, noise_scheduler)
+    
+    text_encoder = text_encoder_cls.from_pretrained(
+        args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision
+    )
+
+    text_encoder.text_model.embeddings.forward = embedding_forward.__get__(text_encoder.text_model.embeddings)
+
+    
+    embedding_manager_config = OmegaConf.load(args.embedding_manager_config)
+    experiment_name = args.experiment_name
+
+    Embedding_Manager = EmbeddingManagerId_adain(   
+            tokenizer,
+            text_encoder,
+            device = accelerator.device,
+            training = True,
+            num_embeds_per_token = embedding_manager_config.model.personalization_config.params.num_embeds_per_token,            
+            token_dim = embedding_manager_config.model.personalization_config.params.token_dim,
+            mlp_depth = embedding_manager_config.model.personalization_config.params.mlp_depth,
+            loss_type = embedding_manager_config.model.personalization_config.params.loss_type,
+            input_dim = embedding_manager_config.model.personalization_config.params.input_dim,
+            experiment_name = experiment_name,
+    )
+    
+    Embedding_Manager.name_projection_layer.apply(weights_init_normal)
+    
+    Embedding_D = Embedding_discriminator(embedding_manager_config.model.personalization_config.params.token_dim * 2, dropout_rate = 0.2)
+    Embedding_D.apply(weights_init_normal)
+    
+    if args.pretrained_embedding_manager_path is not None:
+        epoch = args.pretrained_embedding_manager_epoch
+        embedding_manager_path = os.path.join(args.pretrained_embedding_manager_path, "embeddings_manager-{}.pt".format(epoch))
+        Embedding_Manager.load(embedding_manager_path)
+        embedding_D_path = os.path.join(args.pretrained_embedding_manager_path, "embedding_D-{}.pt".format(epoch))        
+        Embedding_D = torch.load(embedding_D_path)
+    
+    for param in Embedding_Manager.trainable_projection_parameters():
+        param.requires_grad = True
+    Embedding_D.requires_grad = True
+        
+    text_encoder.requires_grad_(False)
+
+
+    # Check that all trainable models are in full precision
+    low_precision_error_string = (
+        "Please make sure to always have all model weights in full float32 precision when starting training - even if"
+        " doing mixed precision training. copy of the weights should still be float32."
+    )
+
+    if accelerator.unwrap_model(text_encoder).dtype != torch.float32:
+        raise ValueError(
+            f"Text encoder loaded as datatype {accelerator.unwrap_model(text_encoder).dtype}."
+            f" {low_precision_error_string}"
+        )
+
+    # Enable TF32 for faster training on Ampere GPUs,
+    # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices
+    if args.allow_tf32:
+        torch.backends.cuda.matmul.allow_tf32 = True
+
+    if args.scale_lr:
+        args.learning_rate = (
+            args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes
+        )
+
+    # Use 8-bit Adam for lower memory usage or to fine-tune the model in 16GB GPUs
+    if args.use_8bit_adam:
+        try:
+            import bitsandbytes as bnb
+        except ImportError:
+            raise ImportError(
+                "To use 8-bit Adam, please install the bitsandbytes library: `pip install bitsandbytes`."
+            )
+
+        optimizer_class = bnb.optim.AdamW8bit
+    else:
+        optimizer_class = torch.optim.AdamW
+
+
+    projection_params_to_optimize = Embedding_Manager.trainable_projection_parameters()
+    optimizer_projection = optimizer_class(
+        projection_params_to_optimize,
+        lr=args.learning_rate,
+        betas=(args.adam_beta1, args.adam_beta2),
+        weight_decay=args.adam_weight_decay,
+        eps=args.adam_epsilon,
+    )
+
+    discriminator_params_to_optimize = list(Embedding_D.parameters())
+    optimizer_discriminator = optimizer_class(
+        discriminator_params_to_optimize,
+        lr=args.learning_rate,
+        betas=(args.adam_beta1, args.adam_beta2),
+        weight_decay=args.adam_weight_decay,
+        eps=args.adam_epsilon,
+    )
+
+
+    train_dataset = FaceIdDataset(
+        experiment_name = experiment_name
+    )
+    
+    print("dataset_length", train_dataset._length)
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset,
+        batch_size=args.train_batch_size,
+        shuffle=True,
+        num_workers=accelerator.num_processes,
+    )
+
+    # Scheduler and math around the number of training steps.
+    overrode_max_train_steps = False
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    if args.max_train_steps is None:
+        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
+        overrode_max_train_steps = True
+    
+    lr_scheduler_proj = get_scheduler(
+        args.lr_scheduler,
+        optimizer=optimizer_projection,
+        num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps,
+        num_training_steps=args.max_train_steps * args.gradient_accumulation_steps,
+        num_cycles=args.lr_num_cycles,
+        power=args.lr_power,
+    )
+
+    lr_scheduler_disc = get_scheduler(
+        args.lr_scheduler,
+        optimizer=optimizer_discriminator,
+        num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps,
+        num_training_steps=args.max_train_steps * args.gradient_accumulation_steps,
+        num_cycles=args.lr_num_cycles,
+        power=args.lr_power,
+    )    
+    
+    Embedding_Manager, optimizer_projection, optimizer_discriminator, train_dataloader, lr_scheduler_proj, lr_scheduler_disc = accelerator.prepare(
+        Embedding_Manager, optimizer_projection, optimizer_discriminator, train_dataloader, lr_scheduler_proj, lr_scheduler_disc
+    )
+
+
+    # For mixed precision training we cast the text_encoder and vae weights to half-precision
+    # as these models are only used for inference, keeping weights in full precision is not required.
+    weight_dtype = torch.float32
+    if accelerator.mixed_precision == "fp16":
+        weight_dtype = torch.float16
+    elif accelerator.mixed_precision == "bf16":
+        weight_dtype = torch.bfloat16
+
+    # Move vae and unet to device and cast to weight_dtype
+    text_encoder.to(accelerator.device, dtype=weight_dtype)
+    Embedding_Manager.to(accelerator.device, dtype=weight_dtype)
+    Embedding_D.to(accelerator.device, dtype=weight_dtype)
+
+    # We need to recalculate our total training steps as the size of the training dataloader may have changed.
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    if overrode_max_train_steps:
+        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
+
+    # Afterwards we recalculate our number of training epochs
+    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+
+    # We need to initialize the trackers we use, and also store our configuration.
+    # The trackers initializes automatically on the main process.
+    if accelerator.is_main_process:
+        accelerator.init_trackers("identity_space", config=vars(args))
+    
+    
+    # Train!
+    total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+
+    logger.info("***** Running training *****")
+    logger.info(f"  Num examples = {len(train_dataset)}")
+    logger.info(f"  Num batches each epoch = {len(train_dataloader)}")
+    logger.info(f"  Num Epochs = {args.num_train_epochs}")
+    logger.info(f"  Instantaneous batch size per device = {args.train_batch_size}")
+    logger.info(f"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
+    logger.info(f"  Gradient Accumulation steps = {args.gradient_accumulation_steps}")
+    logger.info(f"  Total optimization steps = {args.max_train_steps}")
+    global_step = 0
+    first_epoch = 0
+
+    # Potentially load in the weights and states from a previous save
+    if args.resume_from_checkpoint:
+        if args.resume_from_checkpoint != "latest":
+            path = os.path.basename(args.resume_from_checkpoint)
+        else:
+            # Get the mos recent checkpoint
+            dirs = os.listdir(args.output_dir)
+            dirs = [d for d in dirs if d.startswith("checkpoint")]
+            dirs = sorted(dirs, key=lambda x: int(x.split("-")[1]))
+            path = dirs[-1] if len(dirs) > 0 else None
+
+        if path is None:
+            accelerator.print(
+                f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run."
+            )
+            args.resume_from_checkpoint = None
+        else:
+            accelerator.print(f"Resuming from checkpoint {path}")
+            accelerator.load_state(os.path.join(args.output_dir, path))
+            global_step = int(path.split("-")[1])
+
+            resume_global_step = global_step * args.gradient_accumulation_steps
+            first_epoch = global_step // num_update_steps_per_epoch
+            resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps)
+
+    # Only show the progress bar once on each machine.
+    progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process)
+    progress_bar.set_description("Steps")
+
+    num_iter = 0 
+    # trained_images_num = 0
+    for epoch in range(first_epoch, args.num_train_epochs): 
+        print("=====================================")
+        print("epoch:", epoch)
+        print("=====================================")
+        Embedding_Manager.train()  
+        for step, batch in enumerate(train_dataloader):  
+                 
+            # Skip steps until we reach the resumed step 
+            if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step:
+                if step % args.gradient_accumulation_steps == 0:
+                    progress_bar.update(1)
+                continue
+
+            random_embeddings = torch.randn(1, 1, args.input_dim).to(accelerator.device)
+            random_embeddings = random_embeddings.repeat(args.train_batch_size, 1, 1)
+            
+            encoder_hidden_states, other_return_dict, positions_list = encode_prompt(batch["caption"], 
+                                                                                    batch["name"], 
+                                                                                    text_encoder, tokenizer, 
+                                                                                    Embedding_Manager, 
+                                                                                    is_train=True, 
+                                                                                    random_embeddings = random_embeddings, 
+                                                                                    timesteps = 0)
+
+            name_embeddings = other_return_dict["name_embeddings"]    
+            adained_total_embedding = other_return_dict["adained_total_embedding"]
+            fake_emb = adained_total_embedding
+            
+            criterionGAN = GANLoss().to(accelerator.device)
+            
+            set_requires_grad(Embedding_D, True)
+            optimizer_discriminator.zero_grad(set_to_none=args.set_grads_to_none)
+            # fake            
+            pred_fake = Embedding_D(fake_emb.detach()) 
+            loss_D_fake = criterionGAN(pred_fake[0], False)                   
+            
+            # Real
+            random_noise = torch.rand_like(name_embeddings) * 0.005   
+            real_name_embeddings = random_noise + name_embeddings
+            pred_real = Embedding_D(real_name_embeddings)  
+            loss_D_real = criterionGAN(pred_real[0], True)             
+            
+            loss_D = (loss_D_fake + loss_D_real) * 0.5
+            accelerator.backward(loss_D)
+            if accelerator.sync_gradients:
+                accelerator.clip_grad_norm_(discriminator_params_to_optimize, args.max_grad_norm)   
+            optimizer_discriminator.step()
+
+            set_requires_grad(Embedding_D, False)  
+            optimizer_projection.zero_grad(set_to_none=args.set_grads_to_none)
+            pred_fake = Embedding_D(fake_emb)
+             
+            loss_G_GAN = criterionGAN(pred_fake[0], True) 
+            
+            num_embeddings = encoder_hidden_states.size(0)   
+            loss_consistency = 0.0
+            for i in range(num_embeddings):
+                position1 = positions_list[i]
+                name_embedding1 = torch.cat([encoder_hidden_states[i][position1], encoder_hidden_states[i][position1 + 1]], dim=0)
+                for j in range(i + 1, num_embeddings):
+                    position2 = positions_list[j]
+                    name_embedding2 = torch.cat([encoder_hidden_states[j][position2], encoder_hidden_states[j][position2 + 1]], dim=0)
+                    loss_consistency += F.mse_loss(name_embedding1, name_embedding2)
+            
+            loss_consistency /= (num_embeddings * (num_embeddings - 1)) / 2
+            
+            loss = loss_G_GAN * args.l_gan_lambda + loss_consistency * args.l_consis_lambda
+            
+            accelerator.backward(loss)  
+            
+            if accelerator.sync_gradients: 
+                accelerator.clip_grad_norm_(projection_params_to_optimize, args.max_grad_norm)
+            optimizer_projection.step()
+            lr_scheduler_proj.step()
+            lr_scheduler_disc.step()
+
+            num_iter += 1
+
+            # Checks if the accelerator has performed an optimization step behind the scenes
+            if accelerator.sync_gradients:
+                progress_bar.update(1)
+                if global_step % args.checkpointing_steps == 0:
+                    if accelerator.is_main_process:
+                        save_path = os.path.join(args.output_dir, f"embeddings_manager-{global_step}.pt")
+                        # accelerator.save_state(save_path)
+                        try:
+                            Embedding_Manager.save(save_path)
+                        except:
+                            Embedding_Manager.module.save(save_path)
+                        
+                        save_path_d = os.path.join(args.output_dir, f"embedding_D-{global_step}.pt")
+                        Embedding_D.save(save_path_d)
+
+                        logger.info(f"Saved state to {save_path}")  
+                    
+                global_step += 1
+
+            adained_total_embeddings_max_min = (round(adained_total_embedding.max().detach().item(), 4),
+                                               round(adained_total_embedding.min().detach().item(), 4))  
+            
+            logs = {"m1": adained_total_embeddings_max_min,
+                    "l_G_GAN": loss_G_GAN.detach().item(),
+                    "l_consistency": loss_consistency.detach().item(),
+                    "l_D_real": loss_D_real.detach().item(),
+                    "l_D_fake": loss_D_fake.detach().item(),                     
+                    "loss": loss.detach().item(), 
+                    }
+            progress_bar.set_postfix(**logs)
+            accelerator.log(logs, step=global_step)
+            
+            if global_step >= args.max_train_steps:
+                break
+
+    # Create the pipeline using the trained modules and save it.
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

training_weight/man_GAN/embeddings_manager-7000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23ad2b0e562fac58f51b6deb1fb129b1317b76aa98bdf5b70e870c7a5ed38862
+size 17356032

training_weight/normal_GAN/embeddings_manager-10000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c0051eccaa1435d0ec678dc8d8e7130d09849b48aa7b67f51aee9aa4bad71cb
+size 17356044

training_weight/woman_GAN/embeddings_manager-6000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fc0dc612520375801ec596354479c194baee2f090a50f7ae7cc9e41479eb3f
+size 17356032

utils.py

@@ -0,0 +1,237 @@
+from PIL import Image
+
+import torch
+import torch.nn.functional as F
+
+from transformers.modeling_outputs import BaseModelOutputWithPooling
+from transformers.models.clip.modeling_clip import _make_causal_mask, _expand_mask
+from torch import autograd
+import accelerate
+import torch.nn as nn
+
+from PIL import Image
+import numpy as np
+
+def set_requires_grad(nets, requires_grad=False):
+    """Set requies_grad=Fasle for all the networks to avoid unnecessary computations
+    Parameters:
+        nets (network list)   -- a list of networks
+        requires_grad (bool)  -- whether the networks require gradients or not
+    """
+    if not isinstance(nets, list):
+        nets = [nets]
+    for net in nets:
+        if net is not None:
+            for param in net.parameters():
+                param.requires_grad = requires_grad
+
+def discriminator_r1_loss_accelerator(accelerator, real_pred, real_w):
+    grad_real, = accelerate.gradient(
+        outputs=real_pred.sum(), inputs=real_w, create_graph=True #, only_inputs=True
+    )
+    grad_penalty = grad_real.pow(2).reshape(grad_real.shape[0], -1).sum(1).mean()
+
+    return grad_penalty
+
+class GANLoss(nn.Module):
+    def __init__(self, use_lsgan=True, target_real_label=1.0, target_fake_label=0.0):
+        super(GANLoss, self).__init__()
+        self.register_buffer('real_label', torch.tensor(target_real_label))
+        self.register_buffer('fake_label', torch.tensor(target_fake_label))
+        if use_lsgan:
+            self.loss = nn.MSELoss()
+        else:
+            self.loss = nn.BCEWithLogitsLoss()
+
+    def get_target_tensor(self, input, target_is_real):
+        if target_is_real:
+            target_tensor = self.real_label
+        else:
+            target_tensor = self.fake_label
+        return target_tensor.expand_as(input)
+
+    def __call__(self, input, target_is_real):
+        target_tensor = self.get_target_tensor(input, target_is_real)
+        return self.loss(input, target_tensor)
+
+
+
+def discriminator_r1_loss(real_pred, real_w):
+    grad_real, = autograd.grad(
+        outputs=real_pred.sum(), inputs=real_w, create_graph=True #, only_inputs=True
+    )
+    grad_penalty = grad_real.pow(2).reshape(grad_real.shape[0], -1).sum(1).mean()
+
+    return grad_penalty
+
+def add_noise_return_paras(
+    self,
+    original_samples: torch.FloatTensor,
+    noise: torch.FloatTensor,
+    timesteps: torch.IntTensor,
+) -> torch.FloatTensor:
+    # Make sure alphas_cumprod and timestep have same device and dtype as original_samples
+    alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device, dtype=original_samples.dtype)
+    timesteps = timesteps.to(original_samples.device)
+
+    sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
+    sqrt_alpha_prod = sqrt_alpha_prod.flatten()
+    while len(sqrt_alpha_prod.shape) < len(original_samples.shape):
+        sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
+
+    sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
+    sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
+    while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape):
+        sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
+
+    noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
+    return noisy_samples, sqrt_alpha_prod, sqrt_one_minus_alpha_prod
+
+
+def text_encoder_forward(
+    text_encoder = None,
+    input_ids = None,
+    name_batch = None,
+    attention_mask = None,
+    position_ids = None,
+    output_attentions = None,
+    output_hidden_states = None,
+    return_dict = None,
+    embedding_manager = None,
+    only_embedding=False,
+    random_embeddings = None,
+    timesteps = None,
+):
+    output_attentions = output_attentions if output_attentions is not None else text_encoder.config.output_attentions
+    output_hidden_states = (
+        output_hidden_states if output_hidden_states is not None else text_encoder.config.output_hidden_states
+    )
+    return_dict = return_dict if return_dict is not None else text_encoder.config.use_return_dict
+
+    if input_ids is None:
+        raise ValueError("You have to specify either input_ids")
+
+    input_shape = input_ids.size()
+    input_ids = input_ids.view(-1, input_shape[-1])
+    
+    hidden_states, other_return_dict = text_encoder.text_model.embeddings(input_ids=input_ids, 
+                                                                          position_ids=position_ids,
+                                                                          name_batch = name_batch,
+                                                                          embedding_manager=embedding_manager,
+                                                                          only_embedding=only_embedding,
+                                                                          random_embeddings = random_embeddings,
+                                                                          timesteps = timesteps,
+                                                                          )
+    if only_embedding:
+        return hidden_states
+
+
+    causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device)
+    if attention_mask is not None:
+        attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
+
+    encoder_outputs = text_encoder.text_model.encoder(
+        inputs_embeds=hidden_states,
+        attention_mask=attention_mask,
+        causal_attention_mask=causal_attention_mask,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+    )
+
+    last_hidden_state = encoder_outputs[0]
+    last_hidden_state = text_encoder.text_model.final_layer_norm(last_hidden_state)
+
+    if text_encoder.text_model.eos_token_id == 2:
+        pooled_output = last_hidden_state[
+            torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+            input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1),
+        ]
+    else:
+        pooled_output = last_hidden_state[
+            torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device),
+            (input_ids.to(dtype=torch.int, device=last_hidden_state.device) == text_encoder.text_model.eos_token_id)
+            .int()
+            .argmax(dim=-1),
+        ]
+
+    if not return_dict:
+        return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+    return BaseModelOutputWithPooling(
+        last_hidden_state=last_hidden_state,
+        pooler_output=pooled_output,
+        hidden_states=encoder_outputs.hidden_states,
+        attentions=encoder_outputs.attentions,
+    )[0], other_return_dict  
+
+
+def downsampling(img: torch.tensor, w: int, h: int) -> torch.tensor:
+    return F.interpolate(
+        img.unsqueeze(0).unsqueeze(1),
+        size=(w, h),
+        mode="bilinear",
+        align_corners=True,
+    ).squeeze()
+
+
+def image_grid(images, rows=2, cols=2):
+    w, h = images[0].size
+    grid = Image.new('RGB', size=(cols * w, rows * h))
+
+    for i, img in enumerate(images):
+        grid.paste(img, box=(i % cols * w, i // cols * h))
+    return grid
+
+
+def latents_to_images(vae, latents, scale_factor=0.18215):
+    """
+    Decode latents to PIL images.
+    """
+    scaled_latents = 1.0 / scale_factor * latents.clone()
+    images = vae.decode(scaled_latents).sample
+    images = (images / 2 + 0.5).clamp(0, 1)
+    images = images.detach().cpu().permute(0, 2, 3, 1).numpy()
+
+    if images.ndim == 3:
+        images = images[None, ...]
+    images = (images * 255).round().astype("uint8")
+    pil_images = [Image.fromarray(image) for image in images]
+
+    return pil_images
+
+
+def merge_and_save_images(output_images):
+    image_size = output_images[0].size
+
+    merged_width = len(output_images) * image_size[0]
+    merged_height = image_size[1]
+
+    merged_image = Image.new('RGB', (merged_width, merged_height), (255, 255, 255))
+
+    for i, image in enumerate(output_images):
+        merged_image.paste(image, (i * image_size[0], 0))
+
+    return merged_image
+
+class GANLoss(nn.Module):
+    def __init__(self, use_lsgan=True, target_real_label=1.0, target_fake_label=0.0):
+        super(GANLoss, self).__init__()
+        self.register_buffer('real_label', torch.tensor(target_real_label))
+        self.register_buffer('fake_label', torch.tensor(target_fake_label))
+
+        if use_lsgan:
+            self.loss = nn.MSELoss() 
+        else:
+            self.loss = nn.BCELoss()
+
+    def get_target_tensor(self, input, target_is_real): 
+        if target_is_real:
+            target_tensor = self.real_label
+        else:
+            target_tensor = self.fake_label
+        return target_tensor.expand_as(input)
+
+    def __call__(self, input, target_is_real):
+        target_tensor = self.get_target_tensor(input, target_is_real)
+        return self.loss(input, target_tensor)