004T patrickvonplaten commited on
Commit
ceeafaf
0 Parent(s):

Duplicate from kandinsky-community/kandinsky-2-1

Browse files

Co-authored-by: Patrick von Platen <patrickvonplaten@users.noreply.huggingface.co>

.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ prior:
4
+ - kandinsky-community/kandinsky-2-1-prior
5
+ tags:
6
+ - text-to-image
7
+ - kandinsky
8
+ inference: false
9
+ ---
10
+
11
+ # Kandinsky 2.1
12
+
13
+ Kandinsky 2.1 inherits best practices from Dall-E 2 and Latent diffusion while introducing some new ideas.
14
+
15
+ It uses the CLIP model as a text and image encoder, and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.
16
+
17
+ The Kandinsky model is created by [Arseniy Shakhmatov](https://github.com/cene555), [Anton Razzhigaev](https://github.com/razzant), [Aleksandr Nikolich](https://github.com/AlexWortega), [Igor Pavlov](https://github.com/boomb0om), [Andrey Kuznetsov](https://github.com/kuznetsoffandrey) and [Denis Dimitrov](https://github.com/denndimitrov)
18
+
19
+ ## Usage
20
+
21
+ Kandinsky 2.1 is available in diffusers!
22
+
23
+ ```python
24
+ pip install diffusers transformers accelerate
25
+ ```
26
+ ### Text to image
27
+
28
+ ```python
29
+ from diffusers import AutoPipelineForText2Image
30
+ import torch
31
+
32
+ pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16)
33
+ pipe.enable_model_cpu_offload()
34
+
35
+ prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting"
36
+ negative_prompt = "low quality, bad quality"
37
+
38
+ image = pipe(prompt=prompt, negative_prompt=negative_prompt, prior_guidance_scale =1.0, height=768, width=768).images[0]
39
+ image.save("cheeseburger_monster.png")
40
+ ```
41
+
42
+ ![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/cheeseburger.png)
43
+
44
+
45
+ ### Text Guided Image-to-Image Generation
46
+
47
+ ```python
48
+ from diffusers import AutoPipelineForImage2Image
49
+ import torch
50
+ import requests
51
+ from io import BytesIO
52
+ from PIL import Image
53
+ import os
54
+
55
+ pipe = AutoPipelineForImage2Image.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16)
56
+
57
+ pipe.enable_model_cpu_offload()
58
+
59
+ prompt = "A fantasy landscape, Cinematic lighting"
60
+ negative_prompt = "low quality, bad quality"
61
+
62
+ url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
63
+
64
+ response = requests.get(url)
65
+ original_image = Image.open(BytesIO(response.content)).convert("RGB")
66
+ original_image.thumbnail((768, 768))
67
+
68
+ image = pipe(prompt=prompt, image=original_image, strength=0.3).images[0]
69
+ out.images[0].save("fantasy_land.png")
70
+ ```
71
+
72
+ ![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/img2img_fantasyland.png)
73
+
74
+
75
+ ### Interpolate
76
+
77
+ ```python
78
+ from diffusers import KandinskyPriorPipeline, KandinskyPipeline
79
+ from diffusers.utils import load_image
80
+ import PIL
81
+
82
+ import torch
83
+
84
+ pipe_prior = KandinskyPriorPipeline.from_pretrained(
85
+ "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16
86
+ )
87
+ pipe_prior.to("cuda")
88
+
89
+ img1 = load_image(
90
+ "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png"
91
+ )
92
+
93
+ img2 = load_image(
94
+ "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/starry_night.jpeg"
95
+ )
96
+
97
+ # add all the conditions we want to interpolate, can be either text or image
98
+ images_texts = ["a cat", img1, img2]
99
+
100
+ # specify the weights for each condition in images_texts
101
+ weights = [0.3, 0.3, 0.4]
102
+
103
+ # We can leave the prompt empty
104
+ prompt = ""
105
+ prior_out = pipe_prior.interpolate(images_texts, weights)
106
+
107
+ pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16)
108
+ pipe.to("cuda")
109
+
110
+ image = pipe(prompt, **prior_out, height=768, width=768).images[0]
111
+
112
+ image.save("starry_cat.png")
113
+ ```
114
+ ![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/starry_cat.png)
115
+
116
+
117
+ ## Model Architecture
118
+
119
+ ### Overview
120
+ Kandinsky 2.1 is a text-conditional diffusion model based on unCLIP and latent diffusion, composed of a transformer-based image prior model, a unet diffusion model, and a decoder.
121
+
122
+ The model architectures are illustrated in the figure below - the chart on the left describes the process to train the image prior model, the figure in the center is the text-to-image generation process, and the figure on the right is image interpolation.
123
+
124
+ <p float="left">
125
+ <img src="https://raw.githubusercontent.com/ai-forever/Kandinsky-2/main/content/kandinsky21.png"/>
126
+ </p>
127
+
128
+ Specifically, the image prior model was trained on CLIP text and image embeddings generated with a pre-trained [mCLIP model](https://huggingface.co/M-CLIP/XLM-Roberta-Large-Vit-L-14). The trained image prior model is then used to generate mCLIP image embeddings for input text prompts. Both the input text prompts and its mCLIP image embeddings are used in the diffusion process. A [MoVQGAN](https://openreview.net/forum?id=Qb-AoSw4Jnm) model acts as the final block of the model, which decodes the latent representation into an actual image.
129
+
130
+
131
+ ### Details
132
+ The image prior training of the model was performed on the [LAION Improved Aesthetics dataset](https://huggingface.co/datasets/bhargavsdesai/laion_improved_aesthetics_6.5plus_with_images), and then fine-tuning was performed on the [LAION HighRes data](https://huggingface.co/datasets/laion/laion-high-resolution).
133
+
134
+ The main Text2Image diffusion model was trained on the basis of 170M text-image pairs from the [LAION HighRes dataset](https://huggingface.co/datasets/laion/laion-high-resolution) (an important condition was the presence of images with a resolution of at least 768x768). The use of 170M pairs is due to the fact that we kept the UNet diffusion block from Kandinsky 2.0, which allowed us not to train it from scratch. Further, at the stage of fine-tuning, a dataset of 2M very high-quality high-resolution images with descriptions (COYO, anime, landmarks_russia, and a number of others) was used separately collected from open sources.
135
+
136
+
137
+ ### Evaluation
138
+ We quantitatively measure the performance of Kandinsky 2.1 on the COCO_30k dataset, in zero-shot mode. The table below presents FID.
139
+
140
+ FID metric values ​​for generative models on COCO_30k
141
+ | | FID (30k)|
142
+ |:------|----:|
143
+ | eDiff-I (2022) | 6.95 |
144
+ | Image (2022) | 7.27 |
145
+ | Kandinsky 2.1 (2023) | 8.21|
146
+ | Stable Diffusion 2.1 (2022) | 8.59 |
147
+ | GigaGAN, 512x512 (2023) | 9.09 |
148
+ | DALL-E 2 (2022) | 10.39 |
149
+ | GLIDE (2022) | 12.24 |
150
+ | Kandinsky 1.0 (2022) | 15.40 |
151
+ | DALL-E (2021) | 17.89 |
152
+ | Kandinsky 2.0 (2022) | 20.00 |
153
+ | GLIGEN (2022) | 21.04 |
154
+
155
+ For more information, please refer to the upcoming technical report.
156
+
157
+ ## BibTex
158
+ If you find this repository useful in your research, please cite:
159
+ ```
160
+ @misc{kandinsky 2.1,
161
+ title = {kandinsky 2.1},
162
+ author = {Arseniy Shakhmatov, Anton Razzhigaev, Aleksandr Nikolich, Vladimir Arkhipkin, Igor Pavlov, Andrey Kuznetsov, Denis Dimitrov},
163
+ year = {2023},
164
+ howpublished = {},
165
+ }
166
+ ```
ddpm_scheduler/scheduler_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "DDPMScheduler",
3
+ "_diffusers_version": "0.18.0.dev0",
4
+ "beta_end": 0.012,
5
+ "beta_schedule": "linear",
6
+ "beta_start": 0.00085,
7
+ "clip_sample": false,
8
+ "clip_sample_range": 2.0,
9
+ "dynamic_thresholding_ratio": 0.995,
10
+ "num_train_timesteps": 1000,
11
+ "prediction_type": "epsilon",
12
+ "sample_max_value": 2.0,
13
+ "thresholding": true,
14
+ "trained_betas": null,
15
+ "variance_type": "learned_range"
16
+ }
model_index.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "KandinskyPipeline",
3
+ "_diffusers_version": "0.17.0.dev0",
4
+ "text_encoder": [
5
+ "kandinsky",
6
+ "MultilingualCLIP"
7
+ ],
8
+ "tokenizer": [
9
+ "transformers",
10
+ "XLMRobertaTokenizerFast"
11
+ ],
12
+ "scheduler": [
13
+ "diffusers",
14
+ "DDIMScheduler"
15
+ ],
16
+ "unet": [
17
+ "diffusers",
18
+ "UNet2DConditionModel"
19
+ ],
20
+ "movq": [
21
+ "diffusers",
22
+ "VQModel"
23
+ ]
24
+ }
movq/config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "VQModel",
3
+ "_diffusers_version": "0.17.0.dev0",
4
+ "act_fn": "silu",
5
+ "block_out_channels": [
6
+ 128,
7
+ 256,
8
+ 256,
9
+ 512
10
+ ],
11
+ "down_block_types": [
12
+ "DownEncoderBlock2D",
13
+ "DownEncoderBlock2D",
14
+ "DownEncoderBlock2D",
15
+ "AttnDownEncoderBlock2D"
16
+ ],
17
+ "in_channels": 3,
18
+ "latent_channels": 4,
19
+ "layers_per_block": 2,
20
+ "norm_num_groups": 32,
21
+ "norm_type": "spatial",
22
+ "num_vq_embeddings": 16384,
23
+ "out_channels": 3,
24
+ "sample_size": 32,
25
+ "scaling_factor": 0.18215,
26
+ "up_block_types": [
27
+ "AttnUpDecoderBlock2D",
28
+ "UpDecoderBlock2D",
29
+ "UpDecoderBlock2D",
30
+ "UpDecoderBlock2D"
31
+ ],
32
+ "vq_embed_dim": 4
33
+ }
movq/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:772e09739d742ddee6807add2d3c2fd2a32db53896b5d07a92c729d8c879ce59
3
+ size 271492131
movq/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:667319216c14764042b3e7cae7c4d1a309a409a674d413144e9ac65b572c6ea8
3
+ size 271380362
scheduler/scheduler_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "DDIMScheduler",
3
+ "_diffusers_version": "0.17.0.dev0",
4
+ "num_train_timesteps": 1000,
5
+ "beta_schedule": "linear",
6
+ "beta_start": 0.00085,
7
+ "beta_end":0.012,
8
+ "clip_sample" : false,
9
+ "set_alpha_to_one" : false,
10
+ "steps_offset" : 1,
11
+ "prediction_type" : "epsilon",
12
+ "thresholding" : false
13
+ }
text_encoder/config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MultilingualCLIP"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "bos_token_id": 0,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "gelu",
9
+ "hidden_dropout_prob": 0.1,
10
+ "hidden_size": 1024,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 4096,
13
+ "layer_norm_eps": 1e-05,
14
+ "max_position_embeddings": 514,
15
+ "model_type": "xlm-roberta",
16
+ "num_attention_heads": 16,
17
+ "num_hidden_layers": 24,
18
+ "output_past": true,
19
+ "pad_token_id": 1,
20
+ "position_embedding_type": "absolute",
21
+ "transformers_version": "4.17.0.dev0",
22
+ "type_vocab_size": 1,
23
+ "use_cache": true,
24
+ "vocab_size": 250002,
25
+ "numDims": 768,
26
+ "transformerDimensions": 1024
27
+ }
text_encoder/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6fd189d4154f9b69e0c39fc32276b9dd19fe273d865a8f5ae76bfadba69f9e6b
3
+ size 2242765066
text_encoder/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:152bab8bc5813ec882ec634c7301978a0ef29340d4c17b3ae77b6468e017dfbe
3
+ size 2242843757
tokenizer/sentencepiece.bpe.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
3
+ size 5069051
tokenizer/special_tokens_map.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "mask_token": {
6
+ "content": "<mask>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "pad_token": "<pad>",
13
+ "sep_token": "</s>",
14
+ "unk_token": "<unk>"
15
+ }
tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "clean_up_tokenization_spaces": true,
4
+ "cls_token": "<s>",
5
+ "eos_token": "</s>",
6
+ "mask_token": {
7
+ "__type": "AddedToken",
8
+ "content": "<mask>",
9
+ "lstrip": true,
10
+ "normalized": true,
11
+ "rstrip": false,
12
+ "single_word": false
13
+ },
14
+ "model_max_length": 512,
15
+ "pad_token": "<pad>",
16
+ "sep_token": "</s>",
17
+ "tokenizer_class": "XLMRobertaTokenizer",
18
+ "unk_token": "<unk>"
19
+ }
unet/config.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "UNet2DConditionModel",
3
+ "_diffusers_version": "0.17.0.dev0",
4
+ "act_fn": "silu",
5
+ "addition_embed_type": "text_image",
6
+ "addition_embed_type_num_heads": 64,
7
+ "attention_head_dim": 64,
8
+ "block_out_channels": [
9
+ 384,
10
+ 768,
11
+ 1152,
12
+ 1536
13
+ ],
14
+ "center_input_sample": false,
15
+ "class_embed_type": null,
16
+ "class_embeddings_concat": false,
17
+ "conv_in_kernel": 3,
18
+ "conv_out_kernel": 3,
19
+ "cross_attention_dim": 768,
20
+ "cross_attention_norm": null,
21
+ "down_block_types": [
22
+ "ResnetDownsampleBlock2D",
23
+ "SimpleCrossAttnDownBlock2D",
24
+ "SimpleCrossAttnDownBlock2D",
25
+ "SimpleCrossAttnDownBlock2D"
26
+ ],
27
+ "downsample_padding": 1,
28
+ "dual_cross_attention": false,
29
+ "encoder_hid_dim": 1024,
30
+ "encoder_hid_dim_type": "text_image_proj",
31
+ "flip_sin_to_cos": true,
32
+ "freq_shift": 0,
33
+ "in_channels": 4,
34
+ "layers_per_block": 3,
35
+ "mid_block_only_cross_attention": null,
36
+ "mid_block_scale_factor": 1,
37
+ "mid_block_type": "UNetMidBlock2DSimpleCrossAttn",
38
+ "norm_eps": 1e-05,
39
+ "norm_num_groups": 32,
40
+ "num_class_embeds": null,
41
+ "only_cross_attention": false,
42
+ "out_channels": 8,
43
+ "projection_class_embeddings_input_dim": null,
44
+ "resnet_out_scale_factor": 1.0,
45
+ "resnet_skip_time_act": false,
46
+ "resnet_time_scale_shift": "scale_shift",
47
+ "sample_size": 64,
48
+ "time_cond_proj_dim": null,
49
+ "time_embedding_act_fn": null,
50
+ "time_embedding_dim": null,
51
+ "time_embedding_type": "positional",
52
+ "timestep_post_act": null,
53
+ "up_block_types": [
54
+ "SimpleCrossAttnUpBlock2D",
55
+ "SimpleCrossAttnUpBlock2D",
56
+ "SimpleCrossAttnUpBlock2D",
57
+ "ResnetUpsampleBlock2D"
58
+ ],
59
+ "upcast_attention": false,
60
+ "use_linear_projection": false
61
+ }
unet/diffusion_pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf4c63c68860933264eaa290f7f9e308ce5d69a8a3012c26822de98aa056ab30
3
+ size 4914888985
unet/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a963e1be76a02b71fce27ad8321b42f707ba8fe9839f1696aeb4f36a48a7604
3
+ size 4914727722