added image variations

app/api/generate.py

@@ -140,17 +140,16 @@ def image_variations(model: Request, request: ImageVariations, db: Annotated[Ses
     if not current_user.is_active:
         raise HTTPException(status_code=403, detail="Forbidden")
 
-    # prompt = create_prompt(request.prompt, medium=request.medium, style=request.style, artist=request.artist, website=request.website, resolution=request.resolution, additional_details=request.additional_details, color=request.color, lightning=request.lightning)
-    # image = decode_image(request.image)
-    # image.resize((512, 512))
+    prompt = create_prompt(request.prompt, medium=request.medium, style=request.style, artist=request.artist, website=request.website, resolution=request.resolution, additional_details=request.additional_details, color=request.color, lightning=request.lightning)
+    image = decode_image(request.image)
+    image.resize((512, 512))
 
-    # if prompt:
-    #     crud.create_prompt(db=db, user_id=current_user.user_id, prompt=prompt)
+    if prompt:
+        crud.create_prompt(db=db, user_id=current_user.user_id, prompt=prompt)
 
-    # images = model.state.iv_model.generate(pil_image=image, num_samples=request.num_samples, num_inference_steps=request.num_inference_steps,
-    #                                        seed=request.seed, prompt=prompt, scale=request.scale, negative_prompt=request.negative_prompt)
+    images = model.state.iv_model.generate(pil_image=image, num_samples=request.num_samples, num_inference_steps=request.num_inference_steps,
+                                           seed=request.seed, prompt=prompt, scale=request.scale, negative_prompt=request.negative_prompt)
 
-    # images = [encode_image(image) for image in images]
+    images = [encode_image(image) for image in images]
 
-    # return images
-    return ["Image Variations is not supported yet."]
+    return images

app/config.py

@@ -3,7 +3,7 @@ import os
 
 DATABASE_URL = 'sqlite:///./sql_app.db'
 
-SECRET_KEY =  os.environ.get("SECRET_KEY")
+SECRET_KEY = os.environ.get("SECRET_KEY")
 ALGORITHM = os.environ.get("ALGORITHM")
 
 ACCESS_TOKEN_EXPIRE_MINUTES = 30

app/main.py

@@ -6,7 +6,7 @@ from fastapi.security import OAuth2PasswordRequestForm
 from fastapi import APIRouter, FastAPI, HTTPException, Depends
 
 import torch
-# from ip_adapter import IPAdapterXL
+from ip_adapter import IPAdapterXL
 from transformers import AutoModelForImageSegmentation
 from diffusers import AutoPipelineForText2Image, DPMSolverMultistepScheduler, StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler, StableDiffusionXLPipeline
 
@@ -40,20 +40,20 @@ async def lifespan(app: FastAPI):
         "briaai/RMBG-1.4", trust_remote_code=True)
     br_model.to(DEVICE)
 
-    # sdxl_pipe = StableDiffusionXLPipeline.from_pretrained(
-    #     base_model_path,
-    #     torch_dtype=torch.float16,
-    #     add_watermarker=False,
-    # )
-    # iv_model = IPAdapterXL(sdxl_pipe, image_encoder_path, ip_ckpt, DEVICE)
+    sdxl_pipe = StableDiffusionXLPipeline.from_pretrained(
+        base_model_path,
+        torch_dtype=torch.float16,
+        add_watermarker=False,
+    )
+    iv_model = IPAdapterXL(sdxl_pipe, image_encoder_path, ip_ckpt, DEVICE)
 
-    yield {'ti_pipe': ti_pipe, 'ii_pipe': ii_pipe, 'br_model': br_model}  # , 'iv_model': iv_model
+    yield {'ti_pipe': ti_pipe, 'ii_pipe': ii_pipe, 'br_model': br_model, 'iv_model': iv_model}
 
     del ti_pipe
     del ii_pipe
     del br_model
-    # del sdxl_pipe
-    # del iv_model
+    del sdxl_pipe
+    del iv_model
 
 
 app = FastAPI(lifespan=lifespan)
@@ -93,7 +93,7 @@ async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()], db:
     return {"access_token": access_token, "token_type": "bearer"}
 
 
-router.include_router(user.router, prefix="/users")
-router.include_router(prompt.router, prefix="/prompts")
-router.include_router(generate.router, prefix="/generate")
+router.include_router(user.router, prefix="/users", tags=["users"])
+router.include_router(prompt.router, prefix="/prompts", tags=["prompts"])
+router.include_router(generate.router, prefix="/generate", tags=["generate"])
 app.include_router(router)

requirements.txt

@@ -12,4 +12,5 @@ safetensors
 torch
 torchvision
 pillow
-numpy
+einops
+git+https://github.com/tencent-ailab/IP-Adapter.git

sdxl_models/image_encoder/config.json

@@ -0,0 +1,81 @@
+{
+  "architectures": [
+    "CLIPVisionModelWithProjection"
+  ],
+  "_name_or_path": "",
+  "add_cross_attention": false,
+  "architectures": null,
+  "attention_dropout": 0.0,
+  "bad_words_ids": null,
+  "begin_suppress_tokens": null,
+  "bos_token_id": null,
+  "chunk_size_feed_forward": 0,
+  "cross_attention_hidden_size": null,
+  "decoder_start_token_id": null,
+  "diversity_penalty": 0.0,
+  "do_sample": false,
+  "dropout": 0.0,
+  "early_stopping": false,
+  "encoder_no_repeat_ngram_size": 0,
+  "eos_token_id": null,
+  "exponential_decay_length_penalty": null,
+  "finetuning_task": null,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "hidden_act": "gelu",
+  "hidden_size": 1664,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+      },
+  "image_size": 224,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "is_decoder": false,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+      },
+  "layer_norm_eps": 1e-05,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "min_length": 0,
+  "model_type": "clip_vision_model",
+  "no_repeat_ngram_size": 0,
+  "num_attention_heads": 16,
+  "num_beam_groups": 1,
+  "num_beams": 1,
+  "num_channels": 3,
+  "num_hidden_layers": 48,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_scores": false,
+  "pad_token_id": null,
+  "patch_size": 14,
+  "prefix": null,
+  "problem_type": null,
+  "pruned_heads": {},
+  "remove_invalid_values": false,
+  "repetition_penalty": 1.0,
+  "return_dict": true,
+  "return_dict_in_generate": false,
+  "sep_token_id": null,
+  "suppress_tokens": null,
+  "task_specific_params": null,
+  "temperature": 1.0,
+  "tf_legacy_loss": false,
+  "tie_encoder_decoder": false,
+  "tie_word_embeddings": true,
+  "tokenizer_class": null,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torch_dtype": null,
+  "torchscript": false,
+  "transformers_version": "4.24.0",
+  "typical_p": 1.0,
+  "use_bfloat16": false,
+  "projection_dim": 1280
+}

sdxl_models/image_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657723e09f46a7c3957df651601029f66b1748afb12b419816330f16ed45d64d
+size 3689912664

sdxl_models/image_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2999562fbc02f9dc0d9c0acb7cf0970ec3a9b2a578d7d05afe82191d606d2d80
+size 3690112753

sdxl_models/ip-adapter-plus-face_sdxl_vit-h.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50e886d82940b3c5873d80c2b06d8a4b0d0fccec70bc44fd53f16ac3cfd7fc36
+size 1013454761

sdxl_models/ip-adapter-plus-face_sdxl_vit-h.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677ad8860204f7d0bfba12d29e6c31ded9beefdf3e4bbd102518357d31a292c1
+size 847517512

sdxl_models/ip-adapter-plus_sdxl_vit-h.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec70edb7cc8e769c9388d94eeaea3e4526352c9fae793a608782d1d8951fde90
+size 1013454427

sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f5062b8400c94b7159665b21ba5c62acdcd7682262743d7f2aefedef00e6581
+size 847517512

sdxl_models/ip-adapter_sdxl.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7525f2731e9e86d1368e0b68467615d55dda459691965bdd7d37fa3d7fd84c12
+size 702585097

sdxl_models/ip-adapter_sdxl.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba1002529e783604c5f326d49f0122025392d1d20ac8d573b3eeb3e6dea4ebb6
+size 702585376

sdxl_models/ip-adapter_sdxl_vit-h.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b382e2501d0ab3fe2e09312e561a59cd3f21262aff25373700e0cd62c635929
+size 698390793

sdxl_models/ip-adapter_sdxl_vit-h.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebf05d918348aec7abb02a5e9ecef77e0aaea6914a5c4ea13f50d45eb1681831
+size 698391064