index.py

import os
import base64
import numpy as np
from PIL import Image
import io
import requests

import replicate
import gradio as gr

import openai
from openai import OpenAI

from dotenv import load_dotenv, find_dotenv

# Locate the .env file
dotenv_path = find_dotenv()

load_dotenv(dotenv_path)

OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
REPLICATE_API_TOKEN = os.getenv('REPLICATE_API_TOKEN')

client = OpenAI()

def main(img, strength):
    mask = img['layers'][0]

    # Match prompt strength from .4 to 1 (total destruction)
    prompt_strength = round(-0.6 * strength + 1, 2)

    base_image = Image.fromarray(img['background'].astype('uint8'))
    img_base_64 = img_to_base64(base_image)

    if is_transparent(mask) == True:
        mask_base_64 = None
    else:
        mask_img = create_mask_image(mask)
        mask_base_64 = img_to_base64(mask_img)

    prompt = call_openai(img_base_64)
    #prompt = "The image shows a person wearing sleek, over-ear headphones with a matte finish and a cool, light beige color (Pantone 7527 C), captured under soft, diffused natural lighting, emphasizing the smooth and minimalist design of the headphones."

    output_urls = generate_image(prompt, img_base_64, mask_base_64, prompt_strength)
    output_images = [download_image(url) for url in output_urls[3:]]  # Start from the 4th image

    return output_images

def generate_image(prompt, img, mask, prompt_strength):
    input_data = {
        "image": img,
        "prompt": prompt + " expensive",
        "refine": "no_refiner",
        "scheduler": "K_EULER",
        "lora_scale": 0.8,
        "num_outputs": 4,
        "controlnet_1": "edge_canny",
        "controlnet_2": "depth_midas",
        "controlnet_3": "lineart",
        "guidance_scale": 4,
        "apply_watermark": False,
        "negative_prompt":"worst quality, low quality, illustration, 2d, painting, cartoons, sketch, logo",
        "prompt_strength": prompt_strength,
        "sizing_strategy": "controlnet_1_image",
        "controlnet_1_end": 1,
        "controlnet_2_end": 1,
        "controlnet_3_end": 1,
        "controlnet_1_image": img,
        "controlnet_1_start": 0,
        "controlnet_2_image": img,
        "controlnet_2_start": 0,
        "controlnet_3_image": img,
        "controlnet_3_start": 0,
        "num_inference_steps": 30,
        "controlnet_1_conditioning_scale": 0.8,
        "controlnet_2_conditioning_scale": 0.8,
        "controlnet_3_conditioning_scale": 0.75
    }

    if mask is not None:
        input_data["mask"] = mask
    else:
        input_data["prompt_strength"] = .3

    output = replicate.run(
        "fofr/realvisxl-v3-multi-controlnet-lora:90a4a3604cd637cb9f1a2bdae1cfa9ed869362ca028814cdce310a78e27daade",
        input=input_data
    )

    return output

def download_image(url):
    response = requests.get(url)
    img = Image.open(io.BytesIO(response.content))
    return img

def create_mask_image(mask_array):
    # Convert the mask to a numpy array if it's not already
    if not isinstance(mask_array, np.ndarray):
        mask_array = np.array(mask_array)
    
    # Create a new array with the same shape as the mask, but only for RGB channels
    processed_mask = np.zeros((mask_array.shape[0], mask_array.shape[1], 3), dtype=np.uint8)
    
    # Set transparent parts (alpha=0) to black (0, 0, 0)
    transparent_mask = mask_array[:, :, 3] == 0
    processed_mask[transparent_mask] = [0, 0, 0]
    
    # Set black parts (RGB=0, 0, 0 and alpha=255) to white (255, 255, 255)
    black_mask = (mask_array[:, :, :3] == [0, 0, 0]).all(axis=2) & (mask_array[:, :, 3] == 255)
    processed_mask[black_mask] = [255, 255, 255]
    
    return Image.fromarray(processed_mask)

def is_transparent(mask_array):
    return np.all(mask_array[:, :, 3] == 0)

def img_to_base64(img):
    # Extract the format of the image (e.g., JPEG, PNG)
    img_format = img.format if img.format else "PNG"
    
    # Convert the image to bytes
    buffered = io.BytesIO()
    img.save(buffered, format=img_format)
    img_base_64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
    return f"data:image/{img_format.lower()};base64," + img_base_64

def call_openai(image_data):
    try:
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": "Please describe this image in one sentence, with a focus on the material, finish and specific color (color is really important, so provide specific pantone colors), whether the color is warm or cool, and details of the main object in the scene. Mention the type of lighting as well."},
                        {
                            "type": "image_url",
                            "image_url": {
                                "url": image_data,
                            },
                        },
                    ],
                }
            ],
            max_tokens=300,
        )
        return response.choices[0].message.content
    except openai.BadRequestError as e:
        print(e)
        print("e type")
        print(type(e))
        raise gr.Error(f"You uploaded an unsupported image. Please make sure your image is below 20 MB in size and is of one the following formats: ['png', 'jpeg', 'gif', 'webp']")
    except Exception as e:
        raise gr.Error("Unknown Error")

# Define the brush with only black color
black_brush = gr.Brush(colors=["#000000"], default_color="#000000", color_mode="fixed")

# Using the ImageEditor component to enable drawing on the image with limited colors
demo = gr.Interface(
    fn=main,
    inputs=[gr.ImageEditor(brush=black_brush), gr.Slider(0, 1, step=0.025, value=0.5, label="Image Strength")],
    #outputs=[gr.Image(type="pil"), gr.Image(type="pil"), gr.Image(type="pil"), gr.Image(type="pil")]
    outputs=["image", "image", "image", "image"]
)

demo.launch(share=False)