import os import base64 import numpy as np from PIL import Image import io import requests import replicate from flask import Flask, request import gradio as gr import openai from openai import OpenAI from dotenv import load_dotenv, find_dotenv import json # Locate the .env file dotenv_path = find_dotenv() load_dotenv(dotenv_path) OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') REPLICATE_API_TOKEN = os.getenv('REPLICATE_API_TOKEN') client = OpenAI() def main(img): mask = img['layers'][0] base_image = Image.fromarray(img['background'].astype('uint8')) img_base_64 = img_to_base64(base_image) if is_transparent(mask) == True: mask_base_64 = None else: mask_img = create_mask_image(mask) mask_base_64 = img_to_base64(mask_img) prompt = call_openai(img_base_64) output_urls = generate_image(prompt, img_base_64, mask_base_64) output_images = [download_image(url) for url in output_urls[3:]] # Start from the 4th image return output_images def generate_image(prompt, img, mask): input_data = { "image": img, "prompt": prompt + ", expensive", "refine": "no_refiner", "scheduler": "K_EULER", "lora_scale": 0.8, "num_outputs": 4, "controlnet_1": "edge_canny", "controlnet_2": "depth_midas", "controlnet_3": "lineart", "guidance_scale": 7.5, "apply_watermark": False, "negative_prompt":"worst quality, low quality, illustration, 2d, painting, cartoons, sketch", "prompt_strength": 0.75, "sizing_strategy": "controlnet_1_image", "controlnet_1_end": 1, "controlnet_2_end": 1, "controlnet_3_end": 1, "controlnet_1_image": img, "controlnet_1_start": 0, "controlnet_2_image": img, "controlnet_2_start": 0, "controlnet_3_image": img, "controlnet_3_start": 0, "num_inference_steps": 30, "controlnet_1_conditioning_scale": 0.8, "controlnet_2_conditioning_scale": 0.8, "controlnet_3_conditioning_scale": 0.75 } if mask is not None: input_data["mask"] = mask else: input_data["prompt_strength"] = .6 output = replicate.run( "fofr/realvisxl-v3-multi-controlnet-lora:90a4a3604cd637cb9f1a2bdae1cfa9ed869362ca028814cdce310a78e27daade", input=input_data ) return output def download_image(url): response = requests.get(url) img = Image.open(io.BytesIO(response.content)) return img def create_mask_image(mask_array): # Convert the mask to a numpy array if it's not already if not isinstance(mask_array, np.ndarray): mask_array = np.array(mask_array) # Create a new array with the same shape as the mask, but only for RGB channels processed_mask = np.zeros((mask_array.shape[0], mask_array.shape[1], 3), dtype=np.uint8) # Set transparent parts (alpha=0) to black (0, 0, 0) transparent_mask = mask_array[:, :, 3] == 0 processed_mask[transparent_mask] = [0, 0, 0] # Set black parts (RGB=0, 0, 0 and alpha=255) to white (255, 255, 255) black_mask = (mask_array[:, :, :3] == [0, 0, 0]).all(axis=2) & (mask_array[:, :, 3] == 255) processed_mask[black_mask] = [255, 255, 255] return Image.fromarray(processed_mask) def is_transparent(mask_array): return np.all(mask_array[:, :, 3] == 0) def img_to_base64(img): # Extract the format of the image (e.g., JPEG, PNG) img_format = img.format if img.format else "PNG" # Convert the image to bytes buffered = io.BytesIO() img.save(buffered, format=img_format) img_base_64 = base64.b64encode(buffered.getvalue()).decode('utf-8') return f"data:image/{img_format.lower()};base64," + img_base_64 def call_openai(image_data): try: response = client.chat.completions.create( model="gpt-4o", messages=[ { "role": "user", "content": [ {"type": "text", "text": "Please describe this image in one sentence, with a focus on the material and specific color (like pantone level specificity) and details of the main object in the scene. Mention the type of lighting as well."}, { "type": "image_url", "image_url": { "url": image_data, }, }, ], } ], max_tokens=300, ) return response.choices[0].message.content except openai.BadRequestError as e: print(e) print("e type") print(type(e)) raise gr.Error(f"You uploaded an unsupported image. Please make sure your image is below 20 MB in size and is of one the following formats: ['png', 'jpeg', 'gif', 'webp']") except Exception as e: raise gr.Error("Unknown Error") # Define the brush with only black color black_brush = gr.Brush(colors=["#000000"], default_color="#000000", color_mode="fixed") # Using the ImageEditor component to enable drawing on the image with limited colors demo = gr.Interface( fn=main, inputs=gr.ImageEditor(brush=black_brush), outputs=[gr.Image(type="pil"), gr.Image(type="pil"), gr.Image(type="pil"), gr.Image(type="pil")] ) demo.launch(share=False)