# coding: utf-8 import os import gradio as gr import random import torch import cv2 import re import uuid import json import pickle from PIL import Image, ImageDraw, ImageOps, ImageFont import math import numpy as np import argparse import inspect import tempfile from diffusers import StableDiffusionPipeline, StableDiffusionInpaintPipeline, StableDiffusionInstructPix2PixPipeline from diffusers import EulerAncestralDiscreteScheduler, PNDMScheduler from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, DiffusionPipeline, UniPCMultistepScheduler from diffusers import StableDiffusionXLPipeline, AutoPipelineForText2Image from langchain.agents.initialize import initialize_agent from langchain.agents.tools import Tool from langchain.chains.conversation.memory import ConversationBufferMemory from langchain.llms.openai import OpenAI from langchain.llms import AzureOpenAI from sentence_transformers import SentenceTransformer from compel import Compel, ReturnedEmbeddingsType PREFIX = """DiffusionGPT is designed to be able to assist users in generating high-quality images. Human may provide some text prompts to DiffusionGPT. The input prompts will be analyzed by DiffusionGPT to select the most suitable generative model for generating images. Overall, DiffusionGPT is a powerful image generation system that can assist in processing various forms of textual input and match them with the most suitable generative model to accomplish the generation task. TOOLS: ------ DiffusionGPT has access to the following tools:""" FORMAT_INSTRUCTIONS = """To use a tool, please use the following format: ``` Thought: Do I need to use a tool? Yes Action: the action to take, should be one of [{tool_names}] Action Input: the input to the action Observation: the result of the action ``` When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format: ``` Thought: Do I need to use a tool? No {ai_prefix}: [your response here] ``` """ SUFFIX = """You are very strict to the filename correctness and will never fake a file name if it does not exist. You will remember to provide the image file name loyally if it's provided in the last tool observation. Begin! Previous conversation history: {chat_history} New input: {input} DiffusionGPT must use tools to observe images rather than imagination. The thoughts and observations are only visible for DiffusionGPT, DiffusionGPT should remember to repeat important information in the final response for Human. Thought: Do I need to use a tool? {agent_scratchpad} Let's think step by step. """ TOT_PROMPTS = """Identify and behave as five different experts that are appropriate to select one element from the input list that best matches the input prompt. All experts will write down the selection result, then share it with the group. You then analyze all 5 analyses and output the consensus selected element or your best guess matched element. The final selection output MUST be the same as the TEMPLATE: TEMPLATE: ``` Selected: [the selected word] ``` Input list: {search_list} Input prompt: {input} """ PROMPT_PARSE_PROMPTS = """Given the user input text. Please judge the paradigm of the input text, and then recognize the main string of text prompts according to the corresponding form. The output must be same as the TEMPlATE: TEMPLATE: ``` Prompts: [the output prompts] ``` For instance: 1. Input: A dog Prompts: A dog 2. Input: generate an image of a dog Prompts: an image of a dog 3. Input: I want to see a beach Prompts: a beach 4. Input: If you give me a toy, I will laugh very happily Prompts: a toy and a laugh face Input: {inputs} """ TREE_OF_MODEL_PROMPT_SUBJECT = """ You are an information analyst who can analyze and abstract a set of words to abstract some representation categories. Below is a template that can represent the abstracted categories in Subject Dimension belonging to concrete noun: TEMPLATE: ``` Categories: - [Subject] - [Subject] - ... ``` You MUST abstract the categories in a highly abstract manner only from Subject Dimension and ensure the whole number of categories are fewer than 5. Then, You MUST remove the Style-related categories. Please output the categories following the format of TEMPLATE. Input: {input} """ TREE_OF_MODEL_PROMPT_STYLE = """ You are an information analyst who can analyze and summarize a set of words to abstract some representation categories. Below is a template that can represent the the abstracted categories in Style Dimension: TEMPLATE: ``` Categories: - [Style] - [Style] - ... ``` You MUST abstract the categories in a highly abstract manner from only Style dimension and ensure the whole number of categories are fewer than 8. Please output the Categories following the format of TEMPLATE. Input: {input} """ TREE_OF_MODEL_PROMPT_ = """ You are an information analyst who can create a Knowledge Tree according to the input categories. Below is a knowledge tree template: TEMPLATE: ``` Knowledge Tree: - [Subject] - [Style] - ... - [Subject] - ... ``` You MUST place the each Style category as subcategory under the Subject categories based on whether it can be well matched with a specific subject category to form a reasonable scene. Please output the categories following the format of TEMPLATE. Subject Input: {subject} Style Input: {style} """ TREE_OF_MODEL_PROMPT_ADD_MODELS = """ You are an information analyst who can add some input models to an input knowledge tree according to the similarity of the model tags and the categories of the knowledge tree. You need to place each input model into the appropriate subcategory on the tree, one by one. You MUST keep the original content of the knowledge tree. Please output the final knowledge tree. Knowledge Tree Input: {tree} Models Input: {models} Model Tags Input: {model_tags} """ os.makedirs('image', exist_ok=True) from langchain.llms.base import LLM from langchain import PromptTemplate, HuggingFaceHub from langchain.llms import HuggingFacePipeline from transformers import AutoTokenizer, AutoModelForCausalLM def seed_everything(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) return seed def prompts(name, description): def decorator(func): func.name = name func.description = description return func return decorator def cut_dialogue_history(history_memory, keep_last_n_words=500): if history_memory is None or len(history_memory) == 0: return history_memory tokens = history_memory.split() n_tokens = len(tokens) if n_tokens < keep_last_n_words: return history_memory paragraphs = history_memory.split('\n') last_n_tokens = n_tokens while last_n_tokens >= keep_last_n_words: last_n_tokens -= len(paragraphs[0].split(' ')) paragraphs = paragraphs[1:] return '\n' + '\n'.join(paragraphs) class Text2Image: def __init__(self, device): print(f"Initializing Text2Image to {device}") self.device = device self.torch_dtype = torch.float16 if 'cuda' in device else torch.float32 self.llm = OpenAI(temperature=0) if not os.path.exists('model_tree_tot_sdxl.json'): with open('model_data_sdxl.json', 'r') as f: self.model_data_all = json.load(f) model_tags = {model["model_name"]: model["tag"] for model in self.model_data_all} model_tree = self.build_tree(model_tags) model_all_data = {model["model_name"].split(".")[0]: model for model in self.model_data_all} save_model_tree = {} for cate_name, sub_category in model_tree.items(): cate_name = cate_name.lower() temp_category = {} if "Universal" not in sub_category: temp_category["universal"] = [model_all_data["kandinsky"], model_all_data["sd_xl"]] for sec_cate_name, sub_sub_cates in sub_category.items(): sec_cate_name = sec_cate_name.lower() temp_model_list = [] for model_name in sub_sub_cates: model_name = model_name.strip() lower_name = model_name[0].lower() + model_name[1:] if model_name in model_all_data: temp_model_list.append(model_all_data[model_name]) elif lower_name in model_all_data: temp_model_list.append(model_all_data[lower_name]) temp_category[sec_cate_name] = temp_model_list save_model_tree[cate_name] = temp_category # write in json json_data = json.dumps(save_model_tree, indent=2) with open('model_tree_tot_sdxl.json', 'w') as f: f.write(json_data) f.close() with open('model_tree_tot_sdxl.json', 'r') as f: self.model_data = json.load(f) with open('model_data_sdxl.json', 'r') as f: self.model_all_data = json.load(f) self.model_all_data = {model["model_name"]:model for model in self.model_all_data} # Advantage databases with human feedback with open('./VectorDB_HF/prompt_embed_st.pickle', 'rb') as f: self.pt_pairs = pickle.load(f) with open('./VectorDB_HF/prompt2scores_sdxl.json', 'r') as f: self.prompt2scores = json.load(f) self.st_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2') def build_tree(self, model_tags): tags_only = list(model_tags.values()) model_names = list(model_tags.keys()) prompts = TREE_OF_MODEL_PROMPT.format(input=tags_only) prompt1 = TREE_OF_MODEL_PROMPT_SUBJECT.format(input=tags_only) response1 = self.llm(prompt1) prompt2 = TREE_OF_MODEL_PROMPT_STYLE.format(input=tags_only) response2 = self.llm(prompt2) prompt_tree = TREE_OF_MODEL_PROMPT_.format(style=response2, subject=response1) response = self.llm(prompt_tree) tree = response.split("Knowledge Tree:")[1] model_names = [name.split(".")[0] for name in list(model_tags.keys())] prompts = TREE_OF_MODEL_PROMPT_ADD_MODELS.format(model_tags=model_tags, tree=tree, models=model_names) tree = self.llm(prompts) output = {} tree_list = tree.split("\n") for category in tree_list: if category == '': continue if category.startswith("- "): current_key = category[2:] output[current_key] = {} elif category.startswith(" - "): next_key = category[4:] output[current_key][next_key] = [] elif category.startswith(" - "): output[current_key][next_key].append(category[6:]) return output def prompt_parse(self, inputs): prompts = PROMPT_PARSE_PROMPTS.format(inputs=inputs) output = self.llm(prompts) output = output.split("Prompts:")[1] return output.strip() def get_property(self, model_data): properties = [] for model in model_data: name = "model_name:" + model["model_name"] + ", " tag = "tag:" + ",".join(model["tag"]) prop = name + tag + "\n\n" properties.append(prop) return properties def search_one_matched(self, inputs, search_list): tot_prompts = TOT_PROMPTS.format(search_list=search_list, input=inputs) model_name = self.llm(tot_prompts) print(model_name) if "Selected:" in model_name: model_name = model_name.split("Selected:")[-1] for ch in [",", ";", "."]: if ch in model_name: model_name = model_name.split(ch)[0] model_name = model_name.strip().lower() return model_name def select_best_model_with_HF(self, inputs, model_space): text_embed = torch.Tensor(self.st_model.encode([inputs]) ) text_embed /= text_embed.norm(dim=1, keepdim=True) similarity = text_embed @ self.pt_pairs['text_embeds'].T topk_idxs = similarity.topk(5).indices[0,:] topk_model_list = [] model_names_of_tree = [model["model_name"].split(".")[0] for model in model_space] for idx, p in enumerate(topk_idxs): save_prompt_name = self.pt_pairs['prompts'][int(p)][:100].replace('\n','') model_scores = self.prompt2scores[save_prompt_name] model_names = list(model_scores.keys()) reward_scores = [] for name, values in model_scores.items(): reward_scores.append(values['image_reward']) reward_scores = torch.Tensor(reward_scores) topk_model_idx = reward_scores.topk(5).indices.tolist() topk_models = [model_names[i] for i in topk_model_idx] topk_model_list.append(topk_models) prompt1 = f"Please judge whether each name in this list {model_names_of_tree} has highly similar name in the list {topk_model_list}, if yes, output the similar model name, the output MUST be Template: Model: [model name, ...]" intersection_model = self.llm(prompt1) prompts = f"Please select one model name from the following model list {intersection_model} that has the highest frequency and top ranking according to the list {topk_model_list}.\n\n The output MUST be Template: Model: [model name]" selected_model = self.llm(prompts) selected_model = selected_model.split("Model:")[1] return selected_model.strip() def search_model_tree(self, inputs): search_space = self.model_data search_path = [] while not isinstance(search_space, list): search_list = list(search_space.keys()) name = self.search_one_matched(inputs, search_list) search_path.append(name) search_space = search_space[name] candidate_model_data = {} for model in search_space: candidate_model_data[model["model_name"]] = model model_properties = self.get_property(search_space) model_name_pre = self.select_best_model_with_HF(inputs, search_space) all_names = list(self.model_all_data.keys()) all_names = [name + "\n" for name in all_names] prompts = f"Please according to the name of {model_name_pre} and select one element from the list bellow, and ensure the selected element MUST be the same as one of the list {all_names}." model_name = self.llm(prompts).strip('\n') if model_name not in self.model_all_data: model_name = model_name[0].lower() + model_name[1:] selected_model = self.model_all_data[model_name] search_path.append(model_name) return search_path, selected_model def prompt_entension(self, inputs, model): example_prompt = model["example_prompts"][0] example_n_prompt = model["negtive_prompts"][0] prompts = f"Here is a paragraph describing an image. " \ f"{inputs}. " \ f"Please follow the sentence pattern of the example to expand the description of the input paragraph. The output MUST preserve the contents of the input paragraph. Example: {example_prompt}." extended_prompt = self.llm(prompts) return extended_prompt, example_n_prompt def match_id(self, model): model_names = list(self.model_all_data.keys()) prompts = f"Here is a model. " \ f"{model}. " \ f"Please select the model name that best matches the given model from the model name list {model_names}. " \ f"The output must be the same as the word in the list. " matched_name = self.llm(prompts) matched_name = matched_name[2:] return matched_name @prompts(name="Generate Image From User Input Text", description="always useful to generate an image from a user input text and save it to a file. " "The input to this tool MUST be the whole user input text.") def inference(self, inputs): # Prompt Parse original_input = inputs inputs = self.prompt_parse(inputs) # select model tree model_select_path, selected_model_data = self.search_model_tree(inputs) print("Selected model path:", model_select_path) print("Selected model name:", selected_model_data["model_name"]) model_name = selected_model_data["model_name"] model_type = selected_model_data["model_type"] self.pipe_prior = None if "checkpoint" in model_type: if model_name == "kandinsky": self.pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=self.torch_dtype) from kandinsky2 import get_kandinsky2 model_kandinsky = get_kandinsky2('cuda', task_type='text2img', model_version='2.2') elif model_name == "stable-diffusion-xl-base-1.0": self.pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, use_safetensors=True, variant="fp16") elif model_name in list(self.model_all_data.keys()): model_id = "./checkpoints/" + model_name self.pipe = StableDiffusionXLPipeline.from_single_file(model_id, torch_dtype=self.torch_dtype) self.pipe.to(self.device) elif model_type == "lora": base_model = selected_model_data["resources_used"][0] base_model_name = self.match_id(base_model) if base_model_name == "stable-diffusion-xl-base-1.0": self.pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, use_safetensors=True, variant="fp16") else: base_model_id = "./checkpoints/" + base_model_name self.pipe = StableDiffusionXLPipeline.from_single_file(base_model_id, torch_dtype=self.torch_dtype) self.pipe.to(self.device) self.pipe.load_lora_weights("./checkpoints", weight_name=model_name) if selected_model_data["example_prompts"][0] != "None": prompt, n_prompt = self.prompt_entension(inputs, selected_model_data) else: prompt = inputs n_prompt = selected_model_data["negtive_prompts"][0] if n_prompt == "None": n_prompt = "" prompt_embeds = None ## compel for long text compel = Compel(tokenizer=[self.pipe.tokenizer, self.pipe.tokenizer_2] , text_encoder=[self.pipe.text_encoder, self.pipe.text_encoder_2], returned_embeddings_type=ReturnedEmbeddingsType.PENULTIMATE_HIDDEN_STATES_NON_NORMALIZED, requires_pooled=[False, True]) prompt_list = list(prompt.strip().split(".")) n_prompt_list = list(n_prompt.strip().split(".")) prompt = f'{prompt_list}.and()' n_prompt = f'{n_prompt_list}.and()' prompt_embeds, pooled_embeds = compel.build_conditioning_tensor(prompt) negative_conditioning, neg_pooled_embeds = compel.build_conditioning_tensor(n_prompt) [prompt_embeds, negative_conditioning] = compel.pad_conditioning_tensors_to_same_length([prompt_embeds, negative_conditioning]) negative_prompt_embeds = negative_conditioning if prompt_embeds is not None: images = self.pipe(prompt_embeds=prompt_embeds, pooled_prompt_embeds=pooled_embeds, negative_prompt_embeds=negative_prompt_embeds, negative_pooled_prompt_embeds=neg_pooled_embeds).images elif model_name == "kandinsky": images = model_kandinsky.generate_text2img(prompt, decoder_steps=50, batch_size=1, h=1024, w=1024) else: images = self.pipe(prompt, negative_prompt=n_prompt).images image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png") output = images[0] output.save(image_filename) print( f"\nProcessed Text2Image, Input Text: {inputs}, Output Image: {image_filename}") return image_filename class ConversationBot: def __init__(self, load_dict): print(f"Initializing DiffusionGPT, load_dict={load_dict}") self.models = {} # Load Basic Foundation Models for class_name, device in load_dict.items(): self.models[class_name] = globals()[class_name](device=device) # Load Template Foundation Models for class_name, module in globals().items(): if getattr(module, 'template_model', False): template_required_names = {k for k in inspect.signature(module.__init__).parameters.keys() if k!='self'} loaded_names = set([type(e).__name__ for e in self.models.values()]) if template_required_names.issubset(loaded_names): self.models[class_name] = globals()[class_name]( **{name: self.models[name] for name in template_required_names}) print(f"All the Available Functions: {self.models}") self.tools = [] for instance in self.models.values(): for e in dir(instance): if e.startswith('inference'): func = getattr(instance, e) self.tools.append(Tool(name=func.name, description=func.description, func=func)) self.llm = OpenAI(temperature=0) self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output') def init_agent(self, lang): self.memory.clear() #clear previous history place = "Enter text and press enter, or upload an image" label_clear = "Clear" self.agent = initialize_agent( self.tools, self.llm, agent="conversational-react-description", verbose=True, memory=self.memory, return_intermediate_steps=True, agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS, 'suffix': SUFFIX}, handle_parsing_errors="Check your output and make sure it conforms!" ) return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear) def run_text(self, text, state): self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_words=500) res = self.agent({"input": text.strip()}) res['output'] = res['output'].replace("\\", "/") response = re.sub('(image/[-\w]*.png)', lambda m: f'![](file={m.group(0)})*{m.group(0)}*', res['output']) state = state + [(text, response)] print(f"\nProcessed run_text, Input text: {text}\n") return state, state if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--load', type=str, default="Text2Image_cuda:0") args = parser.parse_args() load_dict = {e.split('_')[0].strip(): e.split('_')[1].strip() for e in args.load.split(',')} def init_api(apikey): os.environ['OPENAI_API_KEY'] = apikey global bot bot = ConversationBot(load_dict=load_dict) print('set new api key:', apikey) return None init_api(apikey="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") def inference_warp(prompt): prompt = prompt.strip() global bot state = [] _, state = bot.run_text(prompt, state) print('========>', str(state)) pattern = r"\(file=(.*?)\)" matches = re.findall(pattern, str(state)) if matches: file_path = matches[0] print(file_path) image = Image.open(file_path) return image with gr.Blocks(css="#chatbot .overflow-y-auto{height:1000px}") as demo: state = gr.State([]) with gr.Row(): with gr.Column(): apikey = gr.Textbox(label='apikey', value="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") prompt = gr.Textbox(label='Prompt') run_button = gr.Button('Generate Image') result = gr.Image(label="Generated Image") bot.init_agent("English") run_button.click(fn=inference_warp, inputs=prompt, outputs=result,) apikey.change(fn=init_api, inputs=[apikey]) examples = [ ["a girl with dress and red hat."], ["generate an image of a laughing woman, fashion magazine cover."], ["a cat on the grass."], ["create an illustration of a romantic couple sharing a tender moment under a starry sky."], ["a robot cooking in the kitchen."] ] gr.Examples(examples=examples, inputs=prompt, outputs=result, fn=inference_warp, cache_examples=True, run_on_click=True ) demo.launch(server_name="0.0.0.0", server_port=7862)