introduce control net from diffusers

app.py

@@ -70,27 +70,27 @@ def cut_dialogue_history(history_memory, keep_last_n_words=500):
 class ConversationBot:
     def __init__(self):
         print("Initializing VisualChatGPT")
-        self.llm = OpenAI(temperature=0, openai_api_key="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")
+        self.llm = OpenAI(temperature=0, openai_api_key="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")
         self.edit = ImageEditing(device="cuda:0")
         self.i2t = ImageCaptioning(device="cuda:0")
         self.t2i = T2I(device="cuda:0")
         self.image2canny = image2canny_new()
         self.canny2image = canny2image_new(device="cuda:0")
-        self.image2line = image2line_new()
-        self.line2image = line2image_new(device="cuda:0")
-        self.image2hed = image2hed_new()
-        self.hed2image = hed2image_new(device="cuda:0")
-        self.image2scribble = image2scribble_new()
-        self.scribble2image = scribble2image_new(device="cuda:0")
-        self.image2pose = image2pose_new()
-        self.pose2image = pose2image_new(device="cuda:0")
+        # self.image2line = image2line_new()
+        # self.line2image = line2image_new(device="cuda:0")
+        # self.image2hed = image2hed_new()
+        # self.hed2image = hed2image_new(device="cuda:0")
+        # self.image2scribble = image2scribble_new()
+        # self.scribble2image = scribble2image_new(device="cuda:0")
+        # self.image2pose = image2pose_new()
+        # self.pose2image = pose2image_new(device="cuda:0")
         self.BLIPVQA = BLIPVQA(device="cuda:0")
-        self.image2seg = image2seg_new()
-        self.seg2image = seg2image_new(device="cuda:0")
-        self.image2depth = image2depth_new()
-        self.depth2image = depth2image_new(device="cuda:0")
-        self.image2normal = image2normal_new()
-        self.normal2image = normal2image_new(device="cuda:0")
+        # self.image2seg = image2seg_new()
+        # self.seg2image = seg2image_new(device="cuda:0")
+        # self.image2depth = image2depth_new()
+        # self.depth2image = depth2image_new(device="cuda:0")
+        # self.image2normal = image2normal_new()
+        # self.normal2image = normal2image_new(device="cuda:0")
         self.pix2pix = Pix2Pix(device="cuda:0")
         self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')
         self.tools = [
@@ -106,19 +106,18 @@ class ConversationBot:
             Tool(name="Replace Something From The Photo", func=self.edit.replace_part_of_image,
                  description="useful for when you want to replace an object from the object description or location with another object from its description. "
                              "The input to this tool should be a comma seperated string of three, representing the image_path, the object to be replaced, the object to be replaced with "),
-
-            # Tool(name="Instruct Image Using Text", func=self.pix2pix.inference,
-            #      description="useful for when you want to the style of the image to be like the text. like: make it look like a painting. or make it like a robot. "
-            #                  "The input to this tool should be a comma seperated string of two, representing the image_path and the text. "),
-            # Tool(name="Answer Question About The Image", func=self.BLIPVQA.get_answer_from_question_and_image,
-            #      description="useful for when you need an answer for a question based on an image. like: what is the background color of the last image, how many cats in this figure, what is in this figure. "
-            #                  "The input to this tool should be a comma seperated string of two, representing the image_path and the question"),
-            # Tool(name="Edge Detection On Image", func=self.image2canny.inference,
-            #      description="useful for when you want to detect the edge of the image. like: detect the edges of this image, or canny detection on image, or peform edge detection on this image, or detect the canny image of this image. "
-            #                  "The input to this tool should be a string, representing the image_path"),
-            # Tool(name="Generate Image Condition On Canny Image", func=self.canny2image.inference,
-            #      description="useful for when you want to generate a new real image from both the user desciption and a canny image. like: generate a real image of a object or something from this canny image, or generate a new real image of a object or something from this edge image. "
-            #                  "The input to this tool should be a comma seperated string of two, representing the image_path and the user description. "),
+            Tool(name="Instruct Image Using Text", func=self.pix2pix.inference,
+                 description="useful for when you want to the style of the image to be like the text. like: make it look like a painting. or make it like a robot. "
+                             "The input to this tool should be a comma seperated string of two, representing the image_path and the text. "),
+            Tool(name="Answer Question About The Image", func=self.BLIPVQA.get_answer_from_question_and_image,
+                 description="useful for when you need an answer for a question based on an image. like: what is the background color of the last image, how many cats in this figure, what is in this figure. "
+                             "The input to this tool should be a comma seperated string of two, representing the image_path and the question"),
+            Tool(name="Edge Detection On Image", func=self.image2canny.inference,
+                 description="useful for when you want to detect the edge of the image. like: detect the edges of this image, or canny detection on image, or peform edge detection on this image, or detect the canny image of this image. "
+                             "The input to this tool should be a string, representing the image_path"),
+            Tool(name="Generate Image Condition On Canny Image", func=self.canny2image.inference,
+                 description="useful for when you want to generate a new real image from both the user desciption and a canny image. like: generate a real image of a object or something from this canny image, or generate a new real image of a object or something from this edge image. "
+                             "The input to this tool should be a comma seperated string of two, representing the image_path and the user description. "),
             # Tool(name="Line Detection On Image", func=self.image2line.inference,
             #      description="useful for when you want to detect the straight line of the image. like: detect the straight lines of this image, or straight line detection on image, or peform straight line detection on this image, or detect the straight line image of this image. "
             #                  "The input to this tool should be a string, representing the image_path"),