Upload 7 files

execution.py

@@ -0,0 +1,371 @@
+import os
+import sys
+import copy
+import json
+import threading
+import heapq
+import traceback
+import gc
+
+import torch
+import nodes
+
+def get_input_data(inputs, class_def, unique_id, outputs={}, prompt={}, extra_data={}):
+    valid_inputs = class_def.INPUT_TYPES()
+    input_data_all = {}
+    for x in inputs:
+        input_data = inputs[x]
+        if isinstance(input_data, list):
+            input_unique_id = input_data[0]
+            output_index = input_data[1]
+            if input_unique_id not in outputs:
+                return None
+            obj = outputs[input_unique_id][output_index]
+            input_data_all[x] = obj
+        else:
+            if ("required" in valid_inputs and x in valid_inputs["required"]) or ("optional" in valid_inputs and x in valid_inputs["optional"]):
+                input_data_all[x] = input_data
+
+    if "hidden" in valid_inputs:
+        h = valid_inputs["hidden"]
+        for x in h:
+            if h[x] == "PROMPT":
+                input_data_all[x] = prompt
+            if h[x] == "EXTRA_PNGINFO":
+                if "extra_pnginfo" in extra_data:
+                    input_data_all[x] = extra_data['extra_pnginfo']
+            if h[x] == "UNIQUE_ID":
+                input_data_all[x] = unique_id
+    return input_data_all
+
+def recursive_execute(server, prompt, outputs, current_item, extra_data={}):
+    unique_id = current_item
+    inputs = prompt[unique_id]['inputs']
+    class_type = prompt[unique_id]['class_type']
+    class_def = nodes.NODE_CLASS_MAPPINGS[class_type]
+    if unique_id in outputs:
+        return []
+
+    executed = []
+
+    for x in inputs:
+        input_data = inputs[x]
+
+        if isinstance(input_data, list):
+            input_unique_id = input_data[0]
+            output_index = input_data[1]
+            if input_unique_id not in outputs:
+                executed += recursive_execute(server, prompt, outputs, input_unique_id, extra_data)
+
+    input_data_all = get_input_data(inputs, class_def, unique_id, outputs, prompt, extra_data)
+    if server.client_id is not None:
+        server.last_node_id = unique_id
+        server.send_sync("executing", { "node": unique_id }, server.client_id)
+    obj = class_def()
+
+    nodes.before_node_execution()
+    outputs[unique_id] = getattr(obj, obj.FUNCTION)(**input_data_all)
+    if "ui" in outputs[unique_id]:
+        if server.client_id is not None:
+            server.send_sync("executed", { "node": unique_id, "output": outputs[unique_id]["ui"] }, server.client_id)
+        if "result" in outputs[unique_id]:
+            outputs[unique_id] = outputs[unique_id]["result"]
+    return executed + [unique_id]
+
+def recursive_will_execute(prompt, outputs, current_item):
+    unique_id = current_item
+    inputs = prompt[unique_id]['inputs']
+    will_execute = []
+    if unique_id in outputs:
+        return []
+
+    for x in inputs:
+        input_data = inputs[x]
+        if isinstance(input_data, list):
+            input_unique_id = input_data[0]
+            output_index = input_data[1]
+            if input_unique_id not in outputs:
+                will_execute += recursive_will_execute(prompt, outputs, input_unique_id)
+
+    return will_execute + [unique_id]
+
+def recursive_output_delete_if_changed(prompt, old_prompt, outputs, current_item):
+    unique_id = current_item
+    inputs = prompt[unique_id]['inputs']
+    class_type = prompt[unique_id]['class_type']
+    class_def = nodes.NODE_CLASS_MAPPINGS[class_type]
+
+    is_changed_old = ''
+    is_changed = ''
+    if hasattr(class_def, 'IS_CHANGED'):
+        if unique_id in old_prompt and 'is_changed' in old_prompt[unique_id]:
+            is_changed_old = old_prompt[unique_id]['is_changed']
+        if 'is_changed' not in prompt[unique_id]:
+            input_data_all = get_input_data(inputs, class_def, unique_id, outputs)
+            if input_data_all is not None:
+                is_changed = class_def.IS_CHANGED(**input_data_all)
+                prompt[unique_id]['is_changed'] = is_changed
+        else:
+            is_changed = prompt[unique_id]['is_changed']
+
+    if unique_id not in outputs:
+        return True
+
+    to_delete = False
+    if is_changed != is_changed_old:
+        to_delete = True
+    elif unique_id not in old_prompt:
+        to_delete = True
+    elif inputs == old_prompt[unique_id]['inputs']:
+        for x in inputs:
+            input_data = inputs[x]
+
+            if isinstance(input_data, list):
+                input_unique_id = input_data[0]
+                output_index = input_data[1]
+                if input_unique_id in outputs:
+                    to_delete = recursive_output_delete_if_changed(prompt, old_prompt, outputs, input_unique_id)
+                else:
+                    to_delete = True
+                if to_delete:
+                    break
+    else:
+        to_delete = True
+
+    if to_delete:
+        d = outputs.pop(unique_id)
+        del d
+    return to_delete
+
+class PromptExecutor:
+    def __init__(self, server):
+        self.outputs = {}
+        self.old_prompt = {}
+        self.server = server
+
+    def execute(self, prompt, extra_data={}):
+        nodes.interrupt_processing(False)
+
+        if "client_id" in extra_data:
+            self.server.client_id = extra_data["client_id"]
+        else:
+            self.server.client_id = None
+
+        with torch.inference_mode():
+            for x in prompt:
+                recursive_output_delete_if_changed(prompt, self.old_prompt, self.outputs, x)
+
+            current_outputs = set(self.outputs.keys())
+            executed = []
+            try:
+                to_execute = []
+                for x in prompt:
+                    class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']]
+                    if hasattr(class_, 'OUTPUT_NODE'):
+                        to_execute += [(0, x)]
+
+                while len(to_execute) > 0:
+                    #always execute the output that depends on the least amount of unexecuted nodes first
+                    to_execute = sorted(list(map(lambda a: (len(recursive_will_execute(prompt, self.outputs, a[-1])), a[-1]), to_execute)))
+                    x = to_execute.pop(0)[-1]
+
+                    class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']]
+                    if hasattr(class_, 'OUTPUT_NODE'):
+                        if class_.OUTPUT_NODE == True:
+                            valid = False
+                            try:
+                                m = validate_inputs(prompt, x)
+                                valid = m[0]
+                            except:
+                                valid = False
+                            if valid:
+                                executed += recursive_execute(self.server, prompt, self.outputs, x, extra_data)
+            except Exception as e:
+                print(traceback.format_exc())
+                to_delete = []
+                for o in self.outputs:
+                    if o not in current_outputs:
+                        to_delete += [o]
+                        if o in self.old_prompt:
+                            d = self.old_prompt.pop(o)
+                            del d
+                for o in to_delete:
+                    d = self.outputs.pop(o)
+                    del d
+            else:
+                executed = set(executed)
+                for x in executed:
+                    self.old_prompt[x] = copy.deepcopy(prompt[x])
+            finally:
+                self.server.last_node_id = None
+                if self.server.client_id is not None:
+                    self.server.send_sync("executing", { "node": None }, self.server.client_id)
+
+        gc.collect()
+        if torch.cuda.is_available():
+            if torch.version.cuda: #This seems to make things worse on ROCm so I only do it for cuda
+                torch.cuda.empty_cache()
+                torch.cuda.ipc_collect()
+
+
+def validate_inputs(prompt, item):
+    unique_id = item
+    inputs = prompt[unique_id]['inputs']
+    class_type = prompt[unique_id]['class_type']
+    obj_class = nodes.NODE_CLASS_MAPPINGS[class_type]
+
+    class_inputs = obj_class.INPUT_TYPES()
+    required_inputs = class_inputs['required']
+    for x in required_inputs:
+        if x not in inputs:
+            return (False, "Required input is missing. {}, {}".format(class_type, x))
+        val = inputs[x]
+        info = required_inputs[x]
+        type_input = info[0]
+        if isinstance(val, list):
+            if len(val) != 2:
+                return (False, "Bad Input. {}, {}".format(class_type, x))
+            o_id = val[0]
+            o_class_type = prompt[o_id]['class_type']
+            r = nodes.NODE_CLASS_MAPPINGS[o_class_type].RETURN_TYPES
+            if r[val[1]] != type_input:
+                return (False, "Return type mismatch. {}, {}, {} != {}".format(class_type, x, r[val[1]], type_input))
+            r = validate_inputs(prompt, o_id)
+            if r[0] == False:
+                return r
+        else:
+            if type_input == "INT":
+                val = int(val)
+                inputs[x] = val
+            if type_input == "FLOAT":
+                val = float(val)
+                inputs[x] = val
+            if type_input == "STRING":
+                val = str(val)
+                inputs[x] = val
+
+            if len(info) > 1:
+                if "min" in info[1] and val < info[1]["min"]:
+                    return (False, "Value smaller than min. {}, {}".format(class_type, x))
+                if "max" in info[1] and val > info[1]["max"]:
+                    return (False, "Value bigger than max. {}, {}".format(class_type, x))
+
+            if isinstance(type_input, list):
+                if val not in type_input:
+                    return (False, "Value not in list. {}, {}: {} not in {}".format(class_type, x, val, type_input))
+    return (True, "")
+
+def validate_prompt(prompt):
+    outputs = set()
+    for x in prompt:
+        class_ = nodes.NODE_CLASS_MAPPINGS[prompt[x]['class_type']]
+        if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE == True:
+            outputs.add(x)
+
+    if len(outputs) == 0:
+        return (False, "Prompt has no outputs")
+
+    good_outputs = set()
+    errors = []
+    for o in outputs:
+        valid = False
+        reason = ""
+        try:
+            m = validate_inputs(prompt, o)
+            valid = m[0]
+            reason = m[1]
+        except:
+            valid = False
+            reason = "Parsing error"
+
+        if valid == True:
+            good_outputs.add(x)
+        else:
+            print("Failed to validate prompt for output {} {}".format(o, reason))
+            print("output will be ignored")
+            errors += [(o, reason)]
+
+    if len(good_outputs) == 0:
+        errors_list = "\n".join(set(map(lambda a: "{}".format(a[1]), errors)))
+        return (False, "Prompt has no properly connected outputs\n {}".format(errors_list))
+
+    return (True, "")
+
+
+class PromptQueue:
+    def __init__(self, server):
+        self.server = server
+        self.mutex = threading.RLock()
+        self.not_empty = threading.Condition(self.mutex)
+        self.task_counter = 0
+        self.queue = []
+        self.currently_running = {}
+        self.history = {}
+        server.prompt_queue = self
+
+    def put(self, item):
+        with self.mutex:
+            heapq.heappush(self.queue, item)
+            self.server.queue_updated()
+            self.not_empty.notify()
+
+    def get(self):
+        with self.not_empty:
+            while len(self.queue) == 0:
+                self.not_empty.wait()
+            item = heapq.heappop(self.queue)
+            i = self.task_counter
+            self.currently_running[i] = copy.deepcopy(item)
+            self.task_counter += 1
+            self.server.queue_updated()
+            return (item, i)
+
+    def task_done(self, item_id, outputs):
+        with self.mutex:
+            prompt = self.currently_running.pop(item_id)
+            self.history[prompt[1]] = { "prompt": prompt, "outputs": {} }
+            for o in outputs:
+                if "ui" in outputs[o]:
+                    self.history[prompt[1]]["outputs"][o] = outputs[o]["ui"]
+            self.server.queue_updated()
+
+    def get_current_queue(self):
+        with self.mutex:
+            out = []
+            for x in self.currently_running.values():
+                out += [x]
+            return (out, copy.deepcopy(self.queue))
+
+    def get_tasks_remaining(self):
+        with self.mutex:
+            return len(self.queue) + len(self.currently_running)
+
+    def wipe_queue(self):
+        with self.mutex:
+            self.queue = []
+            self.server.queue_updated()
+
+    def delete_queue_item(self, function):
+        with self.mutex:
+            for x in range(len(self.queue)):
+                if function(self.queue[x]):
+                    if len(self.queue) == 1:
+                        self.wipe_queue()
+                    else:
+                        self.queue.pop(x)
+                        heapq.heapify(self.queue)
+                    self.server.queue_updated()
+                    return True
+        return False
+
+    def get_history(self):
+        with self.mutex:
+            return copy.deepcopy(self.history)
+
+    def wipe_history(self):
+        with self.mutex:
+            self.history = {}
+
+    def delete_history_item(self, id_to_delete):
+        with self.mutex:
+            self.history.pop(id_to_delete, None)

extra_model_paths.yaml.example

@@ -0,0 +1,23 @@
+#Rename this to extra_model_paths.yaml and ComfyUI will load it
+
+#config for a1111 ui
+#all you have to do is change the base_path to where yours is installed
+a111:
+    base_path: path/to/stable-diffusion-webui/
+
+    checkpoints: models/Stable-diffusion
+    configs: models/Stable-diffusion
+    vae: models/VAE
+    loras: models/Lora
+    upscale_models: |
+                  models/ESRGAN
+                  models/SwinIR
+    embeddings: embeddings
+    controlnet: models/ControlNet
+
+#other_ui:
+#    base_path: path/to/ui
+#    checkpoints: models/checkpoints
+
+
+

folder_paths.py

@@ -0,0 +1,69 @@
+import os
+
+supported_ckpt_extensions = set(['.ckpt', '.pth'])
+supported_pt_extensions = set(['.ckpt', '.pt', '.bin', '.pth'])
+try:
+    import safetensors.torch
+    supported_ckpt_extensions.add('.safetensors')
+    supported_pt_extensions.add('.safetensors')
+except:
+    print("Could not import safetensors, safetensors support disabled.")
+
+
+folder_names_and_paths = {}
+
+
+models_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "models")
+folder_names_and_paths["checkpoints"] = ([os.path.join(models_dir, "checkpoints")], supported_ckpt_extensions)
+folder_names_and_paths["configs"] = ([os.path.join(models_dir, "configs")], [".yaml"])
+
+folder_names_and_paths["loras"] = ([os.path.join(models_dir, "loras")], supported_pt_extensions)
+folder_names_and_paths["vae"] = ([os.path.join(models_dir, "vae")], supported_pt_extensions)
+folder_names_and_paths["clip"] = ([os.path.join(models_dir, "clip")], supported_pt_extensions)
+folder_names_and_paths["clip_vision"] = ([os.path.join(models_dir, "clip_vision")], supported_pt_extensions)
+folder_names_and_paths["style_models"] = ([os.path.join(models_dir, "style_models")], supported_pt_extensions)
+folder_names_and_paths["embeddings"] = ([os.path.join(models_dir, "embeddings")], supported_pt_extensions)
+
+folder_names_and_paths["controlnet"] = ([os.path.join(models_dir, "controlnet"), os.path.join(models_dir, "t2i_adapter")], supported_pt_extensions)
+folder_names_and_paths["upscale_models"] = ([os.path.join(models_dir, "upscale_models")], supported_pt_extensions)
+
+
+def add_model_folder_path(folder_name, full_folder_path):
+    global folder_names_and_paths
+    if folder_name in folder_names_and_paths:
+        folder_names_and_paths[folder_name][0].append(full_folder_path)
+
+def get_folder_paths(folder_name):
+    return folder_names_and_paths[folder_name][0][:]
+
+def recursive_search(directory):
+    result = []
+    for root, subdir, file in os.walk(directory, followlinks=True):
+        for filepath in file:
+            #we os.path,join directory with a blank string to generate a path separator at the end.
+            result.append(os.path.join(root, filepath).replace(os.path.join(directory,''),''))
+    return result
+
+def filter_files_extensions(files, extensions):
+    return sorted(list(filter(lambda a: os.path.splitext(a)[-1].lower() in extensions, files)))
+
+
+
+def get_full_path(folder_name, filename):
+    global folder_names_and_paths
+    folders = folder_names_and_paths[folder_name]
+    for x in folders[0]:
+        full_path = os.path.join(x, filename)
+        if os.path.isfile(full_path):
+            return full_path
+
+
+def get_filename_list(folder_name):
+    global folder_names_and_paths
+    output_list = set()
+    folders = folder_names_and_paths[folder_name]
+    for x in folders[0]:
+        output_list.update(filter_files_extensions(recursive_search(x), folders[1]))
+    return sorted(list(output_list))
+
+

main.py

@@ -0,0 +1,162 @@
+import os
+import sys
+import shutil
+
+import threading
+import asyncio
+
+if os.name == "nt":
+    import logging
+    logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())
+
+if __name__ == "__main__":
+    if '--help' in sys.argv:
+        print()
+        print("Valid Command line Arguments:")
+        print("\t--listen [ip]\t\t\tListen on ip or 0.0.0.0 if none given so the UI can be accessed from other computers.")
+        print("\t--port 8188\t\t\tSet the listen port.")
+        print()
+        print("\t--extra-model-paths-config file.yaml\tload an extra_model_paths.yaml file.")
+        print()
+        print()
+        print("\t--dont-upcast-attention\t\tDisable upcasting of attention \n\t\t\t\t\tcan boost speed but increase the chances of black images.\n")
+        print("\t--use-split-cross-attention\tUse the split cross attention optimization instead of the sub-quadratic one.\n\t\t\t\t\tIgnored when xformers is used.")
+        print("\t--use-pytorch-cross-attention\tUse the new pytorch 2.0 cross attention function.")
+        print("\t--disable-xformers\t\tdisables xformers")
+        print("\t--cuda-device 1\t\tSet the id of the cuda device this instance will use.")
+        print()
+        print("\t--highvram\t\t\tBy default models will be unloaded to CPU memory after being used.\n\t\t\t\t\tThis option keeps them in GPU memory.\n")
+        print("\t--normalvram\t\t\tUsed to force normal vram use if lowvram gets automatically enabled.")
+        print("\t--lowvram\t\t\tSplit the unet in parts to use less vram.")
+        print("\t--novram\t\t\tWhen lowvram isn't enough.")
+        print()
+        print("\t--cpu\t\t\tTo use the CPU for everything (slow).")
+        exit()
+
+    if '--dont-upcast-attention' in sys.argv:
+        print("disabling upcasting of attention")
+        os.environ['ATTN_PRECISION'] = "fp16"
+
+    try:
+        index = sys.argv.index('--cuda-device')
+        device = sys.argv[index + 1]
+        os.environ['CUDA_VISIBLE_DEVICES'] = device
+        print("Set cuda device to:", device)
+    except:
+        pass
+
+from nodes import init_custom_nodes
+import execution
+import server
+import folder_paths
+import yaml
+
+def prompt_worker(q, server):
+    e = execution.PromptExecutor(server)
+    while True:
+        item, item_id = q.get()
+        e.execute(item[-2], item[-1])
+        q.task_done(item_id, e.outputs)
+
+async def run(server, address='', port=8188, verbose=True, call_on_start=None):
+    await asyncio.gather(server.start(address, port, verbose, call_on_start), server.publish_loop())
+
+def hijack_progress(server):
+    from tqdm.auto import tqdm
+    orig_func = getattr(tqdm, "update")
+    def wrapped_func(*args, **kwargs):
+        pbar = args[0]
+        v = orig_func(*args, **kwargs)
+        server.send_sync("progress", { "value": pbar.n, "max": pbar.total}, server.client_id)            
+        return v
+    setattr(tqdm, "update", wrapped_func)
+
+def cleanup_temp():
+    temp_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "temp")
+    if os.path.exists(temp_dir):
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+def load_extra_path_config(yaml_path):
+    with open(yaml_path, 'r') as stream:
+        config = yaml.safe_load(stream)
+    for c in config:
+        conf = config[c]
+        if conf is None:
+            continue
+        base_path = None
+        if "base_path" in conf:
+            base_path = conf.pop("base_path")
+        for x in conf:
+            for y in conf[x].split("\n"):
+                if len(y) == 0:
+                    continue
+                full_path = y
+                if base_path is not None:
+                    full_path = os.path.join(base_path, full_path)
+                print("Adding extra search path", x, full_path)
+                folder_paths.add_model_folder_path(x, full_path)
+
+if __name__ == "__main__":
+    cleanup_temp()
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    server = server.PromptServer(loop)
+    q = execution.PromptQueue(server)
+
+    init_custom_nodes()
+    server.add_routes()
+    hijack_progress(server)
+
+    threading.Thread(target=prompt_worker, daemon=True, args=(q,server,)).start()
+    try:
+        address = '0.0.0.0'
+        p_index = sys.argv.index('--listen')
+        try:
+            ip = sys.argv[p_index + 1]
+            if ip[:2] != '--':
+                address = ip
+        except:
+            pass
+    except:
+        address = '127.0.0.1'
+
+    dont_print = False
+    if '--dont-print-server' in sys.argv:
+        dont_print = True
+
+    extra_model_paths_config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "extra_model_paths.yaml")
+    if os.path.isfile(extra_model_paths_config_path):
+        load_extra_path_config(extra_model_paths_config_path)
+
+    if '--extra-model-paths-config' in sys.argv:
+        indices = [(i + 1) for i in range(len(sys.argv) - 1) if sys.argv[i] == '--extra-model-paths-config']
+        for i in indices:
+            load_extra_path_config(sys.argv[i])
+
+    port = 8188
+    try:
+        p_index = sys.argv.index('--port')
+        port = int(sys.argv[p_index + 1])
+    except:
+        pass
+
+    if '--quick-test-for-ci' in sys.argv:
+        exit(0)
+
+    call_on_start = None
+    if "--windows-standalone-build" in sys.argv:
+        def startup_server(address, port):
+            import webbrowser
+            webbrowser.open("http://{}:{}".format(address, port))
+        call_on_start = startup_server
+
+    if os.name == "nt":
+        try:
+            loop.run_until_complete(run(server, address=address, port=port, verbose=not dont_print, call_on_start=call_on_start))
+        except KeyboardInterrupt:
+            pass
+    else:
+        loop.run_until_complete(run(server, address=address, port=port, verbose=not dont_print, call_on_start=call_on_start))
+
+    cleanup_temp()

nodes.py

@@ -0,0 +1,1115 @@
+import torch
+
+import os
+import sys
+import json
+import hashlib
+import copy
+import traceback
+
+from PIL import Image
+from PIL.PngImagePlugin import PngInfo
+import numpy as np
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy"))
+
+
+import comfy.samplers
+import comfy.sd
+import comfy.utils
+
+import comfy.clip_vision
+
+import model_management
+import importlib
+
+import folder_paths
+
+def before_node_execution():
+    model_management.throw_exception_if_processing_interrupted()
+
+def interrupt_processing(value=True):
+    model_management.interrupt_current_processing(value)
+
+MAX_RESOLUTION=8192
+
+class CLIPTextEncode:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"text": ("STRING", {"multiline": True}), "clip": ("CLIP", )}}
+    RETURN_TYPES = ("CONDITIONING",)
+    FUNCTION = "encode"
+
+    CATEGORY = "conditioning"
+
+    def encode(self, clip, text):
+        return ([[clip.encode(text), {}]], )
+
+class ConditioningCombine:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"conditioning_1": ("CONDITIONING", ), "conditioning_2": ("CONDITIONING", )}}
+    RETURN_TYPES = ("CONDITIONING",)
+    FUNCTION = "combine"
+
+    CATEGORY = "conditioning"
+
+    def combine(self, conditioning_1, conditioning_2):
+        return (conditioning_1 + conditioning_2, )
+
+class ConditioningSetArea:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"conditioning": ("CONDITIONING", ),
+                              "width": ("INT", {"default": 64, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "height": ("INT", {"default": 64, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}),
+                              "y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}),
+                              "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+                             }}
+    RETURN_TYPES = ("CONDITIONING",)
+    FUNCTION = "append"
+
+    CATEGORY = "conditioning"
+
+    def append(self, conditioning, width, height, x, y, strength, min_sigma=0.0, max_sigma=99.0):
+        c = []
+        for t in conditioning:
+            n = [t[0], t[1].copy()]
+            n[1]['area'] = (height // 8, width // 8, y // 8, x // 8)
+            n[1]['strength'] = strength
+            n[1]['min_sigma'] = min_sigma
+            n[1]['max_sigma'] = max_sigma
+            c.append(n)
+        return (c, )
+
+class VAEDecode:
+    def __init__(self, device="cpu"):
+        self.device = device
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT", ), "vae": ("VAE", )}}
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "decode"
+
+    CATEGORY = "latent"
+
+    def decode(self, vae, samples):
+        return (vae.decode(samples["samples"]), )
+
+class VAEDecodeTiled:
+    def __init__(self, device="cpu"):
+        self.device = device
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT", ), "vae": ("VAE", )}}
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "decode"
+
+    CATEGORY = "_for_testing"
+
+    def decode(self, vae, samples):
+        return (vae.decode_tiled(samples["samples"]), )
+
+class VAEEncode:
+    def __init__(self, device="cpu"):
+        self.device = device
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "pixels": ("IMAGE", ), "vae": ("VAE", )}}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "encode"
+
+    CATEGORY = "latent"
+
+    def encode(self, vae, pixels):
+        x = (pixels.shape[1] // 64) * 64
+        y = (pixels.shape[2] // 64) * 64
+        if pixels.shape[1] != x or pixels.shape[2] != y:
+            pixels = pixels[:,:x,:y,:]
+        t = vae.encode(pixels[:,:,:,:3])
+
+        return ({"samples":t}, )
+
+
+class VAEEncodeTiled:
+    def __init__(self, device="cpu"):
+        self.device = device
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "pixels": ("IMAGE", ), "vae": ("VAE", )}}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "encode"
+
+    CATEGORY = "_for_testing"
+
+    def encode(self, vae, pixels):
+        x = (pixels.shape[1] // 64) * 64
+        y = (pixels.shape[2] // 64) * 64
+        if pixels.shape[1] != x or pixels.shape[2] != y:
+            pixels = pixels[:,:x,:y,:]
+        t = vae.encode_tiled(pixels[:,:,:,:3])
+
+        return ({"samples":t}, )
+class VAEEncodeForInpaint:
+    def __init__(self, device="cpu"):
+        self.device = device
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "pixels": ("IMAGE", ), "vae": ("VAE", ), "mask": ("MASK", )}}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "encode"
+
+    CATEGORY = "latent/inpaint"
+
+    def encode(self, vae, pixels, mask):
+        x = (pixels.shape[1] // 64) * 64
+        y = (pixels.shape[2] // 64) * 64
+        mask = torch.nn.functional.interpolate(mask[None,None,], size=(pixels.shape[1], pixels.shape[2]), mode="bilinear")[0][0]
+
+        pixels = pixels.clone()
+        if pixels.shape[1] != x or pixels.shape[2] != y:
+            pixels = pixels[:,:x,:y,:]
+            mask = mask[:x,:y]
+
+        #grow mask by a few pixels to keep things seamless in latent space
+        kernel_tensor = torch.ones((1, 1, 6, 6))
+        mask_erosion = torch.clamp(torch.nn.functional.conv2d((mask.round())[None], kernel_tensor, padding=3), 0, 1)
+        m = (1.0 - mask.round())
+        for i in range(3):
+            pixels[:,:,:,i] -= 0.5
+            pixels[:,:,:,i] *= m
+            pixels[:,:,:,i] += 0.5
+        t = vae.encode(pixels)
+
+        return ({"samples":t, "noise_mask": (mask_erosion[0][:x,:y].round())}, )
+
+class CheckpointLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "config_name": (folder_paths.get_filename_list("configs"), ),
+                              "ckpt_name": (folder_paths.get_filename_list("checkpoints"), )}}
+    RETURN_TYPES = ("MODEL", "CLIP", "VAE")
+    FUNCTION = "load_checkpoint"
+
+    CATEGORY = "loaders"
+
+    def load_checkpoint(self, config_name, ckpt_name, output_vae=True, output_clip=True):
+        config_path = folder_paths.get_full_path("configs", config_name)
+        ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
+        return comfy.sd.load_checkpoint(config_path, ckpt_path, output_vae=True, output_clip=True, embedding_directory=folder_paths.get_folder_paths("embeddings"))
+
+class CheckpointLoaderSimple:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ),
+                             }}
+    RETURN_TYPES = ("MODEL", "CLIP", "VAE")
+    FUNCTION = "load_checkpoint"
+
+    CATEGORY = "loaders"
+
+    def load_checkpoint(self, ckpt_name, output_vae=True, output_clip=True):
+        ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
+        out = comfy.sd.load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, embedding_directory=folder_paths.get_folder_paths("embeddings"))
+        return out
+
+class unCLIPCheckpointLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ),
+                             }}
+    RETURN_TYPES = ("MODEL", "CLIP", "VAE", "CLIP_VISION")
+    FUNCTION = "load_checkpoint"
+
+    CATEGORY = "_for_testing/unclip"
+
+    def load_checkpoint(self, ckpt_name, output_vae=True, output_clip=True):
+        ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
+        out = comfy.sd.load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=True, embedding_directory=folder_paths.get_folder_paths("embeddings"))
+        return out
+
+class CLIPSetLastLayer:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "clip": ("CLIP", ),
+                              "stop_at_clip_layer": ("INT", {"default": -1, "min": -24, "max": -1, "step": 1}),
+                              }}
+    RETURN_TYPES = ("CLIP",)
+    FUNCTION = "set_last_layer"
+
+    CATEGORY = "conditioning"
+
+    def set_last_layer(self, clip, stop_at_clip_layer):
+        clip = clip.clone()
+        clip.clip_layer(stop_at_clip_layer)
+        return (clip,)
+
+class LoraLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "model": ("MODEL",),
+                              "clip": ("CLIP", ),
+                              "lora_name": (folder_paths.get_filename_list("loras"), ),
+                              "strength_model": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
+                              "strength_clip": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
+                              }}
+    RETURN_TYPES = ("MODEL", "CLIP")
+    FUNCTION = "load_lora"
+
+    CATEGORY = "loaders"
+
+    def load_lora(self, model, clip, lora_name, strength_model, strength_clip):
+        lora_path = folder_paths.get_full_path("loras", lora_name)
+        model_lora, clip_lora = comfy.sd.load_lora_for_models(model, clip, lora_path, strength_model, strength_clip)
+        return (model_lora, clip_lora)
+
+class TomePatchModel:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "model": ("MODEL",),
+                              "ratio": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}),
+                              }}
+    RETURN_TYPES = ("MODEL",)
+    FUNCTION = "patch"
+
+    CATEGORY = "_for_testing"
+
+    def patch(self, model, ratio):
+        m = model.clone()
+        m.set_model_tomesd(ratio)
+        return (m, )
+
+class VAELoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "vae_name": (folder_paths.get_filename_list("vae"), )}}
+    RETURN_TYPES = ("VAE",)
+    FUNCTION = "load_vae"
+
+    CATEGORY = "loaders"
+
+    #TODO: scale factor?
+    def load_vae(self, vae_name):
+        vae_path = folder_paths.get_full_path("vae", vae_name)
+        vae = comfy.sd.VAE(ckpt_path=vae_path)
+        return (vae,)
+
+class ControlNetLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "control_net_name": (folder_paths.get_filename_list("controlnet"), )}}
+
+    RETURN_TYPES = ("CONTROL_NET",)
+    FUNCTION = "load_controlnet"
+
+    CATEGORY = "loaders"
+
+    def load_controlnet(self, control_net_name):
+        controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
+        controlnet = comfy.sd.load_controlnet(controlnet_path)
+        return (controlnet,)
+
+class DiffControlNetLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "model": ("MODEL",),
+                              "control_net_name": (folder_paths.get_filename_list("controlnet"), )}}
+
+    RETURN_TYPES = ("CONTROL_NET",)
+    FUNCTION = "load_controlnet"
+
+    CATEGORY = "loaders"
+
+    def load_controlnet(self, model, control_net_name):
+        controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
+        controlnet = comfy.sd.load_controlnet(controlnet_path, model)
+        return (controlnet,)
+
+
+class ControlNetApply:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"conditioning": ("CONDITIONING", ),
+                             "control_net": ("CONTROL_NET", ),
+                             "image": ("IMAGE", ),
+                             "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01})
+                             }}
+    RETURN_TYPES = ("CONDITIONING",)
+    FUNCTION = "apply_controlnet"
+
+    CATEGORY = "conditioning"
+
+    def apply_controlnet(self, conditioning, control_net, image, strength):
+        c = []
+        control_hint = image.movedim(-1,1)
+        print(control_hint.shape)
+        for t in conditioning:
+            n = [t[0], t[1].copy()]
+            c_net = control_net.copy().set_cond_hint(control_hint, strength)
+            if 'control' in t[1]:
+                c_net.set_previous_controlnet(t[1]['control'])
+            n[1]['control'] = c_net
+            c.append(n)
+        return (c, )
+
+class CLIPLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "clip_name": (folder_paths.get_filename_list("clip"), ),
+                             }}
+    RETURN_TYPES = ("CLIP",)
+    FUNCTION = "load_clip"
+
+    CATEGORY = "loaders"
+
+    def load_clip(self, clip_name):
+        clip_path = folder_paths.get_full_path("clip", clip_name)
+        clip = comfy.sd.load_clip(ckpt_path=clip_path, embedding_directory=folder_paths.get_folder_paths("embeddings"))
+        return (clip,)
+
+class CLIPVisionLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "clip_name": (folder_paths.get_filename_list("clip_vision"), ),
+                             }}
+    RETURN_TYPES = ("CLIP_VISION",)
+    FUNCTION = "load_clip"
+
+    CATEGORY = "loaders"
+
+    def load_clip(self, clip_name):
+        clip_path = folder_paths.get_full_path("clip_vision", clip_name)
+        clip_vision = comfy.clip_vision.load(clip_path)
+        return (clip_vision,)
+
+class CLIPVisionEncode:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "clip_vision": ("CLIP_VISION",),
+                              "image": ("IMAGE",)
+                             }}
+    RETURN_TYPES = ("CLIP_VISION_OUTPUT",)
+    FUNCTION = "encode"
+
+    CATEGORY = "conditioning"
+
+    def encode(self, clip_vision, image):
+        output = clip_vision.encode_image(image)
+        return (output,)
+
+class StyleModelLoader:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "style_model_name": (folder_paths.get_filename_list("style_models"), )}}
+
+    RETURN_TYPES = ("STYLE_MODEL",)
+    FUNCTION = "load_style_model"
+
+    CATEGORY = "loaders"
+
+    def load_style_model(self, style_model_name):
+        style_model_path = folder_paths.get_full_path("style_models", style_model_name)
+        style_model = comfy.sd.load_style_model(style_model_path)
+        return (style_model,)
+
+
+class StyleModelApply:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"conditioning": ("CONDITIONING", ),
+                             "style_model": ("STYLE_MODEL", ),
+                             "clip_vision_output": ("CLIP_VISION_OUTPUT", ),
+                             }}
+    RETURN_TYPES = ("CONDITIONING",)
+    FUNCTION = "apply_stylemodel"
+
+    CATEGORY = "conditioning/style_model"
+
+    def apply_stylemodel(self, clip_vision_output, style_model, conditioning):
+        cond = style_model.get_cond(clip_vision_output)
+        c = []
+        for t in conditioning:
+            n = [torch.cat((t[0], cond), dim=1), t[1].copy()]
+            c.append(n)
+        return (c, )
+
+class unCLIPConditioning:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {"conditioning": ("CONDITIONING", ),
+                             "clip_vision_output": ("CLIP_VISION_OUTPUT", ),
+                             "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
+                             }}
+    RETURN_TYPES = ("CONDITIONING",)
+    FUNCTION = "apply_adm"
+
+    CATEGORY = "_for_testing/unclip"
+
+    def apply_adm(self, conditioning, clip_vision_output, strength):
+        c = []
+        for t in conditioning:
+            o = t[1].copy()
+            x = (clip_vision_output, strength)
+            if "adm" in o:
+                o["adm"] = o["adm"][:] + [x]
+            else:
+                o["adm"] = [x]
+            n = [t[0], o]
+            c.append(n)
+        return (c, )
+
+
+class EmptyLatentImage:
+    def __init__(self, device="cpu"):
+        self.device = device
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "width": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "height": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "batch_size": ("INT", {"default": 1, "min": 1, "max": 64})}}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "generate"
+
+    CATEGORY = "latent"
+
+    def generate(self, width, height, batch_size=1):
+        latent = torch.zeros([batch_size, 4, height // 8, width // 8])
+        return ({"samples":latent}, )
+
+
+
+class LatentUpscale:
+    upscale_methods = ["nearest-exact", "bilinear", "area"]
+    crop_methods = ["disabled", "center"]
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT",), "upscale_method": (s.upscale_methods,),
+                              "width": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "height": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "crop": (s.crop_methods,)}}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "upscale"
+
+    CATEGORY = "latent"
+
+    def upscale(self, samples, upscale_method, width, height, crop):
+        s = samples.copy()
+        s["samples"] = comfy.utils.common_upscale(samples["samples"], width // 8, height // 8, upscale_method, crop)
+        return (s,)
+
+class LatentRotate:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT",),
+                              "rotation": (["none", "90 degrees", "180 degrees", "270 degrees"],),
+                              }}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "rotate"
+
+    CATEGORY = "latent/transform"
+
+    def rotate(self, samples, rotation):
+        s = samples.copy()
+        rotate_by = 0
+        if rotation.startswith("90"):
+            rotate_by = 1
+        elif rotation.startswith("180"):
+            rotate_by = 2
+        elif rotation.startswith("270"):
+            rotate_by = 3
+
+        s["samples"] = torch.rot90(samples["samples"], k=rotate_by, dims=[3, 2])
+        return (s,)
+
+class LatentFlip:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT",),
+                              "flip_method": (["x-axis: vertically", "y-axis: horizontally"],),
+                              }}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "flip"
+
+    CATEGORY = "latent/transform"
+
+    def flip(self, samples, flip_method):
+        s = samples.copy()
+        if flip_method.startswith("x"):
+            s["samples"] = torch.flip(samples["samples"], dims=[2])
+        elif flip_method.startswith("y"):
+            s["samples"] = torch.flip(samples["samples"], dims=[3])
+
+        return (s,)
+
+class LatentComposite:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples_to": ("LATENT",),
+                              "samples_from": ("LATENT",),
+                              "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
+                              "y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
+                              "feather": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
+                              }}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "composite"
+
+    CATEGORY = "latent"
+
+    def composite(self, samples_to, samples_from, x, y, composite_method="normal", feather=0):
+        x =  x // 8
+        y = y // 8
+        feather = feather // 8
+        samples_out = samples_to.copy()
+        s = samples_to["samples"].clone()
+        samples_to = samples_to["samples"]
+        samples_from = samples_from["samples"]
+        if feather == 0:
+            s[:,:,y:y+samples_from.shape[2],x:x+samples_from.shape[3]] = samples_from[:,:,:samples_to.shape[2] - y, :samples_to.shape[3] - x]
+        else:
+            samples_from = samples_from[:,:,:samples_to.shape[2] - y, :samples_to.shape[3] - x]
+            mask = torch.ones_like(samples_from)
+            for t in range(feather):
+                if y != 0:
+                    mask[:,:,t:1+t,:] *= ((1.0/feather) * (t + 1))
+
+                if y + samples_from.shape[2] < samples_to.shape[2]:
+                    mask[:,:,mask.shape[2] -1 -t: mask.shape[2]-t,:] *= ((1.0/feather) * (t + 1))
+                if x != 0:
+                    mask[:,:,:,t:1+t] *= ((1.0/feather) * (t + 1))
+                if x + samples_from.shape[3] < samples_to.shape[3]:
+                    mask[:,:,:,mask.shape[3]- 1 - t: mask.shape[3]- t] *= ((1.0/feather) * (t + 1))
+            rev_mask = torch.ones_like(mask) - mask
+            s[:,:,y:y+samples_from.shape[2],x:x+samples_from.shape[3]] = samples_from[:,:,:samples_to.shape[2] - y, :samples_to.shape[3] - x] * mask + s[:,:,y:y+samples_from.shape[2],x:x+samples_from.shape[3]] * rev_mask
+        samples_out["samples"] = s
+        return (samples_out,)
+
+class LatentCrop:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT",),
+                              "width": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "height": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 64}),
+                              "x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
+                              "y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
+                              }}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "crop"
+
+    CATEGORY = "latent/transform"
+
+    def crop(self, samples, width, height, x, y):
+        s = samples.copy()
+        samples = samples['samples']
+        x =  x // 8
+        y = y // 8
+
+        #enfonce minimum size of 64
+        if x > (samples.shape[3] - 8):
+            x = samples.shape[3] - 8
+        if y > (samples.shape[2] - 8):
+            y = samples.shape[2] - 8
+
+        new_height = height // 8
+        new_width = width // 8
+        to_x = new_width + x
+        to_y = new_height + y
+        def enforce_image_dim(d, to_d, max_d):
+            if to_d > max_d:
+                leftover = (to_d - max_d) % 8
+                to_d = max_d
+                d -= leftover
+            return (d, to_d)
+
+        #make sure size is always multiple of 64
+        x, to_x = enforce_image_dim(x, to_x, samples.shape[3])
+        y, to_y = enforce_image_dim(y, to_y, samples.shape[2])
+        s['samples'] = samples[:,:,y:to_y, x:to_x]
+        return (s,)
+
+class SetLatentNoiseMask:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "samples": ("LATENT",),
+                              "mask": ("MASK",),
+                              }}
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "set_mask"
+
+    CATEGORY = "latent/inpaint"
+
+    def set_mask(self, samples, mask):
+        s = samples.copy()
+        s["noise_mask"] = mask
+        return (s,)
+
+
+def common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False):
+    latent_image = latent["samples"]
+    noise_mask = None
+    device = model_management.get_torch_device()
+
+    if disable_noise:
+        noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu")
+    else:
+        noise = torch.randn(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, generator=torch.manual_seed(seed), device="cpu")
+
+    if "noise_mask" in latent:
+        noise_mask = latent['noise_mask']
+        noise_mask = torch.nn.functional.interpolate(noise_mask[None,None,], size=(noise.shape[2], noise.shape[3]), mode="bilinear")
+        noise_mask = noise_mask.round()
+        noise_mask = torch.cat([noise_mask] * noise.shape[1], dim=1)
+        noise_mask = torch.cat([noise_mask] * noise.shape[0])
+        noise_mask = noise_mask.to(device)
+
+    real_model = None
+    model_management.load_model_gpu(model)
+    real_model = model.model
+
+    noise = noise.to(device)
+    latent_image = latent_image.to(device)
+
+    positive_copy = []
+    negative_copy = []
+
+    control_nets = []
+    for p in positive:
+        t = p[0]
+        if t.shape[0] < noise.shape[0]:
+            t = torch.cat([t] * noise.shape[0])
+        t = t.to(device)
+        if 'control' in p[1]:
+            control_nets += [p[1]['control']]
+        positive_copy += [[t] + p[1:]]
+    for n in negative:
+        t = n[0]
+        if t.shape[0] < noise.shape[0]:
+            t = torch.cat([t] * noise.shape[0])
+        t = t.to(device)
+        if 'control' in n[1]:
+            control_nets += [n[1]['control']]
+        negative_copy += [[t] + n[1:]]
+
+    control_net_models = []
+    for x in control_nets:
+        control_net_models += x.get_control_models()
+    model_management.load_controlnet_gpu(control_net_models)
+
+    if sampler_name in comfy.samplers.KSampler.SAMPLERS:
+        sampler = comfy.samplers.KSampler(real_model, steps=steps, device=device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
+    else:
+        #other samplers
+        pass
+
+    samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask)
+    samples = samples.cpu()
+    for c in control_nets:
+        c.cleanup()
+
+    out = latent.copy()
+    out["samples"] = samples
+    return (out, )
+
+class KSampler:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"model": ("MODEL",),
+                    "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+                    "steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
+                    "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0}),
+                    "sampler_name": (comfy.samplers.KSampler.SAMPLERS, ),
+                    "scheduler": (comfy.samplers.KSampler.SCHEDULERS, ),
+                    "positive": ("CONDITIONING", ),
+                    "negative": ("CONDITIONING", ),
+                    "latent_image": ("LATENT", ),
+                    "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+                    }}
+
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "sample"
+
+    CATEGORY = "sampling"
+
+    def sample(self, model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0):
+        return common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise)
+
+class KSamplerAdvanced:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"model": ("MODEL",),
+                    "add_noise": (["enable", "disable"], ),
+                    "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+                    "steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
+                    "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0}),
+                    "sampler_name": (comfy.samplers.KSampler.SAMPLERS, ),
+                    "scheduler": (comfy.samplers.KSampler.SCHEDULERS, ),
+                    "positive": ("CONDITIONING", ),
+                    "negative": ("CONDITIONING", ),
+                    "latent_image": ("LATENT", ),
+                    "start_at_step": ("INT", {"default": 0, "min": 0, "max": 10000}),
+                    "end_at_step": ("INT", {"default": 10000, "min": 0, "max": 10000}),
+                    "return_with_leftover_noise": (["disable", "enable"], ),
+                    }}
+
+    RETURN_TYPES = ("LATENT",)
+    FUNCTION = "sample"
+
+    CATEGORY = "sampling"
+
+    def sample(self, model, add_noise, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, start_at_step, end_at_step, return_with_leftover_noise, denoise=1.0):
+        force_full_denoise = True
+        if return_with_leftover_noise == "enable":
+            force_full_denoise = False
+        disable_noise = False
+        if add_noise == "disable":
+            disable_noise = True
+        return common_ksampler(model, noise_seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise, disable_noise=disable_noise, start_step=start_at_step, last_step=end_at_step, force_full_denoise=force_full_denoise)
+
+class SaveImage:
+    def __init__(self):
+        self.output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "output")
+        self.type = "output"
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": 
+                    {"images": ("IMAGE", ),
+                     "filename_prefix": ("STRING", {"default": "ComfyUI"})},
+                "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+                }
+
+    RETURN_TYPES = ()
+    FUNCTION = "save_images"
+
+    OUTPUT_NODE = True
+
+    CATEGORY = "image"
+
+    def save_images(self, images, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):
+        def map_filename(filename):
+            prefix_len = len(os.path.basename(filename_prefix))
+            prefix = filename[:prefix_len + 1]
+            try:
+                digits = int(filename[prefix_len + 1:].split('_')[0])
+            except:
+                digits = 0
+            return (digits, prefix)
+
+        def compute_vars(input):
+            input = input.replace("%width%", str(images[0].shape[1]))
+            input = input.replace("%height%", str(images[0].shape[0]))
+            return input
+
+        filename_prefix = compute_vars(filename_prefix)
+
+        subfolder = os.path.dirname(os.path.normpath(filename_prefix))
+        filename = os.path.basename(os.path.normpath(filename_prefix))
+
+        full_output_folder = os.path.join(self.output_dir, subfolder)
+
+        if os.path.commonpath((self.output_dir, os.path.abspath(full_output_folder))) != self.output_dir:
+            print("Saving image outside the output folder is not allowed.")
+            return {}
+
+        try:
+            counter = max(filter(lambda a: a[1][:-1] == filename and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1
+        except ValueError:
+            counter = 1
+        except FileNotFoundError:
+            os.makedirs(full_output_folder, exist_ok=True)
+            counter = 1
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+        results = list()
+        for image in images:
+            i = 255. * image.cpu().numpy()
+            img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
+            metadata = PngInfo()
+            if prompt is not None:
+                metadata.add_text("prompt", json.dumps(prompt))
+            if extra_pnginfo is not None:
+                for x in extra_pnginfo:
+                    metadata.add_text(x, json.dumps(extra_pnginfo[x]))
+
+            file = f"{filename}_{counter:05}_.png"
+            img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=4)
+            results.append({
+                "filename": file,
+                "subfolder": subfolder,
+                "type": self.type
+            });
+            counter += 1
+
+        return { "ui": { "images": results } }
+
+class PreviewImage(SaveImage):
+    def __init__(self):
+        self.output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "temp")
+        self.type = "temp"
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"images": ("IMAGE", ), },
+                "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+                }
+
+class LoadImage:
+    input_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "input")
+    @classmethod
+    def INPUT_TYPES(s):
+        if not os.path.exists(s.input_dir):
+            os.makedirs(s.input_dir)
+        return {"required":
+                    {"image": (sorted(os.listdir(s.input_dir)), )},
+                }
+
+    CATEGORY = "image"
+
+    RETURN_TYPES = ("IMAGE", "MASK")
+    FUNCTION = "load_image"
+    def load_image(self, image):
+        image_path = os.path.join(self.input_dir, image)
+        i = Image.open(image_path)
+        image = i.convert("RGB")
+        image = np.array(image).astype(np.float32) / 255.0
+        image = torch.from_numpy(image)[None,]
+        if 'A' in i.getbands():
+            mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
+            mask = 1. - torch.from_numpy(mask)
+        else:
+            mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
+        return (image, mask)
+
+    @classmethod
+    def IS_CHANGED(s, image):
+        image_path = os.path.join(s.input_dir, image)
+        m = hashlib.sha256()
+        with open(image_path, 'rb') as f:
+            m.update(f.read())
+        return m.digest().hex()
+
+class LoadImageMask:
+    input_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "input")
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"image": (sorted(os.listdir(s.input_dir)), ),
+                    "channel": (["alpha", "red", "green", "blue"], ),}
+                }
+
+    CATEGORY = "image"
+
+    RETURN_TYPES = ("MASK",)
+    FUNCTION = "load_image"
+    def load_image(self, image, channel):
+        image_path = os.path.join(self.input_dir, image)
+        i = Image.open(image_path)
+        mask = None
+        c = channel[0].upper()
+        if c in i.getbands():
+            mask = np.array(i.getchannel(c)).astype(np.float32) / 255.0
+            mask = torch.from_numpy(mask)
+            if c == 'A':
+                mask = 1. - mask
+        else:
+            mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
+        return (mask,)
+
+    @classmethod
+    def IS_CHANGED(s, image, channel):
+        image_path = os.path.join(s.input_dir, image)
+        m = hashlib.sha256()
+        with open(image_path, 'rb') as f:
+            m.update(f.read())
+        return m.digest().hex()
+
+class ImageScale:
+    upscale_methods = ["nearest-exact", "bilinear", "area"]
+    crop_methods = ["disabled", "center"]
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "image": ("IMAGE",), "upscale_method": (s.upscale_methods,),
+                              "width": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
+                              "height": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
+                              "crop": (s.crop_methods,)}}
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "upscale"
+
+    CATEGORY = "image/upscaling"
+
+    def upscale(self, image, upscale_method, width, height, crop):
+        samples = image.movedim(-1,1)
+        s = comfy.utils.common_upscale(samples, width, height, upscale_method, crop)
+        s = s.movedim(1,-1)
+        return (s,)
+
+class ImageInvert:
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": { "image": ("IMAGE",)}}
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "invert"
+
+    CATEGORY = "image"
+
+    def invert(self, image):
+        s = 1.0 - image
+        return (s,)
+
+
+class ImagePadForOutpaint:
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "image": ("IMAGE",),
+                "left": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}),
+                "top": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}),
+                "right": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}),
+                "bottom": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 64}),
+                "feathering": ("INT", {"default": 40, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE", "MASK")
+    FUNCTION = "expand_image"
+
+    CATEGORY = "image"
+
+    def expand_image(self, image, left, top, right, bottom, feathering):
+        d1, d2, d3, d4 = image.size()
+
+        new_image = torch.zeros(
+            (d1, d2 + top + bottom, d3 + left + right, d4),
+            dtype=torch.float32,
+        )
+        new_image[:, top:top + d2, left:left + d3, :] = image
+
+        mask = torch.ones(
+            (d2 + top + bottom, d3 + left + right),
+            dtype=torch.float32,
+        )
+
+        t = torch.zeros(
+            (d2, d3),
+            dtype=torch.float32
+        )
+
+        if feathering > 0 and feathering * 2 < d2 and feathering * 2 < d3:
+
+            for i in range(d2):
+                for j in range(d3):
+                    dt = i if top != 0 else d2
+                    db = d2 - i if bottom != 0 else d2
+
+                    dl = j if left != 0 else d3
+                    dr = d3 - j if right != 0 else d3
+
+                    d = min(dt, db, dl, dr)
+
+                    if d >= feathering:
+                        continue
+
+                    v = (feathering - d) / feathering
+
+                    t[i, j] = v * v
+
+        mask[top:top + d2, left:left + d3] = t
+
+        return (new_image, mask)
+
+
+NODE_CLASS_MAPPINGS = {
+    "KSampler": KSampler,
+    "CheckpointLoader": CheckpointLoader,
+    "CheckpointLoaderSimple": CheckpointLoaderSimple,
+    "CLIPTextEncode": CLIPTextEncode,
+    "CLIPSetLastLayer": CLIPSetLastLayer,
+    "VAEDecode": VAEDecode,
+    "VAEEncode": VAEEncode,
+    "VAEEncodeForInpaint": VAEEncodeForInpaint,
+    "VAELoader": VAELoader,
+    "EmptyLatentImage": EmptyLatentImage,
+    "LatentUpscale": LatentUpscale,
+    "SaveImage": SaveImage,
+    "PreviewImage": PreviewImage,
+    "LoadImage": LoadImage,
+    "LoadImageMask": LoadImageMask,
+    "ImageScale": ImageScale,
+    "ImageInvert": ImageInvert,
+    "ImagePadForOutpaint": ImagePadForOutpaint,
+    "ConditioningCombine": ConditioningCombine,
+    "ConditioningSetArea": ConditioningSetArea,
+    "KSamplerAdvanced": KSamplerAdvanced,
+    "SetLatentNoiseMask": SetLatentNoiseMask,
+    "LatentComposite": LatentComposite,
+    "LatentRotate": LatentRotate,
+    "LatentFlip": LatentFlip,
+    "LatentCrop": LatentCrop,
+    "LoraLoader": LoraLoader,
+    "CLIPLoader": CLIPLoader,
+    "CLIPVisionEncode": CLIPVisionEncode,
+    "StyleModelApply": StyleModelApply,
+    "unCLIPConditioning": unCLIPConditioning,
+    "ControlNetApply": ControlNetApply,
+    "ControlNetLoader": ControlNetLoader,
+    "DiffControlNetLoader": DiffControlNetLoader,
+    "StyleModelLoader": StyleModelLoader,
+    "CLIPVisionLoader": CLIPVisionLoader,
+    "VAEDecodeTiled": VAEDecodeTiled,
+    "VAEEncodeTiled": VAEEncodeTiled,
+    "TomePatchModel": TomePatchModel,
+    "unCLIPCheckpointLoader": unCLIPCheckpointLoader,
+}
+
+def load_custom_node(module_path):
+    module_name = os.path.basename(module_path)
+    if os.path.isfile(module_path):
+        sp = os.path.splitext(module_path)
+        module_name = sp[0]
+    try:
+        if os.path.isfile(module_path):
+            module_spec = importlib.util.spec_from_file_location(module_name, module_path)
+        else:
+            module_spec = importlib.util.spec_from_file_location(module_name, os.path.join(module_path, "__init__.py"))
+        module = importlib.util.module_from_spec(module_spec)
+        sys.modules[module_name] = module
+        module_spec.loader.exec_module(module)
+        if hasattr(module, "NODE_CLASS_MAPPINGS") and getattr(module, "NODE_CLASS_MAPPINGS") is not None:
+            NODE_CLASS_MAPPINGS.update(module.NODE_CLASS_MAPPINGS)
+        else:
+            print(f"Skip {module_path} module for custom nodes due to the lack of NODE_CLASS_MAPPINGS.")
+    except Exception as e:
+        print(traceback.format_exc())
+        print(f"Cannot import {module_path} module for custom nodes:", e)
+
+def load_custom_nodes():
+    CUSTOM_NODE_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "custom_nodes")
+    possible_modules = os.listdir(CUSTOM_NODE_PATH)
+    if "__pycache__" in possible_modules:
+        possible_modules.remove("__pycache__")
+
+    for possible_module in possible_modules:
+        module_path = os.path.join(CUSTOM_NODE_PATH, possible_module)
+        if os.path.isfile(module_path) and os.path.splitext(module_path)[1] != ".py": continue
+        load_custom_node(module_path)
+
+def init_custom_nodes():
+    load_custom_nodes()
+    load_custom_node(os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_extras"), "nodes_upscale_model.py"))

requirements.txt

@@ -0,0 +1,11 @@
+torch
+torchdiffeq
+torchsde
+einops
+open-clip-torch
+transformers>=4.25.1
+safetensors
+pytorch_lightning
+aiohttp
+accelerate
+pyyaml

server.py

@@ -0,0 +1,294 @@
+import os
+import sys
+import asyncio
+import nodes
+import folder_paths
+import execution
+import uuid
+import json
+import glob
+try:
+    import aiohttp
+    from aiohttp import web
+except ImportError:
+    print("Module 'aiohttp' not installed. Please install it via:")
+    print("pip install aiohttp")
+    print("or")
+    print("pip install -r requirements.txt")
+    sys.exit()
+
+import mimetypes
+
+
+@web.middleware
+async def cache_control(request: web.Request, handler):
+    response: web.Response = await handler(request)
+    if request.path.endswith('.js') or request.path.endswith('.css'):
+        response.headers.setdefault('Cache-Control', 'no-cache')
+    return response
+
+class PromptServer():
+    def __init__(self, loop):
+        PromptServer.instance = self
+
+        mimetypes.init(); 
+        mimetypes.types_map['.js'] = 'application/javascript; charset=utf-8'
+        self.prompt_queue = None
+        self.loop = loop
+        self.messages = asyncio.Queue()
+        self.number = 0
+        self.app = web.Application(client_max_size=20971520, middlewares=[cache_control])
+        self.sockets = dict()
+        self.web_root = os.path.join(os.path.dirname(
+            os.path.realpath(__file__)), "web")
+        routes = web.RouteTableDef()
+        self.routes = routes
+        self.last_node_id = None
+        self.client_id = None
+
+        @routes.get('/ws')
+        async def websocket_handler(request):
+            ws = web.WebSocketResponse()
+            await ws.prepare(request)
+            sid = request.rel_url.query.get('clientId', '')
+            if sid:
+                # Reusing existing session, remove old
+                self.sockets.pop(sid, None)
+            else:
+                sid = uuid.uuid4().hex      
+
+            self.sockets[sid] = ws
+
+            try:
+                # Send initial state to the new client
+                await self.send("status", { "status": self.get_queue_info(), 'sid': sid }, sid)
+                # On reconnect if we are the currently executing client send the current node
+                if self.client_id == sid and self.last_node_id is not None:
+                    await self.send("executing", { "node": self.last_node_id }, sid)
+                    
+                async for msg in ws:
+                    if msg.type == aiohttp.WSMsgType.ERROR:
+                        print('ws connection closed with exception %s' % ws.exception())
+            finally:
+                self.sockets.pop(sid, None)
+            return ws
+
+        @routes.get("/")
+        async def get_root(request):
+            return web.FileResponse(os.path.join(self.web_root, "index.html"))
+
+        @routes.get("/embeddings")
+        def get_embeddings(self):
+            embeddings = folder_paths.get_filename_list("embeddings")
+            return web.json_response(list(map(lambda a: os.path.splitext(a)[0].lower(), embeddings)))
+
+        @routes.get("/extensions")
+        async def get_extensions(request):
+            files = glob.glob(os.path.join(self.web_root, 'extensions/**/*.js'), recursive=True)
+            return web.json_response(list(map(lambda f: "/" + os.path.relpath(f, self.web_root).replace("\\", "/"), files)))
+
+        @routes.post("/upload/image")
+        async def upload_image(request):
+            upload_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "input")
+
+            if not os.path.exists(upload_dir):
+                os.makedirs(upload_dir)
+            
+            post = await request.post()
+            image = post.get("image")
+
+            if image and image.file:
+                filename = image.filename
+                if not filename:
+                    return web.Response(status=400)
+
+                split = os.path.splitext(filename)
+                i = 1
+                while os.path.exists(os.path.join(upload_dir, filename)):
+                    filename = f"{split[0]} ({i}){split[1]}"
+                    i += 1
+
+                filepath = os.path.join(upload_dir, filename)
+
+                with open(filepath, "wb") as f:
+                    f.write(image.file.read())
+                
+                return web.json_response({"name" : filename})
+            else:
+                return web.Response(status=400)
+
+
+        @routes.get("/view")
+        async def view_image(request):
+            if "filename" in request.rel_url.query:
+                type = request.rel_url.query.get("type", "output")
+                if type not in ["output", "input", "temp"]:
+                    return web.Response(status=400)
+
+                output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), type)
+                if "subfolder" in request.rel_url.query:
+                    full_output_dir = os.path.join(output_dir, request.rel_url.query["subfolder"])
+                    if os.path.commonpath((os.path.abspath(full_output_dir), output_dir)) != output_dir:
+                        return web.Response(status=403)
+                    output_dir = full_output_dir
+
+                filename = request.rel_url.query["filename"]
+                filename = os.path.basename(filename)
+                file = os.path.join(output_dir, filename)
+
+                if os.path.isfile(file):
+                    return web.FileResponse(file, headers={"Content-Disposition": f"filename=\"{filename}\""})
+                
+            return web.Response(status=404)
+
+        @routes.get("/prompt")
+        async def get_prompt(request):
+            return web.json_response(self.get_queue_info())
+
+        @routes.get("/object_info")
+        async def get_object_info(request):
+            out = {}
+            for x in nodes.NODE_CLASS_MAPPINGS:
+                obj_class = nodes.NODE_CLASS_MAPPINGS[x]
+                info = {}
+                info['input'] = obj_class.INPUT_TYPES()
+                info['output'] = obj_class.RETURN_TYPES
+                info['output_name'] = obj_class.RETURN_NAMES if hasattr(obj_class, 'RETURN_NAMES') else info['output']
+                info['name'] = x #TODO
+                info['description'] = ''
+                info['category'] = 'sd'
+                if hasattr(obj_class, 'CATEGORY'):
+                    info['category'] = obj_class.CATEGORY
+                out[x] = info
+            return web.json_response(out)
+
+        @routes.get("/history")
+        async def get_history(request):
+            return web.json_response(self.prompt_queue.get_history())
+
+        @routes.get("/queue")
+        async def get_queue(request):
+            queue_info = {}
+            current_queue = self.prompt_queue.get_current_queue()
+            queue_info['queue_running'] = current_queue[0]
+            queue_info['queue_pending'] = current_queue[1]
+            return web.json_response(queue_info)
+
+        @routes.post("/prompt")
+        async def post_prompt(request):
+            print("got prompt")
+            resp_code = 200
+            out_string = ""
+            json_data =  await request.json()
+
+            if "number" in json_data:
+                number = float(json_data['number'])
+            else:
+                number = self.number
+                if "front" in json_data:
+                    if json_data['front']:
+                        number = -number
+
+                self.number += 1
+
+            if "prompt" in json_data:
+                prompt = json_data["prompt"]
+                valid = execution.validate_prompt(prompt)
+                extra_data = {}
+                if "extra_data" in json_data:
+                    extra_data = json_data["extra_data"]
+
+                if "client_id" in json_data:
+                    extra_data["client_id"] = json_data["client_id"]
+                if valid[0]:
+                    self.prompt_queue.put((number, id(prompt), prompt, extra_data))
+                else:
+                    resp_code = 400
+                    out_string = valid[1]
+                    print("invalid prompt:", valid[1])
+
+            return web.Response(body=out_string, status=resp_code)
+        
+        @routes.post("/queue")
+        async def post_queue(request):
+            json_data =  await request.json()
+            if "clear" in json_data:
+                if json_data["clear"]:
+                    self.prompt_queue.wipe_queue()
+            if "delete" in json_data:
+                to_delete = json_data['delete']
+                for id_to_delete in to_delete:
+                    delete_func = lambda a: a[1] == int(id_to_delete)
+                    self.prompt_queue.delete_queue_item(delete_func)
+                    
+            return web.Response(status=200)
+
+        @routes.post("/interrupt")
+        async def post_interrupt(request):
+            nodes.interrupt_processing()
+            return web.Response(status=200)
+
+        @routes.post("/history")
+        async def post_history(request):
+            json_data =  await request.json()
+            if "clear" in json_data:
+                if json_data["clear"]:
+                    self.prompt_queue.wipe_history()
+            if "delete" in json_data:
+                to_delete = json_data['delete']
+                for id_to_delete in to_delete:
+                    self.prompt_queue.delete_history_item(id_to_delete)
+
+            return web.Response(status=200)
+        
+    def add_routes(self):
+        self.app.add_routes(self.routes)
+        self.app.add_routes([
+            web.static('/', self.web_root),
+        ])
+
+    def get_queue_info(self):
+        prompt_info = {}
+        exec_info = {}
+        exec_info['queue_remaining'] = self.prompt_queue.get_tasks_remaining()
+        prompt_info['exec_info'] = exec_info
+        return prompt_info
+
+    async def send(self, event, data, sid=None):
+        message = {"type": event, "data": data}
+       
+        if isinstance(message, str) == False:
+            message = json.dumps(message)
+
+        if sid is None:
+            for ws in self.sockets.values():
+                await ws.send_str(message)
+        elif sid in self.sockets:
+            await self.sockets[sid].send_str(message)
+
+    def send_sync(self, event, data, sid=None):
+        self.loop.call_soon_threadsafe(
+            self.messages.put_nowait, (event, data, sid))
+
+    def queue_updated(self):
+        self.send_sync("status", { "status": self.get_queue_info() })
+
+    async def publish_loop(self):
+        while True:
+            msg = await self.messages.get()
+            await self.send(*msg)
+
+    async def start(self, address, port, verbose=True, call_on_start=None):
+        runner = web.AppRunner(self.app)
+        await runner.setup()
+        site = web.TCPSite(runner, address, port)
+        await site.start()
+
+        if address == '':
+            address = '0.0.0.0'
+        if verbose:
+            print("Starting server\n")
+            print("To see the GUI go to: http://{}:{}".format(address, port))
+        if call_on_start is not None:
+            call_on_start(address, port)
+