import argparse import unittest import os import sys import time import datetime from enum import Enum from typing import List, Tuple import cv2 import requests import numpy as np from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.support import expected_conditions as EC from webdriver_manager.chrome import ChromeDriverManager TIMEOUT = 20 # seconds CWD = os.getcwd() SKI_IMAGE = os.path.join(CWD, "images/ski.jpg") timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") test_result_dir = os.path.join("results", f"test_result_{timestamp}") test_expectation_dir = "expectations" os.makedirs(test_result_dir, exist_ok=True) os.makedirs(test_expectation_dir, exist_ok=True) driver_path = ChromeDriverManager().install() class GenType(Enum): txt2img = "txt2img" img2img = "img2img" def _find_by_xpath(self, driver: webdriver.Chrome, xpath: str) -> "WebElement": return driver.find_element(By.XPATH, xpath) def tab(self, driver: webdriver.Chrome) -> "WebElement": return self._find_by_xpath( driver, f"//*[@id='tabs']/*[contains(@class, 'tab-nav')]//button[text()='{self.value}']", ) def controlnet_panel(self, driver: webdriver.Chrome) -> "WebElement": return self._find_by_xpath( driver, f"//*[@id='tab_{self.value}']//*[@id='controlnet']" ) def generate_button(self, driver: webdriver.Chrome) -> "WebElement": return self._find_by_xpath(driver, f"//*[@id='{self.value}_generate_box']") def prompt_textarea(self, driver: webdriver.Chrome) -> "WebElement": return self._find_by_xpath(driver, f"//*[@id='{self.value}_prompt']//textarea") class SeleniumTestCase(unittest.TestCase): def __init__(self, methodName: str = "runTest") -> None: super().__init__(methodName) self.driver = None self.gen_type = None def setUp(self) -> None: super().setUp() self.driver = webdriver.Chrome(driver_path) self.driver.get(webui_url) wait = WebDriverWait(self.driver, TIMEOUT) wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "#controlnet"))) self.gen_type = GenType.txt2img def tearDown(self) -> None: self.driver.quit() super().tearDown() def select_gen_type(self, gen_type: GenType): gen_type.tab(self.driver).click() self.gen_type = gen_type def set_prompt(self, prompt: str): textarea = self.gen_type.prompt_textarea(self.driver) textarea.clear() textarea.send_keys(prompt) def expand_controlnet_panel(self): controlnet_panel = self.gen_type.controlnet_panel(self.driver) input_image_group = controlnet_panel.find_element( By.CSS_SELECTOR, ".cnet-input-image-group" ) if not input_image_group.is_displayed(): controlnet_panel.click() def enable_controlnet_unit(self): controlnet_panel = self.gen_type.controlnet_panel(self.driver) enable_checkbox = controlnet_panel.find_element( By.CSS_SELECTOR, ".cnet-unit-enabled input[type='checkbox']" ) if not enable_checkbox.is_selected(): enable_checkbox.click() def iterate_preprocessor_types(self, ignore_none: bool = True): dropdown = self.gen_type.controlnet_panel(self.driver).find_element( By.CSS_SELECTOR, f"#{self.gen_type.value}_controlnet_ControlNet-0_controlnet_preprocessor_dropdown", ) index = 0 while True: dropdown.click() options = dropdown.find_elements( By.XPATH, "//ul[contains(@class, 'options')]/li" ) input_element = dropdown.find_element(By.CSS_SELECTOR, "input") if index >= len(options): return option = options[index] index += 1 if "none" in option.text and ignore_none: continue option_text = option.text option.click() yield option_text def select_control_type(self, control_type: str): controlnet_panel = self.gen_type.controlnet_panel(self.driver) control_type_radio = controlnet_panel.find_element( By.CSS_SELECTOR, f'.controlnet_control_type input[value="{control_type}"]' ) control_type_radio.click() time.sleep(3) # Wait for gradio backend to update model/module def set_seed(self, seed: int): seed_input = self.driver.find_element( By.CSS_SELECTOR, f"#{self.gen_type.value}_seed input[type='number']" ) seed_input.clear() seed_input.send_keys(seed) def set_subseed(self, seed: int): show_button = self.driver.find_element( By.CSS_SELECTOR, f"#{self.gen_type.value}_subseed_show input[type='checkbox']", ) if not show_button.is_selected(): show_button.click() subseed_locator = ( By.CSS_SELECTOR, f"#{self.gen_type.value}_subseed input[type='number']", ) WebDriverWait(self.driver, TIMEOUT).until( EC.visibility_of_element_located(subseed_locator) ) subseed_input = self.driver.find_element(*subseed_locator) subseed_input.clear() subseed_input.send_keys(seed) def upload_controlnet_input(self, img_path: str): controlnet_panel = self.gen_type.controlnet_panel(self.driver) image_input = controlnet_panel.find_element( By.CSS_SELECTOR, '.cnet-input-image-group .cnet-image input[type="file"]' ) image_input.send_keys(img_path) def upload_img2img_input(self, img_path: str): image_input = self.driver.find_element( By.CSS_SELECTOR, '#img2img_image input[type="file"]' ) image_input.send_keys(img_path) def generate_image(self, name: str): self.gen_type.generate_button(self.driver).click() progress_bar_locator_visible = EC.visibility_of_element_located( (By.CSS_SELECTOR, f"#{self.gen_type.value}_results .progress") ) WebDriverWait(self.driver, TIMEOUT).until(progress_bar_locator_visible) WebDriverWait(self.driver, TIMEOUT * 10).until_not(progress_bar_locator_visible) generated_imgs = self.driver.find_elements( By.CSS_SELECTOR, f"#{self.gen_type.value}_results #{self.gen_type.value}_gallery img", ) for i, generated_img in enumerate(generated_imgs): # Use requests to get the image content img_content = requests.get(generated_img.get_attribute("src")).content # Save the image content to a file global overwrite_expectation dest_dir = ( test_expectation_dir if overwrite_expectation else test_result_dir ) img_file_name = f"{self.__class__.__name__}_{name}_{i}.png" with open( os.path.join(dest_dir, img_file_name), "wb", ) as img_file: img_file.write(img_content) if not overwrite_expectation: try: img1 = cv2.imread(os.path.join(test_expectation_dir, img_file_name)) img2 = cv2.imread(os.path.join(test_result_dir, img_file_name)) except Exception as e: self.assertTrue(False, f"Get exception reading imgs: {e}") continue self.expect_same_image( img1, img2, diff_img_path=os.path.join( test_result_dir, img_file_name.replace(".png", "_diff.png") ), ) def expect_same_image(self, img1, img2, diff_img_path: str): # Calculate the difference between the two images diff = cv2.absdiff(img1, img2) # Set a threshold to highlight the different pixels threshold = 30 diff_highlighted = np.where(diff > threshold, 255, 0).astype(np.uint8) # Assert that the two images are similar within a tolerance similar = np.allclose(img1, img2, rtol=0.5, atol=1) if not similar: # Save the diff_highlighted image to inspect the differences cv2.imwrite(diff_img_path, diff_highlighted) self.assertTrue(similar) simple_control_types = { "Canny": "canny", "Depth": "depth_midas", "Normal": "normal_bae", "OpenPose": "openpose_full", "MLSD": "mlsd", "Lineart": "lineart_standard (from white bg & black line)", "SoftEdge": "softedge_pidinet", "Scribble": "scribble_pidinet", "Seg": "seg_ofade20k", "Tile": "tile_resample", # Shuffle and Reference are not stable, and expected to fail. # The majority of pixels are same, but some outlier pixels can have big diff. "Shuffle": "shuffle", "Reference": "reference_only", }.keys() class SeleniumTxt2ImgTest(SeleniumTestCase): def setUp(self) -> None: super().setUp() self.select_gen_type(GenType.txt2img) self.set_seed(100) self.set_subseed(1000) def test_simple_control_types(self): """Test simple control types that only requires input image.""" for control_type in simple_control_types: with self.subTest(control_type=control_type): self.expand_controlnet_panel() self.select_control_type(control_type) self.upload_controlnet_input(SKI_IMAGE) self.generate_image(f"{control_type}_ski") class SeleniumImg2ImgTest(SeleniumTestCase): def setUp(self) -> None: super().setUp() self.select_gen_type(GenType.img2img) self.set_seed(100) self.set_subseed(1000) def test_simple_control_types(self): """Test simple control types that only requires input image.""" for control_type in simple_control_types: with self.subTest(control_type=control_type): self.expand_controlnet_panel() self.select_control_type(control_type) self.upload_img2img_input(SKI_IMAGE) self.upload_controlnet_input(SKI_IMAGE) self.generate_image(f"img2img_{control_type}_ski") class SeleniumInpaintTest(SeleniumTestCase): def setUp(self) -> None: super().setUp() def draw_inpaint_mask(self, target_canvas): size = target_canvas.size width = size["width"] height = size["height"] brush_radius = 5 repeat = int(width * 0.1 / brush_radius) trace: List[Tuple[int, int]] = [ (brush_radius, 0), (0, height * 0.2), (brush_radius, 0), (0, -height * 0.2), ] * repeat actions = ActionChains(self.driver) actions.move_to_element(target_canvas) # move to the canvas actions.move_by_offset(*trace[0]) actions.click_and_hold() # click and hold the left mouse button down for stop_point in trace[1:]: actions.move_by_offset(*stop_point) actions.release() # release the left mouse button actions.perform() # perform the action chain def draw_cn_mask(self): canvas = self.gen_type.controlnet_panel(self.driver).find_element( By.CSS_SELECTOR, ".cnet-input-image-group .cnet-image canvas" ) self.draw_inpaint_mask(canvas) def draw_a1111_mask(self): canvas = self.driver.find_element(By.CSS_SELECTOR, "#img2maskimg canvas") self.draw_inpaint_mask(canvas) def test_txt2img_inpaint(self): self.select_gen_type(GenType.txt2img) self.expand_controlnet_panel() self.select_control_type("Inpaint") self.upload_controlnet_input(SKI_IMAGE) self.draw_cn_mask() self.set_seed(100) self.set_subseed(1000) for option in self.iterate_preprocessor_types(): with self.subTest(option=option): self.generate_image(f"{option}_txt2img_ski") def test_img2img_inpaint(self): # Note: img2img inpaint can only use A1111 mask. # ControlNet input is disabled in img2img inpaint. self._test_img2img_inpaint(use_cn_mask=False, use_a1111_mask=True) def _test_img2img_inpaint(self, use_cn_mask: bool, use_a1111_mask: bool): self.select_gen_type(GenType.img2img) self.expand_controlnet_panel() self.select_control_type("Inpaint") self.upload_img2img_input(SKI_IMAGE) # Send to inpaint self.driver.find_element( By.XPATH, f"//*[@id='img2img_copy_to_img2img']//button[text()='inpaint']" ).click() time.sleep(3) # Select latent noise to make inpaint effect more visible. self.driver.find_element( By.XPATH, f"//input[@name='radio-img2img_inpainting_fill' and @value='latent noise']", ).click() self.set_prompt("(coca-cola:2.0)") self.enable_controlnet_unit() self.upload_controlnet_input(SKI_IMAGE) self.set_seed(100) self.set_subseed(1000) prefix = "" if use_cn_mask: self.draw_cn_mask() prefix += "controlnet" if use_a1111_mask: self.draw_a1111_mask() prefix += "A1111" for option in self.iterate_preprocessor_types(): with self.subTest(option=option, mask_prefix=prefix): self.generate_image(f"{option}_{prefix}_img2img_ski") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Your script description.") parser.add_argument( "--overwrite_expectation", action="store_true", help="overwrite expectation" ) parser.add_argument( "--target_url", type=str, default="http://localhost:7860", help="WebUI URL" ) args, unknown_args = parser.parse_known_args() overwrite_expectation = args.overwrite_expectation webui_url = args.target_url sys.argv = sys.argv[:1] + unknown_args unittest.main()