Add application file

.gitignore

@@ -0,0 +1,164 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+output/
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+*.safetensors
+*.json

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,13 +1,35 @@
----
-title: Starline
-emoji: 📊
-colorFrom: pink
-colorTo: red
-sdk: gradio
-sdk_version: 4.33.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# starline
+**St**rict coloring m**a**chine fo**r** **line** drawings.
+
+
+![image](https://github.com/mattyamonaca/starline/assets/48423148/eae07a6e-9c7b-4292-8c70-dac8ec8eeb7b)
+
+
+https://github.com/mattyamonaca/starline/assets/48423148/8199c65c-a19f-42e9-aab7-df5ed6ef5b4c
+
+# Installation
+```
+git clone https://github.com/mattyamonaca/starline.git
+cd starline
+conda create -n starline python=3.10
+conda activate starline
+conda install pytorch torchvision pytorch-cuda=12.1 -c pytorch -c nvidia
+pip install -r requirements.txt
+```
+
+# Usage
+- ```python app.py```
+- Input the line drawing you wish to color (The background should be transparent).
+- Input a prompt describing the color you want to add.
+
+- 背景を透過した状態で線画を入力します
+- 付けたい色を説明するプロンプトを入力します
+
+# Precautions
+- Image size 1024 x 1024 is recommended.
+- Aliasing is a beta version.
+- Areas finely surrounded by line drawings cannot be colored.
+
+- 画像サイズは1024×1024を推奨します
+- エイリアス処理はβ版です。より線画に忠実であることを求める場合は2値線画を推奨します
+- 線画で細かく囲まれた部分は着色できません。着色できない部分は透過した状態で出力されます。

app.py

@@ -0,0 +1,121 @@
+import gradio as gr
+import sys
+from starline import process
+
+from utils import load_cn_model, load_cn_config, randomname
+from convertor import pil2cv, cv2pil
+
+from sd_model import get_cn_pipeline, generate, get_cn_detector
+import cv2
+import os
+import numpy as np
+from PIL import Image
+import zipfile
+import torch
+
+zero = torch.Tensor([0]).cuda()
+
+path = os.getcwd()
+output_dir = f"{path}/output"
+input_dir = f"{path}/input"
+cn_lineart_dir = f"{path}/controlnet/lineart"
+
+load_cn_model(cn_lineart_dir)
+load_cn_config(cn_lineart_dir)
+
+
+def zip_png_files(folder_path):
+    # Zipファイルの名前を設定（フォルダ名と同じにします）
+    zip_path = os.path.join(folder_path, 'output.zip')
+    
+    # zipfileオブジェクトを作成し、書き込みモードで開く
+    with zipfile.ZipFile(zip_path, 'w') as zipf:
+        # フォルダ内のすべてのファイルをループ処理
+        for foldername, subfolders, filenames in os.walk(folder_path):
+            for filename in filenames:
+                # PNGファイルのみを対象にする
+                if filename.endswith('.png'):
+                    # ファイルのフルパスを取得
+                    file_path = os.path.join(foldername, filename)
+                    # zipファイルに追加
+                    zipf.write(file_path, arcname=os.path.relpath(file_path, folder_path))
+
+
+class webui:
+    def __init__(self):
+        self.demo = gr.Blocks()
+
+    def undercoat(self, input_image, pos_prompt, neg_prompt, alpha_th, thickness):
+        org_line_image = input_image
+        image = pil2cv(input_image)
+        image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
+
+        index = np.where(image[:, :, 3] == 0)
+        image[index] = [255, 255, 255, 255]
+        input_image = cv2pil(image)
+
+        pipe = get_cn_pipeline()
+        detectors = get_cn_detector(input_image.resize((1024, 1024), Image.ANTIALIAS))
+        
+
+        gen_image = generate(pipe, detectors, pos_prompt, neg_prompt)
+        color_img, unfinished = process(gen_image.resize((image.shape[1], image.shape[0]), Image.ANTIALIAS) , org_line_image, alpha_th, thickness)
+        color_img.save(f"{output_dir}/color_img.png")
+
+        #color_img = color_img.resize((image.shape[1], image.shape[0]) , Image.ANTIALIAS)
+
+
+        output_img = Image.alpha_composite(color_img, org_line_image)
+        name = randomname(10)
+        os.makedirs(f"{output_dir}/{name}")
+        output_img.save(f"{output_dir}/{name}/output_image.png")
+        org_line_image.save(f"{output_dir}/{name}/line_image.png")
+        color_img.save(f"{output_dir}/{name}/color_image.png")
+        unfinished.save(f"{output_dir}/{name}/unfinished_image.png")
+
+        outputs = [output_img, org_line_image, color_img, unfinished]
+        zip_png_files(f"{output_dir}/{name}")
+        filename = f"{output_dir}/{name}/output.zip"
+
+        return outputs, filename
+
+
+
+    def launch(self, share):
+        with self.demo:
+            with gr.Row():
+                with gr.Column():
+                    input_image = gr.Image(type="pil", image_mode="RGBA")
+
+                    pos_prompt = gr.Textbox(value="1girl, blue hair, pink shirts, bestquality, 4K", max_lines=1000, label="positive prompt")                    
+                    neg_prompt = gr.Textbox(value=" (worst quality, low quality:1.2), (lowres:1.2), (bad anatomy:1.2), (greyscale, monochrome:1.4)", max_lines=1000, label="negative prompt")
+
+                    alpha_th = gr.Slider(maximum = 255, value=100, label = "alpha threshold")
+                    thickness = gr.Number(value=5, label="Thickness of correction area (Odd numbers need to be entered)")
+                    #gr.Slider(maximum = 21, value=3, step=2, label = "Thickness of correction area")
+
+                    submit = gr.Button(value="Start")
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Tab("output"):
+                            output_0 = gr.Gallery(format="png")
+                        output_file = gr.File()
+            submit.click(
+                self.undercoat, 
+                inputs=[input_image, pos_prompt, neg_prompt, alpha_th, thickness], 
+                outputs=[output_0, output_file]
+            )
+
+        self.demo.queue()
+        self.demo.launch(share=share)
+
+
+if __name__ == "__main__":
+    ui = webui()
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "share":
+            ui.launch(share=True)
+        else:
+            ui.launch(share=False)
+    else:
+        ui.launch(share=False)

convertor.py

@@ -0,0 +1,102 @@
+import pandas as pd
+import numpy as np
+from skimage import color 
+from PIL import Image
+
+
+def skimage_rgb2lab(rgb):
+    return color.rgb2lab(rgb.reshape(1,1,3))
+
+
+def rgb2df(img):
+    h, w, _ = img.shape
+    x_l, y_l = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')
+    r, g, b = img[:,:,0], img[:,:,1], img[:,:,2]
+    df = pd.DataFrame({
+        "x_l": x_l.ravel(),
+        "y_l": y_l.ravel(),
+        "r": r.ravel(),
+        "g": g.ravel(),
+        "b": b.ravel(),
+    })
+    return df
+
+def mask2df(mask):
+    h, w = mask.shape
+    x_l, y_l = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')
+    flg = mask.astype(int)
+    df = pd.DataFrame({
+        "x_l_m": x_l.ravel(),
+        "y_l_m": y_l.ravel(),
+        "m_flg": flg.ravel(),
+    })
+    return df
+
+
+def rgba2df(img):
+    h, w, _ = img.shape
+    x_l, y_l = np.meshgrid(np.arange(h), np.arange(w), indexing='ij')
+    r, g, b, a = img[:,:,0], img[:,:,1], img[:,:,2], img[:,:,3]
+    df = pd.DataFrame({
+        "x_l": x_l.ravel(),
+        "y_l": y_l.ravel(),
+        "r": r.ravel(),
+        "g": g.ravel(),
+        "b": b.ravel(),
+        "a": a.ravel()
+    })
+    return df
+
+def hsv2df(img):
+    x_l, y_l = np.meshgrid(np.arange(img.shape[0]), np.arange(img.shape[1]), indexing='ij')
+    h, s, v = np.transpose(img, (2, 0, 1))
+    df = pd.DataFrame({'x_l': x_l.flatten(), 'y_l': y_l.flatten(), 'h': h.flatten(), 's': s.flatten(), 'v': v.flatten()})
+    return df
+
+def df2rgba(img_df):
+    r_img = img_df.pivot_table(index="x_l", columns="y_l",values= "r").reset_index(drop=True).values
+    g_img = img_df.pivot_table(index="x_l", columns="y_l",values= "g").reset_index(drop=True).values
+    b_img = img_df.pivot_table(index="x_l", columns="y_l",values= "b").reset_index(drop=True).values
+    a_img = img_df.pivot_table(index="x_l", columns="y_l",values= "a").reset_index(drop=True).values
+    df_img = np.stack([r_img, g_img, b_img, a_img], 2).astype(np.uint8)
+    return df_img
+
+def df2bgra(img_df):
+    r_img = img_df.pivot_table(index="x_l", columns="y_l",values= "r").reset_index(drop=True).values
+    g_img = img_df.pivot_table(index="x_l", columns="y_l",values= "g").reset_index(drop=True).values
+    b_img = img_df.pivot_table(index="x_l", columns="y_l",values= "b").reset_index(drop=True).values
+    a_img = img_df.pivot_table(index="x_l", columns="y_l",values= "a").reset_index(drop=True).values
+    df_img = np.stack([b_img, g_img, r_img, a_img], 2).astype(np.uint8)
+    return df_img
+
+def df2rgb(img_df):
+    r_img = img_df.pivot_table(index="x_l", columns="y_l",values= "r").reset_index(drop=True).values
+    g_img = img_df.pivot_table(index="x_l", columns="y_l",values= "g").reset_index(drop=True).values
+    b_img = img_df.pivot_table(index="x_l", columns="y_l",values= "b").reset_index(drop=True).values
+    df_img = np.stack([r_img, g_img, b_img], 2).astype(np.uint8)
+    return df_img
+
+def pil2cv(image):
+    new_image = np.array(image, dtype=np.uint8)
+    if new_image.ndim == 2:
+        pass
+    elif new_image.shape[2] == 3:
+        new_image = new_image[:, :, ::-1]
+    elif new_image.shape[2] == 4:
+        new_image = new_image[:, :, [2, 1, 0, 3]]
+    return new_image
+
+def cv2pil(image):
+    new_image = image.copy()
+    if new_image.ndim == 2:
+        pass
+    elif new_image.shape[2] == 3:
+        new_image = new_image[:, :, ::-1]
+    elif new_image.shape[2] == 4:
+        new_image = new_image[:, :, [2, 1, 0, 3]]
+    new_image = Image.fromarray(new_image)
+    return new_image
+
+
+
+

requirements.txt

@@ -0,0 +1,13 @@
+opencv-python==4.7.0.68
+pandas==1.5.3
+scikit-learn==1.2.1
+scikit-image==0.19.3
+Pillow==9.4.0
+tqdm==4.63.0
+diffusers==0.27.2
+gradio==4.32.1
+gradio_client==0.17.0
+transformers==4.40.1
+accelerate==0.21.0
+safetensors==0.4.2
+

sd_model.py

@@ -0,0 +1,66 @@
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL
+import torch
+import spaces
+
+device = "cuda"
+
+def get_cn_pipeline():
+    controlnets = [
+        ControlNetModel.from_pretrained("./controlnet/lineart", torch_dtype=torch.float16, use_safetensors=True),
+        ControlNetModel.from_pretrained("mattyamonaca/controlnet_line2line_xl", torch_dtype=torch.float16)
+    ]
+
+    vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
+    pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+        "cagliostrolab/animagine-xl-3.1", controlnet=controlnets, vae=vae, torch_dtype=torch.float16
+    )
+
+    pipe.enable_model_cpu_offload()
+
+    #if pipe.safety_checker is not None:
+    #    pipe.safety_checker = lambda images, **kwargs: (images, [False])
+    
+    #pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    #pipe.to(device)
+
+    return pipe
+
+def invert_image(img):
+    # 画像を読み込む
+    # 画像をグレースケールに変換（もしもともと白黒でない場合）
+    img = img.convert('L')
+    # 画像の各ピクセルを反転
+    inverted_img = img.point(lambda p: 255 - p)
+    # 反転した画像を保存
+    return inverted_img
+
+
+def get_cn_detector(image):
+    #lineart_anime = LineartAnimeDetector.from_pretrained("lllyasviel/Annotators")
+    #canny = CannyDetector()
+    #lineart_anime_img = lineart_anime(image)
+    #canny_img = canny(image)
+    #canny_img = canny_img.resize((lineart_anime(image).width, lineart_anime(image).height))
+    re_image = invert_image(image)
+    
+    
+    detectors = [re_image, image]
+    print(detectors)
+    return detectors
+
+@spaces.GPU
+def generate(pipe, detectors, prompt, negative_prompt):
+    default_pos = ""
+    default_neg = ""
+    prompt = default_pos + prompt 
+    negative_prompt = default_neg + negative_prompt 
+    print(type(pipe))
+    image = pipe(
+                prompt=prompt,
+                negative_prompt = negative_prompt,
+                image=detectors,
+                num_inference_steps=50,
+                controlnet_conditioning_scale=[1.0, 0.2],
+            ).images[0]
+    return image

starline.py

@@ -0,0 +1,416 @@
+from collections import defaultdict, deque
+
+import cv2
+import numpy as np
+from PIL import Image
+from skimage.color import deltaE_ciede2000, rgb2lab
+from tqdm import tqdm
+
+
+def modify_transparency(img, target_rgb):
+    # 画像を読み込む
+    copy_img = img.copy()
+    data = copy_img.getdata()
+
+    # 新しいピクセルデータを作成
+    new_data = []
+    for item in data:
+        # 指定されたRGB値のピクセルの場合、透明度を255に設定
+        if item[:3] == target_rgb:
+            new_data.append((item[0], item[1], item[2], 255))
+        else:
+            # それ以外の場合、透明度を0に設定
+            new_data.append((item[0], item[1], item[2], 0))
+
+    # 新しいデータを画像に設定し直す
+    copy_img.putdata(new_data)
+    return copy_img
+
+
+def replace_color(image, color_1, color_2, alpha_np):
+    # 画像データを配列に変換
+    data = np.array(image)
+
+    # RGBAモードの画像であるため、形状変更時に4チャネルを考慮
+    original_shape = data.shape
+
+    color_1 = np.array(color_1, dtype=np.uint8)
+    color_2 = np.array(color_2, dtype=np.uint8)
+
+    # 幅優先探索で color_1 の領域を外側から塗りつぶす
+    # color_2 で保護されたオリジナルの線画
+    protected = np.all(data[:, :, :3] == color_2, axis=2)
+    # color_1 で塗られた塗りつぶしたい領域
+    fill_target = np.all(data[:, :, :3] == color_1, axis=2)
+    # すでに塗られている領域
+    colored = (protected | fill_target) == False
+
+    # bfs の始点を列挙
+    # colored をそのまま使ってもいいが、pythonは遅いのでnumpy経由のこの方が速い
+    # 上下左右にシフトした fill_target & colored == True になるやつ
+    adj_r = colored & np.roll(fill_target, -1, axis=0)
+    adj_r[:, -1] = False
+    adj_l = colored & np.roll(fill_target, 1, axis=0)
+    adj_l[:, 0] = False
+    adj_u = colored & np.roll(fill_target, 1, axis=1)
+    adj_u[:, 0] = False
+    adj_d = colored & np.roll(fill_target, -1, axis=1)
+    adj_d[:, -1] = False
+
+    # そのピクセルはすでに塗られていて、上下左右いずれかのピクセルが color_1 であるもの
+    bfs_start = adj_r | adj_l | adj_u | adj_d
+
+    que = deque(
+        zip(*np.where(bfs_start)),
+        maxlen=original_shape[0] * original_shape[1] * 2,
+    )
+
+    with tqdm(total=original_shape[0] * original_shape[1]) as pbar:
+        pbar.update(np.sum(colored) - np.sum(bfs_start) + np.sum(protected))
+        while len(que) > 0:
+            y, x = que.popleft()
+            neighbors = [
+                (x - 1, y),
+                (x + 1, y),
+                (x, y - 1),
+                (x, y + 1),  # 上下左右
+            ]
+            pbar.update(1)
+            # assert not fill_target[y, x] and not protected[y, x]
+            # assert colored[y, x]
+            color = data[y, x, :3]
+
+            for nx, ny in neighbors:
+                if (
+                    nx < 0
+                    or nx >= original_shape[1]
+                    or ny < 0
+                    or ny >= original_shape[0]
+                ):
+                    continue
+                if fill_target[ny, nx]:
+                    fill_target[ny, nx] = False
+                    # colored[ny, nx] = True
+                    data[ny, nx, :3] = color
+                    que.append((ny, nx))
+        pbar.update(pbar.total - pbar.n)
+
+    data[:, :, 3] = 255 - alpha_np
+    return Image.fromarray(data, "RGBA")
+
+
+def recolor_lineart_and_composite(lineart_image, base_image, new_color, alpha_th):
+    """
+    Recolor an RGBA lineart image to a single new color while preserving alpha, and composite it over a base image.
+
+    Args:
+    lineart_image (PIL.Image): The lineart image with RGBA channels.
+    base_image (PIL.Image): The base image to composite onto.
+    new_color (tuple): The new RGB color for the lineart (e.g., (255, 0, 0) for red).
+
+    Returns:
+    PIL.Image: The composited image with the recolored lineart on top.
+    """
+    # Ensure images are in RGBA mode
+    if lineart_image.mode != "RGBA":
+        lineart_image = lineart_image.convert("RGBA")
+    if base_image.mode != "RGBA":
+        base_image = base_image.convert("RGBA")
+
+    # Extract the alpha channel from the lineart image
+    r, g, b, alpha = lineart_image.split()
+
+    alpha_np = np.array(alpha)
+    alpha_np[alpha_np < alpha_th] = 0
+    alpha_np[alpha_np >= alpha_th] = 255
+
+    new_alpha = Image.fromarray(alpha_np)
+
+    # Create a new image using the new color and the alpha channel from the original lineart
+    new_lineart_image = Image.merge(
+        "RGBA",
+        (
+            Image.new("L", lineart_image.size, int(new_color[0])),
+            Image.new("L", lineart_image.size, int(new_color[1])),
+            Image.new("L", lineart_image.size, int(new_color[2])),
+            new_alpha,
+        ),
+    )
+
+    # Composite the new lineart image over the base image
+    composite_image = Image.alpha_composite(base_image, new_lineart_image)
+
+    return composite_image, alpha_np
+
+
+def thicken_and_recolor_lines(base_image, lineart, thickness=3, new_color=(0, 0, 0)):
+    """
+    Thicken the lines of a lineart image, recolor them, and composite onto another image,
+    while preserving the transparency of the original lineart.
+
+    Args:
+    base_image (PIL.Image): The base image to composite onto.
+    lineart (PIL.Image): The lineart image with transparent background.
+    thickness (int): The desired thickness of the lines.
+    new_color (tuple): The new color to apply to the lines (R, G, B).
+
+    Returns:
+    PIL.Image: The image with the recolored and thickened lineart composited on top.
+    """
+    # Ensure both images are in RGBA format
+    if base_image.mode != "RGBA":
+        base_image = base_image.convert("RGBA")
+    if lineart.mode != "RGB":
+        lineart = lineart.convert("RGBA")
+
+    # Convert the lineart image to OpenCV format
+    lineart_cv = np.array(lineart)
+
+    white_pixels = np.sum(lineart_cv == 255)
+    black_pixels = np.sum(lineart_cv == 0)
+
+    lineart_gray = cv2.cvtColor(lineart_cv, cv2.COLOR_RGBA2GRAY)
+
+    if white_pixels > black_pixels:
+        lineart_gray = cv2.bitwise_not(lineart_gray)
+
+    # Thicken the lines using OpenCV
+    kernel = np.ones((thickness, thickness), np.uint8)
+    lineart_thickened = cv2.dilate(lineart_gray, kernel, iterations=1)
+    lineart_thickened = cv2.bitwise_not(lineart_thickened)
+    # Create a new RGBA image for the recolored lineart
+    lineart_recolored = np.zeros_like(lineart_cv)
+    lineart_recolored[:, :, :3] = new_color  # Set new RGB color
+
+    lineart_recolored[:, :, 3] = np.where(
+        lineart_thickened < 250, 255, 0
+    )  # Blend alpha with thickened lines
+
+    # Convert back to PIL Image
+    lineart_recolored_pil = Image.fromarray(lineart_recolored, "RGBA")
+
+    # Composite the thickened and recolored lineart onto the base image
+    combined_image = Image.alpha_composite(base_image, lineart_recolored_pil)
+
+    return combined_image
+
+
+def generate_distant_colors(consolidated_colors, distance_threshold):
+    """
+    Generate new RGB colors that are at least 'distance_threshold' CIEDE2000 units away from given colors.
+
+    Args:
+    consolidated_colors (list of tuples): List of ((R, G, B), count) tuples.
+    distance_threshold (float): The minimum CIEDE2000 distance from the given colors.
+
+    Returns:
+    list of tuples: List of new RGB colors that meet the distance requirement.
+    """
+    # new_colors = []
+    # Convert the consolidated colors to LAB
+    consolidated_lab = [
+        rgb2lab(np.array([color], dtype=np.float32) / 255.0).reshape(3)
+        for color, _ in consolidated_colors
+    ]
+
+    # Try to find a distant color
+    max_attempts = 1000
+    best_dist = 0.0
+    best_color = (0, 0, 0)
+
+    # np.random.seed(42)
+    for _ in range(max_attempts):
+        # Generate a random color in RGB and convert to LAB
+        random_rgb = np.random.randint(0, 256, size=3)
+        random_lab = rgb2lab(np.array([random_rgb], dtype=np.float32) / 255.0).reshape(
+            3
+        )
+        # consolidated_lab にある色からできるだけ遠い色を選びたい
+        min_distance = min(
+            map(
+                lambda base_color_lab: deltaE_ciede2000(base_color_lab, random_lab),
+                consolidated_lab,
+            )
+        )
+        if min_distance > distance_threshold:
+            return tuple(random_rgb)
+        # 閾値以上のものが見つからなかった場合に備えて一番良かったものを覚えておく
+        if best_dist < min_distance:
+            best_dist = min_distance
+            best_color = tuple(random_rgb)
+    return best_color
+
+
+def consolidate_colors(major_colors, threshold):
+    """
+    Consolidate similar colors in the major_colors list based on the CIEDE2000 metric.
+
+    Args:
+    major_colors (list of tuples): List of ((R, G, B), count) tuples.
+    threshold (float): Threshold for CIEDE2000 color difference.
+
+    Returns:
+    list of tuples: Consolidated list of ((R, G, B), count) tuples.
+    """
+    # Convert RGB to LAB
+    colors_lab = [
+        rgb2lab(np.array([[color]], dtype=np.float32) / 255.0).reshape(3)
+        for color, _ in major_colors
+    ]
+    n = len(colors_lab)
+
+    # Find similar colors and consolidate
+    i = 0
+    while i < n:
+        j = i + 1
+        while j < n:
+            delta_e = deltaE_ciede2000(colors_lab[i], colors_lab[j])
+            if delta_e < threshold:
+                # Compare counts and consolidate to the color with the higher count
+                if major_colors[i][1] >= major_colors[j][1]:
+                    major_colors[i] = (
+                        major_colors[i][0],
+                        major_colors[i][1] + major_colors[j][1],
+                    )
+                    major_colors.pop(j)
+                    colors_lab.pop(j)
+                else:
+                    major_colors[j] = (
+                        major_colors[j][0],
+                        major_colors[j][1] + major_colors[i][1],
+                    )
+                    major_colors.pop(i)
+                    colors_lab.pop(i)
+                n -= 1
+                continue
+            j += 1
+        i += 1
+
+    return major_colors
+
+
+def get_major_colors(image, threshold_percentage=0.01):
+    """
+    Analyze an image to find the major RGB values based on a threshold percentage.
+
+    Args:
+    image (PIL.Image): The image to analyze.
+    threshold_percentage (float): The percentage threshold to consider a color as major.
+
+    Returns:
+    list of tuples: A list of (color, count) tuples for colors that are more frequent than the threshold.
+    """
+    # Convert image to RGB if it's not
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+
+    # Count each color
+    color_count = defaultdict(int)
+    for pixel in image.getdata():
+        color_count[pixel] += 1
+
+    # Total number of pixels
+    total_pixels = image.width * image.height
+
+    # Filter colors to find those above the threshold
+    major_colors = [
+        (color, count)
+        for color, count in color_count.items()
+        if (count / total_pixels) >= threshold_percentage
+    ]
+
+    return major_colors
+
+
+def process(image, lineart, alpha_th, thickness):
+    org = image
+    image.save("tmp.png")
+
+    major_colors = get_major_colors(image, threshold_percentage=0.05)
+    major_colors = consolidate_colors(major_colors, 10)
+
+    th = 10
+    threshold_percentage = 0.05
+    while len(major_colors) < 1:
+        threshold_percentage = threshold_percentage - 0.001
+        major_colors = get_major_colors(image, threshold_percentage=threshold_percentage)
+        
+    while len(major_colors) < 1:
+        th = th + 1
+        major_colors = consolidate_colors(major_colors, th)
+
+    new_color_1 = generate_distant_colors(major_colors, 50)
+    image = thicken_and_recolor_lines(
+        org, lineart, thickness=thickness, new_color=new_color_1
+    )
+    
+    major_colors.append((new_color_1, 0))
+    new_color_2 = generate_distant_colors(major_colors, 40)
+    image, alpha_np = recolor_lineart_and_composite(
+        lineart, image, new_color_2, alpha_th
+    )
+    # import time
+    # start = time.time()
+    image = replace_color(image, new_color_1, new_color_2, alpha_np)
+    # end = time.time()
+    # print(f"{end-start} sec")
+    unfinished = modify_transparency(image, new_color_1)
+
+    return image, unfinished
+
+
+def main():
+    import os
+    import sys
+    from argparse import ArgumentParser
+
+    from PIL import Image
+
+    from utils import randomname
+
+    args = ArgumentParser(
+        prog="starline",
+        description="Starline",
+        epilog="Starline",
+    )
+    args.add_argument("-c", "--colored_image", help="colored image", required=True)
+    args.add_argument("-l", "--lineart_image", help="lineart image", required=True)
+    args.add_argument("-o", "--output_dir", help="output directory", default="output")
+    args.add_argument("-a", "--alpha_th", help="alpha threshold", default=100, type=int)
+    args.add_argument("-t", "--thickness", help="line thickness", default=5, type=int)
+
+    args = args.parse_args(sys.argv[1:])
+    colored_image_path = args.colored_image
+    lineart_image_path = args.lineart_image
+    alpha = args.alpha_th
+    thickness = args.thickness
+    output_dir = args.output_dir
+
+    colored_image = Image.open(colored_image_path)
+    lineart_image = Image.open(lineart_image_path)
+    if lineart_image.mode == "P" or lineart_image.mode == "L":
+        # 線画が 1-channel 画像のときの処理
+        # alpha-channel の情報が入力されたと仮定して (透明 -> 0, 不透明 -> 255)
+        # RGB channel はこれを反転させたものにする (透明 -> 白 -> 255, 不透明 -> 黒 -> 0)
+        lineart_image = lineart_image.convert("RGBA")
+        lineart_image = np.array(lineart_image)
+        lineart_image[:, :, 0] = 255 - lineart_image[:, :, 3]
+        lineart_image[:, :, 1] = 255 - lineart_image[:, :, 3]
+        lineart_image[:, :, 2] = 255 - lineart_image[:, :, 3]
+        lineart_image = Image.fromarray(lineart_image)
+        lineart_image = lineart_image.convert("RGBA")
+
+    result_image, unfinished = process(colored_image, lineart_image, alpha, thickness)
+
+    output_image = Image.alpha_composite(result_image, lineart_image)
+
+    name = randomname(10)
+
+    os.makedirs(f"{output_dir}/{name}")
+    output_image.save(f"{output_dir}/{name}/output_image.png")
+    result_image.save(f"{output_dir}/{name}/color_image.png")
+    unfinished.save(f"{output_dir}/{name}/unfinished_image.png")
+
+
+if __name__ == "__main__":
+    main()

utils.py

@@ -0,0 +1,53 @@
+import random
+import string
+import os
+
+import requests
+from tqdm import tqdm
+
+
+def randomname(n):
+   randlst = [random.choice(string.ascii_letters + string.digits) for i in range(n)]
+   return ''.join(randlst)
+
+def load_cn_model(model_dir):
+  folder = model_dir
+  file_name = 'diffusion_pytorch_model.safetensors'
+  url = "https://huggingface.co/kataragi/ControlNet-LineartXL/resolve/main/Katarag_lineartXL-fp16.safetensors"
+
+  file_path = os.path.join(folder, file_name)
+  if not os.path.exists(file_path):
+    response = requests.get(url, stream=True)
+
+    total_size = int(response.headers.get('content-length', 0))
+    with open(file_path, 'wb') as f, tqdm(
+            desc=file_name,
+            total=total_size,
+            unit='iB',
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as bar:
+        for data in response.iter_content(chunk_size=1024):
+            size = f.write(data)
+            bar.update(size)
+
+def load_cn_config(model_dir):
+  folder = model_dir
+  file_name = 'config.json'
+  url = "https://huggingface.co/mattyamonaca/controlnet_line2line_xl/resolve/main/config.json"
+
+  file_path = os.path.join(folder, file_name)
+  if not os.path.exists(file_path):
+    response = requests.get(url, stream=True)
+
+    total_size = int(response.headers.get('content-length', 0))
+    with open(file_path, 'wb') as f, tqdm(
+            desc=file_name,
+            total=total_size,
+            unit='iB',
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as bar:
+        for data in response.iter_content(chunk_size=1024):
+            size = f.write(data)
+            bar.update(size)