Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+.specstory/
+.venv/
+.idea/
+.vscode/
+.pytest_cache/
+.ruff_cache/
+models/
+

README.md

@@ -0,0 +1,85 @@
+---
+title: Pixel GArt
+emoji: 🌖
+colorFrom: red
+colorTo: indigo
+sdk: gradio
+sdk_version: 5.35.0
+app_file: app.py
+pinned: false
+license: mit
+short_description: AI tool for turning sketches into pixel art.
+---
+
+## Introduction
+
+This project is a pixel art generator that leverages Stable Diffusion 1.5 combined with the PixelArtRedmond LoRA to transform hand-drawn sketches into stunning retro-style pixel masterpieces. Designed to unleash creativity, it offers users an easy way to create high-quality pixel art inspired by classic video games.
+
+## 📦 Installation 
+
+### 1. Clone the project
+
+```
+git clone git@github.com:sosiki1997/Pixel_GArt.git
+
+cd Pixel_GArt
+```
+
+### 2. Create and activate conda environment
+
+```
+conda create -n pixel_venv python=3.12
+
+conda activate pixel_venv
+```
+
+### 3. Install dependencies
+
+```
+pip install gradio==3.44.4
+
+pip install -r requirements.txt
+
+pip install torch torchvision opencv-python pillow
+
+pip install git+https://github.com/facebookresearch/segment-anything.git
+```
+
+### 4. Run the server
+
+```
+python -m app.main
+```
+
+### 5. Open in browser
+
+```
+ http://127.0.0.1:7860
+```
+
+<details>
+  <summary>📖 中文说明（点击展开）</summary>
+
+
+本项目是一个像素画生成器，结合了 Stable Diffusion 1.5 和 PixelArtRedmond LoRA，能将手绘草图转化为令人惊艳的复古风格像素艺术。旨在释放创意，让用户轻松创作出高品质的像素艺术作品，灵感源自经典电子游戏。
+
+</details>
+
+<details>
+  <summary>📖 日本語の説明（クリックで展開）</summary>
+
+
+本プロジェクトは、Stable Diffusion 1.5 と PixelArtRedmond LoRA を組み合わせて、手描きのスケッチを圧巻のレトロ風ピクセルアートに変換するジェネレーターです。クラシックゲームにインスパイアされた高品質なピクセルアートを手軽に制作できることを目的としています。
+
+</details>
+
+---
+
+## 🖼️ 出力例（生成画像）
+
+<p align="center">
+  <img src="./readme_img/output_1_snapshot.png" width="600"/>
+  <img src="./readme_img/output_2_snapshot.png" width="600"/>
+  <img src="./readme_img/output_3_snapshot.png" width="600"/>
+  <img src="./readme_img/output_4_snapshot.png" width="600"/>
+</p>

app.py

@@ -0,0 +1,12 @@
+# app.py - Hugging Face 和本地都能用的入口
+
+import sys
+import os
+
+# 把 app 目录加入路径（方便导入 main）
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "app")))
+
+from main import launch_app
+
+# ✅ 无论是否 __main__ 都调用 launch_app（HF Spaces 也能运行）
+launch_app()

app/interface/gradio_ui.py

@@ -0,0 +1,146 @@
+import gradio as gr
+import os
+import sys
+from PIL import Image
+import numpy as np
+import io
+
+# 添加父目录到路径，以便导入 utils 模块
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from utils.image_processing import ImageProcessor
+
+# 导入我们的处理函数
+from utils.process import process_image
+
+def create_gradio_interface(generator):
+    """创建Gradio界面"""
+    
+    def handle_image_opencv(input_image, pixel_size=20):
+        """使用OpenCV处理上传的图像"""
+        if input_image is None:
+            return None, "请上传或绘制图像"
+        
+        try:
+            # 调用OpenCV处理函数
+            result_image = process_image(input_image, pixel_size=pixel_size)
+            return result_image, "処理が成功しました"
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            return None, f"处理失败: {str(e)}"
+    
+    def handle_image_diffusion(input_image, prompt, guidance_scale=7.5):
+        """使用Stable Diffusion处理上传的图像"""
+        if input_image is None:
+            return None, "请上传图像"
+        
+        try:
+            # 将PIL图像转换为字节
+            img_byte_arr = io.BytesIO()
+            input_image.save(img_byte_arr, format='PNG')
+            img_byte_arr = img_byte_arr.getvalue()
+            
+            # 调用generator的generate函数
+            # result_image = generator.generate(img_byte_arr, prompt, guidance_scale=guidance_scale)
+            result_image = generator.generate(prompt=prompt, guidance_scale=guidance_scale)
+
+            return result_image, "処理が成功しました。"
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            return None, f"生成失败: {str(e)}"
+    
+    # 创建Gradio界面
+    with gr.Blocks(title="ドット絵ピクセルアート") as demo:
+        gr.Markdown("# ドット絵ピクセルアート")
+        
+        with gr.Tabs():
+            with gr.TabItem("OpenCV ピクセル化"):
+                with gr.Row():
+                    with gr.Column():
+                        # 输入区域 - 设置固定大小
+                        input_image_opencv = gr.Image(
+                            label="スケッチをアップロードまたは描画", 
+                            type="pil",
+                            height=512,  # 设置固定高度
+                            width=512,   # 设置固定宽度
+                            container=True,  # 使用容器包裹
+                            show_download_button=False,  # 不显示下载按钮
+                            show_label=True,  # 显示标签
+                        )
+                        
+                        pixel_size = gr.Slider(minimum=5, maximum=50, value=20, step=1, 
+                                            label="ピクセルサイズ")
+                        
+                        process_btn_opencv = gr.Button("ドット絵を生成 (OpenCV)")
+                    
+                    with gr.Column():
+                        # 输出区域 - 也设置固定大小
+                        output_image_opencv = gr.Image(
+                            label="生成結果",
+                            height=512,  # 设置固定高度
+                            width=512,   # 设置固定宽度
+                            container=True,  # 使用容器包裹
+                            show_download_button=True,  # 显示下载按钮
+                        )
+                        output_message_opencv = gr.Textbox(label="ステータス")
+            
+            with gr.TabItem("Stable Diffusionで生成"):
+                with gr.Row():
+                    with gr.Column():
+                        # 输入区域
+                        input_image_diffusion = gr.Image(
+                            label="参考画像をアップロード", 
+                            type="pil",
+                            height=512,
+                            width=512,
+                            container=True,
+                            show_download_button=False,
+                        )
+                        
+                        prompt = gr.Textbox(
+                            label="プロンプト", 
+                            placeholder="希望するピクセルアートのスタイルを説明してください...",
+                            value="Pixel Art, PixArFK"
+                        )
+                        
+                        guidance_scale = gr.Slider(
+                            minimum=1.0, 
+                            maximum=15.0, 
+                            value=7.5, 
+                            step=0.5,
+                            label="ガイダンスの強さ（数値が高いほど指定した内容に沿いやすくなります）"
+                        )
+                        
+                        process_btn_diffusion = gr.Button("ドット絵を作成 (Stable Diffusion)")
+                    
+                    with gr.Column():
+                        # 输出区域
+                        output_image_diffusion = gr.Image(
+                            label="生成結果",
+                            height=512,
+                            width=512,
+                            container=True,
+                            show_download_button=True,
+                        )
+                        output_message_diffusion = gr.Textbox(label="ステータス")
+        
+        # 连接OpenCV处理按钮和函数
+        process_btn_opencv.click(
+            fn=handle_image_opencv,
+            inputs=[input_image_opencv, pixel_size],
+            outputs=[output_image_opencv, output_message_opencv]
+        )
+        
+        # 连接Stable Diffusion处理按钮和函数
+        process_btn_diffusion.click(
+            fn=handle_image_diffusion,
+            inputs=[input_image_diffusion, prompt, guidance_scale],
+            outputs=[output_image_diffusion, output_message_diffusion]
+        )
+    
+    return demo
+
+# 启动应用
+if __name__ == "__main__":
+    demo.launch()

app/main.py

@@ -0,0 +1,11 @@
+from models.generator import PixelArtGenerator
+from interface.gradio_ui import create_gradio_interface
+
+# 初始化生成器和 Gradio 界面
+generator = PixelArtGenerator()
+interface = create_gradio_interface(generator)
+
+
+def launch_app():
+    interface.launch()
+

app/utils/extract_subject.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import sys
+import json
+from image_processing import ImageProcessor
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(json.dumps({"error": "No image path provided"}))
+        sys.exit(1)
+        
+    image_path = sys.argv[1]
+    processor = ImageProcessor()
+    
+    try:
+        subject_path, mask = processor.extract_subject(image_path)
+        # 保存掩码为图像
+        mask_path = image_path.replace('.', '_mask.')
+        import cv2
+        import numpy as np
+        cv2.imwrite(mask_path, (mask * 255).astype(np.uint8))
+        
+        print(json.dumps({
+            "subjectPath": subject_path,
+            "maskPath": mask_path
+        }))
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1) 

app/utils/image.py

@@ -0,0 +1,35 @@
+import cv2
+import numpy as np
+from PIL import Image
+
+def process_sketch(
+    image: Image.Image,
+    low_threshold: int = 100,
+    high_threshold: int = 200,
+    bg_color: int = 255
+) -> Image.Image:
+    """处理草图，提取 Canny 边缘"""
+    # 转换为 numpy 数组
+    img_array = np.array(image)
+    
+    # 转换为灰度图
+    gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
+    
+    # 应用高斯模糊减少噪声
+    blurred = cv2.GaussianBlur(gray, (3, 3), 0)
+    
+    # 应用 Canny 边缘检测
+    edges = cv2.Canny(
+        blurred,
+        threshold1=low_threshold,
+        threshold2=high_threshold
+    )
+    
+    # 创建白色背景
+    result = np.full_like(edges, bg_color, dtype=np.uint8)
+    
+    # 将边缘设为黑色
+    result[edges > 0] = 0
+    
+    # 转回 PIL Image
+    return Image.fromarray(result) 

app/utils/image_processing.py

@@ -0,0 +1,176 @@
+import cv2
+import numpy as np
+from PIL import Image
+import torch
+from torchvision import transforms
+from segment_anything import SamPredictor, sam_model_registry
+import os
+
+class ImageProcessor:
+    def __init__(self):
+        # 加载SAM模型用于图像分割
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        # 更新为你下载的模型实际路径
+        model_path = "./models/sam_vit_h_4b8939.pth"  # 修改为你的实际路径
+        
+        # 检查模型文件是否存在
+        if not os.path.exists(model_path):
+            raise FileNotFoundError(f"SAM模型文件未找到: {model_path}")
+            
+        print(f"加载SAM模型: {model_path}")
+        self.sam = sam_model_registry["vit_h"](checkpoint=model_path)
+        self.sam.to(self.device)
+        self.predictor = SamPredictor(self.sam)
+        
+    def extract_subject(self, image_path):
+        """提取图像中的主体对象，使用多点提示和更强的分割策略"""
+        # 读取图像
+        image = cv2.imread(image_path)
+        if image is None:
+            raise ValueError(f"无法读取图像: {image_path}")
+            
+        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        h, w = image.shape[:2]
+        
+        # 设置SAM预测器
+        self.predictor.set_image(image_rgb)
+        
+        # 使用多个点作为提示，覆盖图像的不同区域
+        points = np.array([
+            [w//2, h//2],      # 中心
+            [w//4, h//4],      # 左上
+            [3*w//4, h//4],    # 右上
+            [w//4, 3*h//4],    # 左下
+            [3*w//4, 3*h//4],  # 右下
+        ])
+        
+        print(f"使用多点提示: {points}")
+        
+        # 所有点都标记为前景
+        labels = np.ones(len(points))
+        
+        # 获取掩码
+        masks, scores, _ = self.predictor.predict(
+            point_coords=points,
+            point_labels=labels,
+            multimask_output=True  # 生成多个掩码
+        )
+        
+        # 选择得分最高的掩码
+        best_mask_idx = np.argmax(scores)
+        mask = masks[best_mask_idx]
+        
+        print(f"选择得分最高的掩码: {scores[best_mask_idx]}")
+        
+        # 检查掩码覆盖面积
+        mask_area = np.sum(mask)
+        image_area = h * w
+        coverage = mask_area / image_area
+        print(f"掩码覆盖率: {coverage:.2%}")
+        
+        # 如果掩码覆盖率太小，尝试使用更简单的方法
+        if coverage < 0.05:  # 如果覆盖率小于5%
+            print("掩码覆盖率太小，尝试使用颜色阈值分割")
+            
+            # 使用颜色阈值分割
+            # 转换为HSV颜色空间
+            hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
+            
+            # 定义橙色范围 (松鼠的主要颜色)
+            lower_orange = np.array([10, 100, 100])
+            upper_orange = np.array([25, 255, 255])
+            
+            # 创建掩码
+            color_mask = cv2.inRange(hsv, lower_orange, upper_orange)
+            
+            # 应用形态学操作清理掩码
+            kernel = np.ones((5,5), np.uint8)
+            color_mask = cv2.morphologyEx(color_mask, cv2.MORPH_OPEN, kernel)
+            color_mask = cv2.morphologyEx(color_mask, cv2.MORPH_CLOSE, kernel)
+            
+            # 转换为布尔掩码
+            mask = color_mask > 0
+            
+            # 再次检查掩码覆盖率
+            mask_area = np.sum(mask)
+            coverage = mask_area / image_area
+            print(f"颜色阈值分割后的掩码覆盖率: {coverage:.2%}")
+            
+            # 如果仍然太小，使用简单的矩形区域
+            if coverage < 0.05:
+                print("颜色阈值分割仍然不理想，使用中心区域")
+                mask = np.zeros((h, w), dtype=bool)
+                # 使用图像中心的60%区域
+                h_start, h_end = int(h*0.2), int(h*0.8)
+                w_start, w_end = int(w*0.2), int(w*0.8)
+                mask[h_start:h_end, w_start:w_end] = True
+        
+        # 保存掩码为图像
+        mask_path = os.path.splitext(image_path)[0] + '_mask.png'
+        cv2.imwrite(mask_path, (mask * 255).astype(np.uint8))
+        
+        return image_path, mask
+    
+    def pixelate_subject(self, image_path, mask, pixel_size=20):
+        """对主体进行像素化，并确保边缘也完全像素化"""
+        print("开始像素化处理...")
+        # 读取原始图像
+        image = cv2.imread(image_path)
+        if image is None:
+            raise ValueError(f"无法读取图像: {image_path}")
+            
+        print("图像读取成功，开始处理...")
+        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        h, w = image.shape[:2]
+        
+        # 创建一个透明背景的RGBA图像
+        result = np.zeros((h, w, 4), dtype=np.uint8)
+        
+        # 获取主体区域的边界框
+        y_indices, x_indices = np.where(mask)
+        if len(y_indices) == 0 or len(x_indices) == 0:
+            print("未检测到主体，返回原图")
+            return image_path  # 如果没有检测到主体，返回原图
+            
+        y_min, y_max = np.min(y_indices), np.max(y_indices)
+        x_min, x_max = np.min(x_indices), np.max(x_indices)
+        print(f"主体边界框: ({x_min}, {y_min}) - ({x_max}, {y_max})")
+        
+        # 提取主体区域
+        subject = image_rgb[y_min:y_max, x_min:x_max]
+        subject_mask = mask[y_min:y_max, x_min:x_max]
+        
+        # 创建一个完整的RGBA图像，包含主体和透明背景
+        subject_rgba = np.zeros((subject.shape[0], subject.shape[1], 4), dtype=np.uint8)
+        subject_rgba[:,:,:3] = subject
+        subject_rgba[:,:,3] = (subject_mask * 255).astype(np.uint8)
+        
+        print("对整个主体(包括边缘)进行像素化...")
+        # 对整个RGBA图像进行像素化处理
+        h_sub, w_sub = subject_rgba.shape[:2]
+        print(f"主体尺寸: {w_sub}x{h_sub}, 像素大小: {pixel_size}")
+        
+        # 确保目标尺寸至少为1x1
+        target_w = max(1, w_sub // pixel_size)
+        target_h = max(1, h_sub // pixel_size)
+        print(f"目标尺寸: {target_w}x{target_h}")
+        
+        # 先缩小
+        temp = cv2.resize(subject_rgba, (target_w, target_h), interpolation=cv2.INTER_LINEAR)
+        # 再放大，使用最近邻插值以保持像素化效果
+        pixelated_rgba = cv2.resize(temp, (w_sub, h_sub), interpolation=cv2.INTER_NEAREST)
+        
+        print("将处理后的主体放回原位置...")
+        # 将处理后的主体放回原位置
+        result[y_min:y_max, x_min:x_max] = pixelated_rgba
+        
+        # 保存结果为PNG（支持透明度）
+        result_path = os.path.splitext(image_path)[0] + '_pixelated.png'
+        
+        print(f"保存结果到: {result_path}")
+        # 使用PIL保存RGBA图像
+        pil_image = Image.fromarray(result)
+        pil_image.save(result_path, format='PNG')
+        
+        print("像素化处理完成!")
+        return result_path 

app/utils/pixelate_subject.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import sys
+import json
+import cv2
+import numpy as np
+from image_processing import ImageProcessor
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print(json.dumps({"error": "Missing arguments"}))
+        sys.exit(1)
+        
+    subject_path = sys.argv[1]
+    mask_path = sys.argv[2]
+    pixel_size = int(sys.argv[3]) if len(sys.argv) > 3 else 20
+    
+    # 读取掩码
+    mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
+    mask = mask > 0  # 二值化
+    
+    processor = ImageProcessor()
+    
+    try:
+        result_path = processor.pixelate_subject(subject_path, mask, pixel_size)
+        print(json.dumps({
+            "resultPath": result_path
+        }))
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1) 

app/utils/process.py

@@ -0,0 +1,61 @@
+from PIL import Image
+# 确保导入路径正确
+from utils.image_processing import ImageProcessor
+import time
+import gradio as gr
+import traceback
+
+# 初始化图像处理器
+try:
+    print("初始化 ImageProcessor...")
+    image_processor = ImageProcessor()
+    print("ImageProcessor 初始化成功")
+except Exception as e:
+    print(f"初始化 ImageProcessor 失败: {str(e)}")
+    traceback.print_exc()
+
+def process_image(input_image, *args, **kwargs):
+    progress = gr.Progress()
+    
+    print("开始处理输入图片...")
+    progress(0, desc="开始处理...")
+    
+    # 保存输入图像到临时文件
+    temp_input_path = "temp_input.png"
+    input_image.save(temp_input_path)
+    
+    try:
+        print("使用SAM提取主体...")
+        progress(0.3, desc="提取主体...")
+        # 使用SAM提取主体
+        _, mask = image_processor.extract_subject(temp_input_path)
+        
+        print("对主体进行像素化...")
+        progress(0.6, desc="像素化处理...")
+        # 对主体进行像素化
+        pixel_size = kwargs.get('pixel_size', 20)
+        result_path = image_processor.pixelate_subject(temp_input_path, mask, pixel_size)
+        
+        # 读取结果图像
+        progress(0.9, desc="生成最终图像...")
+        result_image = Image.open(result_path)
+        
+        print("完成主体提取和像素化!")
+        progress(1.0, desc="处理完成!")
+        return result_image
+    except Exception as e:
+        traceback.print_exc()
+        print(f"处理失败: {str(e)}")
+        progress(1.0, desc="处理失败")
+        
+        # 如果失败，继续使用原来的处理流程
+        print("回退到原始处理流程...")
+        
+        print("提取边缘...")
+        # 原来的边缘提取代码
+        
+        print("开始生成图片...")
+        # 原来的图片生成代码
+        
+        # ... 其余原始处理代码 ...
+        return None 

requirements.txt

@@ -0,0 +1,36 @@
+# 核心 AI 与图像处理
+torch
+torchvision
+torchaudio
+transformers
+diffusers
+accelerate
+safetensors
+scipy
+tqdm
+einops
+opencv-python
+Pillow
+numpy
+kornia
+
+# Gradio UI
+gradio
+
+# CLIP interrogator
+clip-interrogator
+
+# FastAPI 和 Web API（可选）
+fastapi
+uvicorn
+python-multipart
+
+# 通用工具
+requests
+pydantic
+packaging
+
+openai
+huggingface-hub
+
+git+https://github.com/facebookresearch/segment-anything.git