Upload 37 files

.gitattributes

@@ -46,3 +46,4 @@ static/videos/shiba.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/steve.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/teaser.mp4 filter=lfs diff=lfs merge=lfs -text
 static/videos/toby.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth/ebsynth.dll filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,190 @@
+### macOS template
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+

.idea/.gitignore

@@ -0,0 +1,8 @@
+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,12 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="scripts.*" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (venv)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/sd-webui-mov2mov.iml" filepath="$PROJECT_DIR$/.idea/sd-webui-mov2mov.iml" />
+    </modules>
+  </component>
+</project>

.idea/sd-webui-mov2mov.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (venv)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

CHANGELOG.md

@@ -0,0 +1,20 @@
+### 2023/9/30
+1. automatic video fps parsing
+2. Video editing features.
+   1. Customizable selection of keyframes or automatic generation of keyframes.
+   2. Backtrack keyframe tag.
+   3. automatically synthesize video based on keyframes via Ezsynth(https://github.com/Trentonom0r3/Ezsynth).
+   4. Currently, only the Windows system is supported. If your system does not support it, you can close this tab.
+
+### 2023/9/24
+1. Move the tab behind img2img.
+2. Fix the issue of video synthesis failure on the Mac system.
+3. Fix the problem of refiner not taking effect
+
+### 2023/9/23
+1. Fixed the issue where the tab is not displayed in the sd1.6 version.
+2. Inference of video width and height.
+3. Support for Refiner.
+4. Temporarily removed modnet functionality.
+5. Temporarily removed the function to add prompts frame by frame (ps: I believe there's a better approach, will add in the next version).
+6. Changed video synthesis from ffmpeg to imageio.

CHANGELOG_CN.md

@@ -0,0 +1,26 @@
+### 2023/9/30
+1. 自动解析视频的fps
+2. 视频编辑功能.
+   1. 可以自定义选择关键帧或者自动生成关键帧
+   2. 反推关键帧tag
+   3. 通过Ezsynth(https://github.com/Trentonom0r3/Ezsynth)自动根据关键帧合成视频.
+   4. 目前只有windows系统可以使用,如果您系统不支持,可以关闭该选项卡.
+
+### 2023/9/26
+1. 编辑关键帧
+2. 自动反推关键帧
+
+### 2023/9/24
+1. 移动选项卡至img2img后面
+2. 修复mac系统下,视频合成失败的问题
+3. 修复refiner不生效的问题
+
+
+### 2023/9/23
+
+1. 修复sd1.6版本选项卡不显示的问题.
+2. 推理视频宽高
+3. 支持Refiner
+4. 暂时移除modnet功能.
+5. 暂时移除逐帧添加prompt功能(ps:我觉得有更好的方式,下个版本添加)
+6. 修改合成视频的ffmpeg为imageio

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Scholar0
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,27 +1,65 @@
----
-title: Mov2Mov
-emoji: 🧠
-colorFrom: yellow
-colorTo: indigo
-sdk: static
-pinned: false
-license: mit
-short_description: Mov2Mov
----
-
-# Nerfies
-
-This is the repository that contains source code for the [Nerfies website](https://nerfies.github.io).
-
-If you find Nerfies useful for your work please cite:
-```
-@article{park2021nerfies
-  author    = {Park, Keunhong and Sinha, Utkarsh and Barron, Jonathan T. and Bouaziz, Sofien and Goldman, Dan B and Seitz, Steven M. and Martin-Brualla, Ricardo},
-  title     = {Nerfies: Deformable Neural Radiance Fields},
-  journal   = {ICCV},
-  year      = {2021},
-}
-```
-
-# Website License
-<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
+[English](README.md) | [中文简体](README_CN.md)
+
+## Mov2mov This is the Mov2mov plugin for Automatic1111/stable-diffusion-webui.
+
+![img.png](images/2.jpg)
+![img1.png](images/1.png)
+
+
+
+Features:
+- Directly process frames from videos
+- Package into a video after processing
+- Video Editing(beta)
+  - Dramatically reduce video flicker by keyframe compositing!
+  - You can customize the keyframe selection or auto-generate keyframes.
+  - Backpropel keyframe tag
+  - Currently only available for windows, if your system does not support, you can turn off this tab.
+
+Also, mov2mov will work better with the [bg-mask](https://github.com/Scholar01/sd-webui-bg-mask) plugin 😃
+
+# Table of Contents
+
+
+- [Table of Contents](#table-of-contents)
+  - [Usage Regulations](#usage-regulations)
+  - [Installation](#installation)
+  - [Change Log](#change-log)
+  - [Instructions](#instructions)
+  - [Thanks](#thanks)
+## Usage Regulations
+
+1. Please resolve the authorization issues of the video source on your own. Any problems caused by using unauthorized videos for conversion must be borne by the user. It has nothing to do with mov2mov!
+2. Any video made with mov2mov and published on video platforms must clearly specify the source of the video used for conversion in the description. For example, if you use someone else's video and convert it through AI, you must provide a clear link to the original video; if you use your own video, you must also state this in the description.
+3. All copyright issues caused by the input source must be borne by the user. Note that many videos explicitly state that they cannot be reproduced or copied!
+4. Please strictly comply with national laws and regulations to ensure that the content is legal and compliant. Any legal responsibility caused by using this plugin must be borne by the user. It has nothing to do with mov2mov!
+
+## Installation
+
+1. Open the Extensions tab.
+2. Click on Install from URL.
+3. Enter the URL for the extension's git repository.
+4. Click Install.
+5. Restart WebUI.
+
+
+
+## Change Log
+
+[Change Log](CHANGELOG.md)
+
+
+
+## Instructions
+
+- Video tutorials:
+  - [https://www.bilibili.com/video/BV1Mo4y1a7DF](https://www.bilibili.com/video/BV1Mo4y1a7DF)
+  - [https://www.bilibili.com/video/BV1rY4y1C7Q5](https://www.bilibili.com/video/BV1rY4y1C7Q5)
+- QQ channel: [https://pd.qq.com/s/akxpjjsgd](https://pd.qq.com/s/akxpjjsgd)
+- Discord: [https://discord.gg/hUzF3kQKFW](https://discord.gg/hUzF3kQKFW)
+
+## Thanks
+
+- modnet-entry: [https://github.com/RimoChan/modnet-entry](https://github.com/RimoChan/modnet-entry)
+- MODNet: [https://github.com/ZHKKKe/MODNet](https://github.com/ZHKKKe/MODNet)
+- Ezsynth: [https://github.com/Trentonom0r3/Ezsynth](https://github.com/Trentonom0r3/Ezsynth)

README_CN.md

@@ -0,0 +1,63 @@
+[English](README.md) | [中文简体](README_CN.md)
+
+# Mov2mov 适用于Automatic1111/stable-diffusion-webui 的 Mov2mov 插件。
+
+![img.png](images/2.jpg)
+![img1.png](images/1.png)
+
+
+功能：
+- 直接从视频逐帧处理
+- 处理完成后打包成视频
+- 视频编辑(beta)
+  - 通过关键帧合成的方式,大幅度减少视频闪烁!
+  - 可以自定义选择关键帧或者自动生成关键帧
+  - 反推关键帧tag
+  - 目前只有windows系统可以使用,如果您系统不支持,可以关闭该选项卡.
+
+
+另外,mov2mov与[bg-mask](https://github.com/Scholar01/sd-webui-bg-mask)插件一起工作会更好😃
+
+# 目录
+
+- [Mov2mov 适用于Automatic1111/stable-diffusion-webui 的 Mov2mov 插件。](#mov2mov-适用于automatic1111stable-diffusion-webui-的-mov2mov-插件)
+- [目录](#目录)
+  - [使用规约](#使用规约)
+  - [安装方法](#安装方法)
+  - [更新日志](#更新日志)
+  - [说明](#说明)
+  - [感谢](#感谢)
+
+## 使用规约
+
+1. 请自行解决视频来源的授权问题，任何由于使用非授权视频进行转换造成的问题，需自行承担全部责任和一切后果，于mov2mov无关！
+2. 任何发布到视频平台的基于mov2mov制作的视频，都必须要在简介中明确指明用于转换的视频来源。例如：使用他人发布的视频，通过ai进行转换的，必须要给出明确的原视频链接；若使用的是自己/自己的视频，也必须在简介加以说明。
+3. 由输入源造成的侵权问题需自行承担全部责任和一切后果。注意，很多视频明确指出不可转载，复制！
+4. 请严格遵守国家相关法律法规，确保内容合法合规。任何由于使用本插件造成的法律责任，需自行承担全部责任和一切后果，于mov2mov无关！
+
+## 安装方法
+
+1. 打开扩展（Extension）标签。
+2. 点击从网址安装（Install from URL）
+3. 在扩展的 git 仓库网址（URL for extension's git repository）处输入 
+4. 点击安装（Install）
+5. 重启 WebUI
+
+
+## 更新日志
+
+[更新日志](CHANGELOG_CN.md)
+
+## 说明
+
+- 视频教程:
+  - [https://www.bilibili.com/video/BV1Mo4y1a7DF](https://www.bilibili.com/video/BV1Mo4y1a7DF)
+  - [https://www.bilibili.com/video/BV1rY4y1C7Q5](https://www.bilibili.com/video/BV1rY4y1C7Q5)
+- QQ频道: [https://pd.qq.com/s/akxpjjsgd](https://pd.qq.com/s/akxpjjsgd)
+- Discord: [https://discord.gg/hUzF3kQKFW](https://discord.gg/hUzF3kQKFW)
+
+## 感谢
+
+- modnet-entry: [https://github.com/RimoChan/modnet-entry](https://github.com/RimoChan/modnet-entry)
+- MODNet: [https://github.com/ZHKKKe/MODNet](https://github.com/ZHKKKe/MODNet)
+- Ezsynth: [https://github.com/Trentonom0r3/Ezsynth](https://github.com/Trentonom0r3/Ezsynth)

ebsynth/__init__.py

@@ -0,0 +1,8 @@
+
+from .ebsynth_generate import EbsynthGenerate, Keyframe, Sequence, EbSynthTask
+
+AFTER_DETAILER = "ADetailer"
+
+__all__ = [
+    "EbsynthGenerate", "Keyframe", "Sequence", "EbSynthTask"
+]

ebsynth/_ebsynth.py

@@ -0,0 +1,224 @@
+# fork for Ezsynth(https://github.com/Trentonom0r3/Ezsynth)
+
+import os
+import sys
+from ctypes import *
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+libebsynth = None
+cached_buffer = {}
+
+EBSYNTH_BACKEND_CPU = 0x0001
+EBSYNTH_BACKEND_CUDA = 0x0002
+EBSYNTH_BACKEND_AUTO = 0x0000
+EBSYNTH_MAX_STYLE_CHANNELS = 8
+EBSYNTH_MAX_GUIDE_CHANNELS = 24
+EBSYNTH_VOTEMODE_PLAIN = 0x0001  # weight = 1
+EBSYNTH_VOTEMODE_WEIGHTED = 0x0002  # weight = 1/(1+error)
+
+
+def _normalize_img_shape(img):
+    img_len = len(img.shape)
+    if img_len == 2:
+        sh, sw = img.shape
+        sc = 0
+    elif img_len == 3:
+        sh, sw, sc = img.shape
+
+    if sc == 0:
+        sc = 1
+        img = img[..., np.newaxis]
+    return img
+
+
+def run(img_style, guides,
+        patch_size=5,
+        num_pyramid_levels=-1,
+        num_search_vote_iters=6,
+        num_patch_match_iters=4,
+        stop_threshold=5,
+        uniformity_weight=3500.0,
+        extraPass3x3=False,
+        ):
+    if patch_size < 3:
+        raise ValueError("patch_size is too small")
+    if patch_size % 2 == 0:
+        raise ValueError("patch_size must be an odd number")
+    if len(guides) == 0:
+        raise ValueError("at least one guide must be specified")
+
+    global libebsynth
+    if libebsynth is None:
+        if sys.platform[0:3] == 'win':
+            libebsynth_path = str(Path(__file__).parent / 'ebsynth.dll')
+            libebsynth = CDLL(libebsynth_path)
+        else:
+            # todo: implement for linux
+            pass
+
+        if libebsynth is not None:
+            libebsynth.ebsynthRun.argtypes = ( \
+                c_int,
+                c_int,
+                c_int,
+                c_int,
+                c_int,
+                c_void_p,
+                c_void_p,
+                c_int,
+                c_int,
+                c_void_p,
+                c_void_p,
+                POINTER(c_float),
+                POINTER(c_float),
+                c_float,
+                c_int,
+                c_int,
+                c_int,
+                POINTER(c_int),
+                POINTER(c_int),
+                POINTER(c_int),
+                c_int,
+                c_void_p,
+                c_void_p
+            )
+
+    if libebsynth is None:
+        return img_style
+
+    img_style = _normalize_img_shape(img_style)
+    sh, sw, sc = img_style.shape
+    t_h, t_w, t_c = 0, 0, 0
+
+    if sc > EBSYNTH_MAX_STYLE_CHANNELS:
+        raise ValueError(f"error: too many style channels {sc}, maximum number is {EBSYNTH_MAX_STYLE_CHANNELS}")
+
+    guides_source = []
+    guides_target = []
+    guides_weights = []
+
+    for i in range(len(guides)):
+        source_guide, target_guide, guide_weight = guides[i]
+        source_guide = _normalize_img_shape(source_guide)
+        target_guide = _normalize_img_shape(target_guide)
+        s_h, s_w, s_c = source_guide.shape
+        nt_h, nt_w, nt_c = target_guide.shape
+
+        if s_h != sh or s_w != sw:
+            raise ValueError("guide source and style resolution must match style resolution.")
+
+        if t_c == 0:
+            t_h, t_w, t_c = nt_h, nt_w, nt_c
+        elif nt_h != t_h or nt_w != t_w:
+            raise ValueError("guides target resolutions must be equal")
+
+        if s_c != nt_c:
+            raise ValueError("guide source and target channels must match exactly.")
+
+        guides_source.append(source_guide)
+        guides_target.append(target_guide)
+
+        guides_weights += [guide_weight / s_c] * s_c
+
+    guides_source = np.concatenate(guides_source, axis=-1)
+    guides_target = np.concatenate(guides_target, axis=-1)
+    guides_weights = (c_float * len(guides_weights))(*guides_weights)
+
+    styleWeight = 1.0
+    style_weights = [styleWeight / sc for i in range(sc)]
+    style_weights = (c_float * sc)(*style_weights)
+
+    maxPyramidLevels = 0
+    for level in range(32, -1, -1):
+        if min(min(sh, t_h) * pow(2.0, -level), \
+               min(sw, t_w) * pow(2.0, -level)) >= (2 * patch_size + 1):
+            maxPyramidLevels = level + 1
+            break
+
+    if num_pyramid_levels == -1:
+        num_pyramid_levels = maxPyramidLevels
+    num_pyramid_levels = min(num_pyramid_levels, maxPyramidLevels)
+
+    num_search_vote_iters_per_level = (c_int * num_pyramid_levels)(*[num_search_vote_iters] * num_pyramid_levels)
+    num_patch_match_iters_per_level = (c_int * num_pyramid_levels)(*[num_patch_match_iters] * num_pyramid_levels)
+    stop_threshold_per_level = (c_int * num_pyramid_levels)(*[stop_threshold] * num_pyramid_levels)
+
+    buffer = cached_buffer.get((t_h, t_w, sc), None)
+    if buffer is None:
+        buffer = create_string_buffer(t_h * t_w * sc)
+        cached_buffer[(t_h, t_w, sc)] = buffer
+
+    libebsynth.ebsynthRun(EBSYNTH_BACKEND_AUTO,  # backend
+                          sc,  # numStyleChannels
+                          guides_source.shape[-1],  # numGuideChannels
+                          sw,  # sourceWidth
+                          sh,  # sourceHeight
+                          img_style.tobytes(),
+                          # sourceStyleData (width * height * numStyleChannels) bytes, scan-line order
+                          guides_source.tobytes(),
+                          # sourceGuideData (width * height * numGuideChannels) bytes, scan-line order
+                          t_w,  # targetWidth
+                          t_h,  # targetHeight
+                          guides_target.tobytes(),
+                          # targetGuideData (width * height * numGuideChannels) bytes, scan-line order
+                          None,
+                          # targetModulationData (width * height * numGuideChannels) bytes, scan-line order; pass NULL to switch off the modulation
+                          style_weights,  # styleWeights (numStyleChannels) floats
+                          guides_weights,  # guideWeights (numGuideChannels) floats
+                          uniformity_weight,
+                          # uniformityWeight reasonable values are between 500-15000, 3500 is a good default
+                          patch_size,  # patchSize odd sizes only, use 5 for 5x5 patch, 7 for 7x7, etc.
+                          EBSYNTH_VOTEMODE_WEIGHTED,  # voteMode use VOTEMODE_WEIGHTED for sharper result
+                          num_pyramid_levels,  # numPyramidLevels
+
+                          num_search_vote_iters_per_level,
+                          # numSearchVoteItersPerLevel how many search/vote iters to perform at each level (array of ints, coarse first, fine last)
+                          num_patch_match_iters_per_level,
+                          # numPatchMatchItersPerLevel how many Patch-Match iters to perform at each level (array of ints, coarse first, fine last)
+                          stop_threshold_per_level,
+                          # stopThresholdPerLevel stop improving pixel when its change since last iteration falls under this threshold
+                          1 if extraPass3x3 else 0,
+                          # extraPass3x3 perform additional polishing pass with 3x3 patches at the finest level, use 0 to disable
+                          None,  # outputNnfData (width * height * 2) ints, scan-line order; pass NULL to ignore
+                          buffer  # outputImageData  (width * height * numStyleChannels) bytes, scan-line order
+                          )
+
+    return np.frombuffer(buffer, dtype=np.uint8).reshape((t_h, t_w, sc)).copy()
+
+
+# transfer color from source to target
+def color_transfer(img_source, img_target):
+    guides = [(cv2.cvtColor(img_source, cv2.COLOR_BGR2GRAY),
+               cv2.cvtColor(img_target, cv2.COLOR_BGR2GRAY),
+               1)]
+    h, w, c = img_source.shape
+    result = []
+    for i in range(c):
+        result += [
+            run(img_source[..., i:i + 1], guides=guides,
+                patch_size=11,
+                num_pyramid_levels=40,
+                num_search_vote_iters=6,
+                num_patch_match_iters=4,
+                stop_threshold=5,
+                uniformity_weight=500.0,
+                extraPass3x3=True,
+                )
+
+        ]
+    return np.concatenate(result, axis=-1)
+
+
+def task(img_style, guides):
+    return run(img_style,
+               guides,
+               patch_size=5,
+               num_pyramid_levels=6,
+               num_search_vote_iters=12,
+               num_patch_match_iters=6,
+               uniformity_weight=3500.0,
+               extraPass3x3=False
+               )

ebsynth/ebsynth.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a2499714514ea5016a04145d653012bfa6e74906293e0bae5a51ac155dba4a6
+size 28829696

ebsynth/ebsynth_generate.py

@@ -0,0 +1,143 @@
+import cv2
+import numpy as np
+from PIL.Image import Image
+from dataclasses import dataclass, field
+from toolz import groupby
+
+
+@dataclass
+class Keyframe:
+    num: int
+    image: np.ndarray = field(repr=False)
+    prompt: str = field(repr=False)
+
+
+@dataclass
+class Sequence:
+    start: int
+    keyframe: Keyframe
+    end: int
+    # 当前序列的所有帧
+    frames: dict[int, np.ndarray] = field(default_factory=dict, repr=False)
+    # 序列生成的所有帧
+    generate_frames: dict[int, np.ndarray] = field(default_factory=dict, repr=False)
+
+
+@dataclass
+class EbSynthTask:
+    style: np.ndarray = field(repr=False)
+    source: np.ndarray = field(repr=False)
+    target: np.ndarray = field(repr=False)
+    frame_num: int
+    key_frame_num: int
+    weight: float = field(default=1.0, repr=False)
+
+
+class EbsynthGenerate:
+    def __init__(self, keyframes: list[Keyframe], frames: list[np.ndarray], fps: int):
+        self.keyframes = keyframes
+        self.frames = frames
+        self.fps = fps
+        self.sequences = []
+        self.setup_sequences()
+
+    def setup_sequences(self):
+        """
+        初始化序列,在这个阶段,frame_num对应的帧就已经处理好了,在后面使用不需要再处理frame-1了
+        """
+        self.sequences.clear()
+        all_frames = len(self.frames)
+        left_frame = 1
+        for i, keyframe in enumerate(self.keyframes):
+            right_frame = self.keyframes[i + 1].num if i + 1 < len(self.keyframes) else all_frames
+            frames = {}
+            for frame_num in range(left_frame, right_frame + 1):
+                frames[frame_num] = self.frames[frame_num - 1]
+            sequence = Sequence(left_frame, keyframe, right_frame, frames)
+            self.sequences.append(sequence)
+            left_frame = keyframe.num
+        return self.sequences
+
+    def get_tasks(self, weight: float = 4.0) -> list[EbSynthTask]:
+        tasks = []
+        for i, sequence in enumerate(self.sequences):
+            frames = sequence.frames.items()
+            source = sequence.frames[sequence.keyframe.num]
+            style = sequence.keyframe.image
+            for frame_num, frame in frames:
+                target = frame
+                task = EbSynthTask(style, source, target, frame_num, sequence.keyframe.num, weight)
+                tasks.append(task)
+        return tasks
+
+    def append_generate_frames(self, key_frames_num, frame_num, generate_frames):
+        """
+
+        Args:
+            key_frames_num:  用于定位sequence
+            frame_num: key
+            generate_frames: value
+
+        Returns:
+
+        """
+        for sequence in self.sequences:
+            if sequence.keyframe.num == key_frames_num:
+                sequence.generate_frames[frame_num] = generate_frames
+                break
+        else:
+            raise ValueError(f'not found key frame num {key_frames_num}')
+
+    def merge_sequences(self):
+        # 存储合并后的结果
+        merged_frames = []
+        border = 1
+        for i in range(len(self.sequences)):
+            current_seq = self.sequences[i]
+            next_seq = self.sequences[i + 1] if i + 1 < len(self.sequences) else None
+
+            # 如果存在下一个序列
+            if next_seq:
+                # 获取两个序列的帧交集
+                common_frames_nums = set(current_seq.frames.keys()).intersection(
+                    set(range(next_seq.start + border, next_seq.end)) if i > 0 else set(
+                        range(next_seq.start, next_seq.end)))
+
+                for j, frame_num in enumerate(common_frames_nums):
+                    # 从两个序列中获取帧并合并
+                    frame1 = current_seq.generate_frames[frame_num]
+                    frame2 = next_seq.generate_frames[frame_num]
+
+                    weight = float(j) / float(len(common_frames_nums))
+                    merged_frame = cv2.addWeighted(frame1, 1 - weight, frame2, weight, 0)
+                    merged_frames.append((frame_num, merged_frame))
+
+            # 如果没有下一个序列
+            else:
+                # 添加与前一序列的差集帧到结果中
+                if i > 0:
+                    prev_seq = self.sequences[i - 1]
+                    difference_frames_nums = set(current_seq.frames.keys()) - set(prev_seq.frames.keys())
+                else:
+                    difference_frames_nums = set(current_seq.frames.keys())
+
+                for frame_num in difference_frames_nums:
+                    merged_frames.append((frame_num, current_seq.generate_frames[frame_num]))
+
+        # group_merged_frames = groupby(lambda x: x[0], merged_frames)
+        # merged_frames.clear()
+        # # 取出value长度大于1的元素
+        # for key, value in group_merged_frames.items():
+        #     if len(value) > 1:
+        #         # 将value中的所有元素合并
+        #         merged_frame = value[0][1]
+        #         for i in range(1, len(value)):
+        #             merged_frame = cv2.addWeighted(merged_frame, weight, value[i][1], 1 - weight, 0)
+        #         merged_frames.append((key, merged_frame))
+        #     else:
+        #         merged_frames.append((key, value[0][1]))
+        result = []
+        for i, frame in sorted(merged_frames, key=lambda x: x[0]):
+            result.append(frame)
+
+        return result

install.py

@@ -0,0 +1,19 @@
+import os
+import platform
+import launch
+
+if not launch.is_installed("cv2"):
+    print('Installing requirements for Mov2mov')
+    launch.run_pip("install opencv-python", "requirements for opencv")
+
+if platform.system() == 'Windows':
+    if not launch.is_installed('imageio'):
+        print('Installing requirements for Mov2mov')
+        launch.run_pip("install imageio", "requirements for imageio")
+    if not launch.is_installed('imageio-ffmpeg'):
+        print('Installing requirements for Mov2mov')
+        launch.run_pip("install imageio-ffmpeg", "requirements for imageio-ffmpeg")
+else:
+    if not launch.is_installed('ffmpeg'):
+        print('Installing requirements for Mov2mov')
+        launch.run_pip("install ffmpeg", "requirements for ffmpeg")

javascript/m2m_ui.js

@@ -0,0 +1,48 @@
+function submit_mov2mov() {
+    rememberGallerySelection('mov2mov_gallery')
+    showSubmitButtons('mov2mov', false)
+    showResultVideo('mov2mov', false)
+
+    var id = randomId()
+    requestProgress(id, gradioApp().getElementById('mov2mov_gallery_container'), gradioApp().getElementById('mov2mov_gallery'), function () {
+        showSubmitButtons('mov2mov', true)
+        showResultVideo('mov2mov', true)
+    })
+
+    var res = create_submit_args(arguments)
+    res[0] = id
+    return res
+}
+
+function showResultVideo(tabname, show) {
+    gradioApp().getElementById(tabname + '_video').style.display = show ? "block" : "none"
+    gradioApp().getElementById(tabname + '_gallery').style.display = show ? "none" : "block"
+
+}
+
+
+function showModnetModels() {
+    var check = arguments[0]
+    gradioApp().getElementById('mov2mov_modnet_model').style.display = check ? "block" : "none"
+    gradioApp().getElementById('mov2mov_merge_background').style.display = check ? "block" : "none"
+    return []
+}
+
+function switchModnetMode() {
+    let mode = arguments[0]
+
+    if (mode === 'Clear' || mode === 'Origin' || mode === 'Green' || mode === 'Image') {
+        gradioApp().getElementById('modnet_background_movie').style.display = "none"
+        gradioApp().getElementById('modnet_background_image').style.display = "block"
+    } else {
+        gradioApp().getElementById('modnet_background_movie').style.display = "block"
+        gradioApp().getElementById('modnet_background_image').style.display = "none"
+    }
+
+    return []
+}
+
+
+function copy_from(type) {
+    return []
+}

requirements.txt

@@ -0,0 +1,3 @@
+opencv-python
+imageio
+imageio-ffmpeg

scripts/m2m_config.py

@@ -0,0 +1,2 @@
+mov2mov_outpath_samples = 'outputs/mov2mov-images'
+mov2mov_output_dir = 'outputs/mov2mov-videos'

scripts/m2m_hook.py

@@ -0,0 +1,63 @@
+from collections import defaultdict
+
+def patch(key, obj, field, replacement):
+    """Replaces a function in a module or a class.
+
+    Also stores the original function in this module, possible to be retrieved via original(key, obj, field).
+    If the function is already replaced by this caller (key), an exception is raised -- use undo() before that.
+
+    Arguments:
+        key: identifying information for who is doing the replacement. You can use __name__.
+        obj: the module or the class
+        field: name of the function as a string
+        replacement: the new function
+
+    Returns:
+        the original function
+    """
+
+    patch_key = (obj, field)
+    if patch_key in originals[key]:
+        raise RuntimeError(f"patch for {field} is already applied")
+
+    original_func = getattr(obj, field)
+    originals[key][patch_key] = original_func
+
+    setattr(obj, field, replacement)
+
+    return original_func
+
+
+def undo(key, obj, field):
+    """Undoes the peplacement by the patch().
+
+    If the function is not replaced, raises an exception.
+
+    Arguments:
+        key: identifying information for who is doing the replacement. You can use __name__.
+        obj: the module or the class
+        field: name of the function as a string
+
+    Returns:
+        Always None
+    """
+
+    patch_key = (obj, field)
+
+    if patch_key not in originals[key]:
+        raise RuntimeError(f"there is no patch for {field} to undo")
+
+    original_func = originals[key].pop(patch_key)
+    setattr(obj, field, original_func)
+
+    return None
+
+
+def original(key, obj, field):
+    """Returns the original function for the patch created by the patch() function"""
+    patch_key = (obj, field)
+
+    return originals[key].get(patch_key, None)
+
+
+originals = defaultdict(dict)

scripts/m2m_ui.py

@@ -0,0 +1,622 @@
+import importlib
+import os
+import platform
+import shutil
+import subprocess as sp
+import sys
+
+import gradio as gr
+
+import modules
+import modules.scripts as scripts
+from modules import (
+    script_callbacks,
+    shared,
+    call_queue,
+    sd_samplers,
+    ui_prompt_styles,
+    sd_models,
+)
+from modules.call_queue import wrap_gradio_gpu_call
+from modules.images import image_data
+from modules.shared import opts
+from modules.ui import (
+    ordered_ui_categories,
+    create_sampler_and_steps_selection,
+    switch_values_symbol,
+    create_override_settings_dropdown,
+    detect_image_size_symbol,
+    plaintext_to_html,
+    paste_symbol,
+    clear_prompt_symbol,
+    restore_progress_symbol,
+)
+from modules.ui_common import (
+    folder_symbol,
+    update_generation_info,
+    create_refresh_button,
+)
+from modules.ui_components import (
+    ResizeHandleRow,
+    FormRow,
+    ToolButton,
+    FormGroup,
+    InputAccordion,
+)
+from scripts import m2m_hook as patches
+from scripts import m2m_util
+from scripts import mov2mov
+from scripts.mov2mov import scripts_mov2mov
+from scripts.m2m_config import mov2mov_outpath_samples, mov2mov_output_dir
+from scripts.movie_editor import MovieEditor
+
+id_part = "mov2mov"
+
+
+def save_video(video):
+    path = "logs/movies"
+    if not os.path.exists(path):
+        os.makedirs(path, exist_ok=True)
+    index = len([path for path in os.listdir(path) if path.endswith(".mp4")]) + 1
+    video_path = os.path.join(path, str(index).zfill(5) + ".mp4")
+    shutil.copyfile(video, video_path)
+    filename = os.path.relpath(video_path, path)
+    return gr.File.update(value=video_path, visible=True), plaintext_to_html(
+        f"Saved: {filename}"
+    )
+
+
+class Toprow:
+    """Creates a top row UI with prompts, generate button, styles, extra little buttons for things, and enables some functionality related to their operation"""
+
+    def __init__(self, is_img2img, id_part=None):
+        if not id_part:
+            id_part = "img2img" if is_img2img else "txt2img"
+        self.id_part = id_part
+
+        with gr.Row(elem_id=f"{id_part}_toprow", variant="compact"):
+            with gr.Column(elem_id=f"{id_part}_prompt_container", scale=6):
+                with gr.Row():
+                    with gr.Column(scale=80):
+                        with gr.Row():
+                            self.prompt = gr.Textbox(
+                                label="Prompt",
+                                elem_id=f"{id_part}_prompt",
+                                show_label=False,
+                                lines=3,
+                                placeholder="Prompt (press Ctrl+Enter or Alt+Enter to generate)",
+                                elem_classes=["prompt"],
+                            )
+                            self.prompt_img = gr.File(
+                                label="",
+                                elem_id=f"{id_part}_prompt_image",
+                                file_count="single",
+                                type="binary",
+                                visible=False,
+                            )
+
+                with gr.Row():
+                    with gr.Column(scale=80):
+                        with gr.Row():
+                            self.negative_prompt = gr.Textbox(
+                                label="Negative prompt",
+                                elem_id=f"{id_part}_neg_prompt",
+                                show_label=False,
+                                lines=3,
+                                placeholder="Negative prompt (press Ctrl+Enter or Alt+Enter to generate)",
+                                elem_classes=["prompt"],
+                            )
+
+            self.button_interrogate = None
+            self.button_deepbooru = None
+            if is_img2img:
+                with gr.Column(scale=1, elem_classes="interrogate-col"):
+                    self.button_interrogate = gr.Button(
+                        "Interrogate\nCLIP", elem_id="interrogate"
+                    )
+                    self.button_deepbooru = gr.Button(
+                        "Interrogate\nDeepBooru", elem_id="deepbooru"
+                    )
+
+            with gr.Column(scale=1, elem_id=f"{id_part}_actions_column"):
+                with gr.Row(
+                    elem_id=f"{id_part}_generate_box", elem_classes="generate-box"
+                ):
+                    self.interrupt = gr.Button(
+                        "Interrupt",
+                        elem_id=f"{id_part}_interrupt",
+                        elem_classes="generate-box-interrupt",
+                    )
+                    self.skip = gr.Button(
+                        "Skip",
+                        elem_id=f"{id_part}_skip",
+                        elem_classes="generate-box-skip",
+                    )
+                    self.submit = gr.Button(
+                        "Generate", elem_id=f"{id_part}_generate", variant="primary"
+                    )
+
+                    self.skip.click(
+                        fn=lambda: shared.state.skip(),
+                        inputs=[],
+                        outputs=[],
+                    )
+
+                    self.interrupt.click(
+                        fn=lambda: shared.state.interrupt(),
+                        inputs=[],
+                        outputs=[],
+                    )
+
+                with gr.Row(elem_id=f"{id_part}_tools"):
+                    self.paste = ToolButton(value=paste_symbol, elem_id="paste")
+
+                    self.clear_prompt_button = ToolButton(
+                        value=clear_prompt_symbol, elem_id=f"{id_part}_clear_prompt"
+                    )
+                    self.restore_progress_button = ToolButton(
+                        value=restore_progress_symbol,
+                        elem_id=f"{id_part}_restore_progress",
+                        visible=False,
+                    )
+
+                    self.token_counter = gr.HTML(
+                        value="<span>0/75</span>",
+                        elem_id=f"{id_part}_token_counter",
+                        elem_classes=["token-counter"],
+                    )
+                    self.token_button = gr.Button(
+                        visible=False, elem_id=f"{id_part}_token_button"
+                    )
+                    self.negative_token_counter = gr.HTML(
+                        value="<span>0/75</span>",
+                        elem_id=f"{id_part}_negative_token_counter",
+                        elem_classes=["token-counter"],
+                    )
+                    self.negative_token_button = gr.Button(
+                        visible=False, elem_id=f"{id_part}_negative_token_button"
+                    )
+
+                    self.clear_prompt_button.click(
+                        fn=lambda *x: x,
+                        _js="confirm_clear_prompt",
+                        inputs=[self.prompt, self.negative_prompt],
+                        outputs=[self.prompt, self.negative_prompt],
+                    )
+
+                self.ui_styles = ui_prompt_styles.UiPromptStyles(
+                    id_part, self.prompt, self.negative_prompt
+                )
+
+        self.prompt_img.change(
+            fn=modules.images.image_data,
+            inputs=[self.prompt_img],
+            outputs=[self.prompt, self.prompt_img],
+            show_progress=False,
+        )
+
+
+def create_output_panel(tabname, outdir):
+    def open_folder(f):
+        if not os.path.exists(f):
+            print(
+                f'Folder "{f}" does not exist. After you create an image, the folder will be created.'
+            )
+            return
+        elif not os.path.isdir(f):
+            print(
+                f"""
+WARNING
+An open_folder request was made with an argument that is not a folder.
+This could be an error or a malicious attempt to run code on your computer.
+Requested path was: {f}
+""",
+                file=sys.stderr,
+            )
+            return
+
+        if not shared.cmd_opts.hide_ui_dir_config:
+            path = os.path.normpath(f)
+            if platform.system() == "Windows":
+                os.startfile(path)
+            elif platform.system() == "Darwin":
+                sp.Popen(["open", path])
+            elif "microsoft-standard-WSL2" in platform.uname().release:
+                sp.Popen(["wsl-open", path])
+            else:
+                sp.Popen(["xdg-open", path])
+
+    with gr.Column(variant="panel", elem_id=f"{tabname}_results"):
+        with gr.Group(elem_id=f"{tabname}_gallery_container"):
+            result_gallery = gr.Gallery(
+                label="Output",
+                show_label=False,
+                elem_id=f"{tabname}_gallery",
+                columns=4,
+                preview=True,
+                height=shared.opts.gallery_height or None,
+            )
+            result_video = gr.PlayableVideo(
+                label="Output Video", show_label=False, elem_id=f"{tabname}_video"
+            )
+
+        generation_info = None
+        with gr.Column():
+            with gr.Row(
+                elem_id=f"image_buttons_{tabname}", elem_classes="image-buttons"
+            ):
+                open_folder_button = ToolButton(
+                    folder_symbol,
+                    elem_id=f"{tabname}_open_folder",
+                    visible=not shared.cmd_opts.hide_ui_dir_config,
+                    tooltip="Open images output directory.",
+                )
+
+                if tabname != "extras":
+                    save = ToolButton(
+                        "💾",
+                        elem_id=f"save_{tabname}",
+                        tooltip=f"Save the image to a dedicated directory ({shared.opts.outdir_save}).",
+                    )
+
+            open_folder_button.click(
+                fn=lambda: open_folder(shared.opts.outdir_samples or outdir),
+                inputs=[],
+                outputs=[],
+            )
+
+            download_files = gr.File(
+                None,
+                file_count="multiple",
+                interactive=False,
+                show_label=False,
+                visible=False,
+                elem_id=f"download_files_{tabname}",
+            )
+
+            with gr.Group():
+                html_info = gr.HTML(
+                    elem_id=f"html_info_{tabname}", elem_classes="infotext"
+                )
+                html_log = gr.HTML(
+                    elem_id=f"html_log_{tabname}", elem_classes="html-log"
+                )
+
+                generation_info = gr.Textbox(
+                    visible=False, elem_id=f"generation_info_{tabname}"
+                )
+                if tabname == "txt2img" or tabname == "img2img" or tabname == "mov2mov":
+                    generation_info_button = gr.Button(
+                        visible=False, elem_id=f"{tabname}_generation_info_button"
+                    )
+                    generation_info_button.click(
+                        fn=update_generation_info,
+                        _js="function(x, y, z){ return [x, y, selected_gallery_index()] }",
+                        inputs=[generation_info, html_info, html_info],
+                        outputs=[html_info, html_info],
+                        show_progress=False,
+                    )
+
+                save.click(
+                    fn=call_queue.wrap_gradio_call(save_video),
+                    inputs=[result_video],
+                    outputs=[
+                        download_files,
+                        html_log,
+                    ],
+                    show_progress=False,
+                )
+
+            return result_gallery, result_video, generation_info, html_info, html_log
+
+
+def create_refiner():
+    with InputAccordion(
+        False, label="Refiner", elem_id=f"{id_part}_enable"
+    ) as enable_refiner:
+        with gr.Row():
+            refiner_checkpoint = gr.Dropdown(
+                label="Checkpoint",
+                elem_id=f"{id_part}_checkpoint",
+                choices=sd_models.checkpoint_tiles(),
+                value="",
+                tooltip="switch to another model in the middle of generation",
+            )
+            create_refresh_button(
+                refiner_checkpoint,
+                sd_models.list_models,
+                lambda: {"choices": sd_models.checkpoint_tiles()},
+                f"{id_part}_checkpoint_refresh",
+            )
+
+            refiner_switch_at = gr.Slider(
+                value=0.8,
+                label="Switch at",
+                minimum=0.01,
+                maximum=1.0,
+                step=0.01,
+                elem_id=f"{id_part}_switch_at",
+                tooltip="fraction of sampling steps when the switch to refiner model should happen; 1=never, 0.5=switch in the middle of generation",
+            )
+    return enable_refiner, refiner_checkpoint, refiner_switch_at
+
+
+def on_ui_tabs():
+    scripts_mov2mov.initialize_scripts(is_img2img=True)
+
+    # with gr.Blocks(analytics_enabled=False) as mov2mov_interface:
+    with gr.TabItem(
+        "mov2mov", id=f"tab_{id_part}", elem_id=f"tab_{id_part}"
+    ) as mov2mov_interface:
+        toprow = Toprow(is_img2img=False, id_part=id_part)
+        dummy_component = gr.Label(visible=False)
+        with gr.Tab(
+            "Generation", id=f"{id_part}_generation"
+        ) as mov2mov_generation_tab, ResizeHandleRow(equal_height=False):
+            with gr.Column(variant="compact", elem_id="mov2mov_settings"):
+                with gr.Tabs(elem_id=f"mode_{id_part}"):
+                    init_mov = gr.Video(
+                        label="Video for mov2mov",
+                        elem_id=f"{id_part}_mov",
+                        show_label=False,
+                        source="upload",
+                    )
+
+                with FormRow():
+                    resize_mode = gr.Radio(
+                        label="Resize mode",
+                        elem_id=f"{id_part}_resize_mode",
+                        choices=[
+                            "Just resize",
+                            "Crop and resize",
+                            "Resize and fill",
+                            "Just resize (latent upscale)",
+                        ],
+                        type="index",
+                        value="Just resize",
+                    )
+                scripts_mov2mov.prepare_ui()
+
+                for category in ordered_ui_categories():
+                    if category == "sampler":
+                        steps, sampler_name = create_sampler_and_steps_selection(
+                            sd_samplers.visible_sampler_names(), id_part
+                        )
+                    elif category == "dimensions":
+                        with FormRow():
+                            with gr.Column(elem_id=f"{id_part}_column_size", scale=4):
+                                with gr.Tabs():
+                                    with gr.Tab(
+                                        label="Resize to",
+                                        elem_id=f"{id_part}_tab_resize_to",
+                                    ) as tab_scale_to:
+                                        with FormRow():
+                                            with gr.Column(
+                                                elem_id=f"{id_part}_column_size",
+                                                scale=4,
+                                            ):
+                                                width = gr.Slider(
+                                                    minimum=64,
+                                                    maximum=2048,
+                                                    step=8,
+                                                    label="Width",
+                                                    value=512,
+                                                    elem_id=f"{id_part}_width",
+                                                )
+                                                height = gr.Slider(
+                                                    minimum=64,
+                                                    maximum=2048,
+                                                    step=8,
+                                                    label="Height",
+                                                    value=512,
+                                                    elem_id=f"{id_part}_height",
+                                                )
+                                            with gr.Column(
+                                                elem_id=f"{id_part}_dimensions_row",
+                                                scale=1,
+                                                elem_classes="dimensions-tools",
+                                            ):
+                                                res_switch_btn = ToolButton(
+                                                    value=switch_values_symbol,
+                                                    elem_id=f"{id_part}_res_switch_btn",
+                                                )
+                                                detect_image_size_btn = ToolButton(
+                                                    value=detect_image_size_symbol,
+                                                    elem_id=f"{id_part}_detect_image_size_btn",
+                                                )
+                    elif category == "denoising":
+                        denoising_strength = gr.Slider(
+                            minimum=0.0,
+                            maximum=1.0,
+                            step=0.01,
+                            label="Denoising strength",
+                            value=0.75,
+                            elem_id=f"{id_part}_denoising_strength",
+                        )
+
+                        noise_multiplier = gr.Slider(
+                            minimum=0,
+                            maximum=1.5,
+                            step=0.01,
+                            label="Noise multiplier",
+                            elem_id=f"{id_part}_noise_multiplier",
+                            value=1,
+                        )
+                        with gr.Row(elem_id=f"{id_part}_frames_setting"):
+                            movie_frames = gr.Slider(
+                                minimum=10,
+                                maximum=60,
+                                step=1,
+                                label="Movie FPS",
+                                elem_id=f"{id_part}_movie_frames",
+                                value=30,
+                            )
+                            max_frames = gr.Number(
+                                label="Max FPS",
+                                value=-1,
+                                elem_id=f"{id_part}_max_frames",
+                            )
+
+                    elif category == "cfg":
+                        with gr.Row():
+                            cfg_scale = gr.Slider(
+                                minimum=1.0,
+                                maximum=30.0,
+                                step=0.5,
+                                label="CFG Scale",
+                                value=7.0,
+                                elem_id=f"{id_part}_cfg_scale",
+                            )
+                            image_cfg_scale = gr.Slider(
+                                minimum=0,
+                                maximum=3.0,
+                                step=0.05,
+                                label="Image CFG Scale",
+                                value=1.5,
+                                elem_id=f"{id_part}_image_cfg_scale",
+                                visible=False,
+                            )
+
+                    elif category == "checkboxes":
+                        with FormRow(elem_classes="checkboxes-row", variant="compact"):
+                            pass
+
+                    elif category == "accordions":
+                        with gr.Row(
+                            elem_id=f"{id_part}_accordions", elem_classes="accordions"
+                        ):
+                            scripts_mov2mov.setup_ui_for_section(category)
+
+                    elif category == "override_settings":
+                        with FormRow(elem_id=f"{id_part}_override_settings_row") as row:
+                            override_settings = create_override_settings_dropdown(
+                                "mov2mov", row
+                            )
+
+                    elif category == "scripts":
+                        editor = MovieEditor(id_part, init_mov, movie_frames)
+                        editor.render()
+                        with FormGroup(elem_id=f"{id_part}_script_container"):
+                            custom_inputs = scripts_mov2mov.setup_ui()
+
+                    if category not in {"accordions"}:
+                        scripts_mov2mov.setup_ui_for_section(category)
+
+            (
+                mov2mov_gallery,
+                result_video,
+                generation_info,
+                html_info,
+                html_log,
+            ) = create_output_panel(id_part, opts.mov2mov_output_dir)
+
+            res_switch_btn.click(
+                fn=None,
+                _js="function(){switchWidthHeight('mov2mov')}",
+                inputs=None,
+                outputs=None,
+                show_progress=False,
+            )
+
+            # calc video size
+            detect_image_size_btn.click(
+                fn=calc_video_w_h,
+                inputs=[init_mov, width, height],
+                outputs=[width, height],
+            )
+
+            mov2mov_args = dict(
+                fn=wrap_gradio_gpu_call(mov2mov.mov2mov, extra_outputs=[None, "", ""]),
+                _js="submit_mov2mov",
+                inputs=[
+                    dummy_component,
+                    toprow.prompt,
+                    toprow.negative_prompt,
+                    toprow.ui_styles.dropdown,
+                    init_mov,
+                    steps,
+                    sampler_name,
+                    cfg_scale,
+                    image_cfg_scale,
+                    denoising_strength,
+                    height,
+                    width,
+                    resize_mode,
+                    override_settings,
+                    # refiner
+                    # enable_refiner, refiner_checkpoint, refiner_switch_at,
+                    # mov2mov params
+                    noise_multiplier,
+                    movie_frames,
+                    max_frames,
+                    # editor
+                    editor.gr_enable_movie_editor,
+                    editor.gr_df,
+                    editor.gr_eb_weight,
+                ]
+                + custom_inputs,
+                outputs=[
+                    result_video,
+                    generation_info,
+                    html_info,
+                    html_log,
+                ],
+                show_progress=False,
+            )
+
+            toprow.submit.click(**mov2mov_args)
+
+    return [(mov2mov_interface, "mov2mov", f"{id_part}_tabs")]
+
+
+def calc_video_w_h(video, width, height):
+    if not video:
+        return width, height
+
+    return m2m_util.calc_video_w_h(video)
+
+
+def on_ui_settings():
+    section = ("mov2mov", "Mov2Mov")
+    shared.opts.add_option(
+        "mov2mov_outpath_samples",
+        shared.OptionInfo(
+            mov2mov_outpath_samples, "Mov2Mov output path for image", section=section
+        ),
+    )
+    shared.opts.add_option(
+        "mov2mov_output_dir",
+        shared.OptionInfo(
+            mov2mov_output_dir, "Mov2Mov output path for video", section=section
+        ),
+    )
+
+
+img2img_toprow: gr.Row = None
+
+
+def block_context_init(self, *args, **kwargs):
+    origin_block_context_init(self, *args, **kwargs)
+
+    if self.elem_id == "tab_img2img":
+        self.parent.__enter__()
+        on_ui_tabs()
+        self.parent.__exit__()
+
+
+def on_app_reload():
+    global origin_block_context_init
+    if origin_block_context_init:
+        patches.undo(__name__, obj=gr.blocks.BlockContext, field="__init__")
+        origin_block_context_init = None
+
+
+origin_block_context_init = patches.patch(
+    __name__,
+    obj=gr.blocks.BlockContext,
+    field="__init__",
+    replacement=block_context_init,
+)
+script_callbacks.on_before_reload(on_app_reload)
+script_callbacks.on_ui_settings(on_ui_settings)
+# script_callbacks.on_ui_tabs(on_ui_tabs)

scripts/m2m_util.py

@@ -0,0 +1,121 @@
+import os.path
+import platform
+import cv2
+import numpy
+import imageio
+
+
+def calc_video_w_h(video_path):
+    cap = cv2.VideoCapture(video_path)
+
+    if not cap.isOpened():
+        raise ValueError("Can't open video file")
+
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+    cap.release()
+
+    return width, height
+
+
+def get_mov_frame_count(file):
+    if file is None:
+        return None
+    cap = cv2.VideoCapture(file)
+
+    if not cap.isOpened():
+        return None
+
+    frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    cap.release()
+    return frames
+
+
+def get_mov_fps(file):
+    if file is None:
+        return None
+    cap = cv2.VideoCapture(file)
+
+    if not cap.isOpened():
+        return None
+
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    cap.release()
+    return fps
+
+
+def get_mov_all_images(file, frames, rgb=False):
+    if file is None:
+        return None
+    cap = cv2.VideoCapture(file)
+
+    if not cap.isOpened():
+        return None
+
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    if frames > fps:
+        print('Waring: The set number of frames is greater than the number of video frames')
+        frames = int(fps)
+
+    skip = fps // frames
+    count = 1
+    fs = 1
+    image_list = []
+    while (True):
+        flag, frame = cap.read()
+        if not flag:
+            break
+        else:
+            if fs % skip == 0:
+                if rgb:
+                    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                image_list.append(frame)
+                count += 1
+        fs += 1
+    cap.release()
+    return image_list
+
+
+def images_to_video(images, frames, out_path):
+    if platform.system() == 'Windows':
+        # Use imageio with the 'libx264' codec on Windows
+        return images_to_video_imageio(images, frames, out_path, 'libx264')
+    elif platform.system() == 'Darwin':
+        # Use cv2 with the 'avc1' codec on Mac
+        return images_to_video_cv2(images, frames, out_path, 'avc1')
+    else:
+        # Use cv2 with the 'mp4v' codec on other operating systems as it's the most widely supported
+        return images_to_video_cv2(images, frames, out_path, 'mp4v')
+
+
+def images_to_video_imageio(images, frames, out_path, codec):
+    # 判断out_path是否存在,不存在则创建
+    if not os.path.exists(os.path.dirname(out_path)):
+        os.makedirs(os.path.dirname(out_path), exist_ok=True)
+
+    with imageio.v2.get_writer(out_path, format='ffmpeg', mode='I', fps=frames, codec=codec) as writer:
+        for img in images:
+            writer.append_data(numpy.asarray(img))
+    return out_path
+
+
+def images_to_video_cv2(images, frames, out_path, codec):
+    if len(images) <= 0:
+        return None
+    # 判断out_path是否存在,不存在则创建
+    if not os.path.exists(os.path.dirname(out_path)):
+        os.makedirs(os.path.dirname(out_path), exist_ok=True)
+
+    fourcc = cv2.VideoWriter_fourcc(*codec)
+    if len(images) > 0:
+        img = images[0]
+        img_width, img_height = img.size
+        w = img_width
+        h = img_height
+    video = cv2.VideoWriter(out_path, fourcc, frames, (w, h))
+    for image in images:
+        img = cv2.cvtColor(numpy.asarray(image), cv2.COLOR_RGB2BGR)
+        video.write(img)
+    video.release()
+    return out_path

scripts/module_ui_extensions.py

@@ -0,0 +1,56 @@
+import gradio
+from modules import script_callbacks, ui_components
+from scripts import m2m_hook as patches
+
+
+elem_ids = []
+
+
+def fix_elem_id(component, **kwargs):
+    if "elem_id" not in kwargs:
+        return None
+    elem_id = kwargs["elem_id"]
+    if not elem_id:
+        return None
+    if elem_id not in elem_ids:
+        elem_ids.append(elem_id)
+    else:
+        elem_id = elem_id + "_" + str(elem_ids.count(elem_id))
+        elem_ids.append(elem_id)
+
+    return elem_id
+
+
+def IOComponent_init(self, *args, **kwargs):
+    elem_id = fix_elem_id(self, **kwargs)
+    if elem_id:
+        kwargs.pop("elem_id")
+        res = original_IOComponent_init(self, elem_id=elem_id, *args, **kwargs)
+    else:
+        res = original_IOComponent_init(self, *args, **kwargs)
+    return res
+
+
+def InputAccordion_init(self, *args, **kwargs):
+    elem_id = fix_elem_id(self, **kwargs)
+    if elem_id:
+        kwargs.pop("elem_id")
+        res = original_InputAccordion_init(self, elem_id=elem_id, *args, **kwargs)
+    else:
+        res = original_InputAccordion_init(self, *args, **kwargs)
+    return res
+
+
+original_IOComponent_init = patches.patch(
+    __name__,
+    obj=gradio.components.IOComponent,
+    field="__init__",
+    replacement=IOComponent_init,
+)
+
+original_InputAccordion_init = patches.patch(
+    __name__,
+    obj=ui_components.InputAccordion,
+    field="__init__",
+    replacement=InputAccordion_init,
+)

scripts/mov2mov.py

@@ -0,0 +1,302 @@
+import os.path
+import platform
+import time
+
+import PIL.Image
+from tqdm import tqdm
+
+import modules
+
+import cv2
+import numpy as np
+import pandas
+from PIL import Image
+from modules import shared, processing
+from modules.generation_parameters_copypaste import create_override_settings_dict
+from modules.processing import StableDiffusionProcessingImg2Img, process_images, Processed
+from modules.shared import opts, state
+from modules.ui import plaintext_to_html
+import modules.scripts as scripts
+
+from scripts.m2m_util import get_mov_all_images, images_to_video
+from scripts.m2m_config import mov2mov_outpath_samples, mov2mov_output_dir
+import modules
+from ebsynth import EbsynthGenerate, Keyframe
+
+scripts_mov2mov = scripts.ScriptRunner()
+
+
+def check_data_frame(df: pandas.DataFrame):
+    # 删除df的frame值为0的行
+    df = df[df['frame'] > 0]
+
+    # 判断df是否为空
+    if len(df) <= 0:
+        return False
+
+    return True
+
+
+def save_video(images, fps, extension='.mp4'):
+    if not os.path.exists(shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir)):
+        os.makedirs(shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir), exist_ok=True)
+
+    r_f = extension
+
+    print(f'Start generating {r_f} file')
+
+    video = images_to_video(images, fps,
+                            os.path.join(shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir),
+                                         str(int(time.time())) + r_f, ))
+    print(f'The generation is complete, the directory::{video}')
+
+    return video
+
+
+def process_mov2mov(p, mov_file, movie_frames, max_frames, resize_mode, w, h, args):
+    processing.fix_seed(p)
+    images = get_mov_all_images(mov_file, movie_frames)
+    if not images:
+        print('Failed to parse the video, please check')
+        return
+
+    print(f'The video conversion is completed, images:{len(images)}')
+    if max_frames == -1 or max_frames > len(images):
+        max_frames = len(images)
+
+    max_frames = int(max_frames)
+
+    p.do_not_save_grid = True
+    state.job_count = max_frames  # * p.n_iter
+    generate_images = []
+    for i, image in enumerate(images):
+        if i >= max_frames:
+            break
+
+        state.job = f"{i + 1} out of {max_frames}"
+        if state.skipped:
+            state.skipped = False
+
+        if state.interrupted:
+            break
+
+        # 存一张底图
+        img = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), 'RGB')
+
+        p.init_images = [img] * p.batch_size
+        proc = scripts_mov2mov.run(p, *args)
+        if proc is None:
+            print(f'current progress: {i + 1}/{max_frames}')
+            processed = process_images(p)
+            # 只取第一张
+            gen_image = processed.images[0]
+            generate_images.append(gen_image)
+
+    video = save_video(generate_images, movie_frames)
+
+    return video
+
+
+def process_keyframes(p, mov_file, fps, df, args):
+    processing.fix_seed(p)
+    images = get_mov_all_images(mov_file, fps)
+    if not images:
+        print('Failed to parse the video, please check')
+        return
+
+    # 通过宽高,缩放模式,预处理图片
+    images = [PIL.Image.fromarray(image) for image in images]
+    images = [modules.images.resize_image(p.resize_mode, image, p.width, p.height) for image in images]
+    images = [np.asarray(image) for image in images]
+
+    default_prompt = p.prompt
+    max_frames = len(df)
+
+    p.do_not_save_grid = True
+    state.job_count = max_frames  # * p.n_iter
+    generate_images = []
+
+    for i, row in df.iterrows():
+        p.prompt = default_prompt + row['prompt']
+        frame = images[row['frame'] - 1]
+
+        state.job = f"{i + 1} out of {max_frames}"
+        if state.skipped:
+            state.skipped = False
+
+        if state.interrupted:
+            break
+
+        img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB), 'RGB')
+        p.init_images = [img]
+        proc = scripts_mov2mov.run(p, *args)
+        if proc is None:
+            print(f'current progress: {i + 1}/{max_frames}')
+            processed = process_images(p)
+            gen_image = processed.images[0]
+
+            if gen_image.height != p.height or gen_image.width != p.width:
+                print(f'Warning: The generated image size is inconsistent with the original image size, '
+                      f'please check the configuration parameters')
+                gen_image = gen_image.resize((p.width, p.height))
+
+            keyframe = Keyframe(row['frame'], np.asarray(gen_image), row['prompt'])
+            generate_images.append(keyframe)
+
+    # 由于生成图片可能会产生像素偏差,这里再对齐一次宽高
+    images = [PIL.Image.fromarray(image) for image in images]
+    images = [image.resize(p.width, p.height) if image.width != p.width or image.height != p.height else image for image
+              in images]
+    images = [np.asarray(image) for image in images]
+
+    return generate_images, images
+
+
+def process_mov2mov_ebsynth(p, eb_generate, weight=4.0):
+    from ebsynth._ebsynth import task as EbsyncthRun
+    tasks = eb_generate.get_tasks(weight)
+    tasks_len = len(tasks)
+    state.job_count = tasks_len  # * p.n_iter
+
+    for i, task in tqdm(enumerate(tasks)):
+
+        state.job = f"{i + 1} out of {tasks_len}"
+        if state.skipped:
+            state.skipped = False
+
+        if state.interrupted:
+            break
+
+        result = EbsyncthRun(task.style, [(task.source, task.target, task.weight)])
+        eb_generate.append_generate_frames(task.key_frame_num, task.frame_num, result)
+        state.nextjob()
+
+    print(f'Start merge frames')
+    result = eb_generate.merge_sequences()
+    video = save_video(result, eb_generate.fps)
+    return video
+
+
+def mov2mov(id_task: str,
+            prompt,
+            negative_prompt,
+            prompt_styles,
+            mov_file,
+            steps,
+            sampler_name,
+            cfg_scale,
+            image_cfg_scale,
+            denoising_strength,
+            height,
+            width,
+            resize_mode,
+            override_settings_texts,
+
+            # refiner
+            # enable_refiner, refiner_checkpoint, refiner_switch_at,
+            # mov2mov params
+
+            noise_multiplier,
+            movie_frames,
+            max_frames,
+            # editor
+            enable_movie_editor,
+            df: pandas.DataFrame,
+            eb_weight,
+
+            *args):
+    if not mov_file:
+        raise Exception('Error！ Please add a video file!')
+
+    override_settings = create_override_settings_dict(override_settings_texts)
+    assert 0. <= denoising_strength <= 1., 'can only work with strength in [0.0, 1.0]'
+    mask_blur = 4
+    inpainting_fill = 1
+    inpaint_full_res = False
+    inpaint_full_res_padding = 32
+    inpainting_mask_invert = 0
+
+    p = StableDiffusionProcessingImg2Img(
+        sd_model=shared.sd_model,
+        outpath_samples=shared.opts.data.get("mov2mov_outpath_samples", mov2mov_outpath_samples),
+        outpath_grids=opts.outdir_grids or opts.outdir_img2img_grids,
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        styles=prompt_styles,
+        sampler_name=sampler_name,
+        batch_size=1,
+        n_iter=1,
+        steps=steps,
+        cfg_scale=cfg_scale,
+        width=width,
+        height=height,
+        init_images=[None],
+        mask=None,
+
+        mask_blur=mask_blur,
+        inpainting_fill=inpainting_fill,
+        resize_mode=resize_mode,
+        denoising_strength=denoising_strength,
+        image_cfg_scale=image_cfg_scale,
+        inpaint_full_res=inpaint_full_res,
+        inpaint_full_res_padding=inpaint_full_res_padding,
+        inpainting_mask_invert=inpainting_mask_invert,
+        override_settings=override_settings,
+        initial_noise_multiplier=noise_multiplier
+
+    )
+
+    p.scripts = scripts_mov2mov
+    p.script_args = args
+    # print('script_args', args)
+    #
+    # if not enable_refiner or refiner_checkpoint in (None, "", "None"):
+    #     p.refiner_checkpoint = None
+    #     p.refiner_switch_at = None
+    # else:
+    #     p.refiner_checkpoint = refiner_checkpoint
+    #     p.refiner_switch_at = refiner_switch_at
+
+    if shared.cmd_opts.enable_console_prompts:
+        print(f"\nmov2mov: {prompt}", file=shared.progress_print_out)
+
+    p.extra_generation_params["Mask blur"] = mask_blur
+
+    if not enable_movie_editor:
+        print(f'\nStart parsing the number of mov frames')
+        generate_video = process_mov2mov(p, mov_file, movie_frames, max_frames, resize_mode, width, height, args)
+        processed = Processed(p, [], p.seed, "")
+    else:
+        # editor
+        if platform.system() != 'Windows':
+            raise Exception('The Movie Editor is currently only supported on Windows')
+
+        # check df no frame
+        if not check_data_frame(df):
+            raise Exception('Please add a frame in the Movie Editor or disable it')
+
+        # sort df for index
+        df = df.sort_values(by='frame').reset_index(drop=True)
+
+        # generate keyframes
+        print(f'Start generate keyframes')
+        keyframes, frames = process_keyframes(p, mov_file, movie_frames, df, args)
+        eb_generate = EbsynthGenerate(keyframes, frames, movie_frames)
+        print(f'\nStart generate frames')
+
+        generate_video = process_mov2mov_ebsynth(p, eb_generate, weight=eb_weight)
+
+        processed = Processed(p, [], p.seed, "")
+    p.close()
+
+    shared.total_tqdm.clear()
+
+    generation_info_js = processed.js()
+    if opts.samples_log_stdout:
+        print(generation_info_js)
+
+    if opts.do_not_show_images:
+        processed.images = []
+
+    return generate_video, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(
+        processed.comments, classname="comments")

scripts/movie_editor.py

@@ -0,0 +1,345 @@
+import platform
+
+import gradio as gr
+import pandas
+from PIL import Image
+from tqdm import tqdm
+
+from modules import shared, deepbooru
+from modules.ui_components import InputAccordion, ToolButton
+from scripts import m2m_util
+
+
+class MovieEditor:
+    def __init__(self, id_part, gr_movie: gr.Video, gr_fps: gr.Slider):
+        self.gr_eb_weight = None
+        self.gr_df = None
+        self.gr_keyframe = None
+        self.gr_frame_number = None
+        self.gr_frame_image = None
+        self.gr_movie = gr_movie
+        self.gr_fps = gr_fps
+        self.gr_enable_movie_editor = None
+
+        self.is_windows = platform.system() == "Windows"
+        self.id_part = id_part
+        self.frames = []
+        self.frame_count = 0
+        self.selected_data_frame = -1
+
+    def render(self):
+        id_part = self.id_part
+        with InputAccordion(
+            True, label="Movie Editor", elem_id=f"{id_part}_editor_accordion"
+        ):
+            gr.HTML(
+                "<div style='color:red;font-weight: bold;border: 2px solid yellow;padding: 10px;font-size: 20px;'>"
+                "This feature is in beta version!!! <br>"
+                "It only supports Windows!!! <br>"
+                "Make sure you have installed the ControlNet and IP-Adapter models."
+                "</div>"
+            )
+
+            self.gr_enable_movie_editor = gr.Checkbox(
+                label="Enable Movie Editor",
+                elem_id=f"{id_part}_editor_enable",
+            )
+
+            self.gr_frame_image = gr.Image(
+                label="Frame",
+                elem_id=f"{id_part}_video_frame",
+                source="upload",
+                visible=False,
+                height=480,
+            )
+
+            # key frame tabs
+            with gr.Tabs(elem_id=f"{id_part}_keyframe_tabs"):
+                with gr.TabItem("Custom", id=f"{id_part}_keyframe_tab_custom"):
+                    with gr.Row():
+                        self.gr_frame_number = gr.Slider(
+                            label="Frame number",
+                            elem_id=f"{id_part}_video_frame_number",
+                            step=1,
+                            maximum=0,
+                            minimum=0,
+                        )
+
+                with gr.TabItem("Auto", elem_id=f"{id_part}_keyframe_tab_auto"):
+                    with gr.Row():
+                        key_frame_interval = gr.Slider(
+                            label="Key frame interval",
+                            elem_id=f"{id_part}_video_keyframe_interval",
+                            step=1,
+                            maximum=100,
+                            minimum=0,
+                            value=2,
+                        )
+                        key_frame_interval_generate = ToolButton(
+                            "♺",
+                            elem_id=f"{id_part}_video_editor_auto_keyframe",
+                            visible=True,
+                            tooltip="generate",
+                        )
+
+            with gr.Row(elem_id=f"{id_part}_keyframe_custom_container"):
+                add_keyframe = ToolButton(
+                    "✚",
+                    elem_id=f"{id_part}_video_editor_add_keyframe",
+                    visible=True,
+                    tooltip="Add keyframe",
+                )
+                remove_keyframe = ToolButton(
+                    "✖",
+                    elem_id=f"{id_part}_video_editor_remove_keyframe",
+                    visible=True,
+                    tooltip="Remove selected keyframe",
+                )
+
+                clear_keyframe = ToolButton(
+                    "🗑",
+                    elem_id=f"{id_part}_video_editor_clear_keyframe",
+                    visible=True,
+                    tooltip="Clear keyframe",
+                )
+
+            with gr.Row():
+                data_frame = gr.Dataframe(
+                    headers=["id", "frame", "prompt"],
+                    datatype=["number", "number", "str"],
+                    row_count=1,
+                    col_count=(3, "fixed"),
+                    max_rows=None,
+                    height=480,
+                    elem_id=f"{id_part}_video_editor_custom_data_frame",
+                )
+                self.gr_df = data_frame
+
+            with gr.Row():
+                interrogate = gr.Button(
+                    value="Clip Interrogate Keyframe",
+                    size="sm",
+                    elem_id=f"{id_part}_video_editor_interrogate",
+                )
+                deepbooru = gr.Button(
+                    value="Deepbooru Keyframe",
+                    size="sm",
+                    elem_id=f"{id_part}_video_editor_deepbooru",
+                )
+
+            with gr.Row():
+                self.gr_eb_weight = gr.Slider(
+                    label="EbSynth weight",
+                    elem_id=f"{id_part}_video_eb_weight",
+                    step=0.1,
+                    maximum=10,
+                    minimum=0,
+                    value=4.0,
+                )
+
+        self.gr_movie.change(
+            fn=self.movie_change,
+            inputs=[self.gr_movie],
+            outputs=[self.gr_frame_image, self.gr_frame_number, self.gr_fps],
+            show_progress=True,
+        )
+
+        self.gr_frame_number.change(
+            fn=self.movie_frame_change,
+            inputs=[self.gr_movie, self.gr_frame_number],
+            outputs=[self.gr_frame_image],
+            show_progress=True,
+        )
+
+        self.gr_fps.change(
+            fn=self.fps_change,
+            inputs=[self.gr_movie, self.gr_fps],
+            outputs=[self.gr_frame_image, self.gr_frame_number],
+            show_progress=True,
+        )
+
+        data_frame.select(self.data_frame_select, data_frame, self.gr_frame_number)
+
+        add_keyframe.click(
+            fn=self.add_keyframe_click,
+            inputs=[data_frame, self.gr_frame_number],
+            outputs=[data_frame],
+            show_progress=False,
+        )
+        remove_keyframe.click(
+            fn=self.remove_keyframe_click,
+            inputs=[data_frame],
+            outputs=[data_frame],
+            show_progress=False,
+        )
+
+        clear_keyframe.click(
+            fn=lambda df: df.drop(df.index, inplace=True),
+            inputs=[data_frame],
+            outputs=[data_frame],
+            show_progress=False,
+        )
+
+        key_frame_interval_generate.click(
+            fn=self.key_frame_interval_generate_click,
+            inputs=[data_frame, key_frame_interval],
+            outputs=[data_frame],
+            show_progress=True,
+        )
+
+        interrogate.click(
+            fn=self.interrogate_keyframe,
+            inputs=[data_frame],
+            outputs=[data_frame],
+            show_progress=True,
+        )
+
+        deepbooru.click(
+            fn=self.deepbooru_keyframe,
+            inputs=[data_frame],
+            outputs=[data_frame],
+            show_progress=True,
+        )
+
+    def interrogate_keyframe(self, data_frame: pandas.DataFrame):
+        """
+        Interrogate key frame
+        """
+        bar = tqdm(total=len(data_frame))
+        for index, row in data_frame.iterrows():
+            if row["frame"] <= 0:
+                continue
+            bar.set_description(f'Interrogate key frame {row["frame"]}')
+            frame = row["frame"] - 1
+            image = self.frames[frame]
+            image = Image.fromarray(image)
+            prompt = shared.interrogator.interrogate(image.convert("RGB"))
+            data_frame.at[index, "prompt"] = prompt
+            bar.update(1)
+
+        return data_frame
+
+    def deepbooru_keyframe(self, data_frame: pandas.DataFrame):
+        """
+        Deepbooru key frame
+
+        """
+        bar = tqdm(total=len(data_frame))
+        for index, row in data_frame.iterrows():
+            if row["frame"] <= 0:
+                continue
+            bar.set_description(f'Interrogate key frame {row["frame"]}')
+            frame = row["frame"] - 1
+            image = self.frames[frame]
+            image = Image.fromarray(image)
+            prompt = deepbooru.model.tag(image)
+            data_frame.at[index, "prompt"] = prompt
+            bar.update(1)
+
+        return data_frame
+
+    def data_frame_select(self, event: gr.SelectData, data_frame: pandas.DataFrame):
+        row, col = event.index
+        self.selected_data_frame = row
+        row = data_frame.iloc[row]
+        frame = row["frame"]
+        if 0 < frame <= self.frame_count:
+            return int(frame)
+        else:
+            return 0
+
+    def add_keyframe_click(self, data_frame: pandas.DataFrame, gr_frame_number: int):
+        """
+        Add a key frame to the data frame
+        """
+        if gr_frame_number < 1:
+            return data_frame
+
+        data_frame = data_frame[data_frame["frame"] > 0]
+
+        if gr_frame_number in data_frame["frame"].values:
+            return data_frame
+
+        row = {"id": len(data_frame), "frame": gr_frame_number, "prompt": ""}
+        data_frame.loc[len(data_frame)] = row
+
+        data_frame = data_frame.sort_values(by="frame").reset_index(drop=True)
+
+        data_frame["id"] = range(len(data_frame))
+
+        return data_frame
+
+    def remove_keyframe_click(self, data_frame: pandas.DataFrame):
+        """
+        Remove the selected key frame
+        """
+        if self.selected_data_frame < 0:
+            return data_frame
+
+        data_frame = data_frame.drop(self.selected_data_frame)
+
+        data_frame = data_frame.sort_values(by="frame").reset_index(drop=True)
+
+        data_frame["id"] = range(len(data_frame))
+
+        return data_frame
+
+    def key_frame_interval_generate_click(
+        self, data_frame: pandas.DataFrame, key_frame_interval: int
+    ):
+        if key_frame_interval < 1:
+            return data_frame
+
+        # 按照key_frame_interval的间隔添加关键帧
+        for i in range(0, self.frame_count, key_frame_interval):
+            data_frame = self.add_keyframe_click(data_frame, i + 1)
+
+        # 添加最后一帧
+        data_frame = self.add_keyframe_click(data_frame, self.frame_count)
+
+        return data_frame
+
+    def movie_change(self, movie_path):
+        if not movie_path:
+            return (
+                gr.Image.update(visible=False),
+                gr.Slider.update(maximum=0, minimum=0),
+                gr.Slider.update(),
+            )
+        fps = m2m_util.get_mov_fps(movie_path)
+        self.frames = m2m_util.get_mov_all_images(movie_path, fps, True)
+
+        self.frame_count = len(self.frames)
+        return (
+            gr.Image.update(visible=True),
+            gr.Slider.update(maximum=self.frame_count, minimum=0, value=0),
+            gr.Slider.update(maximum=fps, minimum=0, value=fps),
+        )
+
+    def movie_frame_change(self, movie_path, frame_number):
+        if not movie_path:
+            return gr.Image.update(visible=False)
+
+        if frame_number <= 0:
+            return gr.Image.update(
+                visible=True, label=f"Frame: {frame_number}", value=None
+            )
+
+        return gr.Image.update(
+            visible=True,
+            label=f"Frame: {frame_number}",
+            value=self.frames[frame_number - 1],
+        )
+
+    def fps_change(self, movie_path, fps):
+        if not movie_path:
+            return gr.Image.update(visible=False), gr.Slider.update(
+                maximum=0, minimum=0
+            )
+
+        self.frames = m2m_util.get_mov_all_images(movie_path, fps, True)
+        self.frame_count = len(self.frames)
+        return (
+            gr.Image.update(visible=True),
+            gr.Slider.update(maximum=self.frame_count, minimum=0, value=0),
+        )

style.css

@@ -0,0 +1,172 @@
+#mov2mov_interrupt, #mov2mov_skip {
+    position: absolute;
+    width: 50%;
+    height: 100%;
+    background: #b4c0cc;
+    display: none;
+}
+
+/*#mov2mov_modnet_model, #mov2mov_merge_background {*/
+/*    display: none;*/
+/*}*/
+
+#modnet_background_movie {
+    display: none;
+}
+
+
+#mov2mov_video {
+    display: none;
+}
+
+#mov2mov_generate {
+    min-height: 7.5em;
+}
+
+@media screen and (min-width: 2500px) {
+    #mov2mov_gallery {
+        min-height: 768px;
+    }
+}
+
+#mov2mov_gallery img {
+    object-fit: scale-down;
+}
+
+#mov2mov_tools {
+    gap: 0.4em;
+}
+
+#mov2mov_actions_column {
+    margin: 0.35rem 0.75rem 0.35rem 0;
+}
+
+
+#mov2mov_actions_column {
+    gap: 0;
+    margin-right: .75rem;
+}
+
+
+#mov2mov_styles_row {
+    gap: 0.25em;
+    margin-top: 0.3em;
+}
+
+#mov2mov_styles_row > button {
+    margin: 0;
+}
+
+#mov2mov_styles {
+    padding: 0;
+}
+
+#mov2mov_styles > label > div {
+    min-height: 3.2em;
+}
+
+#mov2mov_extra_networks .search {
+    display: inline-block;
+    max-width: 16em;
+    margin: 0.3em;
+    align-self: center;
+}
+
+#mov2mov_extra_view {
+    width: auto;
+}
+
+#mov2mov_preview {
+    position: absolute;
+    width: 320px;
+    left: 0;
+    right: 0;
+    margin-left: auto;
+    margin-right: auto;
+    margin-top: 34px;
+    z-index: 100;
+    border: none;
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
+
+@media screen and (min-width: 768px) {
+    #mov2mov_preview {
+        position: absolute;
+    }
+}
+
+@media screen and (max-width: 767px) {
+    #mov2mov_preview {
+        position: relative;
+    }
+}
+
+#mov2mov_preview div.left-0.top-0 {
+    display: none;
+}
+
+
+#mov2mov_interrupt, #mov2mov_skip {
+    position: absolute;
+    width: 50%;
+    height: 100%;
+    background: #b4c0cc;
+    display: none;
+}
+
+#mov2mov_interrupt {
+    left: 0;
+    border-radius: 0.5rem 0 0 0.5rem;
+}
+
+#mov2mov_skip {
+    right: 0;
+    border-radius: 0 0.5rem 0.5rem 0;
+}
+
+#mov2mov_mov video {
+    height: 480px;
+    max-height: 480px;
+    min-height: 480px;
+}
+
+#mov2mov_video video {
+    height: 480px;
+    max-height: 480px;
+    min-height: 480px;
+}
+
+#mov2mov_video_frame img {
+    height: 480px;
+    max-height: 480px;
+    min-height: 480px;
+}
+
+
+#mov2mov_checkboxes {
+    margin-bottom: 0.5em;
+    margin-left: 0em;
+}
+
+#mov2mov_checkboxes > div {
+    flex: 0;
+    white-space: nowrap;
+    min-width: auto;
+}
+
+#mov2mov_extra_view {
+    width: auto;
+}
+
+/* dataframe */
+#mov2mov_video_editor_custom_data_frame button {
+    display: none;
+}
+
+#mov2mov_keyframe_custom_container>div {
+    display: flex;
+    justify-content: center;
+    /*增加间距*/
+    gap: 1.5em;
+}

tests/ebsynth/ebsynth_generate_test.py

@@ -0,0 +1,114 @@
+import os
+
+import numpy as np
+import unittest
+import cv2
+
+import importlib
+
+utils = importlib.import_module("extensions.sd-webui-mov2mov.tests.utils", "utils")
+utils.setup_test_env()
+
+from ebsynth.ebsynth_generate import EbsynthGenerate, Keyframe, Sequence
+
+
+class EbsynthGenerateTestCase(unittest.TestCase):
+    def get_image(self, folder, name):
+        return cv2.imread(os.path.join(os.path.dirname(__file__), 'images', folder, name))
+
+    def test_sequences(self):
+        # 模拟100帧的视频
+        keyframes = [
+            Keyframe(1, None, None),
+            Keyframe(10, None, None),
+            Keyframe(20, None, None),
+            Keyframe(30, None, None),
+
+        ]
+        frames = [np.zeros((100, 100, 3))] * 40
+
+        eb_generate = EbsynthGenerate(keyframes, frames, 24)
+        eb_generate.setup_sequences()
+        self.assertEqual(len(eb_generate.sequences), 4)
+        self.assertEqual(eb_generate.sequences[0].start, 1)
+        self.assertEqual(eb_generate.sequences[0].keyframe.num, 1)
+        self.assertEqual(eb_generate.sequences[0].end, 10)
+
+        self.assertEqual(eb_generate.sequences[1].start, 1)
+        self.assertEqual(eb_generate.sequences[1].keyframe.num, 10)
+        self.assertEqual(eb_generate.sequences[1].end, 20)
+
+        self.assertEqual(eb_generate.sequences[2].start, 10)
+        self.assertEqual(eb_generate.sequences[2].keyframe.num, 20)
+        self.assertEqual(eb_generate.sequences[2].end, 30)
+
+        self.assertEqual(eb_generate.sequences[3].start, 20)
+        self.assertEqual(eb_generate.sequences[3].keyframe.num, 30)
+        self.assertEqual(eb_generate.sequences[3].end, 40)
+
+        self.assertEqual(list(eb_generate.sequences[0].frames.keys()), [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+        self.assertEqual(list(eb_generate.sequences[1].frames.keys()), [i for i in range(1, 21)])
+        self.assertEqual(list(eb_generate.sequences[2].frames.keys()), [i for i in range(10, 31)])
+        self.assertEqual(list(eb_generate.sequences[3].frames.keys()), [i for i in range(20, 41)])
+
+        keyframes = [
+            Keyframe(1, None, None),
+            Keyframe(3, None, None),
+            Keyframe(5, None, None),
+        ]
+        frames = [np.zeros((100, 100, 3))] * 10
+
+        eb_generate = EbsynthGenerate(keyframes, frames, 24)
+        eb_generate.setup_sequences()
+
+        self.assertEqual(len(eb_generate.sequences), 3)
+        self.assertEqual(eb_generate.sequences[0].start, 1)
+        self.assertEqual(eb_generate.sequences[0].keyframe.num, 1)
+        self.assertEqual(eb_generate.sequences[0].end, 3)
+
+        self.assertEqual(eb_generate.sequences[1].start, 1)
+        self.assertEqual(eb_generate.sequences[1].keyframe.num, 3)
+        self.assertEqual(eb_generate.sequences[1].end, 5)
+
+        self.assertEqual(eb_generate.sequences[2].start, 3)
+        self.assertEqual(eb_generate.sequences[2].keyframe.num, 5)
+        self.assertEqual(eb_generate.sequences[2].end, 10)
+
+        keyframes = [
+            Keyframe(1, None, None),
+            Keyframe(3, None, None),
+            Keyframe(5, None, None),
+        ]
+        frames = [np.zeros((100, 100, 3))] * 5
+
+        eb_generate = EbsynthGenerate(keyframes, frames, 24)
+        eb_generate.setup_sequences()
+
+        self.assertEqual(len(eb_generate.sequences), 3)
+        self.assertEqual(eb_generate.sequences[0].start, 1)
+        self.assertEqual(eb_generate.sequences[0].keyframe.num, 1)
+        self.assertEqual(eb_generate.sequences[0].end, 3)
+
+        self.assertEqual(eb_generate.sequences[1].start, 1)
+        self.assertEqual(eb_generate.sequences[1].keyframe.num, 3)
+        self.assertEqual(eb_generate.sequences[1].end, 5)
+
+        self.assertEqual(eb_generate.sequences[2].start, 3)
+        self.assertEqual(eb_generate.sequences[2].keyframe.num, 5)
+        self.assertEqual(eb_generate.sequences[2].end, 5)
+
+    def test_get_guides(self):
+        keyframes = [Keyframe(i, self.get_image('keys', f'{i:04d}.png'), '') for i in range(1, 72, 10)]
+        frames = [self.get_image('video', f'{i:04d}.png') for i in range(0, 73)]
+
+        eb_generate = EbsynthGenerate(keyframes, frames, 24)
+        tasks = eb_generate.get_tasks(4.0)
+        num = 0
+        for sequence in eb_generate.sequences:
+            for i in range(sequence.start, sequence.end + 1):
+                self.assertEqual(tasks[num].frame_num, i)
+                num += 1
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/ebsynth/ebsynth_test.py

@@ -0,0 +1,87 @@
+import importlib
+import os
+import unittest
+
+import cv2
+
+utils = importlib.import_module("extensions.sd-webui-mov2mov.tests.utils", "utils")
+utils.setup_test_env()
+
+from ebsynth.ebsynth_generate import EbsynthGenerate, Keyframe, Sequence
+from ebsynth._ebsynth import task as EbsyncthRun
+from scripts import m2m_util
+
+
+class MyTestCase(unittest.TestCase):
+    def get_image(self, folder, name):
+        return cv2.imread(os.path.join(os.path.dirname(__file__), 'images', folder, name))
+
+    def setUp(self) -> None:
+        self.keyframes = [Keyframe(i, self.get_image('keys', f'{i:04d}.png'), '') for i in range(1, 72, 10)]
+        frames = [self.get_image('video', f'{i:04d}.png') for i in range(0, 72)]
+
+        self.eb_generate = EbsynthGenerate(self.keyframes, frames, 24)
+
+    def test_keyframes(self):
+        for i, sequence in enumerate(self.eb_generate.sequences):
+            self.assertEqual(sequence.keyframe.num, self.keyframes[i].num)
+            self.assertTrue((sequence.keyframe.image == self.keyframes[i].image).all())
+
+    def test_task(self):
+        """
+        测试生成的任务是否正确
+
+        Returns:
+
+        """
+
+        tasks = self.eb_generate.get_tasks(4.0)
+        for task in tasks:
+            result = EbsyncthRun(task.style, [(task.source, task.target, task.weight)])
+            dir_name = os.path.join(os.path.dirname(__file__), 'images', 'test', f'out_{task.key_frame_num}')
+            if not os.path.exists(dir_name):
+                os.mkdir(dir_name)
+            cv2.imwrite(os.path.join(dir_name, f'{task.frame_num:04d}.png'), result)
+
+    def test_merge(self):
+        """
+        测试merge是否正确
+
+        """
+
+        def get_sequence(keyframe_num):
+            for sequence in self.eb_generate.sequences:
+                if sequence.keyframe.num == keyframe_num:
+                    return sequence
+            else:
+                raise ValueError(f'not found key frame num {keyframe_num}')
+
+        # 模拟结果
+        test_dir = os.path.join(os.path.dirname(__file__), 'images', 'test')
+
+        # 获取out_{keyframe}文件夹
+        for keyframe in self.keyframes:
+            out_dir = os.path.join(test_dir, f'out_{keyframe.num:04d}')
+            # 获取out_{keyframe}文件夹下的所有文件,并且按照 {i:04d}.png 的顺序添加到eb_generate.generate_frames
+            sequence = get_sequence(keyframe.num)
+            for i in range(sequence.start, sequence.end + 1):
+                self.eb_generate.append_generate_frames(keyframe.num, i,
+                                                        cv2.imread(os.path.join(out_dir, f'{i:04d}.png')))
+        # 测试merge
+        result = self.eb_generate.merge_sequences(0.4)
+
+        if not os.path.exists(os.path.join(test_dir, 'merge_1')):
+            os.mkdir(os.path.join(test_dir, 'merge_1'))
+
+        frames = []
+
+        for i, frame in enumerate(result):
+            if frame is not None:
+                cv2.imwrite(os.path.join(test_dir, 'merge_1', f'{i:04d}.png'), frame)
+                frames.append(frame)
+        m2m_util.images_to_video(frames, self.eb_generate.fps,
+                                 os.path.join(test_dir, 'merge_1', f'm.mp4'))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/utils.py

@@ -0,0 +1,9 @@
+import sys
+
+import os
+
+
+def setup_test_env():
+    ext_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    if ext_root not in sys.path:
+        sys.path.append(ext_root)