hysts HF staff commited on
Commit
3da52d6
1 Parent(s): 0ed19ce
Files changed (9) hide show
  1. .dockerignore +1 -0
  2. .gitignore +162 -0
  3. .pre-commit-config.yaml +50 -0
  4. .vscode/settings.json +21 -0
  5. Dockerfile +54 -0
  6. README.md +3 -5
  7. app.py +85 -0
  8. requirements.txt +15 -0
  9. style.css +10 -0
.dockerignore ADDED
@@ -0,0 +1 @@
 
 
1
+ models
.gitignore ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models
2
+
3
+ # Byte-compiled / optimized / DLL files
4
+ __pycache__/
5
+ *.py[cod]
6
+ *$py.class
7
+
8
+ # C extensions
9
+ *.so
10
+
11
+ # Distribution / packaging
12
+ .Python
13
+ build/
14
+ develop-eggs/
15
+ dist/
16
+ downloads/
17
+ eggs/
18
+ .eggs/
19
+ lib/
20
+ lib64/
21
+ parts/
22
+ sdist/
23
+ var/
24
+ wheels/
25
+ share/python-wheels/
26
+ *.egg-info/
27
+ .installed.cfg
28
+ *.egg
29
+ MANIFEST
30
+
31
+ # PyInstaller
32
+ # Usually these files are written by a python script from a template
33
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
34
+ *.manifest
35
+ *.spec
36
+
37
+ # Installer logs
38
+ pip-log.txt
39
+ pip-delete-this-directory.txt
40
+
41
+ # Unit test / coverage reports
42
+ htmlcov/
43
+ .tox/
44
+ .nox/
45
+ .coverage
46
+ .coverage.*
47
+ .cache
48
+ nosetests.xml
49
+ coverage.xml
50
+ *.cover
51
+ *.py,cover
52
+ .hypothesis/
53
+ .pytest_cache/
54
+ cover/
55
+
56
+ # Translations
57
+ *.mo
58
+ *.pot
59
+
60
+ # Django stuff:
61
+ *.log
62
+ local_settings.py
63
+ db.sqlite3
64
+ db.sqlite3-journal
65
+
66
+ # Flask stuff:
67
+ instance/
68
+ .webassets-cache
69
+
70
+ # Scrapy stuff:
71
+ .scrapy
72
+
73
+ # Sphinx documentation
74
+ docs/_build/
75
+
76
+ # PyBuilder
77
+ .pybuilder/
78
+ target/
79
+
80
+ # Jupyter Notebook
81
+ .ipynb_checkpoints
82
+
83
+ # IPython
84
+ profile_default/
85
+ ipython_config.py
86
+
87
+ # pyenv
88
+ # For a library or package, you might want to ignore these files since the code is
89
+ # intended to run in multiple environments; otherwise, check them in:
90
+ # .python-version
91
+
92
+ # pipenv
93
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
94
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
95
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
96
+ # install all needed dependencies.
97
+ #Pipfile.lock
98
+
99
+ # poetry
100
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
101
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
102
+ # commonly ignored for libraries.
103
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
104
+ #poetry.lock
105
+
106
+ # pdm
107
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
108
+ #pdm.lock
109
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
110
+ # in version control.
111
+ # https://pdm.fming.dev/#use-with-ide
112
+ .pdm.toml
113
+
114
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
115
+ __pypackages__/
116
+
117
+ # Celery stuff
118
+ celerybeat-schedule
119
+ celerybeat.pid
120
+
121
+ # SageMath parsed files
122
+ *.sage.py
123
+
124
+ # Environments
125
+ .env
126
+ .venv
127
+ env/
128
+ venv/
129
+ ENV/
130
+ env.bak/
131
+ venv.bak/
132
+
133
+ # Spyder project settings
134
+ .spyderproject
135
+ .spyproject
136
+
137
+ # Rope project settings
138
+ .ropeproject
139
+
140
+ # mkdocs documentation
141
+ /site
142
+
143
+ # mypy
144
+ .mypy_cache/
145
+ .dmypy.json
146
+ dmypy.json
147
+
148
+ # Pyre type checker
149
+ .pyre/
150
+
151
+ # pytype static type analyzer
152
+ .pytype/
153
+
154
+ # Cython debug symbols
155
+ cython_debug/
156
+
157
+ # PyCharm
158
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
159
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
160
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
161
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162
+ #.idea/
.pre-commit-config.yaml ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v4.4.0
4
+ hooks:
5
+ - id: check-executables-have-shebangs
6
+ - id: check-json
7
+ - id: check-merge-conflict
8
+ - id: check-shebang-scripts-are-executable
9
+ - id: check-toml
10
+ - id: check-yaml
11
+ - id: end-of-file-fixer
12
+ - id: mixed-line-ending
13
+ args: ["--fix=lf"]
14
+ - id: requirements-txt-fixer
15
+ - id: trailing-whitespace
16
+ - repo: https://github.com/myint/docformatter
17
+ rev: v1.7.5
18
+ hooks:
19
+ - id: docformatter
20
+ args: ["--in-place"]
21
+ - repo: https://github.com/pycqa/isort
22
+ rev: 5.12.0
23
+ hooks:
24
+ - id: isort
25
+ args: ["--profile", "black"]
26
+ - repo: https://github.com/pre-commit/mirrors-mypy
27
+ rev: v1.5.1
28
+ hooks:
29
+ - id: mypy
30
+ args: ["--ignore-missing-imports"]
31
+ additional_dependencies: ["types-python-slugify", "types-requests", "types-PyYAML"]
32
+ - repo: https://github.com/psf/black
33
+ rev: 23.7.0
34
+ hooks:
35
+ - id: black
36
+ language_version: python3.10
37
+ args: ["--line-length", "119"]
38
+ - repo: https://github.com/kynan/nbstripout
39
+ rev: 0.6.1
40
+ hooks:
41
+ - id: nbstripout
42
+ args: ["--extra-keys", "metadata.interpreter metadata.kernelspec cell.metadata.pycharm"]
43
+ - repo: https://github.com/nbQA-dev/nbQA
44
+ rev: 1.7.0
45
+ hooks:
46
+ - id: nbqa-black
47
+ - id: nbqa-pyupgrade
48
+ args: ["--py37-plus"]
49
+ - id: nbqa-isort
50
+ args: ["--float-to-top"]
.vscode/settings.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "[python]": {
3
+ "editor.defaultFormatter": "ms-python.black-formatter",
4
+ "editor.formatOnType": true,
5
+ "editor.codeActionsOnSave": {
6
+ "source.organizeImports": true
7
+ }
8
+ },
9
+ "black-formatter.args": [
10
+ "--line-length=119"
11
+ ],
12
+ "isort.args": ["--profile", "black"],
13
+ "flake8.args": [
14
+ "--max-line-length=119"
15
+ ],
16
+ "ruff.args": [
17
+ "--line-length=119"
18
+ ],
19
+ "editor.formatOnSave": true,
20
+ "files.insertFinalNewline": true
21
+ }
Dockerfile ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
2
+ ENV DEBIAN_FRONTEND=noninteractive
3
+ RUN apt-get update && \
4
+ apt-get upgrade -y && \
5
+ apt-get install -y --no-install-recommends \
6
+ git \
7
+ git-lfs \
8
+ wget \
9
+ curl \
10
+ # python build dependencies \
11
+ build-essential \
12
+ libssl-dev \
13
+ zlib1g-dev \
14
+ libbz2-dev \
15
+ libreadline-dev \
16
+ libsqlite3-dev \
17
+ libncursesw5-dev \
18
+ xz-utils \
19
+ tk-dev \
20
+ libxml2-dev \
21
+ libxmlsec1-dev \
22
+ libffi-dev \
23
+ liblzma-dev \
24
+ # gradio dependencies \
25
+ ffmpeg && \
26
+ apt-get clean && \
27
+ rm -rf /var/lib/apt/lists/*
28
+
29
+ RUN useradd -m -u 1000 user
30
+ USER user
31
+ ENV HOME=/home/user \
32
+ PATH=/home/user/.local/bin:${PATH}
33
+ WORKDIR ${HOME}/app
34
+
35
+ RUN curl https://pyenv.run | bash
36
+ ENV PATH=${HOME}/.pyenv/shims:${HOME}/.pyenv/bin:${PATH}
37
+ ARG PYTHON_VERSION=3.10.12
38
+ RUN pyenv install ${PYTHON_VERSION} && \
39
+ pyenv global ${PYTHON_VERSION} && \
40
+ pyenv rehash && \
41
+ pip install --no-cache-dir -U pip setuptools wheel
42
+
43
+ COPY --chown=1000 ./requirements.txt /tmp/requirements.txt
44
+ RUN pip install -r /tmp/requirements.txt
45
+
46
+ COPY --chown=1000 . ${HOME}/app
47
+ ENV PYTHONPATH=${HOME}/app \
48
+ PYTHONUNBUFFERED=1 \
49
+ GRADIO_ALLOW_FLAGGING=never \
50
+ GRADIO_NUM_PORTS=1 \
51
+ GRADIO_SERVER_NAME=0.0.0.0 \
52
+ GRADIO_THEME=huggingface \
53
+ SYSTEM=spaces
54
+ CMD ["python", "app.py"]
README.md CHANGED
@@ -1,11 +1,9 @@
1
  ---
2
- title: I2VGen XL
3
- emoji: 📉
4
  colorFrom: green
5
  colorTo: pink
6
- sdk: gradio
7
- sdk_version: 3.41.0
8
- app_file: app.py
9
  pinned: false
10
  ---
11
 
 
1
  ---
2
+ title: I2VGen-XL
3
+ emoji: 🔥
4
  colorFrom: green
5
  colorTo: pink
6
+ sdk: docker
 
 
7
  pinned: false
8
  ---
9
 
app.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ import os
4
+ import pathlib
5
+ import tempfile
6
+
7
+ import gradio as gr
8
+ import torch
9
+ from huggingface_hub import snapshot_download
10
+ from modelscope.outputs import OutputKeys
11
+ from modelscope.pipelines import pipeline
12
+
13
+ DESCRIPTION = """# I2VGen-XL
14
+
15
+ I2VGen-XL can generate videos that are semantically similar to the input image and text. The generated videos are high-definition (1280 * 720), wide-screen (16:9), temporally coherent, and have good texture.
16
+ """
17
+
18
+ if torch.cuda.is_available():
19
+ model_cache_dir = os.getenv("MODEL_CACHE_DIR", "./models")
20
+
21
+ image2video_model_dir = pathlib.Path(model_cache_dir) / "MS-Image2Video"
22
+ snapshot_download(repo_id="damo-vilab/MS-Image2Video", repo_type="model", local_dir=image2video_model_dir)
23
+ image_to_video_pipe = pipeline(
24
+ task="image-to-video", model=image2video_model_dir.as_posix(), model_revision="v1.1.0", device="cuda:0"
25
+ )
26
+
27
+ video2video_model_dir = pathlib.Path(model_cache_dir) / "MS-Vid2Vid-XL"
28
+ snapshot_download(repo_id="damo-vilab/MS-Vid2Vid-XL", repo_type="model", local_dir=video2video_model_dir)
29
+ video_to_video_pipe = pipeline(
30
+ task="video-to-video", model=video2video_model_dir.as_posix(), model_revision="v1.1.0", device="cuda:0"
31
+ )
32
+ else:
33
+ image_to_video_pipe = None
34
+ video_to_video_pipe = None
35
+
36
+
37
+ def image_to_video(image_path: str) -> str:
38
+ output_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
39
+ image_to_video_pipe(image_path, output_video=output_file.name)[OutputKeys.OUTPUT_VIDEO]
40
+ return output_file.name
41
+
42
+
43
+ def video_to_video(video_path: str, text: str) -> str:
44
+ p_input = {"video_path": video_path, "text": text}
45
+ output_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
46
+ video_to_video_pipe(p_input, output_video=output_file.name)[OutputKeys.OUTPUT_VIDEO]
47
+ return output_file.name
48
+
49
+
50
+ with gr.Blocks(css="style.css") as demo:
51
+ gr.Markdown(DESCRIPTION)
52
+ with gr.Box():
53
+ gr.Markdown('Step 1: Upload an image and click the "Generate video" button.')
54
+ with gr.Row():
55
+ with gr.Column():
56
+ input_image = gr.Image(label="Input image", type="filepath", height=300)
57
+ i2v_button = gr.Button("Generate video")
58
+ with gr.Column():
59
+ output_video_1 = gr.Video(label="Output video 1", interactive=False, height=300)
60
+ with gr.Box():
61
+ gr.Markdown(
62
+ 'Step 2: Add an English text description of the video content and click the "Generate high-resolution video" button.'
63
+ )
64
+ with gr.Row():
65
+ with gr.Column():
66
+ text_description = gr.Textbox(label="Text description")
67
+ v2v_button = gr.Button("Generate high-resolution video")
68
+ with gr.Column():
69
+ output_video_2 = gr.Video(label="Output video 2", height=300)
70
+
71
+ i2v_button.click(
72
+ fn=image_to_video,
73
+ inputs=input_image,
74
+ outputs=output_video_1,
75
+ api_name="image-to-video",
76
+ )
77
+ v2v_button.click(
78
+ fn=video_to_video,
79
+ inputs=[output_video_1, text_description],
80
+ outputs=output_video_2,
81
+ api_name="video-to-video",
82
+ )
83
+
84
+ if __name__ == "__main__":
85
+ demo.queue(max_size=10, api_open=False).launch()
requirements.txt ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ easydict==1.10
2
+ einops==0.6.1
3
+ fairscale==0.4.13
4
+ gradio==3.41.0
5
+ huggingface_hub==0.16.4
6
+ imageio==2.31.1
7
+ modelscope==1.8.4
8
+ open_clip_torch==2.20.0
9
+ opencv-python-headless==4.8.0.76
10
+ pytorch-lightning==2.0.7
11
+ rotary-embedding-torch==0.2.7
12
+ scipy==1.11.2
13
+ torch==2.0.1
14
+ torchsde==0.2.5
15
+ xformers==0.0.20
style.css ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ h1 {
2
+ text-align: center;
3
+ }
4
+
5
+ #duplicate-button {
6
+ margin: auto;
7
+ color: #fff;
8
+ background: #1565c0;
9
+ border-radius: 100vh;
10
+ }