ganeshblank commited on
Commit
dd6463f
β€’
1 Parent(s): 45b0642

requirement

Browse files
Files changed (6) hide show
  1. .gitattributes +35 -35
  2. .gitignore +21 -0
  3. LICENSE +21 -0
  4. README.md +142 -13
  5. inference.py +49 -0
  6. requirements.txt +22 -0
.gitattributes CHANGED
@@ -1,35 +1,35 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ **/__pycache__/
4
+ *.py[cod]
5
+ **/*.py[cod]
6
+ *$py.class
7
+
8
+ # Model weights
9
+ **/*.pth
10
+ **/*.onnx
11
+
12
+ pretrained_weights/*.md
13
+ pretrained_weights/docs
14
+
15
+ # Ipython notebook
16
+ *.ipynb
17
+
18
+ # Temporary files or benchmark resources
19
+ animations/*
20
+ tmp/*
21
+ .vscode/launch.json
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2024 Kuaishou Visual Generation and Interaction Center
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
README.md CHANGED
@@ -1,13 +1,142 @@
1
- ---
2
- title: Liveportrait Video
3
- emoji: πŸ“š
4
- colorFrom: indigo
5
- colorTo: indigo
6
- sdk: gradio
7
- sdk_version: 4.38.1
8
- app_file: app.py
9
- pinned: false
10
- license: mit
11
- ---
12
-
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <h1 align="center">This is the modification of LivePortrait: Efficient Portrait Animation with Stitching and Retargeting Control for allowing video as a source </h1>
2
+
3
+
4
+ <br>
5
+ <div align="center">
6
+ <!-- <a href='LICENSE'><img src='https://img.shields.io/badge/license-MIT-yellow'></a> -->
7
+ <a href='https://arxiv.org/pdf/2407.03168'><img src='https://img.shields.io/badge/arXiv-LivePortrait-red'></a>
8
+ <a href='https://liveportrait.github.io'><img src='https://img.shields.io/badge/Project-LivePortrait-green'></a>
9
+ <a href ='https://github.com/KwaiVGI/LivePortrait'>Official Liveportrait</a>
10
+ </div>
11
+ <br>
12
+
13
+ <p align="center">
14
+ <img src="./assets/docs/showcase2.gif" alt="showcase">
15
+ <br>
16
+ πŸ”₯ For more results, visit LivePortrait <a href="https://liveportrait.github.io/"><strong>homepage</strong></a> πŸ”₯
17
+ </p>
18
+
19
+
20
+ ## πŸ”₯ Getting Started
21
+ ### 1. Clone the code and prepare the environment
22
+ ```bash
23
+ git clone https://github.com/KwaiVGI/LivePortrait
24
+ cd LivePortrait
25
+
26
+ # create env using conda
27
+ conda create -n LivePortrait python==3.9.18
28
+ conda activate LivePortrait
29
+ # install dependencies with pip
30
+ pip install -r requirements.txt
31
+ ```
32
+
33
+ ### 2. Download pretrained weights
34
+
35
+ Download the pretrained weights from HuggingFace:
36
+ ```bash
37
+ # you may need to run `git lfs install` first
38
+ git clone https://huggingface.co/KwaiVGI/liveportrait pretrained_weights
39
+ ```
40
+
41
+ Or, download all pretrained weights from [Google Drive](https://drive.google.com/drive/folders/1UtKgzKjFAOmZkhNK-OYT0caJ_w2XAnib) or [Baidu Yun](https://pan.baidu.com/s/1MGctWmNla_vZxDbEp2Dtzw?pwd=z5cn). We have packed all weights in one directory 😊. Unzip and place them in `./pretrained_weights` ensuring the directory structure is as follows:
42
+ ```text
43
+ pretrained_weights
44
+ β”œβ”€β”€ insightface
45
+ β”‚ └── models
46
+ β”‚ └── buffalo_l
47
+ β”‚ β”œβ”€β”€ 2d106det.onnx
48
+ β”‚ └── det_10g.onnx
49
+ └── liveportrait
50
+ β”œβ”€β”€ base_models
51
+ β”‚ β”œβ”€β”€ appearance_feature_extractor.pth
52
+ β”‚ β”œβ”€β”€ motion_extractor.pth
53
+ β”‚ β”œβ”€β”€ spade_generator.pth
54
+ β”‚ └── warping_module.pth
55
+ β”œβ”€β”€ landmark.onnx
56
+ └── retargeting_models
57
+ └── stitching_retargeting_module.pth
58
+ ```
59
+
60
+ ### 3. Inference πŸš€
61
+
62
+ #### Fast hands-on
63
+ ```bash
64
+ python inference.py
65
+ ```
66
+
67
+ If the script runs successfully, you will get an output mp4 file named `animations/s6--d0_concat.mp4`. This file includes the following results: driving video, input image, and generated result.
68
+
69
+ <p align="center">
70
+ <img src="./assets/docs/inference.gif" alt="image">
71
+ </p>
72
+
73
+ Or, you can change the input by specifying the `-s` and `-d` arguments:
74
+
75
+ ```bash
76
+ python inference.py -s assets/examples/source/s9.jpg -d assets/examples/driving/d0.mp4
77
+
78
+ # disable pasting back to run faster
79
+ python inference.py -s assets/examples/source/s9.jpg -d assets/examples/driving/d0.mp4 --no_flag_pasteback
80
+
81
+ # more options to see
82
+ python inference.py -h
83
+ ```
84
+
85
+ For video: you can change the input by specifying the `-sd` and `-d` arguments:
86
+
87
+ ```bash
88
+ python inference.py -sd assets/examples/driving/d3.mp4 -d assets/examples/driving/d0.mp4 -vd True
89
+
90
+ # disable pasting back to run faster
91
+ python inference.py -sd assets/examples/driving/d3.mp4 -d assets/examples/driving/d0.mp4 -vd True --no_flag_pasteback
92
+
93
+ ```
94
+ #### Driving video auto-cropping
95
+
96
+ πŸ“• To use your own driving video, we **recommend**:
97
+ - Crop it to a **1:1** aspect ratio (e.g., 512x512 or 256x256 pixels), or enable auto-cropping by `--flag_crop_driving_video`.
98
+ - Focus on the head area, similar to the example videos.
99
+ - Minimize shoulder movement.
100
+ - Make sure the first frame of driving video is a frontal face with **neutral expression**.
101
+
102
+ Below is a auto-cropping case by `--flag_crop_driving_video`:
103
+ ```bash
104
+ python inference.py -s assets/examples/source/s9.jpg -d assets/examples/driving/d13.mp4 --flag_crop_driving_video
105
+ ```
106
+
107
+ If you find the results of auto-cropping is not well, you can modify the `--scale_crop_video`, `--vy_ratio_crop_video` options to adjust the scale and offset, or do it manually.
108
+
109
+ #### Template making
110
+ You can also use the `.pkl` file auto-generated to speed up the inference, and **protect privacy**, such as:
111
+ ```bash
112
+ python inference.py -s assets/examples/source/s9.jpg -d assets/examples/driving/d5.pkl
113
+ ```
114
+
115
+ **Discover more interesting results on our [Homepage](https://liveportrait.github.io)** 😊
116
+
117
+ ### 4. Gradio interface πŸ€—
118
+
119
+ We also provide a Gradio interface for a better experience, just run by:
120
+
121
+ ```bash
122
+ python app.py
123
+ ```
124
+
125
+ You can specify the `--server_port`, `--share`, `--server_name` arguments to satisfy your needs!
126
+
127
+
128
+
129
+
130
+ ## Acknowledgements
131
+ We would like to thank the contributors of [FOMM](https://github.com/AliaksandrSiarohin/first-order-model), [Open Facevid2vid](https://github.com/zhanglonghao1992/One-Shot_Free-View_Neural_Talking_Head_Synthesis), [SPADE](https://github.com/NVlabs/SPADE), [InsightFace](https://github.com/deepinsight/insightface), [LivePortrait](https://github.com/KwaiVGI/LivePortrait) repositories, for their open research and contributions.
132
+
133
+ ## Citation πŸ’–
134
+
135
+ ```bibtex
136
+ @article{guo2024liveportrait,
137
+ title = {LivePortrait: Efficient Portrait Animation with Stitching and Retargeting Control},
138
+ author = {Guo, Jianzhu and Zhang, Dingyun and Liu, Xiaoqiang and Zhong, Zhizhou and Zhang, Yuan and Wan, Pengfei and Zhang, Di},
139
+ journal = {arXiv preprint arXiv:2407.03168},
140
+ year = {2024}
141
+ }
142
+ ```
inference.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding: utf-8
2
+
3
+ import os.path as osp
4
+ import tyro
5
+ from src.config.argument_config import ArgumentConfig
6
+ from src.config.inference_config import InferenceConfig
7
+ from src.config.crop_config import CropConfig
8
+ from src.live_portrait_pipeline import LivePortraitPipeline
9
+
10
+
11
+ def partial_fields(target_class, kwargs):
12
+ return target_class(**{k: v for k, v in kwargs.items() if hasattr(target_class, k)})
13
+
14
+
15
+ def fast_check_args(args: ArgumentConfig):
16
+ if not osp.exists(args.source_image):
17
+ raise FileNotFoundError(f"source image not found: {args.source_image}")
18
+ if not osp.exists(args.driving_info):
19
+ raise FileNotFoundError(f"driving info not found: {args.driving_info}")
20
+
21
+
22
+ def main():
23
+ # set tyro theme
24
+ tyro.extras.set_accent_color("bright_cyan")
25
+ args = tyro.cli(ArgumentConfig)
26
+
27
+ # fast check the args
28
+ fast_check_args(args)
29
+
30
+ # specify configs for inference
31
+ inference_cfg = partial_fields(InferenceConfig, args.__dict__) # use attribute of args to initial InferenceConfig
32
+ crop_cfg = partial_fields(CropConfig, args.__dict__) # use attribute of args to initial CropConfig
33
+
34
+ live_portrait_pipeline = LivePortraitPipeline(
35
+ inference_cfg=inference_cfg,
36
+ crop_cfg=crop_cfg
37
+ )
38
+
39
+ # run
40
+ if args.flag_svideo:
41
+ live_portrait_pipeline.execute_source_video(args)
42
+
43
+ else:
44
+ live_portrait_pipeline.execute(args)
45
+
46
+
47
+
48
+ if __name__ == '__main__':
49
+ main()
requirements.txt ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ --extra-index-url https://download.pytorch.org/whl/cu118
2
+ torch==2.3.0
3
+ torchvision==0.18.0
4
+ torchaudio==2.3.0
5
+
6
+ numpy==1.26.4
7
+ pyyaml==6.0.1
8
+ opencv-python==4.10.0.84
9
+ scipy==1.13.1
10
+ imageio==2.34.2
11
+ lmdb==1.4.1
12
+ tqdm==4.66.4
13
+ rich==13.7.1
14
+ ffmpeg-python==0.2.0
15
+ onnxruntime-gpu==1.18.0
16
+ onnx==1.16.1
17
+ scikit-image==0.24.0
18
+ albumentations==1.4.10
19
+ matplotlib==3.9.0
20
+ imageio-ffmpeg==0.5.1
21
+ tyro==0.8.5
22
+ gradio==4.37.1