Upload folder using huggingface_hub

app.py

@@ -4,7 +4,7 @@ from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
 from diffusers.utils import export_to_video
 import os
 
-
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def generate_video(prompt):
     # load pipeline
     pipe = DiffusionPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b", torch_dtype=torch.float16,

aps.py

@@ -0,0 +1,22 @@
+from huggingface_hub import snapshot_download
+
+from modelscope.pipelines import pipeline
+from modelscope.outputs import OutputKeys
+import pathlib
+import gradio as gr
+
+
+def video_gen(prompt):
+    model_dir = pathlib.Path('weights')
+    snapshot_download('damo-vilab/modelscope-damo-text-to-video-synthesis',
+                       repo_type='model', local_dir=model_dir)
+
+    pipe = pipeline('text-to-video-synthesis', model_dir.as_posix())
+    prompt = {
+            'text': 'A panda eating bamboo on a rock.',
+        }
+    output_video_path = pipe(prompt,)[OutputKeys.OUTPUT_VIDEO]
+    return output_video_path
+
+demo = gr.Interface(fn=video_gen, inputs="text", outputs="video")
+demo.launch(share=True)

weights/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

weights/README.md

@@ -0,0 +1,99 @@
+---
+license: cc-by-nc-4.0
+pipeline_tag: text-to-video
+---
+
+The original repo is [here](https://modelscope.cn/models/damo/text-to-video-synthesis/summary). 
+
+**We Are Hiring!** (Based in Beijing / Hangzhou, China.)
+
+If you're looking for an exciting challenge and the opportunity to work with cutting-edge technologies in AIGC and large-scale pretraining, then we are the place for you. We are looking for talented, motivated and creative individuals to join our team. If you are interested, please send your CV to us.
+
+EMAIL: yingya.zyy@alibaba-inc.com
+
+This model is based on a multi-stage text-to-video generation diffusion model, which inputs a description text and returns a video that matches the text description. Only English input is supported.
+
+## Model Description
+
+The text-to-video generation diffusion model consists of three sub-networks: text feature extraction, text feature-to-video latent space diffusion model, and video latent space to video visual space. The overall model parameters are about 1.7 billion. Support English input. The diffusion model adopts the Unet3D structure, and realizes the function of video generation through the iterative denoising process from the pure Gaussian noise video.
+
+**This model is meant for research purposes. Please look at the [model limitations and biases](#model-limitations-and-biases) and [misuse, malicious use and excessive use](#misuse-malicious-use-and-excessive-use) sections.**
+
+**How to expect the model to be used and where it is applicable**
+
+This model has a wide range of applications and can reason and generate videos based on arbitrary English text descriptions.
+
+## How to use
+
+ 
+The model has been launched on [ModelScope Studio](https://modelscope.cn/studios/damo/text-to-video-synthesis/summary) and [huggingface](https://huggingface.co/spaces/damo-vilab/modelscope-text-to-video-synthesis), you can experience it directly; you can also refer to [Colab page](https://colab.research.google.com/drive/1uW1ZqswkQ9Z9bp5Nbo5z59cAn7I0hE6R?usp=sharing#scrollTo=bSluBq99ObSk) to build it yourself.
+In order to facilitate the experience of the model, users can refer to the [Aliyun Notebook Tutorial](https://modelscope.cn/headlines/detail/26) to quickly develop this Text-to-Video model.
+
+This demo requires about 16GB CPU RAM and 16GB GPU RAM. Under the ModelScope framework, the current model can be used by calling a simple Pipeline, where the input must be in dictionary format, the legal key value is 'text', and the content is a short text. This model currently only supports inference on the GPU. Enter specific code examples as follows:
+
+
+### Operating environment (Python Package)
+
+```
+pip install modelscope==1.4.2
+pip install open_clip_torch
+pip install pytorch-lightning
+```
+
+### Code example (Demo Code)
+
+```python
+from huggingface_hub import snapshot_download
+
+from modelscope.pipelines import pipeline
+from modelscope.outputs import OutputKeys
+import pathlib
+
+model_dir = pathlib.Path('weights')
+snapshot_download('damo-vilab/modelscope-damo-text-to-video-synthesis',
+                   repo_type='model', local_dir=model_dir)
+
+pipe = pipeline('text-to-video-synthesis', model_dir.as_posix())
+test_text = {
+        'text': 'A panda eating bamboo on a rock.',
+    }
+output_video_path = pipe(test_text,)[OutputKeys.OUTPUT_VIDEO]
+print('output_video_path:', output_video_path)
+```
+
+### View results
+
+The above code will display the save path of the output video, and the current encoding format can be played normally with [VLC player](https://www.videolan.org/vlc/).
+
+The output mp4 file can be viewed by [VLC media player](https://www.videolan.org/vlc/). Some other media players may not view it normally.
+
+## Model limitations and biases
+
+* The model is trained based on public data sets such as Webvid, and the generated results may have deviations related to the distribution of training data.
+* This model cannot achieve perfect film and television quality generation.
+* The model cannot generate clear text.
+* The model is mainly trained with English corpus and does not support other languages ​​at the moment**.
+* The performance of this model needs to be improved on complex compositional generation tasks.
+
+## Misuse, Malicious Use and Excessive Use
+
+* The model was not trained to realistically represent people or events, so using it to generate such content is beyond the model's capabilities.
+* It is prohibited to generate content that is demeaning or harmful to people or their environment, culture, religion, etc.
+* Prohibited for pornographic, violent and bloody content generation.
+* Prohibited for error and false information generation.
+
+## Training data
+
+The training data includes [LAION5B](https://huggingface.co/datasets/laion/laion2B-en), [ImageNet](https://www.image-net.org/), [Webvid](https://m-bain.github.io/webvid-dataset/) and other public datasets. Image and video filtering is performed after pre-training such as aesthetic score, watermark score, and deduplication.
+
+## Citation
+
+```bibtex
+    @InProceedings{VideoFusion,
+        author    = {Luo, Zhengxiong and Chen, Dayou and Zhang, Yingya and Huang, Yan and Wang, Liang and Shen, Yujun and Zhao, Deli and Zhou, Jingren and Tan, Tieniu},
+        title     = {VideoFusion: Decomposed Diffusion Models for High-Quality Video Generation},
+        booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
+        month     = {June},
+        year      = {2023}
+    }
+```

weights/VQGAN_autoencoder.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88ecb782561455673c4b78d05093494b9c539fc6bfc08f3a9a4a0dd7b0b10f36
+size 5214865159

weights/configuration.json

@@ -0,0 +1,34 @@
+{   "framework": "pytorch",
+    "task": "text-to-video-synthesis",
+    "model": {
+        "type": "latent-text-to-video-synthesis",
+        "model_args": {
+            "ckpt_clip": "open_clip_pytorch_model.bin",
+            "ckpt_unet": "text2video_pytorch_model.pth",
+            "ckpt_autoencoder": "VQGAN_autoencoder.pth",
+            "max_frames": 16,
+            "tiny_gpu": 1
+        },
+        "model_cfg": {
+            "unet_in_dim": 4,
+            "unet_dim": 320,
+            "unet_y_dim": 768,
+            "unet_context_dim": 1024,
+            "unet_out_dim": 4,
+            "unet_dim_mult": [1, 2, 4, 4],
+            "unet_num_heads": 8,
+            "unet_head_dim": 64,
+            "unet_res_blocks": 2,
+            "unet_attn_scales": [1, 0.5, 0.25],
+            "unet_dropout": 0.1,
+            "temporal_attention": "True",
+            "num_timesteps": 1000,
+            "mean_type": "eps",
+            "var_type": "fixed_small",
+            "loss_type": "mse"
+        }
+    },
+    "pipeline": {
+        "type": "latent-text-to-video-synthesis"
+    }
+}

weights/text2video_pytorch_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e139fb08e50a6c072127d9ecef4fe8e91dbdf24edad23a4e1a7c569f0ca3488
+size 5645549049