bluestarburst commited on
Commit
f248e7b
1 Parent(s): fdf716b

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. AnimatedDiff_Colab.ipynb +112 -3
  2. handler.py +9 -2
AnimatedDiff_Colab.ipynb CHANGED
@@ -646,7 +646,41 @@
646
  },
647
  {
648
  "cell_type": "code",
649
- "execution_count": 12,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
650
  "metadata": {
651
  "colab": {
652
  "base_uri": "https://localhost:8080/"
@@ -654,13 +688,88 @@
654
  "id": "kpUetphRI9D3",
655
  "outputId": "d0bba9b2-b93c-476a-9909-94e9f7d5239b"
656
  },
657
- "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
658
  "source": [
659
  "# %cd /content/AnimateDiff-SceneFusion\n",
660
  "# !python -m torch.distributed.run --nnodes=1 --nproc_per_node=1 train.py --config configs/training/training.yaml\n",
661
  "\n",
662
  "# !accelerate launch train.py --config=configs/training/training.yaml\n",
663
- "!accelerate launch train.py --config=configs/{PROJECT}.yaml --python={3.10}"
664
  ]
665
  },
666
  {
 
646
  },
647
  {
648
  "cell_type": "code",
649
+ "execution_count": 15,
650
+ "metadata": {},
651
+ "outputs": [
652
+ {
653
+ "name": "stdout",
654
+ "output_type": "stream",
655
+ "text": [
656
+ "Requirement already satisfied: scipy in c:\\users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages (1.11.4)\n",
657
+ "Collecting ffmpeg\n",
658
+ " Downloading ffmpeg-1.4.tar.gz (5.1 kB)\n",
659
+ " Preparing metadata (setup.py): started\n",
660
+ " Preparing metadata (setup.py): finished with status 'done'\n",
661
+ "Collecting ffmpeg-python\n",
662
+ " Downloading ffmpeg_python-0.2.0-py3-none-any.whl (25 kB)\n",
663
+ "Requirement already satisfied: numpy<1.28.0,>=1.21.6 in c:\\users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages (from scipy) (1.26.0)\n",
664
+ "Collecting future (from ffmpeg-python)\n",
665
+ " Using cached future-0.18.3-py3-none-any.whl\n",
666
+ "Building wheels for collected packages: ffmpeg\n",
667
+ " Building wheel for ffmpeg (setup.py): started\n",
668
+ " Building wheel for ffmpeg (setup.py): finished with status 'done'\n",
669
+ " Created wheel for ffmpeg: filename=ffmpeg-1.4-py3-none-any.whl size=6089 sha256=8fb011d5946d5a48b6a2391b3b8c21b5476fb1298d9d99378a174c78764604be\n",
670
+ " Stored in directory: c:\\users\\texas\\appdata\\local\\pip\\cache\\wheels\\8e\\7a\\69\\cd6aeb83b126a7f04cbe7c9d929028dc52a6e7d525ff56003a\n",
671
+ "Successfully built ffmpeg\n",
672
+ "Installing collected packages: ffmpeg, future, ffmpeg-python\n",
673
+ "Successfully installed ffmpeg-1.4 ffmpeg-python-0.2.0 future-0.18.3\n"
674
+ ]
675
+ }
676
+ ],
677
+ "source": [
678
+ "!pip install scipy ffmpeg ffmpeg-python"
679
+ ]
680
+ },
681
+ {
682
+ "cell_type": "code",
683
+ "execution_count": 17,
684
  "metadata": {
685
  "colab": {
686
  "base_uri": "https://localhost:8080/"
 
688
  "id": "kpUetphRI9D3",
689
  "outputId": "d0bba9b2-b93c-476a-9909-94e9f7d5239b"
690
  },
691
+ "outputs": [
692
+ {
693
+ "name": "stdout",
694
+ "output_type": "stream",
695
+ "text": [
696
+ "loaded temporal unet's pretrained weights from models/StableDiffusion/unet ...\n",
697
+ "### missing keys: 560; \n",
698
+ "### unexpected keys: 0;\n",
699
+ "### Temporal Module Parameters: 417.1376 M\n",
700
+ "optimizer values 0.0003 0.9 0.999 0.01 1e-08\n"
701
+ ]
702
+ },
703
+ {
704
+ "name": "stderr",
705
+ "output_type": "stream",
706
+ "text": [
707
+ "The following values were not passed to `accelerate launch` and had defaults used instead:\n",
708
+ "\t`--num_processes` was set to a value of `1`\n",
709
+ "\t`--num_machines` was set to a value of `1`\n",
710
+ "\t`--mixed_precision` was set to a value of `'no'`\n",
711
+ "\t`--dynamo_backend` was set to a value of `'no'`\n",
712
+ "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n",
713
+ "11/24/2023 22:12:30 - INFO - __main__ - Distributed environment: NO\n",
714
+ "Num processes: 1\n",
715
+ "Process index: 0\n",
716
+ "Local process index: 0\n",
717
+ "Device: cuda\n",
718
+ "\n",
719
+ "Mixed precision type: fp16\n",
720
+ "\n",
721
+ "{'variance_type', 'prediction_type'} was not found in config. Values will be initialized to default values.\n",
722
+ "{'use_linear_projection', 'use_inflated_groupnorm', 'num_class_embeds', 'upcast_attention', 'resnet_time_scale_shift', 'mid_block_type', 'class_embed_type', 'dual_cross_attention', 'only_cross_attention'} was not found in config. Values will be initialized to default values.\n",
723
+ "{'prediction_type'} was not found in config. Values will be initialized to default values.\n",
724
+ "Traceback (most recent call last):\n",
725
+ " File \"c:\\Users\\texas\\Desktop\\School\\SceneFusion\\AnimateDiff-SceneFusion\\train.py\", line 457, in <module>\n",
726
+ " main(**OmegaConf.load(args.config))\n",
727
+ " File \"c:\\Users\\texas\\Desktop\\School\\SceneFusion\\AnimateDiff-SceneFusion\\train.py\", line 257, in main\n",
728
+ " unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(\n",
729
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1288, in prepare\n",
730
+ " result = tuple(\n",
731
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1289, in <genexpr>\n",
732
+ " self._prepare_one(obj, first_pass=True, device_placement=d) for obj, d in zip(args, device_placement)\n",
733
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1094, in _prepare_one\n",
734
+ " return self.prepare_model(obj, device_placement=device_placement)\n",
735
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\accelerator.py\", line 1387, in prepare_model\n",
736
+ " model = model.to(self.device)\n",
737
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 989, in to\n",
738
+ " return self._apply(convert)\n",
739
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 641, in _apply\n",
740
+ " module._apply(fn)\n",
741
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 641, in _apply\n",
742
+ " module._apply(fn)\n",
743
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 641, in _apply\n",
744
+ " module._apply(fn)\n",
745
+ " [Previous line repeated 2 more times]\n",
746
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 664, in _apply\n",
747
+ " param_applied = fn(param)\n",
748
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\torch\\nn\\modules\\module.py\", line 987, in convert\n",
749
+ " return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)\n",
750
+ "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 86.00 MiB (GPU 0; 4.00 GiB total capacity; 3.34 GiB already allocated; 0 bytes free; 3.44 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n",
751
+ "Traceback (most recent call last):\n",
752
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\runpy.py\", line 196, in _run_module_as_main\n",
753
+ " return _run_code(code, main_globals, None,\n",
754
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\runpy.py\", line 86, in _run_code\n",
755
+ " exec(code, run_globals)\n",
756
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\Scripts\\accelerate.exe\\__main__.py\", line 7, in <module>\n",
757
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\commands\\accelerate_cli.py\", line 47, in main\n",
758
+ " args.func(args)\n",
759
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\commands\\launch.py\", line 994, in launch_command\n",
760
+ " simple_launcher(args)\n",
761
+ " File \"C:\\Users\\texas\\anaconda3\\envs\\animatediff\\lib\\site-packages\\accelerate\\commands\\launch.py\", line 636, in simple_launcher\n",
762
+ " raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n",
763
+ "subprocess.CalledProcessError: Command '['C:\\\\Users\\\\texas\\\\anaconda3\\\\envs\\\\animatediff\\\\python.exe', 'train.py', '--config=configs/test.yaml']' returned non-zero exit status 1.\n"
764
+ ]
765
+ }
766
+ ],
767
  "source": [
768
  "# %cd /content/AnimateDiff-SceneFusion\n",
769
  "# !python -m torch.distributed.run --nnodes=1 --nproc_per_node=1 train.py --config configs/training/training.yaml\n",
770
  "\n",
771
  "# !accelerate launch train.py --config=configs/training/training.yaml\n",
772
+ "!accelerate launch train.py --config=configs/{PROJECT}.yaml"
773
  ]
774
  },
775
  {
handler.py CHANGED
@@ -10,6 +10,8 @@ import os
10
  from diffusers.utils.import_utils import is_xformers_available
11
  from typing import Any
12
  import torch
 
 
13
  from einops import rearrange
14
 
15
  from animatediff.models.unet import UNet3DConditionModel
@@ -52,7 +54,6 @@ class EndpointHandler():
52
  # huggingface download motion module from bluestarburst/AnimateDiff-SceneFusion/models/Motion_Module/mm_sd_v15.ckpt
53
 
54
  motion_module = hf_hub_download(repo_id="bluestarburst/AnimateDiff-SceneFusion", filename="models/Motion_Module/mm_sd_v15.ckpt")
55
-
56
 
57
  self.pipeline = load_weights(
58
  self.pipeline,
@@ -65,11 +66,17 @@ class EndpointHandler():
65
  lora_alpha = 0.8,
66
  ).to("cuda")
67
 
68
- def __call__(self, prompt, negative_prompt, steps, guidance_scale):
69
  """
70
  __call__ method will be called once per request. This can be used to
71
  run inference.
72
  """
 
 
 
 
 
 
73
  vids = self.pipeline(
74
  prompt=prompt,
75
  negative_prompt=negative_prompt,
 
10
  from diffusers.utils.import_utils import is_xformers_available
11
  from typing import Any
12
  import torch
13
+ import torchvision
14
+ import numpy as np
15
  from einops import rearrange
16
 
17
  from animatediff.models.unet import UNet3DConditionModel
 
54
  # huggingface download motion module from bluestarburst/AnimateDiff-SceneFusion/models/Motion_Module/mm_sd_v15.ckpt
55
 
56
  motion_module = hf_hub_download(repo_id="bluestarburst/AnimateDiff-SceneFusion", filename="models/Motion_Module/mm_sd_v15.ckpt")
 
57
 
58
  self.pipeline = load_weights(
59
  self.pipeline,
 
66
  lora_alpha = 0.8,
67
  ).to("cuda")
68
 
69
+ def __call__(self, data : Any):
70
  """
71
  __call__ method will be called once per request. This can be used to
72
  run inference.
73
  """
74
+
75
+ prompt = data.pop("prompt", "")
76
+ negative_prompt = data.pop("negative_prompt", "easynegative,bad_construction,bad_structure,bad_wail,bad_windows,blurry,cloned_window,cropped,deformed,disfigured,error,extra_windows,extra_chimney,extra_door,extra_structure,extra_frame,fewer_digits,fused_structure,gross_proportions,jpeg_artifacts,long_roof,low_quality,structure_limbs,missing_windows,missing_doors,missing_roofs,mutated_structure,mutation,normal_quality,out_of_frame,owres,poorly_drawn_structure,poorly_drawn_house,signature,text,too_many_windows,ugly,username,uta,watermark,worst_quality")
77
+ steps = data.pop("steps", 25)
78
+ guidance_scale = data.pop("guidance_scale", 12.5)
79
+
80
  vids = self.pipeline(
81
  prompt=prompt,
82
  negative_prompt=negative_prompt,