fcakyon commited on
Commit
c130fd5
1 Parent(s): 47d09f8

Upload 4 files

Browse files
Files changed (4) hide show
  1. README.md +6 -8
  2. app.py +186 -0
  3. requirements.txt +7 -0
  4. utils.py +51 -0
README.md CHANGED
@@ -1,13 +1,11 @@
1
  ---
2
  title: Video Classification
3
- emoji: 👁
4
- colorFrom: purple
5
- colorTo: indigo
6
  sdk: gradio
7
- sdk_version: 3.12.0
8
  app_file: app.py
9
  pinned: false
10
- license: openrail
11
- ---
12
-
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: Video Classification
3
+ emoji: 📽
4
+ colorFrom: blue
5
+ colorTo: pink
6
  sdk: gradio
7
+ sdk_version: 2.9.1
8
  app_file: app.py
9
  pinned: false
10
+ license: apache-2.0
11
+ ---
 
 
app.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gradio as gr
3
+ from utils import (
4
+ create_gif_from_video_file,
5
+ download_youtube_video,
6
+ get_num_total_frames,
7
+ )
8
+ from transformers import pipeline
9
+ from huggingface_hub import HfApi, ModelSearchArguments, ModelFilter
10
+
11
+ FRAME_SAMPLING_RATE = 4
12
+ DEFAULT_MODEL = "fcakyon/timesformer-base-finetuned-k600"
13
+
14
+ VALID_VIDEOCLASSIFICATION_MODELS = [
15
+ "MCG-NJU/videomae-large-finetuned-kinetics",
16
+ "facebook/timesformer-base-finetuned-k400",
17
+ "fcakyon/timesformer-large-finetuned-k400",
18
+ "MCG-NJU/videomae-base-finetuned-kinetics",
19
+ "fcakyon/timesformer-base-finetuned-k600", # currently facebook version is broken
20
+ "fcakyon/timesformer-large-finetuned-k600",
21
+ "facebook/timesformer-hr-finetuned-k400",
22
+ "fcakyon/timesformer-hr-finetuned-k600", # currently facebook version is broken
23
+ "facebook/timesformer-base-finetuned-ssv2",
24
+ "fcakyon/timesformer-large-finetuned-ssv2",
25
+ "facebook/timesformer-hr-finetuned-ssv2",
26
+ "MCG-NJU/videomae-base-finetuned-ssv2",
27
+ "MCG-NJU/videomae-base-short-finetuned-kinetics",
28
+ "MCG-NJU/videomae-base-short-ssv2",
29
+ "MCG-NJU/videomae-base-short-finetuned-ssv2",
30
+ "sayakpaul/videomae-base-finetuned-ucf101-subset",
31
+ "nateraw/videomae-base-finetuned-ucf101",
32
+ "MCG-NJU/videomae-base-ssv2",
33
+ "zahrav/videomae-base-finetuned-ucf101-subset",
34
+ ]
35
+
36
+
37
+ pipe = pipeline(
38
+ task="video-classification",
39
+ model=DEFAULT_MODEL,
40
+ top_k=5,
41
+ frame_sampling_rate=FRAME_SAMPLING_RATE,
42
+ )
43
+
44
+
45
+ examples = [
46
+ ["https://www.youtube.com/watch?v=huAJ9dC5lmI"],
47
+ ["https://www.youtube.com/watch?v=wvcWt6u5HTg"],
48
+ ["https://www.youtube.com/watch?v=-3kZSi5qjRM"],
49
+ ["https://www.youtube.com/watch?v=-6usjfP8hys"],
50
+ ["https://www.youtube.com/watch?v=B8OdMwVwyXc"],
51
+ ["https://www.youtube.com/watch?v=B9ea7YyCP6E"],
52
+ ["https://www.youtube.com/watch?v=BBkpaeJBKmk"],
53
+ ["https://www.youtube.com/watch?v=BBqU8Apee_g"],
54
+ ["https://www.youtube.com/watch?v=BDHub0gBGtc"],
55
+ ["https://www.youtube.com/watch?v=I7cwq6_4QtM"],
56
+ ["https://www.youtube.com/watch?v=Z0mJDXpNhYA"],
57
+ ["https://www.youtube.com/watch?v=QkQQjFGnZlg"],
58
+ ["https://www.youtube.com/watch?v=IQaoRUQif14"],
59
+ ]
60
+
61
+
62
+ def get_video_model_names():
63
+ model_args = ModelSearchArguments()
64
+ filter = ModelFilter(
65
+ task=model_args.pipeline_tag.VideoClassification,
66
+ library=model_args.library.Transformers,
67
+ )
68
+ api = HfApi()
69
+ video_models = list(
70
+ iter(api.list_models(filter=filter, sort="downloads", direction=-1))
71
+ )
72
+ video_models = [video_model.id for video_model in video_models]
73
+ return video_models
74
+
75
+
76
+ def select_model(model_name):
77
+ global pipe
78
+ pipe = pipeline(
79
+ task="video-classification",
80
+ model=model_name,
81
+ top_k=5,
82
+ frame_sampling_rate=FRAME_SAMPLING_RATE,
83
+ )
84
+
85
+
86
+ def predict(youtube_url_or_file_path):
87
+
88
+ if youtube_url_or_file_path.startswith("http"):
89
+ video_path = download_youtube_video(youtube_url_or_file_path)
90
+ else:
91
+ video_path = youtube_url_or_file_path
92
+
93
+ # rearrange sampling rate based on video length and model input length
94
+ num_total_frames = get_num_total_frames(video_path)
95
+ num_model_input_frames = pipe.model.config.num_frames
96
+ if num_total_frames < FRAME_SAMPLING_RATE * num_model_input_frames:
97
+ frame_sampling_rate = num_total_frames // num_model_input_frames
98
+ else:
99
+ frame_sampling_rate = FRAME_SAMPLING_RATE
100
+
101
+ gif_path = create_gif_from_video_file(
102
+ video_path, frame_sampling_rate=frame_sampling_rate, save_path="video.gif"
103
+ )
104
+
105
+ # run inference
106
+ results = pipe(videos=video_path, frame_sampling_rate=frame_sampling_rate)
107
+
108
+ os.remove(video_path)
109
+
110
+ label_to_score = {result["label"]: result["score"] for result in results}
111
+
112
+ return label_to_score, gif_path
113
+
114
+
115
+ app = gr.Blocks()
116
+ with app:
117
+ gr.Markdown(
118
+ "# **<p align='center'>Video Classification with HuggingFace Transformers</p>**"
119
+ )
120
+ gr.Markdown(
121
+ """
122
+ <p style='text-align: center'>
123
+ Perform video classification with <a href='https://huggingface.co/models?pipeline_tag=video-classification&library=transformers' target='_blank'>HuggingFace Transformers video models</a>.
124
+ <br> For zero-shot classification, you can use the <a href='https://huggingface.co/spaces/fcakyon/zero-shot-video-classification' target='_blank'>zero-shot classification demo</a>.
125
+ </p>
126
+ """
127
+ )
128
+ gr.Markdown(
129
+ """
130
+ <p style='text-align: center'>
131
+ Follow me for more!
132
+ <br> <a href='https://twitter.com/fcakyon' target='_blank'>twitter</a> | <a href='https://github.com/fcakyon' target='_blank'>github</a> | <a href='https://www.linkedin.com/in/fcakyon/' target='_blank'>linkedin</a> | <a href='https://fcakyon.medium.com/' target='_blank'>medium</a>
133
+ </p>
134
+ """
135
+ )
136
+
137
+ with gr.Row():
138
+ with gr.Column():
139
+ model_names_dropdown = gr.Dropdown(
140
+ choices=VALID_VIDEOCLASSIFICATION_MODELS,
141
+ label="Model:",
142
+ show_label=True,
143
+ value=DEFAULT_MODEL,
144
+ )
145
+ model_names_dropdown.change(fn=select_model, inputs=model_names_dropdown)
146
+ with gr.Tab(label="Youtube URL"):
147
+ gr.Markdown("### **Provide a Youtube video URL**")
148
+ youtube_url = gr.Textbox(label="Youtube URL:", show_label=True)
149
+ youtube_url_predict_btn = gr.Button(value="Predict")
150
+ with gr.Tab(label="Local File"):
151
+ gr.Markdown("### **Upload a video file**")
152
+ video_file = gr.Video(label="Video File:", show_label=True)
153
+ local_video_predict_btn = gr.Button(value="Predict")
154
+ with gr.Column():
155
+ video_gif = gr.Image(
156
+ label="Input Clip",
157
+ show_label=True,
158
+ )
159
+ with gr.Column():
160
+ predictions = gr.Label(
161
+ label="Predictions:", show_label=True, num_top_classes=5
162
+ )
163
+
164
+ gr.Markdown("**Examples:**")
165
+ gr.Examples(
166
+ examples,
167
+ youtube_url,
168
+ [predictions, video_gif],
169
+ fn=predict,
170
+ cache_examples=False,
171
+ )
172
+
173
+ youtube_url_predict_btn.click(
174
+ predict, inputs=youtube_url, outputs=[predictions, video_gif]
175
+ )
176
+ local_video_predict_btn.click(
177
+ predict, inputs=video_file, outputs=[predictions, video_gif]
178
+ )
179
+ gr.Markdown(
180
+ """
181
+ \n Demo created by: <a href=\"https://github.com/fcakyon\">fcakyon</a>.
182
+ <br> Powered by <a href='https://huggingface.co/models?pipeline_tag=video-classification&library=transformers' target='_blank'>HuggingFace Transformers video models</a> .
183
+ """
184
+ )
185
+
186
+ app.launch()
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ gradio
2
+ torch
3
+ decord
4
+ pytube
5
+ imageio
6
+ transformers @ git+https://github.com/huggingface/transformers.git@799cea64ac1029d66e9e58f18bc6f47892270723
7
+ huggingface-hub
utils.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ from pytube import YouTube
3
+ import numpy as np
4
+ from decord import VideoReader, cpu
5
+ import imageio
6
+
7
+
8
+ def download_youtube_video(url: str):
9
+ yt = YouTube(url)
10
+
11
+ streams = yt.streams.filter(file_extension="mp4")
12
+ file_path = streams[0].download()
13
+ return file_path
14
+
15
+
16
+ def sample_frames_from_video_file(
17
+ file_path: str, num_frames: int = 16, frame_sampling_rate=1
18
+ ):
19
+ videoreader = VideoReader(file_path)
20
+ videoreader.seek(0)
21
+
22
+ # sample frames
23
+ start_idx = 0
24
+ end_idx = num_frames * frame_sampling_rate - 1
25
+ indices = np.linspace(start_idx, end_idx, num=num_frames, dtype=np.int64)
26
+ frames = videoreader.get_batch(indices).asnumpy()
27
+
28
+ return frames
29
+
30
+
31
+ def get_num_total_frames(file_path: str):
32
+ videoreader = VideoReader(file_path)
33
+ videoreader.seek(0)
34
+ return len(videoreader)
35
+
36
+
37
+ def convert_frames_to_gif(frames, save_path: str = "frames.gif"):
38
+ converted_frames = frames.astype(np.uint8)
39
+ Path(save_path).parent.mkdir(parents=True, exist_ok=True)
40
+ imageio.mimsave(save_path, converted_frames, fps=8)
41
+ return save_path
42
+
43
+
44
+ def create_gif_from_video_file(
45
+ file_path: str,
46
+ num_frames: int = 16,
47
+ frame_sampling_rate: int = 1,
48
+ save_path: str = "frames.gif",
49
+ ):
50
+ frames = sample_frames_from_video_file(file_path, num_frames, frame_sampling_rate)
51
+ return convert_frames_to_gif(frames, save_path)