Duplicate from fcakyon/video-classification

Co-authored-by: Fatih <fcakyon@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,12 @@
+---
+title: Video Classification
+emoji: 📽
+colorFrom: pink
+colorTo: blue
+sdk: gradio
+sdk_version: 3.12.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+duplicated_from: fcakyon/video-classification
+---

app.py

@@ -0,0 +1,184 @@
+import os
+import gradio as gr
+from utils import (
+    create_gif_from_video_file,
+    download_youtube_video,
+    get_num_total_frames,
+)
+from transformers import pipeline
+from huggingface_hub import HfApi, ModelSearchArguments, ModelFilter
+
+FRAME_SAMPLING_RATE = 4
+DEFAULT_MODEL = "facebook/timesformer-base-finetuned-k400"
+
+VALID_VIDEOCLASSIFICATION_MODELS = [
+    "MCG-NJU/videomae-large-finetuned-kinetics",
+    "facebook/timesformer-base-finetuned-k400",
+    "fcakyon/timesformer-large-finetuned-k400",
+    "MCG-NJU/videomae-base-finetuned-kinetics",
+    "facebook/timesformer-base-finetuned-k600",
+    "fcakyon/timesformer-large-finetuned-k600",
+    "facebook/timesformer-hr-finetuned-k400",
+    "facebook/timesformer-hr-finetuned-k600",
+    "facebook/timesformer-base-finetuned-ssv2",
+    "fcakyon/timesformer-large-finetuned-ssv2",
+    "facebook/timesformer-hr-finetuned-ssv2",
+    "MCG-NJU/videomae-base-finetuned-ssv2",
+    "MCG-NJU/videomae-base-short-finetuned-kinetics",
+    "MCG-NJU/videomae-base-short-ssv2",
+    "MCG-NJU/videomae-base-short-finetuned-ssv2",
+    "sayakpaul/videomae-base-finetuned-ucf101-subset",
+    "nateraw/videomae-base-finetuned-ucf101",
+    "MCG-NJU/videomae-base-ssv2",
+    "zahrav/videomae-base-finetuned-ucf101-subset",
+]
+
+
+pipe = pipeline(
+    task="video-classification",
+    model=DEFAULT_MODEL,
+    top_k=5,
+    frame_sampling_rate=FRAME_SAMPLING_RATE,
+)
+
+
+examples = [
+    ["https://www.youtube.com/watch?v=huAJ9dC5lmI"],
+    ["https://www.youtube.com/watch?v=wvcWt6u5HTg"],
+    ["https://www.youtube.com/watch?v=-3kZSi5qjRM"],
+    ["https://www.youtube.com/watch?v=-6usjfP8hys"],
+    ["https://www.youtube.com/watch?v=BDHub0gBGtc"],
+    ["https://www.youtube.com/watch?v=B9ea7YyCP6E"],
+    ["https://www.youtube.com/watch?v=BBkpaeJBKmk"],
+    ["https://www.youtube.com/watch?v=BBqU8Apee_g"],
+    ["https://www.youtube.com/watch?v=B8OdMwVwyXc"],
+    ["https://www.youtube.com/watch?v=I7cwq6_4QtM"],
+    ["https://www.youtube.com/watch?v=Z0mJDXpNhYA"],
+    ["https://www.youtube.com/watch?v=QkQQjFGnZlg"],
+    ["https://www.youtube.com/watch?v=IQaoRUQif14"],
+]
+
+
+def get_video_model_names():
+    model_args = ModelSearchArguments()
+    filter = ModelFilter(
+        task=model_args.pipeline_tag.VideoClassification,
+        library=model_args.library.Transformers,
+    )
+    api = HfApi()
+    video_models = list(
+        iter(api.list_models(filter=filter, sort="downloads", direction=-1))
+    )
+    video_models = [video_model.id for video_model in video_models]
+    return video_models
+
+
+def select_model(model_name):
+    global pipe
+    pipe = pipeline(
+        task="video-classification",
+        model=model_name,
+        top_k=5,
+        frame_sampling_rate=FRAME_SAMPLING_RATE,
+    )
+
+
+def predict(youtube_url_or_file_path):
+
+    if youtube_url_or_file_path.startswith("http"):
+        video_path = download_youtube_video(youtube_url_or_file_path)
+    else:
+        video_path = youtube_url_or_file_path
+
+    # rearrange sampling rate based on video length and model input length
+    num_total_frames = get_num_total_frames(video_path)
+    num_model_input_frames = pipe.model.config.num_frames
+    if num_total_frames < FRAME_SAMPLING_RATE * num_model_input_frames:
+        frame_sampling_rate = num_total_frames // num_model_input_frames
+    else:
+        frame_sampling_rate = FRAME_SAMPLING_RATE
+
+    gif_path = create_gif_from_video_file(
+        video_path, frame_sampling_rate=frame_sampling_rate, save_path="video.gif"
+    )
+
+    # run inference
+    results = pipe(videos=video_path, frame_sampling_rate=frame_sampling_rate)
+
+    os.remove(video_path)
+
+    label_to_score = {result["label"]: result["score"] for result in results}
+
+    return label_to_score, gif_path
+
+
+app = gr.Blocks()
+with app:
+    gr.Markdown("# **<p align='center'>Video Classification with 🤗 Transformers</p>**")
+    gr.Markdown(
+        """
+        <p style='text-align: center'>
+        Perform video classification with <a href='https://huggingface.co/models?pipeline_tag=video-classification&library=transformers' target='_blank'>HuggingFace Transformers video models</a>.
+        <br> For zero-shot classification, you can use the <a href='https://huggingface.co/spaces/fcakyon/zero-shot-video-classification' target='_blank'>zero-shot classification demo</a>.
+        </p>
+        """
+    )
+    gr.Markdown(
+        """
+        <p style='text-align: center'>
+        Follow me for more! 
+        <br> <a href='https://twitter.com/fcakyon' target='_blank'>twitter</a> | <a href='https://github.com/fcakyon' target='_blank'>github</a> | <a href='https://www.linkedin.com/in/fcakyon/' target='_blank'>linkedin</a> | <a href='https://fcakyon.medium.com/' target='_blank'>medium</a>
+        </p>
+        """
+    )
+
+    with gr.Row():
+        with gr.Column():
+            model_names_dropdown = gr.Dropdown(
+                choices=VALID_VIDEOCLASSIFICATION_MODELS,
+                label="Model:",
+                show_label=True,
+                value=DEFAULT_MODEL,
+            )
+            model_names_dropdown.change(fn=select_model, inputs=model_names_dropdown)
+            with gr.Tab(label="Youtube URL"):
+                gr.Markdown("### **Provide a Youtube video URL**")
+                youtube_url = gr.Textbox(label="Youtube URL:", show_label=True)
+                youtube_url_predict_btn = gr.Button(value="Predict")
+            with gr.Tab(label="Local File"):
+                gr.Markdown("### **Upload a video file**")
+                video_file = gr.Video(label="Video File:", show_label=True)
+                local_video_predict_btn = gr.Button(value="Predict")
+        with gr.Column():
+            video_gif = gr.Image(
+                label="Input Clip",
+                show_label=True,
+            )
+        with gr.Column():
+            predictions = gr.Label(
+                label="Predictions:", show_label=True, num_top_classes=5
+            )
+
+    gr.Markdown("**Examples:**")
+    gr.Examples(
+        examples,
+        youtube_url,
+        [predictions, video_gif],
+        fn=predict,
+        cache_examples=True,
+    )
+
+    youtube_url_predict_btn.click(
+        predict, inputs=youtube_url, outputs=[predictions, video_gif]
+    )
+    local_video_predict_btn.click(
+        predict, inputs=video_file, outputs=[predictions, video_gif]
+    )
+    gr.Markdown(
+        """
+        \n Demo created by: <a href=\"https://github.com/fcakyon\">fcakyon</a>.
+        <br> Powered by <a href='https://huggingface.co/models?pipeline_tag=video-classification&library=transformers' target='_blank'>HuggingFace Transformers video models</a> .
+        """
+    )
+
+app.launch()

requirements.txt

@@ -0,0 +1,9 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+gradio
+torch==1.13.1+cpu
+torchvision==0.14.1
+decord
+pytube @ git+https://github.com/oncename/pytube.git
+imageio==2.25.1
+transformers
+huggingface-hub

utils.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+from pytube import YouTube
+import numpy as np
+from decord import VideoReader, cpu
+import imageio
+
+
+def download_youtube_video(url: str):
+    yt = YouTube(url)
+
+    streams = yt.streams.filter(file_extension="mp4")
+    file_path = streams[0].download()
+    return file_path
+
+
+def sample_frames_from_video_file(
+    file_path: str, num_frames: int = 16, frame_sampling_rate=1
+):
+    videoreader = VideoReader(file_path)
+    videoreader.seek(0)
+
+    # sample frames
+    start_idx = 0
+    end_idx = num_frames * frame_sampling_rate - 1
+    indices = np.linspace(start_idx, end_idx, num=num_frames, dtype=np.int64)
+    frames = videoreader.get_batch(indices).asnumpy()
+
+    return frames
+
+
+def get_num_total_frames(file_path: str):
+    videoreader = VideoReader(file_path)
+    videoreader.seek(0)
+    return len(videoreader)
+
+
+def convert_frames_to_gif(frames, save_path: str = "frames.gif"):
+    converted_frames = frames.astype(np.uint8)
+    Path(save_path).parent.mkdir(parents=True, exist_ok=True)
+    imageio.mimsave(save_path, converted_frames, fps=8)
+    return save_path
+
+
+def create_gif_from_video_file(
+    file_path: str,
+    num_frames: int = 16,
+    frame_sampling_rate: int = 1,
+    save_path: str = "frames.gif",
+):
+    frames = sample_frames_from_video_file(file_path, num_frames, frame_sampling_rate)
+    return convert_frames_to_gif(frames, save_path)