Repository: gradio-app/gradio

Issue:
Issues when loading/outputting video
### Describe the bug

I'm having weird situations in which calling an inference function directly with a given video works well (I can download and play it), but just wrapping it with a `gr.Interface` outputs an invalid video (even if I download the video, I cannot play it, it gives an error in the demo). I saw @nateraw also had to do some hacks to make it work in a previous demo https://huggingface.co/spaces/nateraw/yolov6/blob/main/app.py#L33.

Adding this before returning makes it work

```python
out_file = tempfile.NamedTemporaryFile(suffix="out.mp4", delete=False)
subprocess.run(f"ffmpeg -y -loglevel quiet -stats -i {output_fname} -c:v libx264 {out_file.name}".split())
return out_file.name
```

### Is there an existing issue for this?

- [X] I have searched the existing issues

### Reproduction

https://huggingface.co/spaces/nateraw/yolov6/blob/main/app.py#L58 without those two lines

### Screenshot

_No response_

### Logs

```shell
None
```


### System Info

```shell
Gradio 3.1.0 and above
```


### Severity

serious, but I can work around it


Discussion:

Resolving patch:
diff --git a/demo/white_noise_vid_not_playable/run.py b/demo/white_noise_vid_not_playable/run.py
new file mode 100644
--- /dev/null
+++ b/demo/white_noise_vid_not_playable/run.py
@@ -0,0 +1,22 @@
+import cv2
+import gradio as gr
+import numpy as np
+
+
+def gif_maker():
+    img_array = []
+    height, width = 50, 50
+    for i in range(30):
+        img_array.append(np.random.randint(0, 255, size=(height, width, 3)).astype(np.uint8))
+    output_file = "test.mp4"
+    out = cv2.VideoWriter(output_file, cv2.VideoWriter_fourcc(*'mp4v'), 15, (height, width))
+    for i in range(len(img_array)):
+        out.write(img_array[i])
+    out.release()
+    return output_file, output_file
+
+
+demo = gr.Interface(gif_maker, inputs=None, outputs=[gr.Video(), gr.File()])
+
+if __name__ == "__main__":
+    demo.launch()
\ No newline at end of file
diff --git a/gradio/components.py b/gradio/components.py
--- a/gradio/components.py
+++ b/gradio/components.py
@@ -1779,7 +1779,11 @@ def stream(
 @document("change", "clear", "play", "pause", "stop", "style")
 class Video(Changeable, Clearable, Playable, IOComponent):
     """
-    Creates an video component that can be used to upload/record videos (as an input) or display videos (as an output).
+    Creates a video component that can be used to upload/record videos (as an input) or display videos (as an output).
+    For the video to be playable in the browser it must have a compatible container and codec combination. Allowed
+    combinations are .mp4 with h264 codec, .ogg with theora codec, and .webm with vp9 codec. If the component detects
+    that the output video would not be playable in the browser it will attempt to convert it to a playable mp4 video.
+    If the conversion fails, the original video is returned.
     Preprocessing: passes the uploaded video as a {str} filepath whose extension can be set by `format`.
     Postprocessing: expects a {str} filepath to a video which is displayed.
     Examples-format: a {str} filepath to a local file that contains the video.
@@ -1919,7 +1923,7 @@ def restore_flagged(self, dir, data, encryption_key):
     def generate_sample(self):
         return deepcopy(media_data.BASE64_VIDEO)
 
-    def postprocess(self, y: str) -> str:
+    def postprocess(self, y: str) -> Optional[Dict[str, str]]:
         """
         Parameters:
             y: path to video
@@ -1929,6 +1933,14 @@ def postprocess(self, y: str) -> str:
         if y is None:
             return None
         returned_format = y.split(".")[-1].lower()
+        if (
+            processing_utils.ffmpeg_installed()
+            and not processing_utils.video_is_playable(y)
+        ):
+            warnings.warn(
+                "Video does not have browser-compatible container or codec. Converting to mp4"
+            )
+            y = processing_utils.convert_video_to_playable_mp4(y)
         if self.format is not None and returned_format != self.format:
             output_file_name = y[0 : y.rindex(".") + 1] + self.format
             ff = FFmpeg(inputs={y: None}, outputs={output_file_name: None})
diff --git a/gradio/processing_utils.py b/gradio/processing_utils.py
--- a/gradio/processing_utils.py
+++ b/gradio/processing_utils.py
@@ -1,13 +1,17 @@
 import base64
+import json
 import mimetypes
 import os
+import pathlib
 import shutil
+import subprocess
 import tempfile
 import warnings
 from io import BytesIO
 
 import numpy as np
 import requests
+from ffmpy import FFmpeg, FFprobe, FFRuntimeError
 from PIL import Image, ImageOps
 
 from gradio import encryptor
@@ -518,3 +522,53 @@ def _scale(a, n, m, copy=True):
 
 def strip_invalid_filename_characters(filename: str) -> str:
     return "".join([char for char in filename if char.isalnum() or char in "._- "])
+
+
+def ffmpeg_installed() -> bool:
+    return shutil.which("ffmpeg") is not None
+
+
+def video_is_playable(video_filepath: str) -> bool:
+    """Determines if a video is playable in the browser.
+
+    A video is playable if it has a playable container and codec.
+        .mp4 -> h264
+        .webm -> vp9
+        .ogg -> theora
+    """
+    try:
+        container = pathlib.Path(video_filepath).suffix.lower()
+        probe = FFprobe(
+            global_options="-show_format -show_streams -select_streams v -print_format json",
+            inputs={video_filepath: None},
+        )
+        output = probe.run(stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+        output = json.loads(output[0])
+        video_codec = output["streams"][0]["codec_name"]
+        return (container, video_codec) in [
+            (".mp4", "h264"),
+            (".ogg", "theora"),
+            (".webm", "vp9"),
+        ]
+    # If anything goes wrong, assume the video can be played to not convert downstream
+    except FFRuntimeError:
+        return True
+
+
+def convert_video_to_playable_mp4(video_path: str) -> str:
+    """Convert the video to mp4. If something goes wrong return the original video."""
+    try:
+        output_path = pathlib.Path(video_path).with_suffix(".mp4")
+        with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+            shutil.copy2(video_path, tmp_file.name)
+            # ffmpeg will automatically use h264 codec (playable in browser) when converting to mp4
+            ff = FFmpeg(
+                inputs={str(tmp_file.name): None},
+                outputs={str(output_path): None},
+                global_options="-y -loglevel quiet",
+            )
+            ff.run()
+    except FFRuntimeError as e:
+        print(f"Error converting video to browser-playable format {str(e)}")
+        output_path = video_path
+    return str(output_path)


Test patch:
diff --git a/test/conftest.py b/test/conftest.py
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,4 +1,14 @@
+import pathlib
+
+import pytest
+
+
 def pytest_configure(config):
     config.addinivalue_line(
         "markers", "flaky: mark test as flaky. Failure will not cause te"
     )
+
+
+@pytest.fixture
+def test_file_dir():
+    return pathlib.Path(pathlib.Path(__file__).parent, "test_files")
diff --git a/test/test_components.py b/test/test_components.py
--- a/test/test_components.py
+++ b/test/test_components.py
@@ -1,6 +1,7 @@
 import json
 import os
 import pathlib
+import shutil
 import tempfile
 import unittest
 from copy import deepcopy
@@ -15,7 +16,7 @@
 from scipy.io import wavfile
 
 import gradio as gr
-from gradio import media_data
+from gradio import media_data, processing_utils
 
 os.environ["GRADIO_ANALYTICS_ENABLED"] = "False"
 
@@ -1914,5 +1915,32 @@ def test_audio_preprocess_can_be_read_by_scipy():
     wavfile.read(output)
 
 
+def test_video_postprocess_converts_to_playable_format(test_file_dir):
+    # This file has a playable container but not playable codec
+    with tempfile.NamedTemporaryFile(suffix="bad_video.mp4") as tmp_not_playable_vid:
+        bad_vid = str(test_file_dir / "bad_video_sample.mp4")
+        assert not processing_utils.video_is_playable(bad_vid)
+        shutil.copy(bad_vid, tmp_not_playable_vid.name)
+        _ = gr.Video().postprocess(tmp_not_playable_vid.name)
+        # The original video gets converted to .mp4 format
+        full_path_to_output = pathlib.Path(tmp_not_playable_vid.name).with_suffix(
+            ".mp4"
+        )
+        assert processing_utils.video_is_playable(str(full_path_to_output))
+
+    # This file has a playable codec but not a playable container
+    with tempfile.NamedTemporaryFile(
+        suffix="playable_but_bad_container.mkv"
+    ) as tmp_not_playable_vid:
+        bad_vid = str(test_file_dir / "playable_but_bad_container.mkv")
+        assert not processing_utils.video_is_playable(bad_vid)
+        shutil.copy(bad_vid, tmp_not_playable_vid.name)
+        _ = gr.Video().postprocess(tmp_not_playable_vid.name)
+        full_path_to_output = pathlib.Path(tmp_not_playable_vid.name).with_suffix(
+            ".mp4"
+        )
+        assert processing_utils.video_is_playable(str(full_path_to_output))
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/test_files/bad_video_sample.mp4 b/test/test_files/bad_video_sample.mp4
new file mode 100644
Binary files /dev/null and b/test/test_files/bad_video_sample.mp4 differ
diff --git a/test/test_files/playable_but_bad_container.mkv b/test/test_files/playable_but_bad_container.mkv
new file mode 100644
Binary files /dev/null and b/test/test_files/playable_but_bad_container.mkv differ
diff --git a/test/test_files/video_sample.ogg b/test/test_files/video_sample.ogg
new file mode 100644
Binary files /dev/null and b/test/test_files/video_sample.ogg differ
diff --git a/test/test_files/video_sample.webm b/test/test_files/video_sample.webm
new file mode 100644
Binary files /dev/null and b/test/test_files/video_sample.webm differ
diff --git a/test/test_processing_utils.py b/test/test_processing_utils.py
--- a/test/test_processing_utils.py
+++ b/test/test_processing_utils.py
@@ -1,8 +1,12 @@
 import os
+import pathlib
+import shutil
 import tempfile
 import unittest
 from copy import deepcopy
+from unittest.mock import patch
 
+import ffmpy
 import matplotlib.pyplot as plt
 import numpy as np
 from PIL import Image
@@ -131,5 +135,48 @@ def test_subclass_conversion(self):
             assert y.dtype == x.dtype
 
 
+def test_video_has_playable_codecs(test_file_dir):
+    assert gr.processing_utils.video_is_playable(
+        str(test_file_dir / "video_sample.mp4")
+    )
+    assert gr.processing_utils.video_is_playable(
+        str(test_file_dir / "video_sample.ogg")
+    )
+    assert gr.processing_utils.video_is_playable(
+        str(test_file_dir / "video_sample.webm")
+    )
+    assert not gr.processing_utils.video_is_playable(
+        str(test_file_dir / "bad_video_sample.mp4")
+    )
+
+
+def test_convert_video_to_playable_mp4(test_file_dir):
+    with tempfile.NamedTemporaryFile(suffix="out.avi") as tmp_not_playable_vid:
+        shutil.copy(
+            str(test_file_dir / "bad_video_sample.mp4"), tmp_not_playable_vid.name
+        )
+        playable_vid = gr.processing_utils.convert_video_to_playable_mp4(
+            tmp_not_playable_vid.name
+        )
+        assert gr.processing_utils.video_is_playable(playable_vid)
+
+
+def raise_ffmpy_runtime_exception():
+    raise ffmpy.FFRuntimeError("", "", "", "")
+
+
+@patch("ffmpy.FFmpeg.run", side_effect=raise_ffmpy_runtime_exception)
+def test_video_conversion_returns_original_video_if_fails(mock_run, test_file_dir):
+    with tempfile.NamedTemporaryFile(suffix="out.avi") as tmp_not_playable_vid:
+        shutil.copy(
+            str(test_file_dir / "bad_video_sample.mp4"), tmp_not_playable_vid.name
+        )
+        playable_vid = gr.processing_utils.convert_video_to_playable_mp4(
+            tmp_not_playable_vid.name
+        )
+        # If the conversion succeeded it'd be .mp4
+        assert pathlib.Path(playable_vid).suffix == ".avi"
+
+
 if __name__ == "__main__":
     unittest.main()