ZachNagengast commited on
Commit
6474363
1 Parent(s): d8c4344

Support mp4s

Browse files
Files changed (2) hide show
  1. app.py +143 -31
  2. demo.mp4 +0 -0
app.py CHANGED
@@ -1,19 +1,58 @@
1
  import gradio as gr
2
  from PIL import Image, ImageDraw, ImageFont, ImageSequence
3
  import numpy as np
 
 
 
4
 
5
- def create_grid(image_file, grid_x, grid_y, font_size, font_color, position, border_size, border_color):
6
- image_file.file.seek(0) # Seek to the beginning of the file
7
 
8
- # Open the image from bytes
9
- img = Image.open(image_file.name)
10
 
11
- total_frames = img.n_frames
12
- selected_frames_count = grid_x * grid_y
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
 
 
 
 
 
 
 
 
 
14
  # Select evenly spaced frames
15
  selected_frames_indices = np.linspace(0, total_frames - 1, selected_frames_count).astype(int)
16
- selected_frames = [img.seek(i) or img.copy() for i in selected_frames_indices]
17
 
18
  # Modify frames by adding border and number
19
  modified_frames = []
@@ -25,9 +64,9 @@ def create_grid(image_file, grid_x, grid_y, font_size, font_color, position, bor
25
 
26
  positions = {
27
  "Top Left": (20, 20),
28
- "Top Right": (img.width - 20 - font_size, 20),
29
- "Bottom Left": (20, img.height - 20 - font_size),
30
- "Bottom Right": (img.width - 20 - font_size, img.height - 20 - font_size)
31
  }
32
 
33
  for i, frame in enumerate(selected_frames):
@@ -55,26 +94,95 @@ def create_grid(image_file, grid_x, grid_y, font_size, font_color, position, bor
55
 
56
  output_info = f"Grid size: {grid_x} x {grid_y}\n\nSelected Frames: {selected_frames_count} / {total_frames} ({selected_frames_count / total_frames * 100:.2f}%)"
57
  return grid_img, output_info
58
- def gif_info(image_file, grid_x, grid_y, font_size, font_color, position, border_size, border_color):
59
- image_file.file.seek(0)
60
- img = Image.open(image_file.name)
61
- total_frames = img.n_frames
62
- frame_rate = img.info.get('duration', 100) / 1000.0 # Convert to seconds
63
 
64
- # Generate grid using create_grid function
65
- grid_img, output_info = create_grid(image_file, grid_x, grid_y, font_size, font_color, position, border_size, border_color)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
 
67
- gif_details = f"![image](/file={image_file.name})\n\n**Total Frames:** {total_frames}\n\n**Frame Rate:** {frame_rate} sec/frame\n\n{output_info}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
 
69
- return grid_img, gif_details
 
 
 
 
 
 
 
 
 
70
 
71
  with gr.Blocks() as app:
72
- gr.Markdown('## GIF Grid Generator')
73
- gr.Markdown('Upload a GIF to generate a grid from its frames. Use the sliders to adjust the grid size and text settings.')
74
  with gr.Row():
75
  with gr.Column():
76
- control_image = gr.File(label="Upload a GIF", type="file", elem_id="file_upload", file_types=[".gif"])
77
- gif_details = gr.Markdown("Upload a gif")
 
 
 
 
 
 
 
 
 
78
  grid_x_slider = gr.Slider(minimum=1, maximum=10, step=1, value=3, label="Grid X Size")
79
  grid_y_slider = gr.Slider(minimum=1, maximum=10, step=1, value=3, label="Grid Y Size")
80
  font_color_dropdown = gr.Dropdown(choices=["Black", "White", "Red", "Green", "Blue"], value="White", label="Numbering Color")
@@ -86,14 +194,18 @@ with gr.Blocks() as app:
86
  result_image = gr.Image(label="Generated Grid")
87
 
88
  # Use .change() method to listen for changes in any of the controls
89
- control_image.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
90
- grid_x_slider.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
91
- grid_y_slider.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
92
- font_size_slider.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
93
- font_color_dropdown.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
94
- position_radio.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
95
- border_size_slider.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
96
- border_color_dropdown.change(gif_info, inputs=[control_image, grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
 
 
 
97
 
98
  if __name__ == "__main__":
 
99
  app.launch()
 
1
  import gradio as gr
2
  from PIL import Image, ImageDraw, ImageFont, ImageSequence
3
  import numpy as np
4
+ import cv2
5
+ import os
6
+ import tempfile
7
 
8
+ global stored_frames
 
9
 
10
+ def load_and_store_frames(image_file):
11
+ global stored_frames
12
 
13
+ # Make sure file exists
14
+ if image_file is None:
15
+ return "File not found", ""
16
+
17
+ print("Loading frames for {}".format(image_file.name))
18
+
19
+ if image_file.name.endswith('.mp4'):
20
+ frames = extract_frames_from_video(image_file.name)
21
+ video_path = image_file.name
22
+ else: # it's a gif
23
+ img = Image.open(image_file.name)
24
+ frames = []
25
+ for i in range(0, img.n_frames):
26
+ img.seek(i)
27
+ frames.append(img.copy())
28
+ # Convert GIF to MP4 for preview
29
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
30
+ tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
31
+ video_path = tmp_file.name
32
+ convert_gif_to_video(image_file.name, tmp_file.name, 1 / (img.info.get('duration', 100) / 1000.0))
33
+
34
+ stored_frames = frames # Store the frames for later use
35
+ return "Frames loaded successfully", video_path
36
+
37
+ def generate_grid(grid_x, grid_y, font_size, font_color, position, border_size, border_color):
38
+ global stored_frames
39
+ print(f"Processing grid with {grid_x} x {grid_y} grid size, font size {font_size}, font color {font_color}, position {position}, border size {border_size}, border color {border_color}")
40
+
41
+ if stored_frames is None:
42
+ load_and_store_frames()
43
+
44
 
45
+ grid_img, output_info = create_grid(stored_frames, grid_x, grid_y, font_size, font_color, position, border_size, border_color)
46
+ details = f"Total Frames: {len(stored_frames)}\n\n{output_info}"
47
+ return grid_img, details
48
+
49
+ def create_grid(frames, grid_x, grid_y, font_size, font_color, position, border_size, border_color):
50
+ total_frames = len(frames)
51
+ selected_frames_count = grid_x * grid_y
52
+
53
  # Select evenly spaced frames
54
  selected_frames_indices = np.linspace(0, total_frames - 1, selected_frames_count).astype(int)
55
+ selected_frames = [frames[i] for i in selected_frames_indices]
56
 
57
  # Modify frames by adding border and number
58
  modified_frames = []
 
64
 
65
  positions = {
66
  "Top Left": (20, 20),
67
+ "Top Right": (frames[0].width - 20 - font_size, 20),
68
+ "Bottom Left": (20, frames[0].height - 20 - font_size),
69
+ "Bottom Right": (frames[0].width - 20 - font_size, frames[0].height - 20 - font_size)
70
  }
71
 
72
  for i, frame in enumerate(selected_frames):
 
94
 
95
  output_info = f"Grid size: {grid_x} x {grid_y}\n\nSelected Frames: {selected_frames_count} / {total_frames} ({selected_frames_count / total_frames * 100:.2f}%)"
96
  return grid_img, output_info
 
 
 
 
 
97
 
98
+ def extract_frames_from_video(video_file):
99
+ """Extract frames from an MP4 video."""
100
+ frames = []
101
+ cap = cv2.VideoCapture(video_file)
102
+ while True:
103
+ ret, frame = cap.read()
104
+ if not ret:
105
+ break
106
+ # Convert BGR format (used by OpenCV) to RGB
107
+ frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
108
+ frames.append(Image.fromarray(frame_rgb))
109
+ cap.release()
110
+ return frames
111
+
112
+
113
+ def convert_gif_to_video(gif_path, output_video_path, frame_rate):
114
+ # Load the gif
115
+ gif = Image.open(gif_path)
116
+ # Define the codec and create VideoWriter object
117
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
118
+ out = cv2.VideoWriter(output_video_path, fourcc, frame_rate, (gif.width, gif.height))
119
+
120
+ # Iterate over the frames of the gif
121
+ for frame_index in range(gif.n_frames):
122
+ gif.seek(frame_index)
123
+ # Convert the PIL Image to an array
124
+ frame_arr = np.array(gif.convert("RGB"))
125
+ # Convert RGB to BGR format
126
+ frame_bgr = cv2.cvtColor(frame_arr, cv2.COLOR_RGB2BGR)
127
+ # Write the frame to the video
128
+ out.write(frame_bgr)
129
 
130
+ out.release()
131
+
132
+ def gif_or_video_info(image_file, grid_x, grid_y, font_size, font_color, position, border_size, border_color):
133
+ image_file.file.seek(0)
134
+ video_path = ""
135
+
136
+ if image_file.name.endswith('.mp4'):
137
+ video_path = image_file.name
138
+ cap = cv2.VideoCapture(image_file.name)
139
+ frame_rate = cap.get(cv2.CAP_PROP_FPS) # Get the actual frame rate of the video
140
+ frames = extract_frames_from_video(image_file.name)
141
+ total_frames = len(frames)
142
+ cap.release()
143
+ else: # it's a gif
144
+ img = Image.open(image_file.name)
145
+ frames = []
146
+ for i in range(0, img.n_frames):
147
+ img.seek(i)
148
+ frames.append(img.copy())
149
+
150
+ total_frames = img.n_frames
151
+ frame_rate = 1 / (img.info.get('duration', 100) / 1000.0) # Convert to seconds
152
+
153
+ # Convert GIF to MP4 and save it to a temp path
154
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
155
+ tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
156
+ video_path = tmp_file.name
157
+ convert_gif_to_video(image_file.name, tmp_file.name, frame_rate)
158
 
159
+ grid_img, output_info = create_grid(frames, grid_x, grid_y, font_size, font_color, position, border_size, border_color)
160
+ details = f"**Total Frames:** {total_frames}\n\n**Frame Rate:** {frame_rate} frames/sec\n\n{output_info}"
161
+
162
+ return grid_img, details, video_path
163
+
164
+ def gif_info(image_file, grid_x, grid_y, font_size, font_color, position, border_size, border_color):
165
+ return gif_or_video_info(image_file, grid_x, grid_y, font_size, font_color, position, border_size, border_color)
166
+
167
+ def mirror(x):
168
+ return x
169
 
170
  with gr.Blocks() as app:
171
+ gr.Markdown('## vid2grid Generator')
172
+ gr.Markdown('Upload a GIF or MP4 to generate a grid from its frames. Use the sliders to adjust the grid size and text settings.\n\nThis is particularly useful for use with multi modal models such as GPT-4V to retrieve descriptions of short videos or gifs, [more example here.](https://twitter.com/zachnagengast/status/1712896232170180651)\n\n **Note:** The grid will be generated only after clicking the "Generate Grid" button.')
173
  with gr.Row():
174
  with gr.Column():
175
+ control_image = gr.File(label="Upload a short MP4 or GIF", type="file", elem_id="file_upload", file_types=[".gif", ".mp4"])
176
+ video_preview = gr.Video(interactive=False, label="Preview", format="mp4")
177
+ gif_details = gr.Markdown("No file found.")
178
+ # gr.Examples(
179
+ # examples=[os.path.join(os.path.dirname(__file__), "demo.mp4")],
180
+ # inputs=[control_image],
181
+ # outputs=[gif_details, video_preview],
182
+ # fn=load_and_store_frames,
183
+ # cache_examples=True,
184
+ # )
185
+ process_button = gr.Button("Generate Grid") # New button to trigger the heavy computation
186
  grid_x_slider = gr.Slider(minimum=1, maximum=10, step=1, value=3, label="Grid X Size")
187
  grid_y_slider = gr.Slider(minimum=1, maximum=10, step=1, value=3, label="Grid Y Size")
188
  font_color_dropdown = gr.Dropdown(choices=["Black", "White", "Red", "Green", "Blue"], value="White", label="Numbering Color")
 
194
  result_image = gr.Image(label="Generated Grid")
195
 
196
  # Use .change() method to listen for changes in any of the controls
197
+ control_image.upload(load_and_store_frames, inputs=[control_image], outputs=[gif_details, video_preview])
198
+
199
+ # grid_x_slider.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details, video_preview])
200
+ # grid_y_slider.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
201
+ # font_size_slider.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
202
+ # font_color_dropdown.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
203
+ # position_radio.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
204
+ # border_size_slider.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
205
+ # border_color_dropdown.change(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
206
+
207
+ process_button.click(generate_grid, inputs=[grid_x_slider, grid_y_slider, font_size_slider, font_color_dropdown, position_radio, border_size_slider, border_color_dropdown], outputs=[result_image, gif_details])
208
 
209
  if __name__ == "__main__":
210
+ stored_frames = None
211
  app.launch()
demo.mp4 ADDED
Binary file (466 kB). View file