init space

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+*.mp4
+.idea/
+__pycache__/

app.py

@@ -0,0 +1,66 @@
+import mediapy
+import gradio as gr
+from utils import load_image
+from interpolator import Interpolator, interpolate_recursively
+
+path = "./smoot.mp4"
+
+interpolator = Interpolator()
+
+
+def predict(image_a, image_b):
+    image1 = load_image(image_a)
+    image2 = load_image(image_b)
+    input_frames = [image1, image2]
+    frames = list(interpolate_recursively(input_frames, interpolator))
+    mediapy.write_video(path, frames, fps=30)
+    return path
+
+
+footer = r"""
+<center>
+<b>
+Demo for <a href='https://www.tensorflow.org/hub/tutorials/tf_hub_delf_module'>DELF</a>
+</b>
+</center>
+"""
+
+coffe = r"""
+<center>
+<a href="https://www.buymeacoffee.com/leonelhs"><img src="https://img.buymeacoffee.com/button-api/?text=Buy me a 
+coffee&emoji=&slug=leonelhs&button_colour=FFDD00&font_colour=000000&font_family=Cookie&outline_colour=000000
+&coffee_colour=ffffff" /></a>
+</center>
+"""
+
+with gr.Blocks(title="DELF") as app:
+    gr.HTML("<center><h1>Match images using DELF</h1></center>")
+    gr.HTML("<center><h3>Neural network and logic for processing images to identify keypoints and their "
+            "descriptors.</h3></center>")
+    with gr.Row(equal_height=False):
+        with gr.Column():
+            input_img_a = gr.Image(type="filepath", label="Input image A")
+            input_img_b = gr.Image(type="filepath", label="Input image B")
+            run_btn = gr.Button(variant="primary")
+        with gr.Column():
+            output_img = gr.Video(format="mp4", label="Interpolate video")
+            gr.ClearButton(components=[input_img_a, input_img_b, output_img], variant="stop")
+
+    run_btn.click(predict, [input_img_a, input_img_b], [output_img])
+
+    with gr.Row():
+        blobs_a = [[f"examples/image_a/{x:02d}.jpg"] for x in range(1, 2)]
+        examples_a = gr.Dataset(components=[input_img_a], samples=blobs_a)
+        examples_a.click(lambda x: x[0], [examples_a], [input_img_a])
+    with gr.Row():
+        blobs_b = [[f"examples/image_b/{x:02d}.jpg"] for x in range(1, 2)]
+        examples_b = gr.Dataset(components=[input_img_b], samples=blobs_b)
+        examples_b.click(lambda x: x[0], [examples_b], [input_img_b])
+
+    with gr.Row():
+        gr.HTML(footer)
+    with gr.Row():
+        gr.HTML(coffe)
+
+app.launch(share=False, debug=True, show_error=True)
+app.queue()

interpolator.py

@@ -0,0 +1,142 @@
+import numpy as np
+import tensorflow as tf
+import tensorflow_hub as hub
+from typing import Generator, List, Iterable
+
+"""A wrapper class for running a frame interpolation based on the FILM model on TFHub
+
+Usage:
+  interpolator = Interpolator()
+  result_batch = interpolator(image_batch_0, image_batch_1, batch_dt)
+  Where image_batch_1 and image_batch_2 are numpy tensors with TF standard
+  (B,H,W,C) layout, batch_dt is the sub-frame time in range [0..1], (B,) layout.
+"""
+
+
+def _pad_to_align(x, align):
+    """Pads image batch x so width and height divide by align.
+
+  Args:
+    x: Image batch to align.
+    align: Number to align to.
+
+  Returns:
+    1) An image padded so width % align == 0 and height % align == 0.
+    2) A bounding box that can be fed readily to tf.image.crop_to_bounding_box
+      to undo the padding.
+  """
+    # Input checking.
+    assert np.ndim(x) == 4
+    assert align > 0, 'align must be a positive number.'
+
+    height, width = x.shape[-3:-1]
+    height_to_pad = (align - height % align) if height % align != 0 else 0
+    width_to_pad = (align - width % align) if width % align != 0 else 0
+
+    bbox_to_pad = {
+        'offset_height': height_to_pad // 2,
+        'offset_width': width_to_pad // 2,
+        'target_height': height + height_to_pad,
+        'target_width': width + width_to_pad
+    }
+    padded_x = tf.image.pad_to_bounding_box(x, **bbox_to_pad)
+    bbox_to_crop = {
+        'offset_height': height_to_pad // 2,
+        'offset_width': width_to_pad // 2,
+        'target_height': height,
+        'target_width': width
+    }
+    return padded_x, bbox_to_crop
+
+
+class Interpolator:
+    """A class for generating interpolated frames between two input frames.
+
+  Uses the Film model from TFHub
+  """
+
+    def __init__(self, times_to_interpolate=6, align: int = 64) -> None:
+        """Loads a saved model.
+
+    Args:
+      align: 'If >1, pad the input size so it divides with this before
+        inference.'
+    """
+        self.times_to_interpolate = times_to_interpolate
+        self._model = hub.load("https://tfhub.dev/google/film/1")
+        self._align = align
+
+    def __call__(self, x0: np.ndarray, x1: np.ndarray,
+                 dt: np.ndarray) -> np.ndarray:
+        """Generates an interpolated frame between given two batches of frames.
+
+    All inputs should be np.float32 datatype.
+
+    Args:
+      x0: First image batch. Dimensions: (batch_size, height, width, channels)
+      x1: Second image batch. Dimensions: (batch_size, height, width, channels)
+      dt: Sub-frame time. Range [0,1]. Dimensions: (batch_size,)
+
+    Returns:
+      The result with dimensions (batch_size, height, width, channels).
+    """
+        if self._align is not None:
+            x0, bbox_to_crop = _pad_to_align(x0, self._align)
+            x1, _ = _pad_to_align(x1, self._align)
+
+        inputs = {'x0': x0, 'x1': x1, 'time': dt[..., np.newaxis]}
+        result = self._model(inputs, training=False)
+        image = result['image']
+
+        if self._align is not None:
+            image = tf.image.crop_to_bounding_box(image, **bbox_to_crop)
+        return image.numpy()
+
+
+def _recursive_generator(
+        frame1: np.ndarray, frame2: np.ndarray, num_recursions: int,
+        interpolator: Interpolator) -> Generator[np.ndarray, None, None]:
+    """Splits halfway to repeatedly generate more frames.
+
+  Args:
+    frame1: Input image 1.
+    frame2: Input image 2.
+    num_recursions: How many times to interpolate the consecutive image pairs.
+    interpolator: The frame interpolator instance.
+
+  Yields:
+    The interpolated frames, including the first frame (frame1), but excluding
+    the final frame2.
+  """
+    if num_recursions == 0:
+        yield frame1
+    else:
+        # Adds the batch dimension to all inputs before calling the interpolator,
+        # and remove it afterwards.
+        time = np.full(shape=(1,), fill_value=0.5, dtype=np.float32)
+        mid_frame = interpolator(np.expand_dims(frame1, axis=0), np.expand_dims(frame2, axis=0), time)[0]
+        yield from _recursive_generator(frame1, mid_frame, num_recursions - 1, interpolator)
+        yield from _recursive_generator(mid_frame, frame2, num_recursions - 1, interpolator)
+
+
+def interpolate_recursively(
+        frames: List[np.ndarray], interpolator: Interpolator) -> Iterable[np.ndarray]:
+    """Generates interpolated frames by repeatedly interpolating the midpoint.
+
+  Args:
+    frames: List of input frames. Expected shape (H, W, 3). The colors should be
+      in the range[0, 1] and in gamma space.
+    num_recursions: Number of times to do recursive midpoint
+      interpolation.
+    interpolator: The frame interpolation model to use.
+
+  Yields:
+    The interpolated frames (including the inputs).
+  """
+    times_to_interpolate = interpolator.times_to_interpolate
+
+    n = len(frames)
+    for i in range(1, n):
+        yield from _recursive_generator(frames[i - 1], frames[i], times_to_interpolate, interpolator)
+    # Separately yield the final frame.
+    yield frames[-1]

requirements.txt

@@ -0,0 +1,4 @@
+tensorflow>=2.15.0
+requests>=2.31.0
+numpy>=1.23.5
+mediapy>=1.2.0

utils.py

@@ -0,0 +1,20 @@
+import numpy as np
+import requests
+import tensorflow as tf
+
+_UINT8_MAX_F = float(np.iinfo(np.uint8).max)
+
+
+def load_image(img_url: str):
+    """Returns an image with shape [height, width, num_channels], with pixels in [0..1] range, and type np.float32."""
+
+    if img_url.startswith("https"):
+        user_agent = {'User-agent': 'Colab Sample (https://tensorflow.org)'}
+        response = requests.get(img_url, headers=user_agent)
+        image_data = response.content
+    else:
+        image_data = tf.io.read_file(img_url)
+
+    image = tf.io.decode_image(image_data, channels=3)
+    image_numpy = tf.cast(image, dtype=tf.float32).numpy()
+    return image_numpy / _UINT8_MAX_F