switching to a cheaper interpolation engine

.env

@@ -1,13 +1,18 @@
 AUTH_REPLICATE_API_TOKEN="<USE YOUR OWN>"
 AUTH_HOTSHOT_XL_API_GRADIO_ACCESS_TOKEN="<USE YOUR OWN>"
+AUTH_INTERPOLATION_API_GRADIO_TOKEN="<USE YOUR OWN>"
 
-# TODO: those could be
+# TODO: support multiple backends
 # VIDEO_HOTSHOT_XL_API_OFFICIAL
-# VIDEO_HOTSHOT_XL_API_NODE
 # VIDEO_HOTSHOT_XL_API_GRADIO
 # VIDEO_HOTSHOT_XL_API_REPLICATE
 VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_GRADIO"
 
+# TODO: support multiple backends
+# STMFNET_REPLICATE
+# FILM_GRADIO
+INTERPOLATION_ENGINE="STMFNET_REPLICATE"
+
 # the official API developed by the Hotshot-XL team
 # note: it isn't released yet
 VIDEO_HOTSHOT_XL_API_OFFICIAL=""
@@ -25,9 +30,13 @@ VIDEO_HOTSHOT_XL_API_GRADIO="https://jbilcke-hf-hotshot-xl-server-1.hf.space"
 VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL="cloneofsimo/hotshot-xl-lora-controlnet"
 VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION="75e26ffd033a59a78954a3d675632f47f7f8470402aec51c255b9f9b7b62568b"
 
+# ----------- INTERPOLATION ------
+
 INTERPOLATION_API_REPLICATE_MODEL="zsxkib/st-mfnet"
 INTERPOLATION_API_REPLICATE_MODEL_VERSION="faa7693430b0a4ac95d1b8e25165673c1d7a7263537a7c4bb9be82a3e2d130fb"
 
+INTERPOLATION_API_GRADIO_URL="https://jbilcke-hf-video-interpolation-server.hf.space"
+
 # ----------- RATE LIMIT -------
 ENABLE_RATE_LIMIT=""
 UPSTASH_REDIS_REST_URL="<USE YOUR OWN>"

src/app/interface/auth-dialog/index.tsx

@@ -0,0 +1,6 @@
+export function AuthDialog() {
+  return (
+    <>
+    </>
+  )
+}

src/app/interface/generate/index.tsx

@@ -9,12 +9,13 @@ import { cn } from "@/lib/utils"
 import { headingFont } from "@/app/interface/fonts"
 import { useCharacterLimit } from "@/lib/useCharacterLimit"
 import { generateAnimation } from "@/app/server/actions/animation"
+import { interpolateVideo } from "@/app/server/actions/interpolation"
 import { getLatestPosts, getPost, postToCommunity } from "@/app/server/actions/community"
 import { getSDXLModels } from "@/app/server/actions/models"
 import { HotshotImageInferenceSize, Post, QualityLevel, QualityOption, SDXLModel } from "@/types"
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
 import { TooltipProvider } from "@radix-ui/react-tooltip"
-import { interpolate } from "@/app/server/actions/interpolate"
+
 import { isRateLimitError } from "@/app/server/utils/isRateLimitError"
 import { useCountdown } from "@/lib/useCountdown"
 
@@ -212,7 +213,7 @@ export function Generate() {
       setRuns(runsRef.current + 1)
 
       try {
-        assetUrl = await interpolate(rawAssetUrl)
+        assetUrl = await interpolateVideo(rawAssetUrl)
 
         if (!assetUrl) {
           throw new Error("invalid interpolated asset url")

src/app/server/actions/animation.ts

@@ -5,8 +5,8 @@ import {Redis} from "@upstash/redis"
 
 import { VideoOptions } from "@/types"
 
-import { generateVideoWithGradioAPI } from "./generateWithGradioApi"
-import { generateVideoWithReplicateAPI } from "./generateWithReplicateAPI"
+import { generateGradio } from "./generateGradio"
+import { generateReplicate } from "./generateReplicate"
 import { filterOutBadWords } from "./censorship"
 
 const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
@@ -102,7 +102,7 @@ export async function generateAnimation({
   try {
 
     if (videoEngine === "VIDEO_HOTSHOT_XL_API_REPLICATE") {
-      return generateVideoWithReplicateAPI({
+      return generateReplicate({
         positivePrompt,
         negativePrompt,
         size,
@@ -144,7 +144,7 @@ export async function generateAnimation({
 
       return content
     } else if (videoEngine === "VIDEO_HOTSHOT_XL_API_GRADIO") {
-      return generateVideoWithGradioAPI({
+      return generateGradio({
         positivePrompt,
         negativePrompt,
         size,

src/app/server/actions/{generateWithGradioApi.ts → generateGradio.ts}

@@ -4,7 +4,7 @@ import { VideoOptions } from "@/types"
 const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
 const accessToken = `${process.env.AUTH_HOTSHOT_XL_API_GRADIO_ACCESS_TOKEN || ""}`
 
-export async function generateVideoWithGradioAPI({
+export async function generateGradio({
   positivePrompt = "",
   negativePrompt = "",
   size = "512x512",

src/app/server/actions/{generateWithReplicateAPI.ts → generateReplicate.ts}

@@ -8,7 +8,7 @@ const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
 const replicateModel = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL || ""}`
 const replicateModelVersion = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION || ""}`
 
-export async function generateVideoWithReplicateAPI({
+export async function generateReplicate({
   positivePrompt = "",
   negativePrompt = "",
   size = "512x512",

src/app/server/actions/interpolateGradio.ts

@@ -0,0 +1,41 @@
+
+const gradioApi = `${process.env.INTERPOLATION_API_GRADIO_URL || ""}`
+const accessToken = `${process.env.AUTH_INTERPOLATION_API_GRADIO_TOKEN || ""}`
+
+export async function interpolateGradio(assetUrl: string): Promise<string> {
+  // we need to remove this header perhaps
+  const videoInBase64 = assetUrl.split("data:video/mp4;base64,").pop()
+
+  const interpolationSteps = 2
+  const nbFramesPerSecond = 32
+
+  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      // Authorization: `Bearer ${token}`,
+    },
+    body: JSON.stringify({
+      fn_index: 0, // <- important!
+      data: [
+        accessToken,
+        videoInBase64,
+        interpolationSteps,
+        nbFramesPerSecond
+      ],
+    }),
+    cache: "no-store",
+    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+    // next: { revalidate: 1 }
+  })
+
+  const { data } = await res.json()
+
+  if (res.status !== 200 || !data[0]?.length) {
+    // This will activate the closest `error.js` Error Boundary
+    throw new Error(`Failed to fetch data (status: ${res.status})`)
+  }
+
+  return data[0]
+}
+

src/app/server/actions/{interpolate.ts → interpolateReplicate.ts}

@@ -1,5 +1,3 @@
-"use server"
-
 import Replicate from "replicate"
 
 import { sleep } from "@/lib/sleep"
@@ -8,7 +6,7 @@ const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
 const replicateModel = `${process.env.INTERPOLATION_API_REPLICATE_MODEL || ""}`
 const replicateModelVersion = `${process.env.INTERPOLATION_API_REPLICATE_MODEL_VERSION || ""}`
 
-export async function interpolate(input: string): Promise<string> {
+export async function interpolateReplicate(input: string): Promise<string> {
   if (!replicateToken) {
     throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use interpolation`)
   }

src/app/server/actions/interpolation.ts

@@ -0,0 +1,25 @@
+"use server"
+
+import { interpolateGradio } from "./interpolateGradio"
+import { interpolateReplicate } from "./interpolateReplicate"
+
+const interpolationEngine = `${process.env.INTERPOLATION_ENGINE || ""}`
+
+export async function interpolateVideo(inputVideo: string): Promise<string> {
+  if (!inputVideo?.length) {
+    throw new Error(`missing input video`)
+  }
+
+  try {
+
+    if (interpolationEngine === "STMFNET_REPLICATE") {
+      return interpolateReplicate(inputVideo)
+    } else if (interpolationEngine === "FILM_GRADIO") {
+      return interpolateGradio(inputVideo)
+    } else {
+      throw new Error(`unsupported interpolation engine "${interpolationEngine}"`)
+    }
+  } catch (err) {
+    throw new Error(`failed to interpolate the video ${err}`)
+  }
+}

src/app/server/actions/models.ts

@@ -22,6 +22,26 @@ export async function getSDXLModels(): Promise<SDXLModel[]> {
   const compatibleModels = content.filter(model => model.is_compatible)
 
   const hardcoded: SDXLModel[] = [
+    {
+      "image": "https://jbilcke-hf-ai-clip-factory.hf.space/images/models/sdxl-starfield.jpg",
+      "title": "sdxl-starfield",
+      "repo": "jbilcke-hf/sdxl-starfield",
+      "trigger_word": "starfield-style",
+      "weights": "pytorch_lora_weights.safetensors",
+      "is_compatible": true,
+      "likes": 0,
+      "downloads": 0
+    },
+    {
+      "image": "https://jbilcke-hf-ai-clip-factory.hf.space/images/models/sdxl-akira.jpg",
+      "title": "sdxl-akira",
+      "repo": "jbilcke-hf/sdxl-akira",
+      "trigger_word": "akira-style",
+      "weights": "pytorch_lora_weights.safetensors",
+      "is_compatible": true,
+      "likes": 0,
+      "downloads": 0
+    },
     {
       "image": "https://jbilcke-hf-ai-clip-factory.hf.space/images/models/sdxl-cyberpunk-2077.jpg",
       "title": "sdxl-cyberpunk-2077",

src/components/ui/dialog.tsx

@@ -11,10 +11,9 @@ const Dialog = DialogPrimitive.Root
 const DialogTrigger = DialogPrimitive.Trigger
 
 const DialogPortal = ({
-  className,
   ...props
 }: DialogPrimitive.DialogPortalProps) => (
-  <DialogPrimitive.Portal className={cn(className)} {...props} />
+  <DialogPrimitive.Portal {...props} />
 )
 DialogPortal.displayName = DialogPrimitive.Portal.displayName