good progress on the AI Stories Factory

package-lock.json

@@ -50,6 +50,7 @@
         "@upstash/redis": "^1.28.3",
         "alchemy-sdk": "^3.2.1",
         "autoprefixer": "10.4.19",
+        "axios": "^1.6.8",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.0",
         "cmdk": "^1.0.0",

package.json

@@ -51,6 +51,7 @@
     "@upstash/redis": "^1.28.3",
     "alchemy-sdk": "^3.2.1",
     "autoprefixer": "10.4.19",
+    "axios": "^1.6.8",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "cmdk": "^1.0.0",

src/app/api/generate/storyboards/generateStoryboard.ts

@@ -0,0 +1,61 @@
+
+import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
+import { generateSeed } from "@/lib/utils/generateSeed"
+import { sleep } from "@/lib/utils/sleep"
+import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
+import { getValidNumber } from "@/lib/utils/getValidNumber"
+
+export async function generateStoryboard({
+  prompt,
+  // negativePrompt,
+  width,
+  height,
+  seed,
+}: {
+  prompt: string
+  // negativePrompt?: string
+  width?: number
+  height?: number
+  seed?: number
+}) {
+  
+  width = getValidNumber(width, 256, 8192, 512)
+  height = getValidNumber(height, 256, 8192, 288)
+
+  // console.log("calling await newRender")
+  prompt = getPositivePrompt(prompt)
+  const negativePrompt = getNegativePrompt()
+
+  let render = await newRender({
+    prompt,
+    negativePrompt,
+    nbFrames: 1,
+    nbFPS: 1,
+    nbSteps: 8,
+    width,
+    height,
+    turbo: true,
+    shouldRenewCache: true,
+    seed: seed || generateSeed()
+  })
+
+  let attempts = 10
+
+  while (attempts-- > 0) {
+    if (render.status === "completed") {
+      return render.assetUrl
+    }
+
+    if (render.status === "error") {
+      console.error(render.error)
+      throw new Error(`failed to generate the image ${render.error}`)
+    }
+
+    await sleep(2000) // minimum wait time
+
+    // console.log("asking getRender")
+    render = await getRender(render.renderId)
+  }
+
+  throw new Error(`failed to generate the image`)
+}

src/app/api/generate/storyboards/route.ts

@@ -5,8 +5,11 @@ import { parseClap } from "@/lib/clap/parseClap"
 import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
 import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
 import { newSegment } from "@/lib/clap/newSegment"
-import { generateImage } from "@/components/interface/latent-engine/resolvers/image/generateImage"
+import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { getToken } from "@/app/api/auth/getToken"
+import { generateSeed } from "@/lib/utils/generateSeed"
+import { getPositivePrompt } from "../../utils/imagePrompts"
+import { generateStoryboard } from "./generateStoryboard"
 
 // a helper to generate storyboards for a Clap
 // this is mostly used by external apps such as the Stories Factory
@@ -25,7 +28,7 @@ export async function POST(req: NextRequest) {
 
   if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
   
-  console.log(`[api/generate/storyboards] detected ${clap.segments} segments`)
+  console.log(`[api/generate/storyboards] detected ${clap.segments.length} segments`)
   
   const shotsSegments = clap.segments.filter(s => s.category === "camera")
   console.log(`[api/generate/storyboards] detected ${shotsSegments.length} shots`)
@@ -73,16 +76,18 @@ export async function POST(req: NextRequest) {
     // TASK 3: GENERATE MISSING STORYBOARD BITMAP
     if (!shotStoryboardSegment.assetUrl) {
       console.log(`[api/generate/storyboards] generating image..`)
-      // note this will do a fetch to AiTube API
-      // which is a bit weird since we are already inside the API, but it works
-      //TODO Julian: maybe we could use an internal function call instead?
-      shotStoryboardSegment.assetUrl = await generateImage({
-        prompt: shotStoryboardSegment.prompt,
-         width: clap.meta.width,
-        height: clap.meta.height,
-        token: jwtToken,
-      })
 
+      try {
+        shotStoryboardSegment.assetUrl = await generateStoryboard({
+          prompt: getPositivePrompt(shotStoryboardSegment.prompt),
+          width: clap.meta.width,
+          height: clap.meta.height,
+        })
+      } catch (err) {
+        console.log(`[api/generate/storyboards] failed to generate an image: ${err}`)
+        throw err
+      }
+   
       console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
     } else {
       console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)

src/components/interface/latent-engine/resolvers/image/generateImage.ts

@@ -1,15 +1,23 @@
 import { aitubeApiUrl } from "../../core/config"
 
+// return an image
+// note that this function can work either on the server-side or the client-side
 export async function generateImage({
   prompt,
   width,
   height,
   token,
+  mode = "data-uri"
 }: {
   prompt: string
   width: number
   height: number
   token: string
+
+  // data-uri are good for small files as they contain all the data, they can be sent over the network,
+  // object-uri are good for large files but can't be sent over the network (they are just a ID)
+  // this also means that object-uri cannot be used on the server-side
+  mode?: "data-uri" | "object-uri"
 }): Promise<string> {
   const requestUri = `${aitubeApiUrl}/api/resolvers/image?t=${
     token
@@ -22,7 +30,28 @@ export async function generateImage({
     encodeURIComponent(prompt)
   }`
   const res = await fetch(requestUri)
-  const blob = await res.blob()
-  const url = URL.createObjectURL(blob)
-  return url
+
+  // will only work on the server-side
+  if (mode === "object-uri") {
+    const blob = await res.blob()
+
+    return URL.createObjectURL(blob)
+  } else {
+    if (typeof window !== "undefined") {
+      const blob = await res.blob()
+
+      // on browser-side
+      const dataURL = await new Promise<string>((resolve, reject) => {
+        var a = new FileReader()
+        a.onload = function(e) { resolve(`${e.target?.result || ""}`) }
+        a.readAsDataURL(blob)
+      })
+      return dataURL 
+    } else {
+      // NodeJS side
+      const buffer = await (res as any).buffer()
+      const contentType = res.headers.get("Content-Type")
+      return "data:" + contentType + ';base64,' + buffer.toString('base64')
+    }
+  }
 }

src/components/interface/latent-engine/resolvers/video/generateVideo.ts

@@ -5,11 +5,17 @@ export async function generateVideo({
   width,
   height,
   token,
+  mode = "data-uri",
 }: {
   prompt: string
   width: number
   height: number
   token: string
+
+  // data-uri are good for small files as they contain all the data, they can be sent over the network,
+  // object-uri are good for large files but can't be sent over the network (they are just a ID)
+  // this also means that object-uri cannot be used on the server-side
+  mode?: "data-uri" | "object-uri"
 }): Promise<string> {
   const requestUri = `${aitubeApiUrl}/api/resolvers/video?t=${
     token
@@ -22,6 +28,24 @@ export async function generateVideo({
   }`
   const res = await fetch(requestUri)
   const blob = await res.blob()
-  const url = URL.createObjectURL(blob)
-  return url
+
+  // will only work on the server-side
+  if (mode === "object-uri") {
+    return URL.createObjectURL(blob)
+  } else {
+    if (typeof window !== "undefined") {
+      // on browser-side
+      const dataURL = await new Promise<string>((resolve, reject) => {
+        var a = new FileReader()
+        a.onload = function(e) { resolve(`${e.target?.result || ""}`) }
+        a.readAsDataURL(blob)
+      })
+      return dataURL 
+    } else {
+      // NodeJS side
+      const contentType = res.headers.get("Content-Type")
+      const buffer = await (res as any).buffer() as Buffer
+      return "data:" + contentType + ';base64,' + buffer.toString('base64')
+    }
+  }
 }

src/components/interface/latent-engine/resolvers/video/index.tsx

@@ -20,6 +20,7 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
       width: clap.meta.width,
       height: clap.meta.height,
       token: useStore.getState().jwtToken,
+      mode: "object-uri" // it's better for videos withing Chrome, apparently
     })
     // console.log(`resolveVideo: generated ${assetUrl}`)
 

src/components/interface/latent-engine/resolvers/video/index_legacy.tsx

@@ -53,6 +53,7 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
         width: clap.meta.width,
         height: clap.meta.height,
         token: useStore.getState().jwtToken,
+        mode: "object-uri" // it's better for videos withing Chrome, apparently
       }))
     }
 

src/components/interface/latent-engine/resolvers/video/index_notSoGood.tsx

@@ -53,6 +53,7 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
         width: clap.meta.width,
         height: clap.meta.height,
         token: useStore.getState().jwtToken,
+        mode: "object-uri" // it's better for videos withing Chrome, apparently
       }))
     }