use @aitube/engine

package-lock.json

@@ -10,6 +10,7 @@
       "dependencies": {
         "@aitube/clap": "0.0.7",
         "@aitube/client": "0.0.7",
+        "@aitube/engine": "^0.0.0",
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.6.7",
         "@jcoreio/async-throttle": "^1.6.0",
@@ -139,6 +140,19 @@
         "typescript": "^5.4.5"
       }
     },
+    "node_modules/@aitube/engine": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.0.tgz",
+      "integrity": "sha512-3yOAXXCUf6pehdB2t1Nt/F4CC2biC9LA3LZK1d7PwmEA4cahP5q5sb2P1633mSrX7ElJEFkXIsKdEz6hltIP2Q==",
+      "dependencies": {
+        "uuid": "^9.0.1",
+        "yaml": "^2.4.1"
+      },
+      "peerDependencies": {
+        "@aitube/clap": "0.0.7",
+        "typescript": "^5.4.5"
+      }
+    },
     "node_modules/@alloc/quick-lru": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",

package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@aitube/clap": "0.0.7",
     "@aitube/client": "0.0.7",
+    "@aitube/engine": "^0.0.0",
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.6.7",
     "@jcoreio/async-throttle": "^1.6.0",

src/app/api/v1/README.md

@@ -12,7 +12,7 @@ This endpoint will generate a .clap (story only) from a prompt
 
 ## /edit
 
-This endpoint will edit a .clap to change models or storyboards
+This endpoint will edit a .clap to change entities, storyboards, videos etc
 
 ## /export
 

src/app/api/v1/edit/dialogues/route.ts

@@ -1,13 +1,11 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { ClapEntity, ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
+import { ClapProject, ClapSegment, getClapAssetSourceType, parseClap, serializeClap } from "@aitube/clap"
 
 import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
 import { getToken } from "@/app/api/auth/getToken"
-
-import { getSpeechBackgroundAudioPrompt } from "@/components/interface/latent-engine/core/prompts/getSpeechBackgroundAudioPrompt"
-import { getSpeechForegroundAudioPrompt } from "@/components/interface/latent-engine/core/prompts/getSpeechForegroundAudioPrompt"
 import { generateSpeechWithParlerTTS } from "@/app/api/generators/speech/generateVoiceWithParlerTTS"
+import { getSpeechBackgroundAudioPrompt } from "@aitube/engine"
 
 // a helper to generate speech for a Clap
 export async function POST(req: NextRequest) {

src/app/api/v1/edit/storyboards/generateStoryboard.ts

@@ -16,7 +16,7 @@ export async function generateStoryboard({
   width?: number
   height?: number
   seed?: number
-}) {
+}): Promise<string> {
   
   width = getValidNumber(width, 256, 8192, 512)
   height = getValidNumber(height, 256, 8192, 288)

src/app/api/v1/edit/storyboards/route.ts

@@ -1,9 +1,9 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { ClapProject, ClapSegment, newSegment, parseClap, serializeClap } from "@aitube/clap"
+import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
+import { getVideoPrompt } from "@aitube/engine"
 
 import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
-import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
 import { getToken } from "@/app/api/auth/getToken"
 
 import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
@@ -81,6 +81,7 @@ export async function POST(req: NextRequest) {
           width: clap.meta.width,
           height: clap.meta.height,
         })
+        shotStoryboardSegment.assetSourceType = getClapAssetSourceType(shotStoryboardSegment.assetUrl)
       } catch (err) {
         console.log(`[api/generate/storyboards] failed to generate an image: ${err}`)
         throw err

src/app/api/v1/edit/videos/generateVideo.ts

@@ -0,0 +1,61 @@
+import { generateSeed, getValidNumber } from "@aitube/clap"
+
+import { newRender, getRender } from "@/app/api/providers/videochain/renderWithVideoChain"
+import { sleep } from "@/lib/utils/sleep"
+import { getNegativePrompt, getPositivePrompt } from "@/app/api/utils/imagePrompts"
+
+export async function generateVideo({
+  prompt,
+  // negativePrompt,
+  width,
+  height,
+  seed,
+}: {
+  prompt: string
+  // negativePrompt?: string
+  width?: number
+  height?: number
+  seed?: number
+}): Promise<string> {
+  
+  // we want to keep it vertical
+  width = getValidNumber(width, 256, 8192, 288)
+  height = getValidNumber(height, 256, 8192, 512)
+
+  // console.log("calling await newRender")
+  prompt = getPositivePrompt(prompt)
+  const negativePrompt = getNegativePrompt()
+
+  let render = await newRender({
+    prompt,
+    negativePrompt,
+    nbFrames: 80,
+    nbFPS: 24,
+    nbSteps: 4,
+    width,
+    height,
+    turbo: true,
+    shouldRenewCache: true,
+    seed: seed || generateSeed()
+  })
+
+  let attempts = 10
+
+  while (attempts-- > 0) {
+    if (render.status === "completed") {
+      return render.assetUrl
+    }
+
+    if (render.status === "error") {
+      console.error(render.error)
+      throw new Error(`failed to generate the video file ${render.error}`)
+    }
+
+    await sleep(2000) // minimum wait time
+
+    // console.log("asking getRender")
+    render = await getRender(render.renderId)
+  }
+
+  throw new Error(`failed to generate the video file`)
+}

src/app/api/v1/edit/videos/route.ts

@@ -0,0 +1,103 @@
+import { NextResponse, NextRequest } from "next/server"
+
+import { ClapProject, ClapSegment, getClapAssetSourceType, newSegment, parseClap, serializeClap } from "@aitube/clap"
+import { getVideoPrompt } from "@aitube/engine"
+
+import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
+import { getToken } from "@/app/api/auth/getToken"
+import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
+
+import { generateVideo } from "./generateVideo"
+
+
+// a helper to generate videos for a Clap
+// this is mostly used by external apps such as the Stories Factory
+// this function will:
+//
+// - add missing videos to the shots
+// - add missing video prompts
+// - add missing video files
+export async function POST(req: NextRequest) {
+
+  const jwtToken = await getToken({ user: "anonymous" })
+
+  const blob = await req.blob()
+
+  const clap: ClapProject = await parseClap(blob)
+
+  if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
+  
+  console.log(`[api/generate/videos] detected ${clap.segments.length} segments`)
+  
+  const shotsSegments: ClapSegment[] = clap.segments.filter(s => s.category === "camera")
+  console.log(`[api/generate/videos] detected ${shotsSegments.length} shots`)
+  
+  if (shotsSegments.length > 32) {
+    throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
+  }
+
+  for (const shotSegment of shotsSegments) {
+
+    const shotSegments: ClapSegment[] = clap.segments.filter(s =>
+      startOfSegment1IsWithinSegment2(s, shotSegment)
+    )
+
+    const shotVideoSegments: ClapSegment[] = shotSegments.filter(s =>
+      s.category === "video"
+    )
+
+    let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
+    
+    console.log(`[api/generate/videos] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
+  
+    // TASK 1: GENERATE MISSING VIDEO SEGMENT
+    if (!shotVideoSegment) {
+      shotVideoSegment = newSegment({
+        track: 1,
+        startTimeInMs: shotSegment.startTimeInMs,
+        endTimeInMs: shotSegment.endTimeInMs,
+        assetDurationInMs: shotSegment.assetDurationInMs,
+        category: "video",
+        prompt: "",
+        assetUrl: "",
+        outputType: "video"
+      })
+      console.log(`[api/generate/videos] generated video segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
+    }
+
+    // TASK 2: GENERATE MISSING VIDEO PROMPT
+    if (shotVideoSegment && !shotVideoSegment?.prompt) {
+      // video is missing, let's generate it
+      shotVideoSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
+      console.log(`[api/generate/videos] generating video prompt: ${shotVideoSegment.prompt}`)
+    }
+
+    // TASK 3: GENERATE MISSING VIDEO FILE
+    if (shotVideoSegment && !shotVideoSegment.assetUrl) {
+      console.log(`[api/generate/videos] generating video file..`)
+
+      try {
+        shotVideoSegment.assetUrl = await generateVideo({
+          prompt: getPositivePrompt(shotVideoSegment.prompt),
+          width: clap.meta.width,
+          height: clap.meta.height,
+        })
+        shotVideoSegment.assetSourceType = getClapAssetSourceType(shotVideoSegment.assetUrl)
+      } catch (err) {
+        console.log(`[api/generate/videos] failed to generate a video file: ${err}`)
+        throw err
+      }
+   
+      console.log(`[api/generate/videos] generated video files: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
+    } else {
+      console.log(`[api/generate/videos] there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
+    }
+  }
+
+  console.log(`[api/generate/videos] returning the clap augmented with videos`)
+
+  return new NextResponse(await serializeClap(clap), {
+    status: 200,
+    headers: new Headers({ "content-type": "application/x-gzip" }),
+  })
+}

src/components/interface/latent-engine/core/prompts/getCharacterPrompt.ts

@@ -1,26 +0,0 @@
-import { ClapEntity } from "@aitube/clap"
-
-export function getCharacterPrompt(entity: ClapEntity): string {
-
-  let characterPrompt = ""
-  if (entity.description) {
-    characterPrompt = [
-      // the label (character name) can help making the prompt more unique
-      // this might backfires however, if the name is
-      // something like "SUN", "SILVER" etc
-      // I'm not sure stable diffusion really needs this,
-      // so let's skip it for now (might still be useful for locations, though)
-      // we also want to avoid triggering "famous people" (BARBOSSA etc)
-      // entity.label,
-
-      entity.description
-    ].join(", ")
-  } else {
-    characterPrompt = [
-      entity.gender !== "object" ? entity.gender : "",
-      entity.age ? `aged ${entity.age}yo` : '',
-      entity.label ? `named ${entity.label}` : '',
-    ].map(i => i.trim()).filter(i => i).join(", ")
-  }
-  return characterPrompt
-}

src/components/interface/latent-engine/core/prompts/getCharacterReferencePrompt.ts

@@ -1,29 +0,0 @@
-import { ClapEntity } from "@aitube/clap";
-
-import { getCharacterPrompt } from "./getCharacterPrompt";
-
-/**
- * Return a prompt for a "formal" picture, centered, neutral etc
- * @param entity 
- * @returns 
- */
-export function getCharacterReferencePrompt(entity: ClapEntity) {
-  const characterPrompt = [
-    `beautiful`,
-    `close-up`,
-    `photo portrait`,
-    `id photo`,
-    getCharacterPrompt(entity),
-    `neutral expression`,
-    `neutral background`,
-    `frontal`,
-    `photo studio`,
-    `crisp`,
-    `sharp`,
-    `intricate details`,
-    `centered`,
-    // `aligned`
-  ].map(i => i.trim()).filter(i => i).join(", ")
-  
-  return characterPrompt
-}

src/components/interface/latent-engine/core/prompts/getSpeechBackgroundAudioPrompt.ts

@@ -1,52 +0,0 @@
-import { ClapEntity, ClapSegment } from "@aitube/clap"
-
-import { getCharacterPrompt } from "./getCharacterPrompt"
-
-/**
- * Construct an audio background for a voice from a list of active segments
- * 
- * @param segments 
- * @returns 
- */
-export function getSpeechBackgroundAudioPrompt(
-  segments: ClapSegment[] = [],
-  entitiesById: Record<string, ClapEntity> = {},
-  extraPositivePrompt: string[] = [] // "clear sound, high quality" etc
-): string {
-  return segments
-    .filter(({ category, outputType }) => (
-      category === "dialogue" ||
-      category === "weather" ||
-      category === "location"
-    ))
-    .sort((a, b) => b.label.localeCompare(a.label))
-    .map(segment => {
-      const entity: ClapEntity | undefined = entitiesById[segment?.entityId || ""] || undefined
-      
-      if (segment.category === "dialogue") {
-        // if we can't find the entity then we are unable
-        // to make any assumption about the gender, age and voice timbre
-        if (!entity) {
-          return `person, speaking normally`
-        }
-
-        const characterPrompt = getCharacterPrompt(entity)
-
-        return `${characterPrompt}, speaking normally`
-        
-      } else if (segment.category === "location") {
-        // the location is part of the background noise
-        // but this might produce unexpected results - we'll see!
-        return segment.prompt
-      } else if (segment.category === "weather") {
-        // the weather is part of the background noise
-        // here too this might produce weird and unexpected results 🍿
-        return segment.prompt
-      }
-      // ignore the rest
-      return ""
-    })
-    .filter(x => x)
-    .concat([ ...extraPositivePrompt ])
-    .join(". ")
-}

src/components/interface/latent-engine/core/prompts/getSpeechForegroundAudioPrompt.ts

@@ -1,20 +0,0 @@
-import {  ClapSegment } from "@aitube/clap"
-
-/**
- * Construct an audio foreground for a voice from a list of active segments
- * 
- * This is the "dialogue" prompt, ie. the actual spoken words,
- * so we don't need to do anything fancy here, we only use the raw text
- * 
- * @param segments 
- * @returns 
- */
-export function getSpeechForegroundAudioPrompt(
-  segments: ClapSegment[] = []
-): string {
-  return segments
-    .filter(({ category }) => category === "dialogue")
-    .sort((a, b) => b.label.localeCompare(a.label))
-    .map(({ prompt }) => prompt).filter(x => x)
-    .join(". ")
-}

src/components/interface/latent-engine/core/prompts/getVideoPrompt.ts

@@ -1,90 +0,0 @@
-import { ClapEntity, ClapSegment } from "@aitube/clap"
-
-import { deduplicatePrompt } from "../../utils/prompting/deduplicatePrompt"
-
-import { getCharacterPrompt } from "./getCharacterPrompt"
-
-/**
- * Construct a video prompt from a list of active segments
- * 
- * @param segments 
- * @returns 
- */
-export function getVideoPrompt(
-  segments: ClapSegment[] = [],
-  entitiesIndex: Record<string, ClapEntity> = {},
-  extraPositivePrompt: string[] = []
-): string {
-
-  // console.log("entitiesIndex:", entitiesIndex)
-
-  // to construct the video we need to collect all the segments describing it
-  // we ignore unrelated categories (music, dialogue) or non-prompt items (eg. an audio sample)
-  const tmp = segments
-    .filter(({ category, outputType }) => {
-      if (outputType === "audio") {
-        return false
-      }
-
-      if (
-        category === "character" ||
-        category === "location" ||
-        category === "time" ||
-        category === "era" ||
-        category === "lighting" ||
-        category === "weather" ||
-        category === "action" ||
-        category === "style" ||
-        category === "camera" ||
-        category === "generic"
-      ) {
-        return true
-      }
-      return false
-    })
-
-  tmp.sort((a, b) => b.label.localeCompare(a.label))
-
-  let videoPrompt = tmp.map(segment => {
-    const entity: ClapEntity | undefined = entitiesIndex[segment?.entityId || ""] || undefined
-    
-    if (segment.category === "dialogue") {
-
-      // if we can't find the entity, then we are unable
-      // to make any assumption about the gender, age or appearance
-      if (!entity) {
-        console.log("ERROR: this is a dialogue, but couldn't find the entity!")
-        return `portrait of a person speaking, blurry background, bokeh`
-      }
-
-      const characterTrigger = entity?.triggerName || ""
-      const characterLabel = entity?.label || ""
-      const characterDescription = entity?.description || ""
-      const dialogueLine = segment?.prompt || ""
-     
-      const characterPrompt = getCharacterPrompt(entity)
-
-      // in the context of a video, we some something additional:
-      // we create a "bokeh" style
-      return `portrait of a person speaking, blurry background, bokeh, ${characterPrompt}`
-      
-    } else if (segment.category === "location") {
-  
-      // if we can't find the location's entity, we default to returning the prompt
-      if (!entity) {
-        console.log("ERROR: this is a location, but couldn't find the entity!")
-        return segment.prompt
-      }
-
-      return entity.description
-    } else {
-      return segment.prompt
-    }
-  }).filter(x => x)
-
-  videoPrompt = videoPrompt.concat([
-    ...extraPositivePrompt
-  ])
-
-  return deduplicatePrompt(videoPrompt.join(", "))
-}

src/components/interface/latent-engine/core/useLatentEngine.ts

@@ -1,7 +1,8 @@
 
 import { create } from "zustand"
 
-import { ClapEntity, ClapProject, ClapSegment, newClap, parseClap } from "@aitube/clap"
+import { ClapProject, ClapSegment, newClap, parseClap } from "@aitube/clap"
+import { getVideoPrompt } from "@aitube/engine"
 
 import { LatentEngineStore } from "./types"
 import { resolveSegments } from "../resolvers/resolveSegments"
@@ -10,13 +11,10 @@ import { fetchLatentClap } from "./generators/fetchLatentClap"
 import { InteractiveSegmenterResult, MPMask } from "@mediapipe/tasks-vision"
 import { segmentFrame } from "@/lib/on-device-ai/segmentFrameOnClick"
 import { drawSegmentation } from "../utils/canvas/drawSegmentation"
-import { filterImage } from "@/lib/on-device-ai/filterImage"
 import { getZIndexDepth } from "../utils/data/getZIndexDepth"
 import { getSegmentStartAt } from "../utils/data/getSegmentStartAt"
-import { getSegmentId } from "../utils/data/getSegmentId"
 import { getElementsSortedByStartAt } from "../utils/data/getElementsSortedByStartAt"
 import { getSegmentEndAt } from "../utils/data/getSegmentEndAt"
-import { getVideoPrompt } from "./prompts/getVideoPrompt"
 import { setZIndexDepthId } from "../utils/data/setZIndexDepth"
 import { setSegmentStartAt } from "../utils/data/setSegmentStartAt"
 import { setSegmentEndAt } from "../utils/data/setSegmentEndAt"