aitube client 0.0.19

package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.0",
       "dependencies": {
         "@aitube/clap": "0.0.14",
-        "@aitube/client": "0.0.17",
+        "@aitube/client": "0.0.19",
         "@aitube/engine": "0.0.4",
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.6.7",
@@ -130,9 +130,9 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.17",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.17.tgz",
-      "integrity": "sha512-waRA1k2pqKI7uOXUnBs6y056JY2h7LO+kzKDcHBiNSAyC0ZvSvP7VqTia2fxpF99rik6HgmC3N0AslObP4T6Zw==",
+      "version": "0.0.19",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.19.tgz",
+      "integrity": "sha512-SKsctZEHRmS3Z63jOUnEp7pGNbmDKyGDTgnoS/GCVciLlWknSC6ht9NJasgEuje06kfgF2Uaj/hWC57JHfoe3A==",
       "dependencies": {
         "query-string": "^9.0.0"
       },
@@ -6709,9 +6709,9 @@
       }
     },
     "node_modules/openai/node_modules/@types/node": {
-      "version": "18.19.32",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.32.tgz",
-      "integrity": "sha512-2bkg93YBSDKk8DLmmHnmj/Rwr18TLx7/n+I23BigFwgexUJoMHZOd8X1OFxuF/W3NN0S2W2E5sVabI5CPinNvA==",
+      "version": "18.19.33",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.33.tgz",
+      "integrity": "sha512-NR9+KrpSajr2qBVp/Yt5TU/rp+b5Mayi3+OlMlcg2cVCfRmcG5PWZ7S4+MG9PZ5gWBoc9Pd0BKSRViuBCRPu0A==",
       "dependencies": {
         "undici-types": "~5.26.4"
       }

package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@aitube/clap": "0.0.14",
-    "@aitube/client": "0.0.17",
+    "@aitube/client": "0.0.19",
     "@aitube/engine": "0.0.4",
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.6.7",

src/app/api/parsers/parseTurbo.ts

@@ -0,0 +1,15 @@
+export function parseTurbo(
+  input?: any,
+  defaultValue: boolean = false
+): boolean {
+  let value = defaultValue
+  
+  try {
+    let maybeTurbo = decodeURIComponent(`${input || value}`).trim()
+
+    value = !!maybeTurbo
+
+  } catch (err) {}
+
+  return value
+}

src/app/api/v1/edit/dialogues/processShot.ts

@@ -16,12 +16,14 @@ export async function processShot({
   shotSegment,
   existingClap,
   newerClap,
-  mode
+  mode,
+  turbo,
 }: {
   shotSegment: ClapSegment
   existingClap: ClapProject
   newerClap: ClapProject
   mode: ClapCompletionMode
+  turbo: boolean
 }): Promise<void> {
 
   const shotSegments: ClapSegment[] = filterSegments(

src/app/api/v1/edit/dialogues/route.ts

@@ -8,6 +8,7 @@ import queryString from "query-string"
 import { parseCompletionMode } from "@/app/api/parsers/parseCompletionMode"
 import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
 import { ClapCompletionMode } from "@aitube/client"
+import { parseTurbo } from "@/app/api/parsers/parseTurbo"
 
 // a helper to generate speech for a Clap
 export async function POST(req: NextRequest) {
@@ -17,6 +18,7 @@ export async function POST(req: NextRequest) {
   const query = (qs || {}).query
   
   const mode = parseCompletionMode(query?.c)
+  const turbo = parseTurbo(query?.t)
   
   const blob = await req.blob()
 
@@ -41,7 +43,8 @@ export async function POST(req: NextRequest) {
       shotSegment,
       existingClap,
       newerClap,
-      mode
+      mode,
+      turbo,
     })
   ))
 

src/app/api/v1/edit/entities/generateImageID.ts

@@ -9,10 +9,12 @@ import { getNegativePrompt, getPositivePrompt } from "@/app/api/utils/imagePromp
 export async function generateImageID({
   prompt,
   // negativePrompt,
+  turbo = false,
   seed,
 }: {
   prompt: string
   // negativePrompt?: string
+  turbo?: boolean
   seed?: number
 }) {
   
@@ -38,8 +40,8 @@ export async function generateImageID({
     // and maybe not use the "turbo" - but I'm not sure
     width,
     height,
-    nbSteps: 8,
-    turbo: true,
+    nbSteps: turbo ? 8 : 25,
+    turbo,
 
     shouldRenewCache: true,
     seed: seed || generateSeed()

src/app/api/v1/edit/entities/index.ts

@@ -11,12 +11,14 @@ export async function editEntities({
   existingClap,
   newerClap,
   entityPrompts = [],
-  mode = ClapCompletionMode.PARTIAL
+  mode = ClapCompletionMode.PARTIAL,
+  turbo = false,
 }: {
   existingClap: ClapProject
   newerClap: ClapProject
   entityPrompts?: ClapEntityPrompt[]
   mode?: ClapCompletionMode
+  turbo?: boolean
 }) {
 
   // note that we can only handle either FULL or PARTIAL
@@ -33,7 +35,8 @@ export async function editEntities({
 
     const entityPromptsWithShots = await generateEntityPrompts({
       prompt: existingClap.meta.description,
-      latentStory: await clapToLatentStory(existingClap)
+      latentStory: await clapToLatentStory(existingClap),
+      turbo,
     })
 
     const allShots = existingClap.segments.filter(s => s.category === "camera")
@@ -137,7 +140,8 @@ export async function editEntities({
     if (!entity.imageId) {
       entity.imageId = await generateImageID({
         prompt: entity.imagePrompt,
-        seed: entity.seed
+        seed: entity.seed,
+        turbo,
       })
       entity.imageSourceType = getClapAssetSourceType(entity.imageId)
       entityHasBeenModified = true

src/app/api/v1/edit/entities/route.ts

@@ -8,6 +8,7 @@ import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
 
 import { editEntities } from "."
 import { ClapCompletionMode } from "@aitube/client"
+import { parseTurbo } from "@/app/api/parsers/parseTurbo"
 
 export async function POST(req: NextRequest) {
   console.log("Hello!")
@@ -17,6 +18,8 @@ export async function POST(req: NextRequest) {
   const query = (qs || {}).query
 
   const mode = parseCompletionMode(query?.c)
+  const turbo = parseTurbo(query?.t)
+  
   // const prompt = parsePrompt(query?.p)
 
   const entityPrompts = parseClapEntityPrompts(query?.e)
@@ -31,7 +34,8 @@ export async function POST(req: NextRequest) {
     existingClap,
     newerClap,
     entityPrompts,
-    mode
+    mode,
+    turbo,
   })
   
   console.log(`[api/edit/entities] returning the newer clap extended with the entities`)

src/app/api/v1/edit/entities/systemPrompt.ts

@@ -5,27 +5,28 @@ The video are meant to be shared on social media platform (Instagram, TikTok, Sn
 Each video is composed of a sequence of shots (a dozen in average), with a voice over and text.
 
 # Task
-You mission is to generate a list of entities/assets (characters, locations etc) associated with each shot.
+You mission is to generate a list of entities/assets associated with each shot.
 
 # Important
 
 - You MUST reply by writing/completing a YAML list of objects.
-- Copy the structure of the examples, but not their content: come up with your own original ideal, you should be creativeç
+- Copy the structure of the examples, but not their content: come up with your own original ideas. Be creative!
 
 # Output schema:
 
 name: name of the entity
-category: can be "character" or "location"
+category: ${
+  // T IS FASTER TO JUST GENERATE CHARACTERS FOR NOW
+  `can only be "character" for now`
+  // can be either "character" or "location"
+}
 image: a description of the entity (you must describe it using a Stable Diffusion prompt - about ~300 chars - using simple descriptive words and adjectives. Describe facts about characters, location, lights, texture, camera orientation, colors, clothes, movements etc. But don't give your opinion, don't talk about the emotions it evokes etc.)
 audio: a textual description of what and how the entity sounds like
 shots: an array containing the shot IDs where the entity is present
 
 # Short example
-
 Given the following inputs:
-
 "A king goes to see a witch to ask if or how he can win an upcoming and challenging battle"
-
 \`\`\`yaml
 - shot: 1
   title: "King Arthus seeks the witch's guidance to win his imminent battle."
@@ -42,13 +43,18 @@ Given the following inputs:
 \`\`\
 
 An example YAML output from the server-side function can be:
-
 \`\`\`yaml
-- name: "Castle's Courtyard"
+${
+// DISABLED: IT IS FASTER TO JUST GENERATE CHARACTERS FOR NOW
+/*
+`- name: "Castle's Courtyard"
   category: "location"
   image: "A medieval castle courtyard, ashlar walls, soldiers and horses, cloudy sky"
   audio: "Background noises of voices, horses, birds, wind, carriages"
-  shots: [1, 2, 3]
+  shots: [1, 2, 3]`
+  */
+ ''
+}
 - name: "King Arthus"
   category: "character"
   image: 1 middle-aged king, pepper-and-salt hair, beared. Dressed in golden armor and a dark purple cape. Majestic, imposing."

src/app/api/v1/edit/storyboards/generateStoryboard.ts

@@ -10,12 +10,14 @@ export async function generateStoryboard({
   width,
   height,
   seed,
+  turbo = false,
 }: {
   prompt: string
   // negativePrompt?: string
   width?: number
   height?: number
   seed?: number
+  turbo?: boolean
 }): Promise<string> {
   
   width = getValidNumber(width, 256, 8192, 512)
@@ -30,10 +32,10 @@ export async function generateStoryboard({
     negativePrompt,
     nbFrames: 1,
     nbFPS: 1,
-    nbSteps: 8,
+    nbSteps: turbo ? 8 : 25,
     width,
     height,
-    turbo: true,
+    turbo,
     shouldRenewCache: true,
     seed: seed || generateSeed()
   })

src/app/api/v1/edit/storyboards/processShot.ts

@@ -19,12 +19,14 @@ export async function processShot({
   shotSegment,
   existingClap,
   newerClap,
-  mode
+  mode,
+  turbo,
 }: {
   shotSegment: ClapSegment
   existingClap: ClapProject
   newerClap: ClapProject
   mode: ClapCompletionMode
+  turbo: boolean
 }): Promise<void> {
 
   const shotSegments: ClapSegment[] = filterSegments(
@@ -82,6 +84,7 @@ export async function processShot({
         prompt: getPositivePrompt(shotStoryboardSegment.prompt),
         width: existingClap.meta.width,
         height: existingClap.meta.height,
+        turbo,
       })
       shotStoryboardSegment.assetSourceType = getClapAssetSourceType(shotStoryboardSegment.assetUrl)
     } catch (err) {

src/app/api/v1/edit/storyboards/route.ts

@@ -7,6 +7,7 @@ import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
 
 import { processShot } from "./processShot"
 import { ClapCompletionMode } from "@aitube/client"
+import { parseTurbo } from "@/app/api/parsers/parseTurbo"
 
 // a helper to generate storyboards for a Clap
 // this is mostly used by external apps such as the Stories Factory
@@ -22,6 +23,7 @@ export async function POST(req: NextRequest) {
   const query = (qs || {}).query
   
   const mode = parseCompletionMode(query?.c)
+  const turbo = parseTurbo(query?.t)
   
   const blob = await req.blob()
 
@@ -47,6 +49,7 @@ export async function POST(req: NextRequest) {
       existingClap,
       newerClap,
       mode,
+      turbo,
     })
   ))
 

src/app/api/v1/edit/videos/generateVideo.ts

@@ -10,12 +10,14 @@ export async function generateVideo({
   width,
   height,
   seed,
+  turbo = false,
 }: {
   prompt: string
   // negativePrompt?: string
   width?: number
   height?: number
   seed?: number
+  turbo?: boolean
 }): Promise<string> {
   
   // we want to keep it vertical
@@ -31,10 +33,10 @@ export async function generateVideo({
     negativePrompt,
     nbFrames: 80,
     nbFPS: 24,
-    nbSteps: 4,
+    nbSteps: turbo ? 4 : 8,
     width,
     height,
-    turbo: true,
+    turbo,
     shouldRenewCache: true,
     seed: seed || generateSeed()
   })

src/app/api/v1/edit/videos/processShot.ts

@@ -19,12 +19,14 @@ export async function processShot({
   shotSegment,
   existingClap,
   newerClap,
-  mode
+  mode,
+  turbo,
 }: {
   shotSegment: ClapSegment
   existingClap: ClapProject
   newerClap: ClapProject
   mode: ClapCompletionMode
+  turbo: boolean
 }): Promise<void> {
   const shotSegments: ClapSegment[] = filterSegments(
     ClapSegmentFilteringMode.START,
@@ -85,7 +87,8 @@ export async function processShot({
       shotVideoSegment.assetUrl = await generateVideo({
         prompt: getPositivePrompt(shotVideoSegment.prompt),
         width: existingClap.meta.width,
-        height: existingClap.meta.height,
+        height: existingClap.meta.height,,
+        turbo,
       })
       shotVideoSegment.assetSourceType = getClapAssetSourceType(shotVideoSegment.assetUrl)
     } catch (err) {

src/app/api/v1/edit/videos/route.ts

@@ -7,6 +7,7 @@ import { parseCompletionMode } from "@/app/api/parsers/parseCompletionMode"
 import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
 
 import { processShot } from "./processShot"
+import { parseTurbo } from "@/app/api/parsers/parseTurbo"
 
 // a helper to generate videos for a Clap
 // this is mostly used by external apps such as the Stories Factory
@@ -22,7 +23,8 @@ export async function POST(req: NextRequest) {
   const query = (qs || {}).query
   
   const mode = parseCompletionMode(query?.c)
-  
+  const turbo = parseTurbo(query?.t)
+
   const blob = await req.blob()
 
   const existingClap: ClapProject = await parseClap(blob)
@@ -46,7 +48,8 @@ export async function POST(req: NextRequest) {
       shotSegment,
       existingClap,
       newerClap,
-      mode
+      mode,
+      turbo,
     })
   ))
 

src/app/api/v1/export/route.ts

@@ -3,6 +3,7 @@ import queryString from "query-string"
 
 import { parseSupportedExportFormat } from "@/app/api/parsers/parseSupportedExportFormat"
 import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
+import { parseTurbo } from "../../parsers/parseTurbo"
 
 // we hide/wrap the micro-service under a unified AiTube API
 export async function POST(req: NextRequest, res: NextResponse) {
@@ -12,6 +13,11 @@ export async function POST(req: NextRequest, res: NextResponse) {
   const query = (qs || {}).query
 
   const format = parseSupportedExportFormat(query?.f)
+  const turbo = parseTurbo(query?.t)
+
+  // the AI Tube Clap Exporter doesn't support turbo mode
+  // this could be implemented by reducing the resolution, for instance
+  // or rather, the non-turbo mode could be the one where we upscale
 
   // let's call our micro-service, which is currently open bar.
   const result = await fetch(