let's add some music

package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.0",
       "dependencies": {
         "@aitube/clap": "0.0.16",
-        "@aitube/client": "0.0.23",
+        "@aitube/client": "0.0.24",
         "@aitube/engine": "0.0.6",
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.6.7",
@@ -130,9 +130,9 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.23",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.23.tgz",
-      "integrity": "sha512-zZeGacE2WWSIO1h+HOQu6ExwWfJ01mzW1SreP3bN67vOmrau+bWRzZmX6Wg7DAHePnjvTkeR01TAiZVXskJOkw==",
+      "version": "0.0.24",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.24.tgz",
+      "integrity": "sha512-9J3PhVabyc/aOcB5j7wF5Fxb4VORB6aoHTTY6Y+ciFi96zp1YcFeYBMmBpIDq/6KqucIJFtANtZDPoy5a/j1Og==",
       "dependencies": {
         "query-string": "^9.0.0"
       },
@@ -4323,9 +4323,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.761",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.761.tgz",
-      "integrity": "sha512-PIbxpiJGx6Bb8dQaonNc6CGTRlVntdLg/2nMa1YhnrwYOORY9a3ZgGN0UQYE6lAcj/lkyduJN7BPt/JiY+jAQQ=="
+      "version": "1.4.762",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.762.tgz",
+      "integrity": "sha512-rrFvGweLxPwwSwJOjIopy3Vr+J3cIPtZzuc74bmlvmBIgQO3VYJDvVrlj94iKZ3ukXUH64Ex31hSfRTLqvjYJQ=="
     },
     "node_modules/elliptic": {
       "version": "6.5.4",
@@ -6691,9 +6691,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.43.0",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.43.0.tgz",
-      "integrity": "sha512-4SMUB/XiqnO5IrEcdzEGGTcHoeXq7D/k82v36zoqSitrMUjenZXGH5JysIH7aF7Wr+gjvq0dT2mV6wLVKA7Seg==",
+      "version": "4.44.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.44.0.tgz",
+      "integrity": "sha512-jVpDIJsBAR83rVbIHPuWRr9UkFc5DaH9ev2kt2IQAhKCs73DBRoFOa5SwtqfN7/CcBdIGBdygpmpc0gsFaV+Ow==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -7922,9 +7922,9 @@
       "integrity": "sha512-NnzSOEKyv4I83qbuKw9ROtJrrT6Z/Xt7I0HiP/e6H6GnpeTDvzwGIGeJ8slai+VwODSHQDooW2CAilJwT9SpRg=="
     },
     "node_modules/styled-components": {
-      "version": "6.1.10",
-      "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-6.1.10.tgz",
-      "integrity": "sha512-4K8IKcn7iOt76riGLjvBhRyNPTkUKTvmnwoRFBOtJLswVvzy2VsoE2KOrfl9FJLQUYbITLJY2wfIZ3tjbkA/Zw==",
+      "version": "6.1.11",
+      "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-6.1.11.tgz",
+      "integrity": "sha512-Ui0jXPzbp1phYij90h12ksljKGqF8ncGx+pjrNPsSPhbUUjWT2tD1FwGo2LF6USCnbrsIhNngDfodhxbegfEOA==",
       "dependencies": {
         "@emotion/is-prop-valid": "1.2.2",
         "@emotion/unitless": "0.8.1",

package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@aitube/clap": "0.0.16",
-    "@aitube/client": "0.0.23",
+    "@aitube/client": "0.0.24",
     "@aitube/engine": "0.0.6",
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.6.7",

src/app/api/generators/music/generateMusicAsBase64.ts

@@ -0,0 +1,72 @@
+import { sleep } from "@/lib/utils/sleep"
+import { generateMusicWithMusicgen } from "./generateMusicWithMusicgen"
+
+// apparently if we ask to generate like 4 minutes of audio, it crashes
+const maxAudioDurationInSec = 120
+
+// generate music
+// this may generate multiple tracks (one after another)
+// if the durationInSec parameter is larger than the max audio duration
+export async function generateMusicAsBase64({
+    prompt,
+    durationInSec,
+    hd = false,
+  }: {
+    prompt: string
+    durationInSec: number
+
+    // use diffusion (increases quality, but requires more RAM)
+    hd?: boolean
+  }): Promise<string[]> {
+  
+  const musicPrompt = prompt || ""
+ 
+  if (durationInSec < 1 || !musicPrompt) { return [] }
+  
+  if (durationInSec > maxAudioDurationInSec) {
+    const halfTheDuration = Math.round(durationInSec / 2)
+  
+    // no! we shouldn't generate them in parallel
+    // or at least, no now, because we only have ONE music server!
+    // const chunks = await Promise.all([
+    //  generateMusic({ video, durationInSec: halfTheDuration })
+    //])
+    // return chunks.reduce((acc, tracks) => ([...acc, ...tracks]), [])
+
+    // instead, let's play it safe and generate them one after another
+    let chunks: string[] = []
+    const partA = await generateMusicAsBase64({ prompt, hd, durationInSec: halfTheDuration })
+    if (partA) { chunks = chunks.concat(partA) }
+
+    const partB = await generateMusicAsBase64({ prompt, hd, durationInSec: halfTheDuration })
+    if (partB) { chunks = chunks.concat(partB) }
+
+    return [...partA, ...partB]
+  }
+
+  let musicTracks: string[] = []
+
+  const musicParams = {
+    prompt: musicPrompt,
+    durationInSec,
+    hd,
+  }
+  try {
+    console.log(`  |- generating ${durationInSec} seconds of music..`)
+    const musicTrack = await generateMusicWithMusicgen(musicParams)
+    if (!musicTrack?.length) { throw new Error("audio is too short to be valid!")}
+    musicTracks.push(musicTrack)
+  } catch (err) {
+    try {
+      await sleep(4000)
+      const musicTrack = await generateMusicWithMusicgen(musicParams)
+      if (!musicTrack?.length) { throw new Error("audio is too short to be valid!")}
+      musicTracks.push(musicTrack)
+    } catch (err2) {
+      console.error(`  |- failed to generate the music (yes, we retried after a delay)`)
+    }
+  }
+
+
+  return musicTracks
+}

src/app/api/generators/music/generateMusicWithMusicgen.ts

@@ -0,0 +1,99 @@
+import { addBase64Header } from "@/lib/data/addBase64Header"
+
+import { tryApiCalls } from "../../utils/tryApiCall"
+import { MusicGenerationParams } from "./types"
+
+const gradioSpaceApiUrl = `https://jbilcke-hf-ai-tube-model-musicgen.hf.space`
+const huggingFaceSpace = "jbilcke-hf/ai-tube-model-musicgen"
+const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
+
+/**
+ * Note: this generates a base64 mp3 file
+ */
+export async function generateMusicWithMusicgen({
+  prompt,
+  durationInSec,
+  hd,
+  debug = false,
+  neverThrow = false,
+}: MusicGenerationParams): Promise<string> {
+
+  const actualFunction = async () => {
+
+    const res = await fetch(gradioSpaceApiUrl + (gradioSpaceApiUrl.endsWith("/") ? "" : "/") + "api/predict", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        // Authorization: `Bearer ${token}`,
+      },
+      body: JSON.stringify({
+        fn_index: 0, // <- important!
+        data: [
+          microserviceApiKey, // string  in 'Secret Token' Textbox component		
+          "facebook/musicgen-stereo-large", // string  in 'Model' Radio component		
+          "", // string  in 'Model Path (custom models)' Textbox component	
+          
+          // can be one of Default or MultiBand_Diffusion
+          // since speed isn't an issue for AI Tube,
+          // we can afford to use the MultiBand Decoder
+          hd ? "MultiBand_Diffusion" : "Default",
+
+          prompt, // string  in 'Input Text' Textbox component
+          null, 	// blob in 'File' Audio component		
+          durationInSec, // number (numeric value between 1 and 300) in 'Duration' Slider component		
+          250, // number  in 'Top-k' Number component		
+          0, // number  in 'Top-p' Number component		
+          1, // number  in 'Temperature' Number component		
+          3, // number  in 'Classifier Free Guidance' Number component
+        ],
+      }),
+      cache: "no-store",
+      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+      // next: { revalidate: 1 }
+    })
+
+    if (res.status !== 200) {
+      throw new Error('Failed to fetch data')
+    }
+
+ 
+    const { data } = await res.json()
+
+    // console.log("data:", data)
+    // Recommendation: handle errors
+    if (res.status !== 200 || !Array.isArray(data)) {
+      // This will activate the closest `error.js` Error Boundary
+      throw new Error(`Failed to fetch data (status: ${res.status})`)
+    }
+    // console.log("data:", data.slice(0, 50))
+  
+    if (!data[0]) {
+      throw new Error(`the returned music was empty`)
+    }
+  
+    console.log("data:", data[0].slice(0, 60))
+    return addBase64Header(data[0] as string, "mp3")
+  }
+
+  try {
+    if (!prompt?.length) {
+      throw new Error(`prompt is too short!`)
+    }
+
+    const result = await tryApiCalls({
+      func: actualFunction,
+      huggingFaceSpace,
+      debug,
+      failureMessage: "failed to generate the music"
+    })
+
+    return result
+  } catch (err) {
+    if (neverThrow) {
+      console.error(`generateVoiceWithMusicgen():`, err)
+      return ""
+    } else {
+      throw err
+    }
+  }
+}

src/app/api/generators/music/types.ts

@@ -0,0 +1,7 @@
+export type MusicGenerationParams = {
+  prompt: string
+  durationInSec: number
+  hd?: boolean
+  debug?: boolean
+  neverThrow?: boolean
+}

src/app/api/generators/speech/generateVoiceWithParlerTTS.ts

@@ -3,7 +3,7 @@ import { tryApiCalls } from "../../utils/tryApiCall"
 
 const gradioSpaceApiUrl = `https://jbilcke-hf-ai-tube-model-parler-tts-mini.hf.space`
 const huggingFaceSpace = "jbilcke-hf/ai-tube-model-parler-tts-mini"
-const apiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
+const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
 
 /**
  * Note: this generates a MP3 file
@@ -34,7 +34,7 @@ export async function generateSpeechWithParlerTTS({
       body: JSON.stringify({
         fn_index: 0, // <- important!
         data: [
-          apiKey,
+          microserviceApiKey,
           text,
           audioId,
         ],

src/app/api/v1/create/index.ts

@@ -8,6 +8,8 @@ import { parseRawStringToYAML } from "@/app/api/parsers/parseRawStringToYAML"
 import { LatentStory } from "@/app/api/v1/types"
 
 import { systemPrompt } from "./systemPrompt"
+import { generateMusicPrompts } from "../edit/music/generateMusicPrompt"
+import { clapToLatentStory } from "../edit/entities/clapToLatentStory"
 
 // a helper to generate Clap stories from a few sentences
 // this is mostly used by external apps such as the Stories Factory
@@ -177,5 +179,18 @@ Output: `
     currentElapsedTimeInMs += defaultSegmentDurationInMs
   }
 
+  // one more thing: music!
+  let musicPrompts: string[] = []
+  
+  try {
+    musicPrompts = await generateMusicPrompts({
+      prompt,
+      latentStory: await clapToLatentStory(clap)
+    })
+  } catch (err) {
+    console.error(`[api/v1/create] failed to generate music prompts`)
+    musicPrompts.push("lofi hiphop loop")
+  }
+
   return clap
 }

src/app/api/v1/edit/dialogues/route.ts

@@ -1,15 +1,13 @@
 import { NextResponse, NextRequest } from "next/server"
-
+import queryString from "query-string"
 import { ClapProject, ClapSegment, ClapSegmentCategory, newClap, parseClap, serializeClap } from "@aitube/clap"
+import { ClapCompletionMode } from "@aitube/client"
 
-
-import { processShot } from "./processShot"
-import queryString from "query-string"
 import { parseCompletionMode } from "@/app/api/parsers/parseCompletionMode"
 import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
-import { ClapCompletionMode } from "@aitube/client"
 import { parseTurbo } from "@/app/api/parsers/parseTurbo"
 
+import { processShot } from "./processShot"
 // a helper to generate speech for a Clap
 export async function POST(req: NextRequest) {
   await throwIfInvalidToken(req.headers.get("Authorization"))

src/app/api/v1/edit/entities/generateEntityPrompts.ts

@@ -1,5 +1,3 @@
-"use server"
-
 import YAML from "yaml"
 import { ClapSegmentCategory, generateSeed } from "@aitube/clap"
 import { ClapEntityPrompt } from "@aitube/client"

src/app/api/v1/edit/music/generateMusic.ts

@@ -0,0 +1,66 @@
+
+import {
+  ClapProject,
+  ClapSegment,
+  getClapAssetSourceType,
+  filterSegments,
+  ClapSegmentFilteringMode,
+  ClapSegmentCategory,
+  newSegment
+} from "@aitube/clap"
+import { ClapCompletionMode } from "@aitube/client"
+import { getSpeechBackgroundAudioPrompt } from "@aitube/engine"
+
+import { generateSpeechWithParlerTTS } from "@/app/api/generators/speech/generateVoiceWithParlerTTS"
+import { getMediaInfo } from "@/app/api/utils/getMediaInfo"
+import { generateMusicWithMusicgen } from "@/app/api/generators/music/generateMusicWithMusicgen"
+
+export async function generateMusic({
+  musicSegment,
+  existingClap,
+  newerClap,
+  mode,
+  turbo,
+}: {
+  musicSegment?: ClapSegment
+  existingClap: ClapProject
+  newerClap: ClapProject
+  mode: ClapCompletionMode
+  turbo: boolean
+}): Promise<void> {
+  if (!musicSegment) {
+    console.log(`generateMusic(): music segment is empty, so skipping music generation.`)
+    return
+  }
+  // for now we do something very basic
+  const prompt = musicSegment.prompt
+  if (!prompt) {
+    console.log(`generateMusic(): music prompt is empty, so skipping music generation.`)
+    return
+  }
+
+  const assetUrl = await generateMusicWithMusicgen({
+    prompt,
+    durationInSec: 10,
+    hd: false,
+    debug: true,
+    neverThrow: true,
+  })
+
+  if (!assetUrl || assetUrl?.length < 30) {
+    console.log(`generateMusic(): generated assetUrl is empty, so music generation failed.`)
+    return
+  }
+
+  if (mode !== ClapCompletionMode.FULL) {
+    console.log(`generateMusic(): adding music to a new clap file`)
+    newerClap.segments.push(newSegment({
+      ...musicSegment,
+      assetUrl,
+    }))
+  } else {
+    console.log(`generateMusic(): overwriting the music inside the existing clap file`)
+    // this will replace the existing clap (normally)
+    musicSegment.assetUrl = assetUrl
+  }
+}

src/app/api/v1/edit/music/generateMusicPrompt.ts

@@ -0,0 +1,77 @@
+
+import YAML from "yaml"
+
+import { predict } from "@/app/api/providers/huggingface/predictWithHuggingFace"
+import { LatentStory } from "@/app/api/v1/types"
+
+import { systemPrompt } from "./systemPrompt"
+
+export async function generateMusicPrompts({
+  prompt = "",
+  latentStory = [],
+  turbo = false,
+}: {
+  prompt?: string
+  latentStory?: LatentStory[]
+  turbo?: boolean
+} = {
+  prompt: "",
+  latentStory: [],
+  turbo: false
+}): Promise<string[]> {
+
+  if (!prompt.length) { throw new Error(`please provide a prompt`) }
+  console.log("generateMusicPrompts(): prompt:", prompt)
+
+
+  if (!latentStory.length) { throw new Error(`please provide a story`) }
+
+  console.log("generateMusicPrompts(): latentStory:", latentStory)
+
+  const userPrompt = `The input story is about: ${prompt}.
+
+The input story is:
+\`\`\`yaml
+${YAML.stringify(
+  // we need to help the LLM by marking the shots with a simple numeric ID
+  latentStory.map((shot, i) => ({
+    shot: i,
+    ...shot,
+  }))
+)}
+\`\`\`
+
+# Output`
+
+  const prefix = "\""
+
+  // we don't need a lot here!
+  const nbMaxNewTokens = 120
+
+  // TODO use streaming for the Hugging Face prediction
+  //
+  // note that a Clap file is actually a YAML stream of documents
+  // so technically we could stream everything from end-to-end
+  // (but I haven't coded the helpers to do this yet)
+  let rawString = await predict({
+    systemPrompt,
+    userPrompt,
+    nbMaxNewTokens,
+    prefix,
+    turbo,
+  })
+
+  // console.log("generateEntityPrompts(): rawString: ", rawString)
+
+  let results: string[] = []
+  
+  // we remove everything after the last ``` (or ``)
+  rawString = rawString.split(/```?/)[0].trim()
+  results.push(rawString)
+
+  if (!Array.isArray(results) || typeof results.at(0) !== "string" || !results) {
+    throw new Error(`failed to generate the output (rawString is: ${rawString})`)
+  }
+
+  return results
+}

src/app/api/v1/edit/music/route.ts

@@ -0,0 +1,57 @@
+import { NextResponse, NextRequest } from "next/server"
+import queryString from "query-string"
+import { ClapProject, ClapSegment, ClapSegmentCategory, newClap, parseClap, serializeClap } from "@aitube/clap"
+import { ClapCompletionMode } from "@aitube/client"
+
+import { parseCompletionMode } from "@/app/api/parsers/parseCompletionMode"
+import { throwIfInvalidToken } from "@/app/api/v1/auth/throwIfInvalidToken"
+import { parseTurbo } from "@/app/api/parsers/parseTurbo"
+
+import { generateMusic } from "./generateMusic"
+
+// a helper to generate speech for a Clap
+export async function POST(req: NextRequest) {
+  await throwIfInvalidToken(req.headers.get("Authorization"))
+
+  const qs = queryString.parseUrl(req.url || "")
+  const query = (qs || {}).query
+  
+  const mode = parseCompletionMode(query?.c)
+  const turbo = parseTurbo(query?.t)
+  
+  const blob = await req.blob()
+
+  const existingClap: ClapProject = await parseClap(blob)
+
+  if (!existingClap?.segments) { throw new Error(`no segment found in the provided clap!`) }
+  
+  // console.log(`[api/edit/dialogues] detected ${existingClap.segments.length} segments`)
+  
+  const musicSegments: ClapSegment[] = existingClap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC)
+  // console.log(`[api/edit/dialogues] detected ${shotsSegments.length} shots`)
+
+  const newerClap = mode === ClapCompletionMode.FULL ? existingClap : newClap({
+    meta: existingClap.meta
+  })
+    
+  if (musicSegments.length > 1) {
+    throw new Error(`Error, only one music track can be generated with the V1 of the AiTube API`)
+  }
+
+  const musicSegment = musicSegments.at(0)
+
+  await generateMusic({
+    musicSegment,
+    existingClap,
+    newerClap,
+    mode,
+    turbo,
+  })
+
+  // console.log(`[api/edit/dialogues] returning the clap augmented with dialogues`)
+
+  return new NextResponse(await serializeClap(newerClap), {
+    status: 200,
+    headers: new Headers({ "content-type": "application/x-gzip" }),
+  })
+}

src/app/api/v1/edit/music/systemPrompt.ts

@@ -0,0 +1,39 @@
+export const systemPrompt: string = `
+You are a backend API engine, designed to generate music prompt output from a story input.
+
+## Prompting guidelines
+
+To create a music prompt, you need to combine styles with moods, plus a few other things.
+1. Please choose a base style among those categories: "Hip Hop and Rap track", "Classic track", "Jazz track", "Electronic and dance track", "Rock'n'Roll track", "Funk track", "Dubstep track", "Afrobeats", "Orchestral track", "Pop track", "Reggae track", "Metal track", "Country track", "Blues track", "Soul track", "R'n'B track", "Disco track", "Trap track", "Ambient track", "Lofi track", "Chill track", etc.
+2. Then choose a vibe: "with an happy vibe", "with a sad vibe", "with an angry vibe", "with a chill vibe", "with a romantic vibe", "with an epic vibe", "with an energetic vibe", "with a dreamy vibe", "with a mysterious vibe", "with a relaxing vibe", "with a dark vibe", "with an upbeat vibe", "with a motivational vibe", "with an inspiring vibe", "with a nostalgic vibe", "with a groovy vibe", "with a cheerful vibe", "with a melancholic vibe", "with a hopeful vibe", etc.
+3. build up a coherent description eg.: "80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums", "a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions bpm: 130", "A cheerful country song with acoustic guitars", "lofi slow bpm electro chill with organic samples" etc.
+
+## Example of input/output
+
+Given the following input story, provided as YAML:
+
+# Input
+
+"A king goes to see a witch to ask if or how he can win an upcoming and challenging battle"
+
+\`\`\`yaml
+- shot: 1
+  comment: "King Arthus seeks the witch's guidance to win his imminent battle."
+  image: "Establishing shot of KING ARTHUS, nervous, wet brown hair. dressed in golden armor and a colorful cape. His face reveals a mix of concern and determination. He's standing in the bright sunshine, inside a castle's courtyard, under cloudy skies. Behind him, a group of soldiers can be seen marching towards the castle gates."
+  voice: "Dark sorceress of the shadows, it is time for you to serve your Lord. Tell me the augur, tell me what you foreknow. Tell me how I will cleave my ennemies to the bone, and ravage them in battle to come up victorious."
+- shot: 2
+  comment: "The witch gives her counsel but warns of an unknown cost."
+  image: "close-up shot of THE WITCH, smiling cunningly, raising a finger while speaking. Background bokeh, dim lightning, menacing, mysterious."
+  voice: "Your Majesty, this will be a bloody battle, but I espy a way to victory for you. But if my advice you follow, victory I foresee, although at a great cost it will be."
+- shot: 3
+  comment: "The words of the witch are sinking in, but King Arthus tries to appear strong"
+  image: "close-up shot on KING ARTHUS, looking concerned, somber, false confidence"
+  voice: "Witch with the wicked tongue, what must be done will be done. I will do everything for my people's sake. Speak now, make know the path to glory."
+\`\`\
+
+As you can see, the theme is medieval, this is for a fantasy movie. So you should generate a music like this:
+
+## Output
+
+"Classical music with a symphonic orchestra, with medieval influences and a chilling mysterious vibe."
+`