afternoon release

package-lock.json

@@ -8,9 +8,9 @@
       "name": "@aitube/clapper",
       "version": "0.0.1",
       "dependencies": {
-        "@aitube/clap": "0.0.26",
-        "@aitube/engine": "0.0.19",
-        "@aitube/timeline": "0.0.25",
+        "@aitube/clap": "0.0.27",
+        "@aitube/engine": "0.0.21",
+        "@aitube/timeline": "0.0.26",
         "@fal-ai/serverless-client": "^0.10.3",
         "@huggingface/hub": "^0.15.1",
         "@huggingface/inference": "^2.7.0",
@@ -106,9 +106,9 @@
       }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.26",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.26.tgz",
-      "integrity": "sha512-7MIzDSZiTTxjKeiFNmuUvvViJV0g8gQp+2uJat99CTCFa9dKKIshYftj+E3FhkBRZ3mWYOQUogOZCQrxtYFy3w==",
+      "version": "0.0.27",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.27.tgz",
+      "integrity": "sha512-3mPciwLyw4FkMI1c2EnumldlCfbVucOTTUZvqcyZVrNUfHIvjy9MJwpg55rCrER3rdfJpduHkK1muk5orA2c0Q==",
       "dependencies": {
         "pure-uuid": "^1.8.1"
       },
@@ -117,23 +117,23 @@
       }
     },
     "node_modules/@aitube/engine": {
-      "version": "0.0.19",
-      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.19.tgz",
-      "integrity": "sha512-VvUG49nKk7SDraE8CipNSW5ijqNJTXXkCOi0X6inLf4TjexBbIphc1BQhD0sa+T4zA4hS3seC9PwKUP/KLwE7A==",
+      "version": "0.0.21",
+      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.21.tgz",
+      "integrity": "sha512-seVvSnzQgUTsb/Vj9Ri6lhZN9LS6Td1D9ytcq9VQgZYSBWRW6zENZdZzOKn0QB8x4MCe+Zdr1NZlZfpukaJa7A==",
       "peerDependencies": {
-        "@aitube/clap": "0.0.26"
+        "@aitube/clap": "0.0.27"
       }
     },
     "node_modules/@aitube/timeline": {
-      "version": "0.0.25",
-      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.25.tgz",
-      "integrity": "sha512-Fv7nblA7dPrOHIUjSoFmZI7GXCyR93/+qUOZQrMwKZygysifdxgVu45C4lEShwabqCKhYZZqF9Eotkyg+NeDaw==",
+      "version": "0.0.26",
+      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.26.tgz",
+      "integrity": "sha512-uFC1oF86g2/kWL6eG5bEy1Z1zJUp27t/zkx35o9cnJ0zgNGYMq5XgvZJ91SYdC8TBvKBRosbZroIOx+7EraCRw==",
       "dependencies": {
         "date-fns": "^3.6.0",
         "react-virtualized-auto-sizer": "^1.0.24"
       },
       "peerDependencies": {
-        "@aitube/clap": "0.0.26",
+        "@aitube/clap": "0.0.27",
         "@radix-ui/react-slider": "^1.1.2",
         "@react-spring/three": "^9.7.3",
         "@react-spring/types": "^9.7.3",

package.json

@@ -10,9 +10,9 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.26",
-    "@aitube/engine": "0.0.19",
-    "@aitube/timeline": "0.0.25",
+    "@aitube/clap": "0.0.27",
+    "@aitube/engine": "0.0.21",
+    "@aitube/timeline": "0.0.26",
     "@fal-ai/serverless-client": "^0.10.3",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/inference": "^2.7.0",

src/app/api/assistant/askAnyAssistant.ts

@@ -112,7 +112,7 @@ export async function askAnyAssistant({
     category: segment.category,
   } as SimplifiedSegmentData))
 
-  console.log("INPUT:", JSON.stringify(inputData, null, 2))
+  // console.log("INPUT:", JSON.stringify(inputData, null, 2))
 
   const chain = chatPrompt.pipe(coerceable).pipe(parser)
 
@@ -143,7 +143,7 @@ export async function askAnyAssistant({
       inputData: JSON.stringify(inputData),
     })
 
-    console.log("OUTPUT:", JSON.stringify(result, null, 2))
+    // console.log("OUTPUT:", JSON.stringify(result, null, 2))
 
     /*
     this whole code doesn't work well actually..
@@ -175,7 +175,6 @@ export async function askAnyAssistant({
     const errObj = err1 as any
     try {
       const keys = Object.keys(errObj)
-      // console.log("keys:", keys)
       if (errObj.llmOutput) {
         return {
           prompt: "",

src/app/api/resolve/providers/comfy/getComfyWorkflow.ts

@@ -32,11 +32,6 @@ export function getComfyWorkflow(request: ResolveRequest) {
     }
     output[`${i}`] = node
   })
-
-  console.log("DEBUG:", {
-    nodes,
-    output
-  })
   
   return JSON.stringify(output)
 }

src/app/api/resolve/providers/huggingface/index.ts

@@ -13,7 +13,6 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
     throw new Error(`Missing API key for "Hugging Face"`)
   }
   
-  console.log(`key: ${request.settings.huggingFaceApiKey}`)
   const hf: HfInferenceEndpoint = new HfInference(request.settings.huggingFaceApiKey)
 
   if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {

src/app/api/resolve/providers/replicate/index.ts

@@ -28,8 +28,6 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
       }
     })
 
-    console.log(`Replicate replied: `, output)
-
     segment.assetUrl = await decodeOutput(output)
     segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
   } catch (err) {

src/app/api/resolve/providers/stabilityai/index.ts

@@ -62,7 +62,6 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
     }
 
     segment.assetUrl = await decodeOutput(content)
-    // console.log(`segment.assetUrl = ${segment.assetUrl.slice(0, 80)}..`)
     segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
   } catch (err) {
     console.error(`failed to call Stability.ai: `, err)

src/app/api/resolve/providers/stabilityai/performRequest.ts

@@ -54,7 +54,6 @@ export async function performRequest({
   )
 
   if (response.status === 200) {
-    console.log("response.data: ", response.data)
     const buffer = Buffer.from(response.data)
     const rawAssetUrl = `data:image/${payload.output_format};base64,${buffer.toString('base64')}`
     const assetUrl = await decodeOutput(rawAssetUrl)

src/app/main.tsx

@@ -39,7 +39,7 @@ function MainContent() {
       NativeTypes.FILE,
     ],
     drop: (item: DroppableThing): void => {
-      console.log("DROP", item)
+      // console.log("DROP", item)
       openFiles(item.files)
     },
     collect: (monitor) => ({

src/components/forms/FormSwitch.tsx

@@ -20,7 +20,6 @@ export function FormSwitch({ label, className, checked, onCheckedChange, horizon
       <Switch
         checked={checked}
         onCheckedChange={(checked) => {
-          console.log("onCheckedChange: " + checked)
           onCheckedChange(!checked)
         }}
       />

src/components/monitor/DynamicPlayer/VideoClipBuffer.tsx

@@ -18,12 +18,9 @@ export function VideoClipBuffer({
   const togglePlayVideo = (play: boolean) => {
     const player = ref.current
     if (!player) { return }
-    // console.log(`togglePlayVideo(${play}) (current status = ${player.paused ? "paused" : "playing"})`)
     if (play && player.paused) {
-      // console.log("playing video..")
       player.play()
     } else if (!play && !player.paused) {
-      // console.log("pausing video..")
       player.pause()
     }
   }

src/components/toolbars/top-menu/ActivitySpinner/index.tsx

@@ -0,0 +1,24 @@
+
+import { CgSpinnerTwoAlt } from "react-icons/cg"
+
+import { cn } from "@/lib/utils"
+
+export function ActivitySpinner({
+  className = "",
+  isBusy = false,
+}: {
+  className?: string
+  isBusy?: boolean
+}) {
+  return (
+    <CgSpinnerTwoAlt
+      className={cn(
+        `w-2 h-2 text-stone-200`,
+        className,
+        isBusy
+        ? 'opacity-100 animate-spin'
+        : 'opacity-0',
+      )}
+    />
+  )
+}

src/components/toolbars/top-menu/IsBusy/index.tsx

@@ -0,0 +1,20 @@
+import { GoDotFill } from "react-icons/go"
+
+import { cn } from "@/lib/utils"
+
+export function IsBusy({
+  nbPendingTasks = 0,
+}: {
+  nbPendingTasks?: number
+}) {
+  return (
+    <GoDotFill
+      className={cn(
+        `ml-[1px] -mt-[9px] w-1.5 h-1.5 text-yellow-400`,
+        nbPendingTasks > 0
+        ? 'opacity-100 animate-pulse'
+        : 'opacity-0'
+      )}
+    />
+  )
+}

src/components/toolbars/top-menu/TopMenuLogo/index.tsx

@@ -0,0 +1,100 @@
+import { MdMovieFilter, MdOutlineMovieFilter } from "react-icons/md"
+import { PiPauseFill, PiPlayFill } from "react-icons/pi"
+import { BiSolidMoviePlay } from "react-icons/bi"
+// import Image from "next/image"
+
+import { useResolver } from "@/controllers/resolver/useResolver"
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
+// import logo from "../../../app/logo-v2.png"
+
+import { ActivitySpinner } from "../ActivitySpinner"
+import { cn } from "@aitube/timeline"
+
+export function TopMenuLogo() {
+  const isPaused = useResolver(s => s.isPaused)
+  const togglePause = useResolver(s => s.togglePause)
+  const isBusyResolving = useResolver(s => s.isBusyResolving)
+
+  // we could display this in the modal, but it is not very informative
+  // it would be more useful to display things like:
+  // nb remaining assets to process per category (10 images, 40 videos..) 
+  // estimated total time
+  // a little timeline of the production rate (a tiny chart)
+  // rendering rate (frames per second etc, kbps etc)
+  const nbRequestsRunningInParallel = useResolver(s => s.nbRequestsRunningInParallel)
+
+  // it doesn't look great when minified like this
+  // <Image src={logo} height={32} alt="Clapper" /> */}
+
+  // we have to disable the button when we are in an intermediary state,
+  // such has a paused resolver with pending tasks
+  // otherwise users get confused, thinking it didn't work
+  // and they might click multiple times
+  const isDisabled = isBusyResolving && isPaused
+
+  return (
+    <Tooltip>
+      <TooltipTrigger className="flex items-center justify-center">
+        <div
+          className={cn(`
+          flex items-center justify-center
+          group
+          `, isDisabled ? "" : "pointer-cursor")}
+          onClick={() => {
+            if (isDisabled) { return }
+            togglePause()
+          }}
+          >
+          {isBusyResolving 
+          ? <div
+             className="
+             flex items-center justify-center
+             w-4 h-4
+             ">
+              <ActivitySpinner
+                isBusy={isBusyResolving}
+                className="absolute opacity-100 group-hover:opacity-0 w-4 h-4 text-yellow-400"
+              />
+              {isPaused
+
+              // imho using the original "logo" (sort of) is a bit more elegant than a simple play button
+              // ? <PiPlayFill className="absolute opacity-0 group-hover:opacity-100 text-yellow-400" />
+              ? <BiSolidMoviePlay className="absolute opacity-0 group-hover:opacity-100 text-yellow-400" />
+
+              : <PiPauseFill className="absolute opacity-0 group-hover:opacity-100 text-yellow-400" />
+
+              }
+            </div>
+          : <BiSolidMoviePlay
+              className=" text-yellow-400"
+            />}
+          <span className="
+          pr-2
+          scale-[88%]
+          text-yellow-400
+          text-lg font-bold tracking-[-0.03em]
+          ">Clapper</span>
+        </div>
+      </TooltipTrigger>
+      <TooltipContent>
+        <div className="text-xs text-yellow-300">
+          {/*
+          <p>{
+          nbRequestsRunningInParallel || 0
+            } remaining task{
+          nbRequestsRunningInParallel > 1 ? 's' : ''
+          }</p>
+          */}
+          {isBusyResolving
+          ? <span>Some tasks are still pending.</span>
+          : <span>No pending task in progress.</span>
+          }
+          <br/>
+          {isPaused
+            ? <span>Click to RESUME generation.</span>
+            : <span>Click to PAUSE generation.</span>}
+        </div>
+      </TooltipContent>
+    </Tooltip>
+  )
+}

src/components/toolbars/top-menu/image/index.tsx

@@ -19,9 +19,14 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
 import { availableComputeProvidersForImages } from "@/components/settings/constants"
 import { ImageGenerationModelList } from "../lists/ImageGenerationModelList"
 import { ImageUpscalingModelList } from "../lists/ImageUpscalingModelList"
+import { IsBusy } from "../IsBusy"
+
 import { ComputeProvider, SettingsCategory } from "@/types"
+import { useRenderer } from "@/controllers/renderer"
+import { useResolver } from "@/controllers/resolver/useResolver"
 
 export function TopMenuImage() {
+  const nbPendingRequestsForImage = useResolver(s => s.nbPendingRequestsForImage)
   const setShowSettings = useUI(s => s.setShowSettings)
   const imageProvider = useSettings(s => s.imageProvider)
   const setImageProvider = useSettings(s => s.setImageProvider)
@@ -33,7 +38,7 @@ export function TopMenuImage() {
   const setImageRenderingStrategy = useSettings((s) => s.setImageRenderingStrategy)
   return (
     <MenubarMenu>
-      <MenubarTrigger>Image</MenubarTrigger>
+      <MenubarTrigger><span>Image</span><IsBusy nbPendingTasks={nbPendingRequestsForImage} /></MenubarTrigger>
       <MenubarContent>
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.IMAGE) }}>Show advanced settings</MenubarItem>

src/components/toolbars/top-menu/index.tsx

@@ -1,30 +1,25 @@
+import { cn } from "@aitube/timeline"
+
 import { Menubar } from "@/components/ui/menubar"
+import { APP_REVISION } from "@/lib/core/constants"
+import { useResolver } from "@/controllers/resolver/useResolver"
 
 import { TopMenuFile } from "./file"
-
 import { TopMenuImage } from "./image"
 import { TopMenuVideo } from "./video"
 import { TopMenuVoice } from "./voice"
 import { TopMenuSound } from "./sound"
 import { TopMenuMusic } from "./music"
 import { TopMenuAssistant } from "./assistant"
-
 import { TopMenuView } from "./view"
-import { cn } from "@aitube/timeline"
-import { APP_REVISION } from "@/lib/core/constants"
-import Image from "next/image"
-import logo from "../../../app/logo-v2.png"
+import { TopMenuLogo } from "./TopMenuLogo"
 
 export function TopMenu() {
+  const isBusyResolving = useResolver(s => s.isBusyResolving)
+
   return (
-    <Menubar className="w-full">
-      {/*
-      it doesn't look great when minified like this
-      <Image src={logo} height={32} alt="Clapper" /> */}
-      <span
-        className="scale-[88%] text-yellow-400 pl-2 text-lg font-bold tracking-[-0.03em] mr-2"
-        ><span>Clapper</span><span className="text-yellow-100">.</span><span
-      className="hidden absolute text-3xs text-red-500/100 tracking-[0.05em] uppercase mt-1.5 -ml-0 -rotate-6  ">ai</span></span>
+    <Menubar className="w-full ml-1">
+      <TopMenuLogo />
       <TopMenuFile />
       <TopMenuImage />
       <TopMenuVideo />

src/components/toolbars/top-menu/music/index.tsx

@@ -19,8 +19,11 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
 import { availableComputeProvidersForMusic } from "@/components/settings/constants"
 import { ComputeProvider, SettingsCategory } from "@/types"
 import { MusicGenerationModelList } from "../lists/MusicGenerationModelList"
+import { useResolver } from "@/controllers/resolver/useResolver"
+import { IsBusy } from "../IsBusy"
 
 export function TopMenuMusic() {
+  const nbPendingRequestsForMusic = useResolver(s => s.nbPendingRequestsForMusic)
   const setShowSettings = useUI(s => s.setShowSettings)
   const musicProvider = useSettings(s => s.musicProvider)
   const setMusicProvider = useSettings(s => s.setMusicProvider)
@@ -31,7 +34,7 @@ export function TopMenuMusic() {
 
   return (
     <MenubarMenu>
-      <MenubarTrigger>Music</MenubarTrigger>
+      <MenubarTrigger><span>Music</span><IsBusy nbPendingTasks={nbPendingRequestsForMusic} /></MenubarTrigger>
       <MenubarContent>
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.MUSIC) }}>Show advanced settings</MenubarItem>

src/components/toolbars/top-menu/sound/index.tsx

@@ -19,8 +19,11 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
 import { availableComputeProvidersForSound } from "@/components/settings/constants"
 import { ComputeProvider, SettingsCategory } from "@/types"
 import { SoundGenerationModelList } from "../lists/SoundGenerationModelList"
+import { useResolver } from "@/controllers/resolver/useResolver"
+import { IsBusy } from "../IsBusy"
 
 export function TopMenuSound() {
+  const nbPendingRequestsForSound = useResolver(s => s.nbPendingRequestsForSound)
   const setShowSettings = useUI(s => s.setShowSettings)
   const soundProvider = useSettings(s => s.soundProvider)
   const setSoundProvider = useSettings(s => s.setSoundProvider)
@@ -30,7 +33,7 @@ export function TopMenuSound() {
   const setSoundRenderingStrategy = useSettings((s) => s.setSoundRenderingStrategy)
   return (
     <MenubarMenu>
-      <MenubarTrigger>Sound</MenubarTrigger>
+      <MenubarTrigger><span>Sound</span><IsBusy nbPendingTasks={nbPendingRequestsForSound} /></MenubarTrigger>
       <MenubarContent>
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.SOUND) }}>Show advanced settings</MenubarItem>

src/components/toolbars/top-menu/video/index.tsx

@@ -20,8 +20,11 @@ import { availableComputeProvidersForVideos } from "@/components/settings/consta
 import { VideoGenerationModelList } from "../lists/VideoGenerationModelList"
 import { VideoUpscalingModelList } from "../lists/VideoUpscalingModelList"
 import { ComputeProvider, SettingsCategory } from "@/types"
+import { useResolver } from "@/controllers/resolver/useResolver"
+import { IsBusy } from "../IsBusy"
 
 export function TopMenuVideo() {
+  const nbPendingRequestsForVideo = useResolver(s => s.nbPendingRequestsForVideo)
   const setShowSettings = useUI(s => s.setShowSettings)
   const videoProvider = useSettings(s => s.videoProvider)
   const setVideoProvider = useSettings(s => s.setVideoProvider)
@@ -33,7 +36,7 @@ export function TopMenuVideo() {
   const setVideoRenderingStrategy = useSettings((s) => s.setVideoRenderingStrategy)
   return (
     <MenubarMenu>
-      <MenubarTrigger>Video</MenubarTrigger>
+      <MenubarTrigger><span>Video</span><IsBusy nbPendingTasks={nbPendingRequestsForVideo} /></MenubarTrigger>
       <MenubarContent>
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.VIDEO) }}>Show advanced settings</MenubarItem>

src/components/toolbars/top-menu/voice/index.tsx

@@ -19,8 +19,11 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
 import { availableComputeProvidersForVoice } from "@/components/settings/constants"
 import { VoiceGenerationModelList } from "../lists/VoiceGenerationModelList"
 import { ComputeProvider, SettingsCategory } from "@/types"
+import { useResolver } from "@/controllers/resolver/useResolver"
+import { IsBusy } from "../IsBusy"
 
 export function TopMenuVoice() {
+  const nbPendingRequestsForVoice = useResolver(s => s.nbPendingRequestsForVoice)
   const setShowSettings = useUI(s => s.setShowSettings)
   const voiceProvider = useSettings(s => s.voiceProvider)
   const setVoiceProvider = useSettings(s => s.setVoiceProvider)
@@ -30,7 +33,7 @@ export function TopMenuVoice() {
   const setVoiceRenderingStrategy = useSettings((s) => s.setVoiceRenderingStrategy)
   return (
     <MenubarMenu>
-      <MenubarTrigger>Voice</MenubarTrigger>
+      <MenubarTrigger><span>Voice</span><IsBusy nbPendingTasks={nbPendingRequestsForVoice} /></MenubarTrigger>
       <MenubarContent>
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.VOICE) }}>Show advanced settings</MenubarItem>

src/controllers/io/parseFileIntoSegments.ts

@@ -1,53 +1,50 @@
 "use client"
 
-import { ClapOutputType, ClapSegment, ClapSegmentCategory, generateSeed, newSegment, UUID } from "@aitube/clap"
+import { ClapAssetSource, ClapOutputType, ClapSegment, ClapSegmentCategory, ClapSegmentStatus, generateSeed, newSegment, UUID } from "@aitube/clap"
 import { findFreeTrack } from "@aitube/timeline"
 
 import { RuntimeSegment } from "@/types"
 
 import { analyzeAudio } from "../audio/analyzeAudio"
 import { ResourceCategory, ResourceType } from "./types"
+import { blobToBase64DataUri } from "@/lib/utils/blobToBase64DataUri"
 
-export async function parseFileIntoSegments({ file, segments }: {
+export async function parseFileIntoSegments({ file }: {
   /**
    * The file to import
    */
   file: File
-
-
-  /**
-   * all existing segments
-   */
-  segments: ClapSegment[]
 }): Promise<ClapSegment[]> {
-  console.log(`filename: ${file.name}`)
-  console.log(`file size: ${file.size} bytes`)
-  console.log(`file type: ${file.type}`)
+  // console.log(`parseFileIntoSegments(): filename = ${file.name}`)
+  // console.log(`parseFileIntoSegments(): file size = ${file.size} bytes`)
+  // console.log(`parseFileIntoSegments(): file type = ${file.type}`)
 
   const extension = file.name.split(".").pop()?.toLowerCase()
 
   console.log("TODO: open a popup to ask if this is a voice character sample, dialogue, music etc")
   
   let type: ResourceType = "misc"
-  let category: ResourceCategory = "misc"
+  let resourceCategory: ResourceCategory = "misc"
 
   const newSegments: ClapSegment[] = []
 
   switch (file.type) {
     case "image/webp":
       type = "image"
-      category = "control_image"
+      resourceCategory = "control_image"
       break;
 
-    case "audio/mpeg":
+    case "audio/mpeg": // this is the "official" one
+    case "audio/mp3": // this is just an alias
     case "audio/wav":
     case "audio/mp4":
+    case "audio/x-mp4": // should be rare, normally is is audio/mp4
     case "audio/m4a": // shouldn't exist
-    case "audio/x-m4a": // should be rare, normallyt is is audio/mp4
+    case "audio/x-m4a": // should be rare, normally is is audio/mp4
     case "audio/webm":
       // for background track, or as an inspiration track, or a voice etc
       type = "audio"
-      category = "background_music"
+      resourceCategory = "background_music"
 
       // TODO: add caption analysis
       const { durationInMs,durationInSteps, bpm, audioBuffer } = await analyzeAudio(file)
@@ -62,17 +59,35 @@ export async function parseFileIntoSegments({ file, segments }: {
       const endTimeInSteps = durationInSteps
       const endTimeInMs = startTimeInMs + durationInMs
 
-      const clapSegment = newSegment({
+      // ok let's stop for a minute there:
+      // if someone drops a .mp3, and assuming we don't yet have the UI to select the category,
+      // do you think it should be a SOUND, a VOICE or a MUSIC by default? 
+      // I expect people will use AI service providers for sound and voice,
+      // maybe in some case music too, but there are also many people
+      // who will want to use their own track eg. to create a music video
+      const category = ClapSegmentCategory.MUSIC
+
+      const assetUrl = await blobToBase64DataUri(file)
+
+      const newSegmentData: Partial<ClapSegment> = {
         prompt: "audio track",
         startTimeInMs, // start time of the segment
         endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
-        track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
+        status: ClapSegmentStatus.COMPLETED,
+        // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
         label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined)
-        category: ClapSegmentCategory.MUSIC,
-      })
+        category,
+        assetUrl,
+        assetDurationInMs: endTimeInMs,
+        assetSourceType: ClapAssetSource.DATA,
+        assetFileFormat: `${file.type}`,
+      }
+
+      const clapSegment = newSegment(newSegmentData)
 
       const audioSegment: RuntimeSegment = {
         ...clapSegment,
+
         outputType: ClapOutputType.AUDIO,
         outputGain: 1,
         audioBuffer,
@@ -81,13 +96,13 @@ export async function parseFileIntoSegments({ file, segments }: {
       // console.log("newSegment:", audioSegment)
 
       // poof! type disappears.. it's magic
-      newSegments.push(audioSegment as ClapSegment)
+      newSegments.push(audioSegment)
       break;
     
     case "text/plain":
       // for dialogue, prompts..
       type = "text"
-      category = "text_prompt"
+      resourceCategory = "text_prompt"
       break;
 
     default:
@@ -116,6 +131,6 @@ export async function parseFileIntoSegments({ file, segments }: {
 
   // Note: uploading is optional, some file type don't need it (eg. text prompt)
 
-  return [...segments, ...newSegments]
+  return newSegments
 }
 

src/controllers/io/parseFilesIntoSegments.ts

@@ -2,21 +2,15 @@
 import { ClapSegment } from "@aitube/clap"
 import { parseFileIntoSegments } from "./parseFileIntoSegments"
 
-export async function parseFilesIntoSegments({ files, segments, }: {
+export async function parseFilesIntoSegments({ files }: {
   /**
    * The files to import
    */
   files: File[]
-
-  /**
-   * all existing segments
-   */
-  segments: ClapSegment[]
 }): Promise<ClapSegment[]> {
   return (
     await Promise.all(files.map(file => parseFileIntoSegments({
       file,
-      segments,
   })))
   ).reduce((acc, segments) => [...acc, ...segments], [])
 }

src/controllers/io/useIO.ts

@@ -31,7 +31,8 @@ export const useIO = create<IOStore>((set, get) => ({
   },
   openFiles: async (files: File[]) => {
     const { openClapBlob, openScreenplay } = get()
-    const segments: ClapSegment[] = useTimeline.getState().segments
+    const timeline: TimelineStore = useTimeline.getState()
+    const { segments, addSegments } = timeline
 
     if (Array.isArray(files)) {
       console.log("user tried to drop some files:", files)
@@ -66,25 +67,17 @@ export const useIO = create<IOStore>((set, get) => ({
       // screenplay files: -> analyze (if there is existing data, show a modal asking to save or not)
       // mp3 file: -> 
       if (isAudioFile) {
-        const newSegments = await parseFileIntoSegments({
-          file,
-          segments,
+        const newSegments = await parseFileIntoSegments({ file })
+
+        console.log("calling  timeline.addSegments with:", newSegments)
+        await timeline.addSegments({
+          segments: newSegments
         })
+        return
       }
 
       const isVideoFile = fileType.startsWith("video/")
   
-
-      // for the moment let's not care of the coordinates at all
-      /*
-      parseFilesIntoSegments({
-        files,
-        columnIndex,
-        rowIndex,
-        segments,
-        segment
-      })
-      */
     }
   },
   openScreenplay: async (projectName: string, fileName: string, fileContent: string | Blob): Promise<void> => {

src/controllers/monitor/useMonitor.ts

@@ -131,7 +131,6 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
       // we force a full state recompute
       // and we also pass jumpedSomewhere=true to indicate that we
       // need a buffer transition
-      // console.log(`forcing a state update`)
       renderLoop(true)
     }
   },

src/controllers/renderer/useRenderLoop.ts

@@ -45,7 +45,6 @@ export function useRenderLoop(): void {
 
     const newTimelineUpdateAtInMs = performance.now()
     const elapsedTimeInMs = newTimelineUpdateAtInMs - lastTimelineUpdateAtInMs
-    // console.log(`TODO: move the timeline cursor according to the elapsed time`)
     setCursorTimestampAtInMs(cursorTimestampAtInMs + elapsedTimeInMs)
     setLastTimelineUpdateAtInMs(newTimelineUpdateAtInMs)
 

src/controllers/renderer/useRenderer.ts

@@ -206,9 +206,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
       }
         
     }
-  
-    // console.log("useRenderer: computeBufferedSegments() returning:", results)
-  
+
     return results
   },
 

src/controllers/resolver/getDefaultResolverState.ts

@@ -3,6 +3,7 @@ import { ResolverState } from "./types"
 export function getDefaultResolverState(): ResolverState {
   const state: ResolverState = {
     isRunning: false,
+    isPaused: false,
 
     defaultParallelismQuotas: {
       video: 1,
@@ -12,15 +13,23 @@ export function getDefaultResolverState(): ResolverState {
       music: 1,
     },
   
-    currentParallelismQuotas: {
-      video: 1,
-      image: 1,
-      voice: 1,
-      sound: 1,
-      music: 1,
-    },
+    // used for UI display, show some metrics
+    currentParallelismQuotaForVideo: 1,
+    currentParallelismQuotaForImage: 1,
+    currentParallelismQuotaForVoice: 1,
+    currentParallelismQuotaForSound: 1,
+    currentParallelismQuotaForMusic: 1,
     
-    nbRequestsRunningInParallel: 0
+    // just some aliases for convenience
+    nbPendingRequestsForVideo: 0,
+    nbPendingRequestsForImage: 0,
+    nbPendingRequestsForVoice: 0,
+    nbPendingRequestsForSound: 0,
+    nbPendingRequestsForMusic: 0,
+
+    nbRequestsRunningInParallel: 0,
+
+    isBusyResolving: false,
   }
   return state
 }

src/controllers/resolver/types.ts

@@ -1,8 +1,17 @@
 import { ClapSegment } from "@aitube/clap"
 
 export type ResolverState = {
+  // this indicate the status of the loop
+  // normally once it is running it is never stopped,
+  // since this is used to check if we have pending tasks
   isRunning: boolean
 
+  // used to "pause" the resolution
+  // request have have already be sent to the API providers
+  // will still be honored, which is why the number of pending
+  // requests won't drop to 0 immediately
+  isPaused: boolean
+
   defaultParallelismQuotas: {
     video: number
     image: number
@@ -12,22 +21,29 @@ export type ResolverState = {
   }
 
   // used for UI display, show some metrics
-  currentParallelismQuotas: {
-    video: number
-    image: number
-    voice: number
-    sound: number
-    music: number
-  }
+  currentParallelismQuotaForVideo: number
+  currentParallelismQuotaForImage: number
+  currentParallelismQuotaForVoice: number
+  currentParallelismQuotaForSound: number
+  currentParallelismQuotaForMusic: number
 
+  // just some aliases for convenience,
   // used for UI display, show some metrics
+  nbPendingRequestsForVideo: number
+  nbPendingRequestsForImage: number
+  nbPendingRequestsForVoice: number
+  nbPendingRequestsForSound: number
+  nbPendingRequestsForMusic: number
   nbRequestsRunningInParallel: number
+  isBusyResolving: boolean
 }
 
 export type ResolverControls = {
   startLoop: () => void
   runLoop: () => Promise<void>
 
+  togglePause: (isPaused?: boolean) => boolean
+
     /**
    * This resolve a segment
    * 

src/controllers/resolver/useResolver.ts

@@ -55,24 +55,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
       }, waitTimeIfNothingToDoInMs)
     }
 
-    // ------- trivial case: maybe we have nothing to do? ------
-
-    const allStrategiesAreOnDemand =
-      imageRenderingStrategy === RenderingStrategy.ON_DEMAND &&
-      videoRenderingStrategy === RenderingStrategy.ON_DEMAND &&
-      soundRenderingStrategy === RenderingStrategy.ON_DEMAND &&
-      voiceRenderingStrategy === RenderingStrategy.ON_DEMAND &&
-      musicRenderingStrategy
-    
-    // nothing to do
-    if (allStrategiesAreOnDemand) {
-      // console.log(`useResolver.runLoop(): all strategies are on-demand only`)
-      return runLoopAgain()
-    }
-
-    // ---------- end of the very trivial case ----------------
-
-
+    // note: do not create a return condition in case all strategies are "on demand"
+    // otherwise we won't be able to get the status of current tasks
 
     // console.log(`useResolver.runLoop()`)
     const timelineState: TimelineStore = useTimeline.getState()
@@ -99,13 +83,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
     //
     // -------------------------------------------------------------------------
 
-    const { defaultParallelismQuotas } = get()
+    const { defaultParallelismQuotas, isPaused } = get()
+
+    let currentParallelismQuotaForVideo = defaultParallelismQuotas.video
+    let currentParallelismQuotaForImage = defaultParallelismQuotas.image
+    let currentParallelismQuotaForVoice = defaultParallelismQuotas.voice
+    let currentParallelismQuotaForSound = defaultParallelismQuotas.sound
+    let currentParallelismQuotaForMusic = defaultParallelismQuotas.music
 
-    // note that we to create a copy here, that way we can modify it
-    const parallelismQuotas = {
-      ...defaultParallelismQuotas,
-    }
-    
     // console.log(`useResolver.runLoop() parallelismQuotas = `, parallelismQuotas)
 
     // we do not need ot get currentParallelismQuotas,
@@ -127,11 +112,13 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           // it is thus vital to deduct it from the parallelism quota,
           // to avoir triggering quota limit on the providers side
           if (s.status === ClapSegmentStatus.IN_PROGRESS) {
-            parallelismQuotas.video = Math.max(0, parallelismQuotas.video - 1)
+            currentParallelismQuotaForVideo = Math.max(0, currentParallelismQuotaForVideo - 1)
           }
           continue
         }
 
+        if (isPaused) { continue }
+
         if (videoRenderingStrategy === RenderingStrategy.ON_DEMAND) {
           continue
         }
@@ -150,8 +137,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           continue
         }
 
-        if (parallelismQuotas.video > 0) {
-          parallelismQuotas.video = Math.max(0, parallelismQuotas.video - 1)
+        if (currentParallelismQuotaForVideo > 0) {
+          currentParallelismQuotaForVideo = Math.max(0, currentParallelismQuotaForVideo - 1)
           segmentsToRender.push(s)
         }
       } else if (s.category === ClapSegmentCategory.STORYBOARD) {
@@ -165,12 +152,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           // it is thus vital to deduct it from the parallelism quota,
           // to avoir triggering quoote limit on the providers side
           if (s.status === ClapSegmentStatus.IN_PROGRESS) {
-            parallelismQuotas.image = Math.max(0, parallelismQuotas.image - 1)
+            currentParallelismQuotaForImage = Math.max(0, currentParallelismQuotaForImage - 1)
           }
           
           continue
         }
         // console.log(`useResolver.runLoop(): found a storyboard segment that has to be generated`)
+        
+        if (isPaused) { continue }
 
         if (imageRenderingStrategy === RenderingStrategy.ON_DEMAND) {
           continue
@@ -192,9 +181,9 @@ export const useResolver = create<ResolverStore>((set, get) => ({
 
         // console.log(`useResolver.runLoop(): strategy is good to go`)
 
-        if (parallelismQuotas.image > 0) {
+        if (currentParallelismQuotaForImage > 0) {
           // console.log(`useResolver.runLoop(): quota is good to go`)
-          parallelismQuotas.image = Math.max(0, parallelismQuotas.image - 1)
+          currentParallelismQuotaForImage = Math.max(0, currentParallelismQuotaForImage - 1)
           segmentsToRender.push(s)
         }
       } else if (s.category === ClapSegmentCategory.DIALOGUE) {
@@ -205,12 +194,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           // it is thus vital to deduct it from the parallelism quota,
           // to avoir triggering quoote limit on the providers side
           if (s.status === ClapSegmentStatus.IN_PROGRESS) {
-            parallelismQuotas.voice = Math.max(0, parallelismQuotas.voice - 1)
+            currentParallelismQuotaForVoice = Math.max(0, currentParallelismQuotaForVoice - 1)
           }
           
           continue
         }
 
+        if (isPaused) { continue }
+
         if (voiceRenderingStrategy === RenderingStrategy.ON_DEMAND) {
           continue
         }
@@ -229,8 +220,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           continue
         }
 
-        if (parallelismQuotas.voice > 0) {
-          parallelismQuotas.voice = Math.max(0, parallelismQuotas.voice - 1)
+        if (currentParallelismQuotaForVoice > 0) {
+          currentParallelismQuotaForVoice = Math.max(0, currentParallelismQuotaForVoice - 1)
           segmentsToRender.push(s)
         }
       } else if (s.category === ClapSegmentCategory.SOUND) {
@@ -241,11 +232,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           // it is thus vital to deduct it from the parallelism quota,
           // to avoir triggering quoote limit on the providers side
           if (s.status === ClapSegmentStatus.IN_PROGRESS) {
-            parallelismQuotas.sound = Math.max(0, parallelismQuotas.sound - 1)
+            currentParallelismQuotaForSound = Math.max(0, currentParallelismQuotaForSound - 1)
           }
           
           continue
         }
+
+        if (isPaused) { continue }
+
         if (soundRenderingStrategy === RenderingStrategy.ON_DEMAND) {
           continue
         }
@@ -264,8 +258,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           continue
         }
 
-        if (parallelismQuotas.sound > 0) {
-          parallelismQuotas.sound = Math.max(0, parallelismQuotas.sound - 1)
+        if (currentParallelismQuotaForSound > 0) {
+          currentParallelismQuotaForSound = Math.max(0, currentParallelismQuotaForSound - 1)
           segmentsToRender.push(s)
         }
       } else if (s.category === ClapSegmentCategory.MUSIC) {
@@ -276,12 +270,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           // it is thus vital to deduct it from the parallelism quota,
           // to avoir triggering quoote limit on the providers side
           if (s.status === ClapSegmentStatus.IN_PROGRESS) {
-            parallelismQuotas.music = Math.max(0, parallelismQuotas.music - 1)
+            currentParallelismQuotaForMusic = Math.max(0, currentParallelismQuotaForMusic - 1)
           }
           
           continue
         }
 
+        if (isPaused) { continue }
+
         if (musicRenderingStrategy === RenderingStrategy.ON_DEMAND) {
           continue
         }
@@ -300,21 +296,13 @@ export const useResolver = create<ResolverStore>((set, get) => ({
           continue
         }
 
-        if (parallelismQuotas.music > 0) {
-          parallelismQuotas.music = Math.max(0, parallelismQuotas.music - 1)
+        if (currentParallelismQuotaForMusic > 0) {
+          currentParallelismQuotaForMusic = Math.max(0, currentParallelismQuotaForMusic - 1)
           segmentsToRender.push(s)
         }
       } // else continue
     }
 
-    if (!segmentsToRender.length) {
-      // nothing to do - this will be the most common case
-      return runLoopAgain()
-    }
-
-    // console.log(`useResolver.runLoop(): firing and forgetting ${segmentsToRender.length} new resolveSegment promises`)
-    // we fire and forget
-    segmentsToRender.forEach(segment => resolveSegment(segment))
 
     // we don't want to do something like this:
     // await Promise.allSettled(segmentsRenderingPromises)
@@ -323,9 +311,56 @@ export const useResolver = create<ResolverStore>((set, get) => ({
     // the idea here is that we don't want to wait for all segments
     // to finish before starting new ones.
 
+    const nbPendingRequestsForVideo = defaultParallelismQuotas.video - currentParallelismQuotaForVideo
+    const nbPendingRequestsForImage = defaultParallelismQuotas.image - currentParallelismQuotaForImage
+    const nbPendingRequestsForVoice = defaultParallelismQuotas.voice - currentParallelismQuotaForVoice
+    const nbPendingRequestsForSound = defaultParallelismQuotas.sound - currentParallelismQuotaForSound
+    const nbPendingRequestsForMusic = defaultParallelismQuotas.music - currentParallelismQuotaForMusic
+
+    const nbRequestsRunningInParallel =
+      nbPendingRequestsForVideo
+      + nbPendingRequestsForImage
+      + nbPendingRequestsForVoice
+      + nbPendingRequestsForSound
+      + nbPendingRequestsForMusic
+    
+    const isBusyResolving = nbRequestsRunningInParallel > 0
+    
+    set({
+      currentParallelismQuotaForVideo,
+      currentParallelismQuotaForImage,
+      currentParallelismQuotaForVoice,
+      currentParallelismQuotaForSound,
+      currentParallelismQuotaForMusic,
+      // just some aliases for convenience
+      nbPendingRequestsForVideo,
+      nbPendingRequestsForImage,
+      nbPendingRequestsForVoice,
+      nbPendingRequestsForSound,
+      nbPendingRequestsForMusic,
+      nbRequestsRunningInParallel,
+      isBusyResolving
+    })
+
+    // console.log(`useResolver.runLoop(): firing and forgetting ${segmentsToRender.length} new resolveSegment promises`)
+    // we fire and forget
+    segmentsToRender.forEach(segment => resolveSegment(segment))
+
     return runLoopAgain()
   },
 
+  
+  togglePause: (isPaused?: boolean): boolean => {
+    const { isPaused: previouslyPaused } = get()
+    if (typeof isPaused === "boolean") {
+      set({ isPaused })
+      return isPaused
+    } else {
+      set({ isPaused: !previouslyPaused })
+      return  !previouslyPaused
+    }
+  },
+
   /**
    * This resolve a segment
    * 

src/lib/core/constants.ts

@@ -4,7 +4,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 
 export const APP_NAME = "Clapper AI"
-export const APP_REVISION = "r20240616-0210"
+export const APP_REVISION = "r20240616-1825"
 
 export const APP_DOMAIN = "Clapper.app"
 export const APP_LINK = "https://clapper.app"

src/lib/hooks/useTitle.ts

@@ -0,0 +1,16 @@
+import { useState, useEffect } from 'react'
+
+export const useTitle = (): {
+  title: string
+  changeTitle: (newTitle: string) => void
+} => {
+  const [title, setTitle] = useState<string>(document.title)
+
+  useEffect(() => {
+    document.title = title
+  }, [title])
+
+  const changeTitle = (newTitle: string) => setTitle(newTitle)
+
+  return { title, changeTitle }
+}