various improvements

next.config.js

@@ -5,7 +5,32 @@ const nextConfig = {
       // a clap file can be quite large - but that's OK
       bodySizeLimit: '32mb'
     }
-  }
+  },
+  async headers() {
+    return [
+      {
+        // matching all API routes
+        source: "/api/:path*",
+        headers: [
+          { key: "Access-Control-Allow-Credentials", value: "true" },
+          { key: "Access-Control-Allow-Origin", value: "*" }, // replace this your actual origin
+          { key: "Access-Control-Allow-Methods", value: "GET,DELETE,PATCH,POST,PUT" },
+          { key: "Access-Control-Allow-Headers", value: "X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version" },
+        ]
+      },
+      {
+        // matching ALL routes
+        source: "/:path*",
+        headers: [
+          // for security reasons, performance.now() not performant unless we disable some CORS stuff
+          //  more context about why, please check the Security paragraph here:
+          // https://developer.mozilla.org/en-US/docs/Web/API/Performance/now#security_requirements
+          { key: "Cross-Origin-Opener-Policy", value: "same-origin" },
+          { key: "Cross-Origin-Embedder-Policy", value: "require-corp" }
+        ]
+      }
+    ]
+}
 }
 
 module.exports = nextConfig

package-lock.json

@@ -1,16 +1,16 @@
 {
-  "name": "@jbilcke-hf/clapper",
-  "version": "0.0.3",
+  "name": "@aitube/clapper",
+  "version": "0.0.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "@jbilcke-hf/clapper",
-      "version": "0.0.3",
+      "name": "@aitube/clapper",
+      "version": "0.0.0",
       "dependencies": {
         "@aitube/clap": "0.0.25",
-        "@aitube/engine": "0.0.16",
-        "@aitube/timeline": "0.0.19",
+        "@aitube/engine": "0.0.17",
+        "@aitube/timeline": "0.0.20",
         "@fal-ai/serverless-client": "^0.10.3",
         "@huggingface/hub": "^0.15.1",
         "@huggingface/inference": "^2.7.0",
@@ -106,17 +106,17 @@
       }
     },
     "node_modules/@aitube/engine": {
-      "version": "0.0.16",
-      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.16.tgz",
-      "integrity": "sha512-vu6PlL3y6DKeeHRfy2N48nQF4Gbebket3PXT7Toc23CfxupuAbZKPr09VT9bTE9v0EpuhCeSW6lPuqWeBtSQew==",
+      "version": "0.0.17",
+      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.17.tgz",
+      "integrity": "sha512-1uC7diXvU1nzpOu5C6SdngerCe2Pn9xi5SEV4VQBbm8plYuiuWeDgzpwqCDcoa6lDz0o3ExQaovBL7nkGre0sg==",
       "peerDependencies": {
         "@aitube/clap": "0.0.25"
       }
     },
     "node_modules/@aitube/timeline": {
-      "version": "0.0.19",
-      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.19.tgz",
-      "integrity": "sha512-CpgpVg875+KIERzrhM6zlejm3AbQ5shdFwSTYqLFtCtcH905EpK1WLbzmbuDWt/s9QYJEIRJOcE9iT78FfCJyw==",
+      "version": "0.0.20",
+      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.20.tgz",
+      "integrity": "sha512-gzis4zAIOXdfcwh8azhWuv4NT+2E8g3/XE33mOxxZEzQIRLTJg/RnoKliFzl6gwPIrt6Z/Z1lW7/kK3HMGdTRQ==",
       "dependencies": {
         "date-fns": "^3.6.0",
         "react-virtualized-auto-sizer": "^1.0.24"
@@ -764,14 +764,15 @@
       }
     },
     "node_modules/@huggingface/tasks": {
-      "version": "0.10.12",
-      "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.12.tgz",
-      "integrity": "sha512-bpW2+ey2SBoEzSvn0eTmE+loLM3/B0Nw2BQQt0xWfafkNrR8xd8Hhc+aH8KdBsWko5YSSUSxb5vS1VenQ6q+/w=="
+      "version": "0.10.14",
+      "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.14.tgz",
+      "integrity": "sha512-8Q3aqTO+ldTTqtK4OfMz/h5DiiMBzUnZKdV0Dq2+JX+UXvqnTDVOk+bJd0QVytJYyNeZgKsj7XQHvEQGyo9cFg=="
     },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.14",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz",
       "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==",
+      "deprecated": "Use @eslint/config-array instead",
       "dependencies": {
         "@humanwhocodes/object-schema": "^2.0.2",
         "debug": "^4.3.1",
@@ -796,7 +797,8 @@
     "node_modules/@humanwhocodes/object-schema": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz",
-      "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA=="
+      "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==",
+      "deprecated": "Use @eslint/object-schema instead"
     },
     "node_modules/@img/sharp-darwin-arm64": {
       "version": "0.33.4",
@@ -3995,9 +3997,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001629",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001629.tgz",
-      "integrity": "sha512-c3dl911slnQhmxUIT4HhYzT7wnBK/XYpGnYLOj4nJBaRiw52Ibe7YxlDaAeRECvA786zCuExhxIUJ2K7nHMrBw==",
+      "version": "1.0.30001632",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001632.tgz",
+      "integrity": "sha512-udx3o7yHJfUxMLkGohMlVHCvFvWmirKh9JAH/d7WOLPetlH+LTL5cocMZ0t7oZx/mdlOWXti97xLZWc8uURRHg==",
       "funding": [
         {
           "type": "opencollective",
@@ -6676,9 +6678,9 @@
       }
     },
     "node_modules/jiti": {
-      "version": "1.21.3",
-      "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.3.tgz",
-      "integrity": "sha512-uy2bNX5zQ+tESe+TiC7ilGRz8AtRGmnJH55NC5S0nSUjvvvM2hJHmefHErugGXN4pNv4Qx7vLsnNw9qJ9mtIsw==",
+      "version": "1.21.6",
+      "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz",
+      "integrity": "sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==",
       "bin": {
         "jiti": "bin/jiti.js"
       }
@@ -6776,9 +6778,9 @@
       }
     },
     "node_modules/langsmith": {
-      "version": "0.1.30",
-      "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.30.tgz",
-      "integrity": "sha512-g8f10H1iiRjCweXJjgM3Y9xl6ApCa1OThDvc0BlSDLVrGVPy1on9wT39vAzYkeadC7oG48p7gfpGlYH3kLkJ9Q==",
+      "version": "0.1.31",
+      "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.31.tgz",
+      "integrity": "sha512-G9zi+84RvUZ7UP/ZC0dx/9SYHk8Bhe9GywUeVBzEyt8M4QeU6FPWT7TEjDSqp/XuPJf5o59Z2QlmNJgMnpUd8Q==",
       "dependencies": {
         "@types/uuid": "^9.0.1",
         "commander": "^10.0.1",
@@ -7475,9 +7477,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.49.1",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.49.1.tgz",
-      "integrity": "sha512-bsFSNhhTNon+g6r4UYPKGLi+PlfP1G9TJGSkZS5nZx+PTwW2YUTlfxXxpOKrPab5auIXJdlYpC/g/wkHGR1xug==",
+      "version": "4.50.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.50.0.tgz",
+      "integrity": "sha512-2ADkNIU6Q589oYHr5pn9k7SbUcrBTK9X0rIXrYqwMVSoqOj1yK9/1OO0ExaWsqOOpD7o58UmRjeKlx9gKAcuKQ==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -7878,9 +7880,9 @@
       }
     },
     "node_modules/postcss-load-config/node_modules/lilconfig": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.1.tgz",
-      "integrity": "sha512-O18pf7nyvHTckunPWCV1XUNXU1piu01y2b7ATJ0ppkUkk8ocqVWBrYjJBCwHDjD/ZWcfyrA0P4gKhzWGi5EINQ==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.2.tgz",
+      "integrity": "sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==",
       "engines": {
         "node": ">=14"
       },
@@ -8421,9 +8423,9 @@
       }
     },
     "node_modules/replicate": {
-      "version": "0.30.1",
-      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.30.1.tgz",
-      "integrity": "sha512-9JZPia6p7UCNfye2kddb2AHcTf3ngyckzq4y5FZkuhhF06AuuOHKN1q6NRSp9iAch7Htmlhm+kRGc+EQbzY8Kw==",
+      "version": "0.30.2",
+      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.30.2.tgz",
+      "integrity": "sha512-U3yDrlwAV/zhaCd5Tcb8IPXqwkzvfXbN+YABM6RGPugtmuo8W5uwxoEQAetWD+xjS5jWLzISBYR5gPNBZPLOQw==",
       "engines": {
         "git": ">=2.11.0",
         "node": ">=18.0.0",
@@ -9796,35 +9798,35 @@
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="
     },
     "node_modules/web-audio-beat-detector": {
-      "version": "8.2.10",
-      "resolved": "https://registry.npmjs.org/web-audio-beat-detector/-/web-audio-beat-detector-8.2.10.tgz",
-      "integrity": "sha512-kVC0NLcuB/gPNr53TJR7zjasZpu3Uf+Ov5fBpHG0XsWaNVS9qZ+HY+J5pd1hD7BnaHH4CuYV7yaiRhjN41GaFQ==",
+      "version": "8.2.11",
+      "resolved": "https://registry.npmjs.org/web-audio-beat-detector/-/web-audio-beat-detector-8.2.11.tgz",
+      "integrity": "sha512-FJ3xvB42aPAV0KekVCasghzFPeLdIky3qQYzAObksPquqybzJE558KZQm1OdETWL7S/Rc1mL4K+Je+gwZp2Vzw==",
       "dependencies": {
-        "@babel/runtime": "^7.24.5",
-        "tslib": "^2.6.2",
-        "web-audio-beat-detector-broker": "^4.1.9",
-        "web-audio-beat-detector-worker": "^5.2.52"
+        "@babel/runtime": "^7.24.7",
+        "tslib": "^2.6.3",
+        "web-audio-beat-detector-broker": "^4.1.10",
+        "web-audio-beat-detector-worker": "^5.2.53"
       }
     },
     "node_modules/web-audio-beat-detector-broker": {
-      "version": "4.1.9",
-      "resolved": "https://registry.npmjs.org/web-audio-beat-detector-broker/-/web-audio-beat-detector-broker-4.1.9.tgz",
-      "integrity": "sha512-4xT0qN8Bb09P8MiCvIvMzHs6fP/uMC+q9oO0VG3A0YjwWGhm97rrXO0vHAUSvE/bUzZyyESpJo4GmpGWqnK9iQ==",
+      "version": "4.1.10",
+      "resolved": "https://registry.npmjs.org/web-audio-beat-detector-broker/-/web-audio-beat-detector-broker-4.1.10.tgz",
+      "integrity": "sha512-mSsZGXtWEp2cC+21GMZ8CPK583rwHtUHxCae84rIrh6i1DCXUuL5ETQZEaIEw2B+3I0WTnEF6mrnHrIjAYvfuw==",
       "dependencies": {
-        "@babel/runtime": "^7.24.5",
+        "@babel/runtime": "^7.24.7",
         "fast-unique-numbers": "^9.0.5",
         "standardized-audio-context": "^25.3.72",
-        "tslib": "^2.6.2",
-        "web-audio-beat-detector-worker": "^5.2.52"
+        "tslib": "^2.6.3",
+        "web-audio-beat-detector-worker": "^5.2.53"
       }
     },
     "node_modules/web-audio-beat-detector-worker": {
-      "version": "5.2.52",
-      "resolved": "https://registry.npmjs.org/web-audio-beat-detector-worker/-/web-audio-beat-detector-worker-5.2.52.tgz",
-      "integrity": "sha512-+W8wHnho7ybSrllvDz5M/LpmjgapdG2Otnn66jYgycPJmj1U8RnLzyhFuonOpFMJc4437IVEEVKBAAXVTVmRzw==",
+      "version": "5.2.53",
+      "resolved": "https://registry.npmjs.org/web-audio-beat-detector-worker/-/web-audio-beat-detector-worker-5.2.53.tgz",
+      "integrity": "sha512-K6gOGN/mxglO5A/IqkjdnKSOs7u8ltb8paVXN9x6DUhSWHJoMta9HqBlFiHZgIKNqDzmYTRS2SiKE+uaukZWzw==",
       "dependencies": {
-        "@babel/runtime": "^7.24.5",
-        "tslib": "^2.6.2"
+        "@babel/runtime": "^7.24.7",
+        "tslib": "^2.6.3"
       }
     },
     "node_modules/web-streams-polyfill": {

package.json

@@ -11,8 +11,8 @@
   },
   "dependencies": {
     "@aitube/clap": "0.0.25",
-    "@aitube/engine": "0.0.16",
-    "@aitube/timeline": "0.0.19",
+    "@aitube/engine": "0.0.17",
+    "@aitube/timeline": "0.0.20",
     "@fal-ai/serverless-client": "^0.10.3",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/inference": "^2.7.0",

src/app/api/{render → resolve}/providers/comfy-comfyicu/index.ts

@@ -1,10 +1,10 @@
 import Replicate from 'replicate'
 
-import { RenderRequest } from "@/types"
+import { ResolveRequest } from "@/types"
 import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
   if (!request.settings.comfyIcuApiKey) {
     throw new Error(`Missing API key for "Comfy.icu"`)
   }

src/app/api/{render → resolve}/providers/comfy-huggingface/index.ts

@@ -1,8 +1,8 @@
-import { RenderRequest } from "@/types"
+import { ResolveRequest } from "@/types"
 import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
   if (!request.settings.huggingFaceApiKey) {
     throw new Error(`Missing API key for "Hugging Face"`)
   }

src/app/api/{render → resolve}/providers/comfy-replicate/index.ts

@@ -1,10 +1,10 @@
 import { ClapSegment, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 
-import { RenderRequest } from "@/types"
+import { ResolveRequest } from "@/types"
 import { getComfyWorkflow } from "../comfy/getComfyWorkflow"
 import { runWorkflow } from "./runWorkflow"
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
   if (!request.settings.replicateApiKey) {
     throw new Error(`Missing API key for "Replicate.com"`)
   }

src/app/api/{render → resolve}/providers/comfy/getComfyWorkflow.ts

@@ -1,10 +1,10 @@
 import { ClapSegmentCategory } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 
-import { ComfyNode, RenderRequest } from "@/types"
+import { ComfyNode, ResolveRequest } from "@/types"
 
 // TODO move this to @aitube/engine or @aitube/engine-comfy
-export function getComfyWorkflow(request: RenderRequest) {
+export function getComfyWorkflow(request: ResolveRequest) {
 
   let comfyWorkflow = "{}"
 

src/app/api/{render → resolve}/providers/falai/index.ts

@@ -1,13 +1,13 @@
 import * as fal from '@fal-ai/serverless-client'
 
-import { FalAiImageSize, RenderRequest } from "@/types"
+import { FalAiImageSize, ResolveRequest } from "@/types"
 import { ClapMediaOrientation, ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 import { decodeOutput } from '@/lib/utils/decodeOutput'
 import { FalAiAudioResponse, FalAiImageResponse, FalAiSpeechResponse, FalAiVideoResponse } from './types'
-import { getRenderRequestPrompts } from '@/lib/utils/getRenderRequestPrompts'
+import { getResolveRequestPrompts } from '@/lib/utils/getResolveRequestPrompts'
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
   if (!request.settings.falAiApiKey) {
     throw new Error(`Missing API key for "Fal.ai"`)
   }
@@ -20,8 +20,8 @@ export async function renderSegment(request: RenderRequest): Promise<ClapSegment
 
   let content = ''
 
-  const prompts = getRenderRequestPrompts(request)
-  
+  const prompts = getResolveRequestPrompts(request)
+
   try {
 
     // for doc see: 
@@ -29,10 +29,11 @@ export async function renderSegment(request: RenderRequest): Promise<ClapSegment
     
     if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
 
-      const visualPrompt = getVideoPrompt(
-        request.segments,
-        request.entities
-      )
+            
+      if (!prompts.positivePrompt) {
+        console.error(`resolveSegment: cannot resolve a storyboard with an empty prompt`)
+        return segment
+      }
 
       const imageSize =
         request.meta.orientation === ClapMediaOrientation.SQUARE
@@ -43,10 +44,22 @@ export async function renderSegment(request: RenderRequest): Promise<ClapSegment
 
       let result: FalAiImageResponse | undefined = undefined
 
-      if (request.settings.falAiModelForImage === "fal-ai/pulid" && request.mainCharacterEntity?.imageId) {
+      if (request.settings.falAiModelForImage === "fal-ai/pulid") {
+        if (!request.mainCharacterEntity?.imageId) {
+          // throw new Error(`you selected model ${request.settings.falAiModelForImage}, but no character was found, so skipping`)
+          // console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/fast-sdxl`)
+          
+          // dirty fix to fallback to a non-face model
+          request.settings.falAiModelForImage = "fal-ai/fast-sdxl"
+        }
+      }
+
+      if (request.settings.falAiModelForImage === "fal-ai/pulid") {
         result = await fal.run(request.settings.falAiModelForImage, {
           input: {
-            prompt: prompts.positivePrompt,
+            reference_images: [{
+              image_url: request.mainCharacterEntity?.imageId
+            }],
             image_size: imageSize,
             num_images: 1,
             sync_mode: true,
@@ -55,12 +68,6 @@ export async function renderSegment(request: RenderRequest): Promise<ClapSegment
         }) as FalAiImageResponse
     
       } else {
-        //throw new Error(`you selected model ${request.settings.falAiModelForImage}, but no character was found, so skipping`)
-        console.log(`warning: user selected model ${request.settings.falAiModelForImage}, but no character was found. Falling back to fal-ai/fast-sdxl`)
-        
-        // dirty fix to fallback to a non-face model
-        request.settings.falAiModelForImage = "fal-ai/fast-sdxl"
-
         result = await fal.run(request.settings.falAiModelForImage, {
           input: {
             prompt: prompts.positivePrompt,

src/app/api/{render → resolve}/providers/huggingface/index.ts

@@ -1,13 +1,13 @@
 import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
 
-import { RenderRequest } from "@/types"
+import { ResolveRequest } from "@/types"
 import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 import { blobToBase64DataUri } from "@/lib/utils/blobToBase64DataUri"
-import { getRenderRequestPrompts } from "@/lib/utils/getRenderRequestPrompts"
+import { getResolveRequestPrompts } from "@/lib/utils/getResolveRequestPrompts"
 import { decodeOutput } from "@/lib/utils/decodeOutput"
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
 
   if (!request.settings.huggingFaceApiKey) {
     throw new Error(`Missing API key for "Hugging Face"`)
@@ -22,7 +22,7 @@ export async function renderSegment(request: RenderRequest): Promise<ClapSegment
   
   const segment: ClapSegment = { ...request.segment }
 
-  const prompts = getRenderRequestPrompts(request)
+  const prompts = getResolveRequestPrompts(request)
   
   try {
     const blob: Blob = await hf.textToImage({

src/app/api/{render → resolve}/providers/modelslab/index.ts

@@ -1,8 +1,8 @@
-import { RenderRequest } from "@/types"
+import { ResolveRequest } from "@/types"
 import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
   if (!request.settings.modelsLabApiKey) {
     throw new Error(`Missing API key for "ModelsLab.com"`)
   }

src/app/api/{render → resolve}/providers/replicate/index.ts

@@ -1,13 +1,13 @@
 import Replicate from 'replicate'
 
-import { RenderRequest } from "@/types"
+import { ResolveRequest } from "@/types"
 import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
 import { getVideoPrompt } from "@aitube/engine"
 import { fetchContentToBase64 } from '@/lib/utils/fetchContentToBase64'
-import { getRenderRequestPrompts } from '@/lib/utils/getRenderRequestPrompts'
+import { getResolveRequestPrompts } from '@/lib/utils/getResolveRequestPrompts'
 import { decodeOutput } from '@/lib/utils/decodeOutput'
 
-export async function renderSegment(request: RenderRequest): Promise<ClapSegment> {
+export async function resolveSegment(request: ResolveRequest): Promise<ClapSegment> {
   if (!request.settings.replicateApiKey) {
     throw new Error(`Missing API key for "Replicate.com"`)
   }
@@ -19,7 +19,7 @@ export async function renderSegment(request: RenderRequest): Promise<ClapSegment
   
   const segment: ClapSegment = { ...request.segment }
 
-  const prompts = getRenderRequestPrompts(request)
+  const prompts = getResolveRequestPrompts(request)
 
   try {
 

src/app/api/{render → resolve}/route.ts

@@ -1,22 +1,22 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { renderSegment as renderSegmentUsingHuggingFace } from "./providers/huggingface"
-import { renderSegment as renderSegmentUsingComfyReplicate } from "./providers/comfy-replicate"
-import { renderSegment as renderSegmentUsingReplicate } from "./providers/replicate"
-import { renderSegment as renderSegmentUsingComfyComfyIcu } from "./providers/comfy-comfyicu"
-import { renderSegment as renderSegmentUsingFalAi } from "./providers/falai"
-import { renderSegment as renderSegmentUsingModelsLab } from "./providers/modelslab"
+import { resolveSegment as resolveSegmentUsingHuggingFace } from "./providers/huggingface"
+import { resolveSegment as resolveSegmentUsingComfyReplicate } from "./providers/comfy-replicate"
+import { resolveSegment as resolveSegmentUsingReplicate } from "./providers/replicate"
+import { resolveSegment as resolveSegmentUsingComfyComfyIcu } from "./providers/comfy-comfyicu"
+import { resolveSegment as resolveSegmentUsingFalAi } from "./providers/falai"
+import { resolveSegment as resolveSegmentUsingModelsLab } from "./providers/modelslab"
 
-import { ComputeProvider, RenderRequest } from "@/types"
+import { ComputeProvider, ResolveRequest } from "@/types"
 import { ClapSegmentCategory } from "@aitube/clap"
 
 export async function POST(req: NextRequest) {
   // do we really need to secure it?
   // I mean.. in the end, the user is using their own credentials,
   // so they cannot siphon free OpenAI, HF, Replicate tokens
-  console.log(`TODO Julian: secure the endpoint`)
+  // console.log(`TODO Julian: secure the endpoint`)
   // await throwIfInvalidToken(req.headers.get("Authorization"))
-  const request = (await req.json()) as RenderRequest
+  const request = (await req.json()) as ResolveRequest
   
   const provider =
     request.segment.category === ClapSegmentCategory.STORYBOARD
@@ -33,25 +33,25 @@ export async function POST(req: NextRequest) {
 
   if (!provider) { throw new Error(`Segments of category ${request.segment.category} are not supported yet`)}
 
-  // console.log(`API RenderRequest = `, request.settings)
-  const renderSegment =
+  // console.log(`API ResolveRequest = `, request.settings)
+  const resolveSegment =
     provider === ComputeProvider.HUGGINGFACE
-    ? renderSegmentUsingHuggingFace
+    ? resolveSegmentUsingHuggingFace
     : provider === ComputeProvider.COMFY_HUGGINGFACE
-    ? renderSegmentUsingComfyReplicate
+    ? resolveSegmentUsingComfyReplicate
     : provider === ComputeProvider.REPLICATE
-    ? renderSegmentUsingReplicate
+    ? resolveSegmentUsingReplicate
     : provider === ComputeProvider.COMFY_COMFYICU
-    ? renderSegmentUsingComfyComfyIcu
+    ? resolveSegmentUsingComfyComfyIcu
     : provider === ComputeProvider.FALAI
-    ? renderSegmentUsingFalAi
+    ? resolveSegmentUsingFalAi
     : provider === ComputeProvider.MODELSLAB
-    ? renderSegmentUsingModelsLab
+    ? resolveSegmentUsingModelsLab
     : null
 
-  if (!renderSegment) { throw new Error(`Provider ${provider} is not supported yet`)}
+  if (!resolveSegment) { throw new Error(`Provider ${provider} is not supported yet`)}
  
-  const segment = await renderSegment(request)
+  const segment = await resolveSegment(request)
 
   return NextResponse.json(segment)
 }

src/components/core/timeline/index.tsx

@@ -1,69 +1,16 @@
-import { useEffect, useTransition } from "react"
-import { ClapSegment } from "@aitube/clap"
-import { ClapTimeline, useTimeline, SegmentRenderer } from "@aitube/timeline"
+import { useEffect } from "react"
+import { ClapTimeline, useTimeline, SegmentResolver } from "@aitube/timeline"
 
 import { cn } from "@/lib/utils"
-import { useSettings } from "@/controllers/settings"
-import { RenderRequest } from "@/types"
 import { useMonitor } from "@/controllers/monitor/useMonitor"
+import { useResolver } from "@/controllers/resolver/useResolver"
 
-const segmentRenderer: SegmentRenderer = async ({
-  segment,
-  segments,
-  entities,
-  speakingCharactersIds,
-  generalCharactersIds,
-  mainCharacterId,
-  mainCharacterEntity,
-  meta,
-}) => {
-
-  const settings = useSettings.getState().getSettings()
-  
-  const request: RenderRequest = {
-    settings,
-    segment,
-    segments,
-    entities,
-    speakingCharactersIds,
-    generalCharactersIds,
-    mainCharacterId,
-    mainCharacterEntity,
-    meta,
-  }
-
-  const res = await fetch("/api/render", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify(request)
-  })
-  console.log(`res:`, res)
-  const newSegment = (await res.json()) as ClapSegment
-  console.log(`newSegment:`, newSegment)
-  return newSegment
-}
 
 export function Timeline() {
-  const [_isPending, startTransition] = useTransition()
-
   const isReady = useTimeline(s => s.isReady)
 
-  const imageRenderingStrategy = useSettings(s => s.imageRenderingStrategy)
-  const setImageRenderingStrategy = useTimeline(s => s.setImageRenderingStrategy)
-  useEffect(() => {
-    if (isReady) setImageRenderingStrategy(imageRenderingStrategy)
-  }, [isReady, setImageRenderingStrategy, imageRenderingStrategy])
-  
-  const videoRenderingStrategy = useSettings(s => s.videoRenderingStrategy)
-  const setVideoRenderingStrategy = useTimeline(s => s.setVideoRenderingStrategy)
-  useEffect(() => {
-    if (isReady) setVideoRenderingStrategy(videoRenderingStrategy)
-  }, [isReady, setVideoRenderingStrategy, videoRenderingStrategy])
-  
-  const getSettings = useSettings(s => s.getSettings)
-  const setSegmentRenderer = useTimeline(s => s.setSegmentRenderer)
+  const resolveSegment: SegmentResolver = useResolver(s => s.resolveSegment)
+  const setSegmentResolver = useTimeline(s => s.setSegmentResolver)
 
   const jumpAt = useMonitor(s => s.jumpAt)
   const checkIfPlaying = useMonitor(s => s.checkIfPlaying)
@@ -73,14 +20,19 @@ export function Timeline() {
   const setIsPlaying = useTimeline(s => s.setIsPlaying)
   const setTogglePlayback = useTimeline(s => s.setTogglePlayback)
 
+  const startLoop = useResolver(s => s.startLoop)
+
   // this is important: we connect the monitor to the timeline
   useEffect(() => {
-    setSegmentRenderer(segmentRenderer)
+    if (!isReady) { return }
+    setSegmentResolver(resolveSegment)
     setJumpAt(jumpAt)
     setIsPlaying(checkIfPlaying)
     setTogglePlayback(togglePlayback)
+    startLoop()
   }, [isReady])
   
+  
   return (
     <ClapTimeline
       showFPS={false}

src/components/monitor/DynamicPlayer/VideoClipBuffer.tsx

@@ -0,0 +1,61 @@
+import { useEffect, useRef } from "react"
+
+import { cn } from "@/lib/utils"
+
+export function VideoClipBuffer({
+  src,
+  className,
+  muted = true,
+  isPlaying = false,
+}: {
+  src?: string;
+  className?: string;
+  muted?: boolean;
+  isPlaying?: boolean;
+}) {
+  const ref = useRef<HTMLVideoElement>(null)
+
+  const togglePlayVideo = (play: boolean) => {
+    const player = ref.current
+    if (!player) { return }
+    // console.log(`togglePlayVideo(${play}) (current status = ${player.paused ? "paused" : "playing"})`)
+    if (play && player.paused) {
+      // console.log("playing video..")
+      player.play()
+    } else if (!play && !player.paused) {
+      // console.log("pausing video..")
+      player.pause()
+    }
+  }
+
+  useEffect(() => {
+    const player = ref.current
+    if (!player) { return }
+    togglePlayVideo(isPlaying)
+  }, [isPlaying])
+
+  if (!src) { return null }
+  
+  return (
+    <video    
+      ref={ref}
+      autoPlay={isPlaying}
+      controls={false}
+      playsInline
+      muted={muted}
+      loop
+      className={cn(
+        `absolute`,
+        `h-full w-full rounded-md overflow-hidden`,
+
+        // iseally we could only use the ease-out and duration-150
+        // to avoid a weird fade to grey,
+        // but the ease out also depends on which video is on top of each other,
+        // in term of z-index, so we should also intervert this
+        `transition-all duration-100 ease-out`,
+        className
+      )}
+      src={src}
+    />
+  )
+}

src/components/monitor/DynamicPlayer/index.tsx

@@ -0,0 +1,113 @@
+"use client"
+
+import { useEffect, useRef, useState } from "react"
+
+import { cn } from "@/lib/utils"
+import { useMonitor } from "@/controllers/monitor/useMonitor"
+import { useTimeline } from "@aitube/timeline"
+
+import { useRequestAnimationFrame } from "@/lib/hooks"
+import { MonitoringMode } from "@/controllers/monitor/types"
+import { VideoClipBuffer } from "./VideoClipBuffer"
+import { useRenderLoop } from "@/controllers/renderer/useRenderLoop"
+import { useRenderer } from "@/controllers/renderer/useRenderer"
+
+export const DynamicPlayer = ({
+  className,
+}: {
+  className?: string
+}) => {  
+  const isPlaying = useMonitor(s => s.isPlaying)
+  const setMonitoringMode = useMonitor(s => s.setMonitoringMode)
+
+  const setCursorTimestampAtInMs = useTimeline(s => s.setCursorTimestampAtInMs)
+
+  // this should only be called once and at only one place in the project!
+  useRenderLoop()
+
+  const { activeVideoSegment, upcomingVideoSegment } = useRenderer(s => s.bufferedSegments)
+
+  const currentVideoUrl = activeVideoSegment?.assetUrl || ""
+
+  // the upcoming video we want to preload (note: we just want to preload it, not display it just yet)
+  const preloadVideoUrl = upcomingVideoSegment?.assetUrl || ""
+
+  const [buffer1Value, setBuffer1Value] = useState("")
+  const [buffer2Value, setBuffer2Value] = useState("")
+  const [activeBufferNumber, setActiveBufferNumber] = useState(1)
+
+  const timeoutRef = useRef<NodeJS.Timeout>()
+
+  const fadeDurationInMs = 300
+
+  useEffect(() => {
+    setMonitoringMode(MonitoringMode.DYNAMIC)
+  }, [])
+
+  useRequestAnimationFrame(() => {
+    const { isPlaying, lastTimelineUpdateAtInMs, setLastTimelineUpdateAtInMs } = useMonitor.getState()
+    if (!isPlaying) { return }
+    const { cursorTimestampAtInMs } = useTimeline.getState()
+    const newTimelineUpdateAtInMs = performance.now()
+    const elapsedTimeInMs = newTimelineUpdateAtInMs - lastTimelineUpdateAtInMs
+    // console.log(`TODO: move the timeline cursor according to the elapsed time`)
+    setCursorTimestampAtInMs(cursorTimestampAtInMs + elapsedTimeInMs)
+    setLastTimelineUpdateAtInMs(newTimelineUpdateAtInMs)
+
+  })
+
+  useEffect(() => {
+    // trivial case: we are at the initial state
+    if (!buffer1Value && !buffer2Value) {
+      setBuffer1Value(currentVideoUrl)
+      setBuffer2Value(preloadVideoUrl)
+      setActiveBufferNumber(1)
+    }
+  }, [buffer1Value, currentVideoUrl, preloadVideoUrl])
+
+
+  // console.log("cursorInSteps:", cursorInSteps)
+  useEffect(() => {
+    /*
+    console.log("ATTENTION: something changed among those: ", {
+      currentVideoUrl, preloadVideoUrl
+    })
+    */
+
+    clearTimeout(timeoutRef.current)
+
+    const newActiveBufferNumber = activeBufferNumber === 1 ? 2 : 1
+    // console.log(`our pre-loaded video should already be available in buffer ${newActiveBufferNumber}`)
+
+    setActiveBufferNumber(newActiveBufferNumber)
+
+    timeoutRef.current = setTimeout(() => {
+      // by now one buffer should be visible, and the other should be hidden
+      // so let's update the invisible one
+      if (newActiveBufferNumber === 2) {
+        setBuffer1Value(preloadVideoUrl)
+      } else {
+        setBuffer2Value(preloadVideoUrl)
+      }
+    }, fadeDurationInMs + 200) // let's add some security in here
+
+    return () => {
+      clearTimeout(timeoutRef.current)
+    }
+  }, [currentVideoUrl, preloadVideoUrl])
+
+  return (
+    <div className={cn(`@container flex flex-col flex-grow w-full`, className)}>
+      <VideoClipBuffer
+        src={buffer1Value}
+        isPlaying={isPlaying}
+        className={cn(activeBufferNumber === 1 ? `opacity-100` : `opacity-0`)}
+      />
+      <VideoClipBuffer
+        src={buffer2Value}
+        isPlaying={isPlaying}
+        className={cn(activeBufferNumber === 2 ? `opacity-100` : `opacity-0`)}
+      />
+    </div>
+  )
+}

src/components/monitor/UniversalPlayer/index.tsx

@@ -2,6 +2,7 @@ import { ClapSegment } from "@aitube/clap"
 import { useTimeline } from "@aitube/timeline"
 
 import { StaticPlayer } from "../../monitor/StaticPlayer"
+import { DynamicPlayer } from "../DynamicPlayer"
 
 // TODO: put this in a separate component eg @aitube-player or @aitube/monitor
 export function UniversalPlayer() {
@@ -9,8 +10,12 @@ export function UniversalPlayer() {
 
   const assetUrl: string = finalVideo?.assetUrl || ""
 
-  console.log('finalVideo:', finalVideo)
+  // console.log('finalVideo:', finalVideo)
   
+  // note: I think we are going to only display the final video in specific cases,
+  // and use the dynamic player 99% the time
+
+  /*
   if (assetUrl) {
     return (
       <div className="
@@ -28,16 +33,17 @@ export function UniversalPlayer() {
       </div>
     )
   }
+    */
 
-  console.log(`TODO: render the scene dynamically`)
 
   return (
     <div className="
-      flex flex-col 
+      flex flex-col flex-grow
       items-center justify-center
-      w-full h-full
-    ">
-    <p>TODO</p>
+      w-full h-[calc(100%-60px)]
+      ">
+      <DynamicPlayer
+      />
     </div>
   )
 }

src/components/monitor/icons/single-icon.tsx

@@ -25,7 +25,7 @@ export function SingleIcon({
 
        // icons is a bit too fat, let's thin them out
        // for a bit of flair we increase the stroke width on group hover
-       thickOnHover ? `stroke-1 group-hover:stroke-2` : ``,
+       thickOnHover ? `stroke-[0.5] group-hover:stroke-[1]` : ``,
       className,
     )}
   />

src/controllers/audio/getDefaultAudioState.ts

@@ -11,7 +11,10 @@ export function getDefaultAudioState(): AudioState {
   }
   
   const state: AudioState = {
+    isPlaying: false,
     isMuted: false,
+    userDefinedGain: 1.0,
+    currentGain: 1.0,
     audioContext,
     currentlyPlaying: [],
   }

src/controllers/audio/startAudioSourceNode.ts

@@ -14,7 +14,7 @@ import { CurrentlyPlayingAudioSource } from "./types"
 export function startAudioSourceNode({
   audioContext,
   segment,
-  elapsedTimeInMs,
+  cursorTimestampAtInMs,
   onEnded
 }: {
   /**
@@ -32,7 +32,7 @@ export function startAudioSourceNode({
    * 
    * This is the position of the playback cursor in the project, in milliseconds (eg. 20000ms)
    */
-  elapsedTimeInMs: number
+  cursorTimestampAtInMs: number
 
   /**
    * Called whenever the audio source will finish playing
@@ -73,7 +73,7 @@ export function startAudioSourceNode({
   gainNode.connect(audioContext.destination)
 
   // make sure we play the segment at a specific time
-  const startTimeInMs = elapsedTimeInMs - segment.startTimeInMs
+  const startTimeInMs = cursorTimestampAtInMs - segment.startTimeInMs
 
   // convert milliseconds to seconds by dividing by 1000
   source.start(audioContext.currentTime, startTimeInMs >= 1000 ? (startTimeInMs / 1000) : 0)
@@ -82,6 +82,7 @@ export function startAudioSourceNode({
     sourceId: UUID(),
     segmentId: segment.id,
     sourceNode: source,
+    originalGain: segment.outputGain,
     gainNode: gainNode,
   }
 

src/controllers/audio/types.ts

@@ -1,3 +1,4 @@
+import { RuntimeSegment } from "@/types"
 
 export type AudioAnalysis = {
   audioBuffer: AudioBuffer
@@ -7,12 +8,22 @@ export type AudioAnalysis = {
 }
 
 export type AudioState = {
+  isPlaying: boolean
   isMuted: boolean
+  userDefinedGain: number
+  currentGain: number
   audioContext: AudioContext // we keep a single audio context
   currentlyPlaying: CurrentlyPlayingAudioSource[]
 }
 
 export type AudioControls = {
+  play: () => void
+  stop: () => void
+  setUserDefinedGain: (userDefinedGain: number) =>  void
+  setCurrentGain: (currentGain: number) => void
+  mute: () => void
+  unmute: () =>  void
+  syncAudioToCurrentCursorPosition: (activeAudioSegments: RuntimeSegment[]) => void
 }
 
 export type AudioStore = AudioState & AudioControls
@@ -39,6 +50,9 @@ export type CurrentlyPlayingAudioSource = {
    */
   sourceNode: AudioScheduledSourceNode
 
+  // the original value that was set to the segment
+  originalGain: number
+
   /**
    * The gain node, to control the volume
    * 

src/controllers/audio/useAudio.ts

@@ -1,10 +1,104 @@
 "use client"
 
 import { create } from "zustand"
+import { TimelineStore, useTimeline } from "@aitube/timeline"
+
+import { RuntimeSegment } from "@/types"
 
 import { AudioStore } from "./types"
 import { getDefaultAudioState } from "./getDefaultAudioState"
+import { startAudioSourceNode } from "./startAudioSourceNode"
 
 export const useAudio = create<AudioStore>((set, get) => ({
   ...getDefaultAudioState(),
+
+  play: () => {
+    // console.log("useAudio: play()")
+    const { isPlaying, currentlyPlaying } = get()
+    if (isPlaying) { return }
+    currentlyPlaying.forEach(p => { p.sourceNode.start() })
+  },
+  stop: () => {
+    // console.log("useAudio: stop()")
+    const { isPlaying, currentlyPlaying } = get()
+    if (isPlaying) { return }
+    currentlyPlaying.forEach(p => { p.sourceNode.stop() })
+    // no need to update currentlyPlaying, it will be automatic
+    // see function playAudioSegment(), below the "source.sourceNode.onended = () => {"
+  },
+
+  setUserDefinedGain: (userDefinedGain: number) => {
+    // console.log(`useAudio: setUserDefinedGain(${userDefinedGain})`)
+    const { setCurrentGain } = get()
+    set({ userDefinedGain })
+    setCurrentGain(userDefinedGain)
+  },
+
+  setCurrentGain: (currentGain: number) => {
+    // console.log(`useAudio: setCurrentGain(${currentGain})`)
+    const { currentlyPlaying } = get()
+    set({ currentGain, isMuted: currentGain === 0 })
+    currentlyPlaying.forEach(p => {
+      p.gainNode.gain.value = p.originalGain * currentGain
+    })
+  },
+
+  mute: () => {
+    // console.log("useAudio: mute()")
+    const { setCurrentGain } = get()
+    setCurrentGain(0)
+  },
+
+  unmute: () => {
+    // console.log("useAudio: unmute()")
+    const { setCurrentGain, userDefinedGain } = get()
+    setCurrentGain(userDefinedGain)
+  },
+
+  /**
+   * This makes sure we are playing what should be played
+   * 
+   * @returns 
+   */
+  syncAudioToCurrentCursorPosition: (activeAudioSegments: RuntimeSegment[]) => {
+    // console.log("useAudio: syncAudioToCurrentCursorPosition()")
+    const { audioContext, currentlyPlaying } = get()
+
+    const timelineStore: TimelineStore = useTimeline.getState()
+    const { cursorTimestampAtInMs } = timelineStore
+
+    const segments: RuntimeSegment[] = activeAudioSegments.filter(s =>
+      !currentlyPlaying.some(p => p.segmentId === s.id)
+    )
+
+    if (!segments.length) {
+      return
+    }
+    // console.log("useAudio: found audio segments that should be playing")
+
+    const newlyStartedAudioSourceNodes = segments.map((segment: RuntimeSegment) =>
+      startAudioSourceNode({
+        audioContext,
+        segment,
+        cursorTimestampAtInMs,
+        onEnded: (sourceId) => {
+          // console.log("useAudio: removing the old source node from the list of playing")
+          // since this callback might be called 30 sec, 3 min, 60 min later,
+          //it is vital to import a fresh store state using useAudio.getState()
+          set({
+            // then we can finally we remove the source from the list, synchronously
+            currentlyPlaying: get().currentlyPlaying.filter(p => p.sourceId !== sourceId)
+          })
+        }
+      })
+    )
+
+    set({
+      currentlyPlaying: [
+         // let's not forget to keep the current samples!
+        ...currentlyPlaying,
+        ...newlyStartedAudioSourceNodes
+      ]
+    })
+  },
 }))

src/controllers/monitor/getDefaultMonitorState.ts

@@ -3,6 +3,7 @@ import { MonitoringMode, MonitorState } from "./types"
 export function getDefaultMonitorState(): MonitorState {
   const state: MonitorState = {
     mode: MonitoringMode.NONE,
+    lastTimelineUpdateAtInMs: 0,
     isPlaying: false,
     staticVideoRef: undefined
   }

src/controllers/monitor/types.ts

@@ -6,6 +6,7 @@ export enum MonitoringMode {
 
 export type MonitorState = {
   mode: MonitoringMode
+  lastTimelineUpdateAtInMs: number
   isPlaying: boolean
   staticVideoRef?: HTMLVideoElement
 }
@@ -34,5 +35,7 @@ export type MonitorControls = {
    * @returns 
    */
   jumpAt: (timeInMs?: number) => void
+
+  setLastTimelineUpdateAtInMs: (lastTimelineUpdateAtInMs: number) => void
 }
 export type MonitorStore = MonitorState & MonitorControls

src/controllers/monitor/useMonitor.ts

@@ -1,8 +1,11 @@
 "use client"
 
 import { create } from "zustand"
+import { ClapSegment } from "@aitube/clap"
 import { useTimeline } from "@aitube/timeline"
 
+
+import { useAudio } from "../audio/useAudio"
 import { MonitoringMode, MonitorStore } from "./types"
 import { getDefaultMonitorState } from "./getDefaultMonitorState"
 
@@ -34,8 +37,13 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
     isPlaying: boolean
   } => {
     const { isPlaying: wasPlaying, mode, staticVideoRef } = get()
+    const { play, stop } = useAudio.getState()
 
     if (mode === MonitoringMode.NONE) {
+      set({
+        isPlaying: false,
+        lastTimelineUpdateAtInMs: performance.now()
+      })
       return {
         wasPlaying: false,
         isPlaying: false
@@ -43,21 +51,29 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
     }
 
     const isPlaying = typeof forcePlaying === "boolean" ? forcePlaying : !wasPlaying
-    
-    set({
-      isPlaying
-    })
+
 
     if (mode === MonitoringMode.STATIC && staticVideoRef) {
       if (isPlaying) {
-        console.log(`previous value = ` + staticVideoRef.currentTime)
+        //  console.log(`previous value = ` + staticVideoRef.currentTime)
         staticVideoRef.play()
       } else {
         staticVideoRef.pause()
       }
     } else if (mode === MonitoringMode.DYNAMIC) {
-      console.log(`TODO Julian: implement dynamic mode`)
+      // console.log(`TODO Julian: implement dynamic mode`)
+      if (isPlaying) {
+        // restart audio
+        play()
+      } else {
+        stop()
+      }
     }
+        
+    set({
+      isPlaying,
+      lastTimelineUpdateAtInMs: performance.now()
+    })
 
     return {
        wasPlaying,
@@ -79,8 +95,13 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
       // console.log("resetting static video current time")
       staticVideoRef.currentTime = timeInMs / 1000
     } else if (mode === MonitoringMode.DYNAMIC) {
-      console.log(`TODO Julian: implement dynamic mode`)
+      // console.log(`TODO Julian: implement jump`)
+      // for audio I think it will be automatic
     }
   },
 
+  setLastTimelineUpdateAtInMs: (lastTimelineUpdateAtInMs: number) => {
+    set({ lastTimelineUpdateAtInMs })
+  },
+
 }))

src/controllers/renderer/constants.ts

@@ -0,0 +1,12 @@
+
+import { DEFAULT_COLUMNS_PER_SLICE, DEFAULT_DURATION_IN_MS_PER_STEP } from "@aitube/timeline"
+
+// how much we look up in the future
+export const blockSizeInMs =
+  DEFAULT_COLUMNS_PER_SLICE // this equals to 4 at the time of writing, but this might become dynamic
+  * DEFAULT_DURATION_IN_MS_PER_STEP // this equals to 500 at the time of writing
+
+  // max refresh rate (eg. 100 ms)
+  // if the user has a fast device you can try faster rates
+  export const ACTIVE_SEGMENTS_REFRESH_RATE_IN_MS = 100
+

src/controllers/renderer/getDefaultBufferedSegments.ts

@@ -0,0 +1,18 @@
+import { BufferedSegments } from "./types"
+
+export function getDefaultBufferedSegments(): BufferedSegments {
+  const result: BufferedSegments = {
+    activeSegmentsCacheKey: "",
+    activeSegments: [],
+    activeVideoSegment: undefined,
+    activeStoryboardSegment: undefined,
+    activeAudioSegments: [],
+
+    upcomingSegmentsCacheKey: "",
+    upcomingSegments: [],
+    upcomingVideoSegment: undefined,
+    upcomingStoryboardSegment: undefined,
+    upcomingAudioSegments: [],
+  }
+  return result
+}

src/controllers/renderer/getDefaultRendererState.ts

@@ -0,0 +1,11 @@
+import { getDefaultBufferedSegments } from "./getDefaultBufferedSegments"
+import { RendererState } from "./types"
+
+export function getDefaultRendererState(): RendererState {
+  const state: RendererState = {
+    bufferedSegments: getDefaultBufferedSegments(),
+
+    // put more stuff here
+  }
+  return state
+}

src/controllers/renderer/getSegmentCacheKey.ts

@@ -0,0 +1,13 @@
+import { ClapSegment } from "@aitube/clap"
+
+export function getSegmentCacheKey(segment: ClapSegment, prefix = "") {
+
+  // we have to be smart here because we can't take the full base64 assetUrl (it might be huge)
+  // so we only use a portion of it
+  
+  return `${prefix}:${
+  segment.id
+  }_${
+  segment.assetUrl.slice(0, 1024)
+  }`
+}

src/controllers/renderer/index.ts

@@ -0,0 +1,17 @@
+export {
+  blockSizeInMs,
+  ACTIVE_SEGMENTS_REFRESH_RATE_IN_MS
+} from "./constants"
+export { getDefaultBufferedSegments } from "./getDefaultBufferedSegments"
+export { getDefaultRendererState } from "./getDefaultRendererState"
+export { getSegmentCacheKey } from "./getSegmentCacheKey"
+export type {
+  ActiveSegments,
+  UpcomingSegments,
+  BufferedSegments,
+  RendererState,
+  RendererControls,
+  RendererStore
+} from "./types"
+export { useRenderer } from "./useRenderer"
+export { useRenderLoop } from "./useRenderLoop"

src/controllers/renderer/types.ts

@@ -0,0 +1,46 @@
+import { ClapSegment } from "@aitube/clap"
+
+export type ActiveSegments = {
+  activeSegmentsCacheKey: string
+  activeSegments: ClapSegment[]
+  activeVideoSegment?: ClapSegment
+  activeStoryboardSegment?: ClapSegment
+  activeAudioSegments: ClapSegment[]
+}
+
+export type UpcomingSegments = {
+  upcomingSegmentsCacheKey: string
+  upcomingSegments: ClapSegment[]
+  upcomingVideoSegment?: ClapSegment
+  upcomingStoryboardSegment?: ClapSegment
+  upcomingAudioSegments: ClapSegment[]
+}
+
+export type BufferedSegments = ActiveSegments & UpcomingSegments
+
+export type RendererState = {
+  bufferedSegments: BufferedSegments
+}
+
+export type RendererControls = {
+  // this will be called at 60 FPS - and yes, it is expensive
+  // we could probably improve things by using a temporal tree index
+
+  /**
+   * Cycle through the segments to see which ones are crossing the current cursor,
+   * then this updates the internal buffer of segments
+   * (this has side effects as it modifies the internal state)
+   * 
+   * @returns 
+   */
+  renderLoop: () => BufferedSegments
+
+  /**
+   * Cycle through the segments to see which ones are crossing the current cursor,
+   * and return it (this doesn't change any state, but it reads the state from various stores)
+   * @returns 
+   */
+  computeBufferedSegments: () => BufferedSegments
+}
+
+export type RendererStore = RendererState & RendererControls

src/controllers/renderer/useRenderLoop.ts

@@ -0,0 +1,23 @@
+import { useRequestAnimationFrame } from "@/lib/hooks"
+import { useRenderer } from "./useRenderer"
+import { useAudio } from "@/controllers/audio/useAudio"
+import { useMonitor } from "../monitor/useMonitor"
+
+/**
+ * Runs a rendering loop
+ * 
+ * Should only be called once!!
+ * @returns 
+ */
+export function useRenderLoop(): void {
+  useRequestAnimationFrame(() => {
+    if (!useMonitor.getState().isPlaying) { return }
+    // this update the internal state of the renderer to make it hold
+    // all the currently visible or hearable items
+    const { activeAudioSegments } = useRenderer.getState().renderLoop()
+
+    // now all we need to do is to update the audio
+    // (well we also need to update the visuals, but it is done in <DynamicPlayer />
+    useAudio.getState().syncAudioToCurrentCursorPosition(activeAudioSegments)
+  })
+}

src/controllers/renderer/useRenderer.ts

@@ -0,0 +1,118 @@
+"use client"
+
+import { create } from "zustand"
+import { ClapOutputType, ClapSegmentCategory } from "@aitube/clap"
+import { TimelineStore, useTimeline } from "@aitube/timeline"
+
+import { RuntimeSegment } from "@/types"
+
+import { BufferedSegments, RendererStore } from "./types"
+import { getDefaultRendererState } from "./getDefaultRendererState"
+import { getSegmentCacheKey } from "./getSegmentCacheKey"
+import { blockSizeInMs } from "./constants"
+import { getDefaultBufferedSegments } from "./getDefaultBufferedSegments"
+
+export const useRenderer = create<RendererStore>((set, get) => ({
+  ...getDefaultRendererState(),
+
+
+  // this will be called at 60 FPS - and yes, it is expensive
+  // we could probably improve things by using a temporal tree index
+  renderLoop: (): BufferedSegments => {
+
+    const { computeBufferedSegments, bufferedSegments } = get()
+
+
+    // note: although useRequestAnimationFrame is called at 60 FPS,
+    // computeBufferedSegments has a throttle since it is expensive
+    const maybeNewBufferedSegments = computeBufferedSegments()
+
+    const activeSegmentsChanged = maybeNewBufferedSegments.activeSegmentsCacheKey !== bufferedSegments.activeSegmentsCacheKey
+    const upcomingSegmentsChanged = maybeNewBufferedSegments.upcomingSegmentsCacheKey !== bufferedSegments.upcomingSegmentsCacheKey
+    
+    if (activeSegmentsChanged || upcomingSegmentsChanged) {
+
+      set({ bufferedSegments: maybeNewBufferedSegments })
+
+      return maybeNewBufferedSegments
+    }
+
+    return bufferedSegments
+  },
+
+  computeBufferedSegments: (): BufferedSegments => {
+    const timelineState: TimelineStore = useTimeline.getState()
+    const { cursorTimestampAtInMs, segments: clapSegments } = timelineState
+    const segments = clapSegments as RuntimeSegment[]
+  
+    const results: BufferedSegments = getDefaultBufferedSegments()
+    // console.log("useRenderer: computeBufferedSegments() called")
+  
+  
+    // we could use a temporal index to keep things efficient here
+    // thiere is this relatively recent algorithm, the IB+ Tree, 
+    // which seems to be good for what we want to do
+    // https://www.npmjs.com/package/i2bplustree
+    // another solution could be to filter the segments into multiple arrays by category,
+    // although we will have to see when those segments are re-computed / synced
+    for (const segment of segments) {
+      const inActiveShot = segment.startTimeInMs <= cursorTimestampAtInMs && cursorTimestampAtInMs < segment.endTimeInMs
+  
+      if (inActiveShot) {
+        const isActiveVideo = segment.category === ClapSegmentCategory.VIDEO && segment.assetUrl
+        // const isActiveStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
+        if (isActiveVideo) {
+          results.activeSegments.push(segment)
+          results.activeVideoSegment = segment
+          results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
+        }
+  
+        const isActiveAudio =
+          // IF this is an audio segment
+          segment.outputType === ClapOutputType.AUDIO &&
+        
+          // AND there is an actual audio buffer attached to it
+          segment.audioBuffer
+  
+        if (isActiveAudio) {
+          results.activeSegments.push(segment)
+          results.activeAudioSegments.push(segment)
+          results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
+        }
+      }
+      
+      const inUpcomingShot =
+        (segment.startTimeInMs <= (cursorTimestampAtInMs + blockSizeInMs))
+        &&
+        ((cursorTimestampAtInMs + blockSizeInMs) < segment.endTimeInMs)
+  
+      if (inUpcomingShot) {
+        const isUpcomingVideo = segment.category === ClapSegmentCategory.VIDEO && segment.assetUrl
+        // const isUpcomingStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
+        if (isUpcomingVideo) {
+          results.upcomingSegments.push(segment)
+          results.upcomingVideoSegment = segment
+          results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
+        }
+  
+        const isUpcomingAudio =
+          // IF this is an audio segment
+          segment.outputType === ClapOutputType.AUDIO &&
+        
+          // AND there is an actual audio buffer attached to it
+          segment.audioBuffer
+  
+        if (isUpcomingAudio) {
+          results.upcomingSegments.push(segment)
+          results.upcomingAudioSegments.push(segment)
+          results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
+        }
+      }
+        
+    }
+  
+    // console.log("useRenderer: computeBufferedSegments() returning:", results)
+  
+    return results
+  },
+}))

src/controllers/resolver/constants.ts

@@ -0,0 +1,3 @@
+// TODO: we should take into account the rate limit of our provider
+// but I think this can be the responsibility of the parent app
+export const DEFAULT_WAIT_TIME_IF_NOTHING_TO_DO_IN_MS = 500

src/controllers/resolver/getDefaultResolverState.ts

@@ -0,0 +1,26 @@
+import { ResolverState } from "./types"
+
+export function getDefaultResolverState(): ResolverState {
+  const state: ResolverState = {
+    isRunning: false,
+
+    defaultParallelismQuotas: {
+      video: 1,
+      image: 1,
+      voice: 1,
+      sound: 1,
+      music: 1,
+    },
+  
+    currentParallelismQuotas: {
+      video: 1,
+      image: 1,
+      voice: 1,
+      sound: 1,
+      music: 1,
+    },
+    
+    nbRequestsRunningInParallel: 0
+  }
+  return state
+}

src/controllers/resolver/types.ts

@@ -0,0 +1,42 @@
+import { ClapSegment } from "@aitube/clap"
+
+export type ResolverState = {
+  isRunning: boolean
+
+  defaultParallelismQuotas: {
+    video: number
+    image: number
+    voice: number
+    sound: number
+    music: number
+  }
+
+  // used for UI display, show some metrics
+  currentParallelismQuotas: {
+    video: number
+    image: number
+    voice: number
+    sound: number
+    music: number
+  }
+
+  // used for UI display, show some metrics
+  nbRequestsRunningInParallel: number
+}
+
+export type ResolverControls = {
+  startLoop: () => void
+  runLoop: () => Promise<void>
+
+    /**
+   * This resolve a segment
+   * 
+   * Note: while we return a clap segment, the original will be replaced, too
+   * 
+   * @param segment 
+   * @returns 
+   */
+  resolveSegment: (segment: ClapSegment) => Promise<ClapSegment>
+}
+
+export type ResolverStore = ResolverState & ResolverControls

src/controllers/resolver/useResolver.ts

@@ -0,0 +1,366 @@
+"use client"
+
+import { create } from "zustand"
+import { ClapEntity, ClapSegment, ClapSegmentCategory, ClapSegmentFilteringMode, ClapSegmentStatus, filterSegments } from "@aitube/clap"
+import { RenderingStrategy, TimelineStore, useTimeline } from "@aitube/timeline"
+
+import { ResolveRequest, RuntimeSegment, SegmentVisibility, SegmentVisibilityPriority } from "@/types"
+
+import { getDefaultResolverState } from "./getDefaultResolverState"
+import { useSettings } from "../settings"
+import { DEFAULT_WAIT_TIME_IF_NOTHING_TO_DO_IN_MS } from "./constants"
+import { ResolverStore } from "./types"
+
+
+export const useResolver = create<ResolverStore>((set, get) => ({
+  ...getDefaultResolverState(),
+
+  startLoop: () => {
+    const {
+      isRunning,
+      runLoop
+    } = get()
+
+    console.log(`useResolver.startLoop() isRunning: ${isRunning}`)
+
+    if (isRunning) { return }
+
+    set({ isRunning: true })
+
+    setTimeout(() => {
+      runLoop()
+    }, 0)
+  },
+
+  /**
+   * A loop which reconstruct a queue at each cycle
+   * 
+   * this has to be dynamic since the user might be moving around 
+   * inside the timeline
+   * @returns 
+   */
+  runLoop: async (): Promise<void> => {
+
+    const { 
+      imageRenderingStrategy,
+      videoRenderingStrategy,
+      soundRenderingStrategy,
+      voiceRenderingStrategy,
+      musicRenderingStrategy,
+    } = useSettings.getState()
+    
+    const runLoopAgain = (waitTimeIfNothingToDoInMs = DEFAULT_WAIT_TIME_IF_NOTHING_TO_DO_IN_MS) => {
+      setTimeout(() => {
+        get().runLoop()
+      }, waitTimeIfNothingToDoInMs)
+    }
+
+    // ------- trivial case: maybe we have nothing to do? ------
+
+    const allStrategiesAreOnDemand =
+      imageRenderingStrategy === RenderingStrategy.ON_DEMAND &&
+      videoRenderingStrategy === RenderingStrategy.ON_DEMAND &&
+      soundRenderingStrategy === RenderingStrategy.ON_DEMAND &&
+      voiceRenderingStrategy === RenderingStrategy.ON_DEMAND &&
+      musicRenderingStrategy
+    
+    // nothing to do
+    if (allStrategiesAreOnDemand) {
+      // console.log(`useResolver.runLoop(): all strategies are on-demand only`)
+      return runLoopAgain()
+    }
+
+    // ---------- end of the very trivial case ----------------
+
+
+
+    // console.log(`useResolver.runLoop()`)
+    const timelineState: TimelineStore = useTimeline.getState()
+    const { visibleSegments, loadedSegments, segments: allSegments, resolveSegment } = timelineState
+
+    // ------------------------------------------------------------------------------------------------
+    //
+    // - we modify the original object in-line to add the visibility setting
+    // - there is a priority order: the info that a segment is "visible" (on screen),
+    //   is more important, which is why it is done after processing the "loaded" segments (the ones that are buffered, because near the sliding window)
+    const tmp: Record<string, RuntimeSegment> = {}
+    for (const s of loadedSegments) { (s as RuntimeSegment).visibility = SegmentVisibility.BUFFERED }
+    for (const s of visibleSegments) { (s as RuntimeSegment).visibility = SegmentVisibility.VISIBLE }
+
+    // sort segments by visibility:
+    // segments visible on screen are show first,
+    // then those nearby, then the hidden ones
+    const segments: RuntimeSegment[] = ([...allSegments] as RuntimeSegment[]).sort((segment1, segment2) => {
+      const priority1 = SegmentVisibilityPriority[segment1.visibility || SegmentVisibility.HIDDEN] || 0
+      const priority2 = SegmentVisibilityPriority[segment2.visibility || SegmentVisibility.HIDDEN] || 0
+      
+      return priority2 - priority1
+    })
+    //
+    // -------------------------------------------------------------------------
+
+    const { defaultParallelismQuotas } = get()
+
+    // note that we to create a copy here, that way we can modify it
+    const parallelismQuotas = {
+      ...defaultParallelismQuotas,
+    }
+    
+    // console.log(`useResolver.runLoop() parallelismQuotas = `, parallelismQuotas)
+
+    // we do not need ot get currentParallelismQuotas,
+    // as we are going to re-compute it
+    // (currentParallelismQuotas is only used in the UI 
+    // to display of the parallel request counter)
+
+    const segmentsToRender: ClapSegment[] = []
+    
+    // the following loop isn't the prettiest, but I think it presents
+    // the dynamic generation logic in a clear way, so let's keep it for now
+    for (const s of segments) {
+
+      if (s.category === ClapSegmentCategory.VIDEO) {
+
+        if (s.status !== ClapSegmentStatus.TO_GENERATE) {
+
+          // this is important: we found an in-progress task!
+          // it is thus vital to deduct it from the parallelism quota,
+          // to avoir triggering quoote limit on the providers side
+          if (s.status === ClapSegmentStatus.IN_PROGRESS) {
+            parallelismQuotas.video = Math.max(0, parallelismQuotas.video - 1)
+          }
+
+          continue
+        }
+        
+        if (videoRenderingStrategy === RenderingStrategy.ON_DEMAND) {
+          continue
+        }
+
+        if (parallelismQuotas.video > 0) {
+          parallelismQuotas.video = Math.max(0, parallelismQuotas.video - 1)
+          segmentsToRender.push(s)
+        }
+      } else if (s.category === ClapSegmentCategory.STORYBOARD) {
+
+        // console.log(`useResolver.runLoop(): found a storyboard segment`)
+
+        if (s.status !== ClapSegmentStatus.TO_GENERATE) {
+          // console.log(`useResolver.runLoop(): found a storyboard segment that is not to_generate`)
+
+          // this is important: we found an in-progress task!
+          // it is thus vital to deduct it from the parallelism quota,
+          // to avoir triggering quoote limit on the providers side
+          if (s.status === ClapSegmentStatus.IN_PROGRESS) {
+            parallelismQuotas.image = Math.max(0, parallelismQuotas.image - 1)
+          }
+          
+          continue
+        }
+        // console.log(`useResolver.runLoop(): found a storyboard segment that has to be generated`)
+
+        if (imageRenderingStrategy === RenderingStrategy.ON_DEMAND) {
+          continue
+        }
+
+        // console.log(`useResolver.runLoop(): strategy is good to go`)
+
+        if (parallelismQuotas.image > 0) {
+          // console.log(`useResolver.runLoop(): quota is good to go`)
+          parallelismQuotas.image = Math.max(0, parallelismQuotas.image - 1)
+          segmentsToRender.push(s)
+        }
+      } else if (s.category === ClapSegmentCategory.DIALOGUE) {
+
+        if (s.status !== ClapSegmentStatus.TO_GENERATE) {
+
+          // this is important: we found an in-progress task!
+          // it is thus vital to deduct it from the parallelism quota,
+          // to avoir triggering quoote limit on the providers side
+          if (s.status === ClapSegmentStatus.IN_PROGRESS) {
+            parallelismQuotas.voice = Math.max(0, parallelismQuotas.voice - 1)
+          }
+          
+          continue
+        }
+
+        if (voiceRenderingStrategy === RenderingStrategy.ON_DEMAND) {
+          continue
+        }
+        if (parallelismQuotas.voice > 0) {
+          parallelismQuotas.voice = Math.max(0, parallelismQuotas.voice - 1)
+          segmentsToRender.push(s)
+        }
+      } else if (s.category === ClapSegmentCategory.SOUND) {
+
+        if (s.status !== ClapSegmentStatus.TO_GENERATE) {
+
+          // this is important: we found an in-progress task!
+          // it is thus vital to deduct it from the parallelism quota,
+          // to avoir triggering quoote limit on the providers side
+          if (s.status === ClapSegmentStatus.IN_PROGRESS) {
+            parallelismQuotas.sound = Math.max(0, parallelismQuotas.sound - 1)
+          }
+          
+          continue
+        }
+        if (soundRenderingStrategy === RenderingStrategy.ON_DEMAND) {
+          continue
+        }
+        if (parallelismQuotas.sound > 0) {
+          parallelismQuotas.sound = Math.max(0, parallelismQuotas.sound - 1)
+          segmentsToRender.push(s)
+        }
+      } else if (s.category === ClapSegmentCategory.MUSIC) {
+
+        if (s.status !== ClapSegmentStatus.TO_GENERATE) {
+
+          // this is important: we found an in-progress task!
+          // it is thus vital to deduct it from the parallelism quota,
+          // to avoir triggering quoote limit on the providers side
+          if (s.status === ClapSegmentStatus.IN_PROGRESS) {
+            parallelismQuotas.music = Math.max(0, parallelismQuotas.music - 1)
+          }
+          
+          continue
+        }
+
+        if (musicRenderingStrategy === RenderingStrategy.ON_DEMAND) {
+          continue
+        }
+        if (parallelismQuotas.music > 0) {
+          parallelismQuotas.music = Math.max(0, parallelismQuotas.music - 1)
+          segmentsToRender.push(s)
+        }
+      } // else continue
+    }
+
+    if (!segmentsToRender.length) {
+      // nothing to do - this will be the most common case
+      return runLoopAgain()
+    }
+
+    // console.log(`useResolver.runLoop(): firing and forgetting ${segmentsToRender.length} new resolveSegment promises`)
+    // we fire and forget
+    segmentsToRender.forEach(segment => resolveSegment(segment))
+
+    // we don't want to do something like this:
+    // await Promise.allSettled(segmentsRenderingPromises)
+    // because that would limit us in terms of parallelism.
+    //
+    // the idea here is that we don't want to wait for all segments
+    // to finish before starting new ones.
+
+    return runLoopAgain()
+  },
+
+  /**
+   * This resolve a segment
+   * 
+   * Note: while we return a clap segment, the original will be replaced, too
+   * 
+   * @param segment 
+   * @returns 
+   */
+  resolveSegment: async (segment: ClapSegment): Promise<ClapSegment> => {
+
+    const settings = useSettings.getState().getSettings()
+  
+    const timelineState: TimelineStore = useTimeline.getState()
+
+    // note: do NOT use the visibleSegments here
+    // that's because resolveSegment is 100% asynchronous,
+    // meaning it might be called on invisible segments too!
+    const { clap, segments: allSegments } = timelineState
+
+    if (!clap?.meta || !allSegments.length) {
+      return segment
+      // throw new Error(`please call setSegmentRender(...) first`)
+    }
+
+    const shotSegments = filterSegments(
+      ClapSegmentFilteringMode.ANY,
+      segment,
+      allSegments
+    )
+
+    if (segment.status === ClapSegmentStatus.IN_PROGRESS) {
+      // console.log(`useResolver.resolveSegment(): warning: this segment is already being generated!`)
+      return segment
+    }
+  
+    segment.status = ClapSegmentStatus.IN_PROGRESS
+    
+    try {
+      const entities = clap.entityIndex || {}
+    
+      const speakingCharactersIds = shotSegments.map(s =>
+        s.category === ClapSegmentCategory.DIALOGUE ? s.entityId : null
+      ).filter(id => id) as string[]
+    
+      const generalCharactersIds = shotSegments.map(s =>
+        s.category === ClapSegmentCategory.CHARACTER ? s.entityId : null
+      ).filter(id => id) as string[]
+    
+      const mainCharacterId: string | undefined = speakingCharactersIds.at(0) || generalCharactersIds.at(0) || undefined
+    
+      const mainCharacterEntity: ClapEntity | undefined = mainCharacterId ? (entities[mainCharacterId] || undefined) : undefined
+
+      const request: ResolveRequest = {
+        settings,
+        segment,
+        segments: shotSegments,
+        entities,
+        speakingCharactersIds,
+        generalCharactersIds,
+        mainCharacterId,
+        mainCharacterEntity,
+        meta: clap.meta,
+      }
+
+      const res = await fetch("/api/resolve", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(request)
+      })
+      // console.log(`useResolver.resolveSegment(): result from /api.render:`, res)
+
+      const newSegmentData = (await res.json()) as ClapSegment
+      // console.log(`useResolver.resolveSegment(): newSegmentData`, newSegmentData)
+
+      const {
+        id,
+        assetUrl,
+        assetDurationInMs,
+        assetFileFormat,
+        assetSourceType,
+        status
+      } = newSegmentData
+
+      // note: this modifies the old object in-place
+      const newSegment = Object.assign(segment, {
+        id,
+        assetUrl,
+        assetDurationInMs,
+        assetFileFormat,
+        assetSourceType,
+        status
+      })
+
+    
+      return newSegment
+    } catch (err) {
+      console.error(`useResolver.resolveSegment(): error: ${err}`)
+
+      // we could do that in a future version to improve error tracking
+      // segment.status = ClapSegmentStatus.ERROR
+    } finally {
+      segment.status = ClapSegmentStatus.COMPLETED
+    }
+
+    return segment
+  }
+
+}))

src/lib/core/constants.ts

@@ -4,7 +4,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 
 export const APP_NAME = "Clapper AI"
-export const APP_REVISION = "r2024-06-08"
+export const APP_REVISION = "r2024-06-11"
 
 export const APP_DOMAIN = "Clapper.app"
 export const APP_LINK = "https://clapper.app"

src/lib/utils/convertToJpeg.ts

@@ -0,0 +1,20 @@
+import sharp from 'sharp'
+
+export function convertToJpeg(imageAsBase64DataUri: string): Promise<string> {
+  const matches = imageAsBase64DataUri.match(/^data:image\/([A-Za-z-+\/]+);base64,(.+)$/)
+  if (!matches || matches.length !== 3) {
+    throw new Error('Invalid input string format')
+  }
+
+  const imageData = Buffer.from(matches[2], 'base64')
+
+  return sharp(imageData)
+    .jpeg({
+      quality: 97
+    })
+    .toBuffer()
+    .then(newImageData => {
+      const base64Image = Buffer.from(newImageData).toString('base64')
+      return `data:image/jpeg;base64,${base64Image}`
+    })
+}

src/lib/utils/decodeOutput.ts

@@ -1,11 +1,23 @@
 import { fetchContentToBase64 } from "./fetchContentToBase64"
+import { convertToJpeg } from "./convertToJpeg"
 
 export async function decodeOutput(input: any): Promise<string> {
   const urlOrBase64 = `${input || ''}`
 
   if (!urlOrBase64) { return '' }
 
-  return urlOrBase64.startsWith("data:")
-  ? urlOrBase64
-  : (await fetchContentToBase64(urlOrBase64))
-}
+  const base64Url =
+    urlOrBase64.startsWith("data:")
+      ? urlOrBase64
+      : (await fetchContentToBase64(urlOrBase64))
+
+  // this step is important since some providers store data as PNG,
+  // which is a unreasonable since a few frames quickly add up to 10 Mb,
+  // we can't afford to have a 20 Gb .clap file
+  //
+  // if you really want to have a pro, Hollywood-grade storyboard storage,
+  // this isn't impossible but then you need to use either file paths or remote URL paths
+  const jpegImageAsBase64 = await convertToJpeg(base64Url)
+
+  return jpegImageAsBase64
+}

src/lib/utils/{getRenderRequestPrompts.ts → getResolveRequestPrompts.ts}

@@ -3,7 +3,7 @@ import { getVideoPrompt } from "@aitube/engine"
 
 import { SettingsState } from "@/controllers/settings"
 
-export function getRenderRequestPrompts({
+export function getResolveRequestPrompts({
   settings,
   segment,
   segments,
@@ -17,6 +17,7 @@ export function getRenderRequestPrompts({
   positivePrompt: string
   negativePrompt: string
 } {
+
   const videoPrompt = getVideoPrompt(
     segments,
     entities
@@ -34,7 +35,7 @@ export function getRenderRequestPrompts({
     : segment.category === ClapSegmentCategory.STORYBOARD
     ? settings.imagePromptSuffix
     : ""
-  ].map(x => x.trim()).join(", ")
+  ].map(x => x.trim()).filter(x => x).join(", ")
 
   const negativePrompt =  [
     segment.category === ClapSegmentCategory.VIDEO
@@ -42,7 +43,7 @@ export function getRenderRequestPrompts({
     : segment.category === ClapSegmentCategory.STORYBOARD
     ? settings.imageNegativePrompt
     : ""
-  ].map(x => x.trim()).join(", ")
+  ].map(x => x.trim()).filter(x => x).join(", ")
 
   return {
     positivePrompt,

src/types.ts

@@ -37,7 +37,7 @@ export enum ComfyIcuAccelerator {
   H100 = "H100"
 }
 
-export type RenderRequest = {
+export type ResolveRequest = {
   settings: SettingsState
 
   // the reference segment to render (eg. storyboard or video)
@@ -128,11 +128,48 @@ export interface ImageSegment {
     score: number;
 }
 
+export enum SegmentVisibility {
+  // the segment is visible, and the user explicitly requested to render it before the others
+  DEMANDED = "DEMANDED",
+
+  // TODO: add some implicit intermediary priority options
+  // such as SELECTED, HOVERED..
+
+  // the segment (or at least a portion of it) is currently visible in the sliding window
+  VISIBLE = "VISIBLE",
+
+  // the segment is hidden, but not too far from the sliding window
+  BUFFERED = "BUFFERED",
+
+  // fully hidden, far from the sliding window
+  HIDDEN = "HIDDEN"
+}
+
+// used for sort
+export const SegmentVisibilityPriority: Record<SegmentVisibility, number> = {
+    // the segment is visible, and the user explicitly requested to render it before the others
+  [SegmentVisibility.DEMANDED]: 3,
+
+  // TODO: add some implicit intermediary priority options
+  // such as SELECTED, HOVERED..
+
+  // the segment (or at least a portion of it) is currently visible in the sliding window
+  [SegmentVisibility.VISIBLE]: 2,
+
+  // the segment is hidden, but not too far from the sliding window
+  [SegmentVisibility.BUFFERED]: 1,
+
+  // fully hidden, far from the sliding window
+  [SegmentVisibility.HIDDEN]: 0
+}
+
 // some data can only exist inside a browser session (eg. AudioBuffer)
 // or at least data that only make sense on client side
 // we could put things like a mouse hover or selected state in here
 export type BrowserOnlySegmentData = {
   audioBuffer?: AudioBuffer
+
+  visibility?: SegmentVisibility
 }
 
 export type RuntimeSegment = ClapSegment & BrowserOnlySegmentData