story + system prompts + bpm + framerate + ratio

packages/api-client/src/api/editClapVideos.ts

@@ -42,7 +42,7 @@ export async function editClapVideos({
     params.t = "true"
   }
   // special trick to not touch the generated
-  // storyboards that are used by pending videos
+  // storyboard images that are used by pending videos
   const idsOfStoryboardsToKeep = clap.segments.map((segment: ClapSegment) => {
     
     const isPendingVideo = (
@@ -53,14 +53,14 @@ export async function editClapVideos({
 
     if (!isPendingVideo) { return undefined }
 
-    const storyboard: ClapSegment | undefined = filterSegments(
+    const storyboardImage: ClapSegment | undefined = filterSegments(
       ClapSegmentFilteringMode.BOTH,
       segment,
       clap.segments,
-      ClapSegmentCategory.STORYBOARD
+      ClapSegmentCategory.IMAGE
     ).at(0)
 
-    return storyboard?.id
+    return storyboardImage?.id
   }).filter((x: any) => x) as string[]
 
   const newClap = await fetchClap(`${aitubeApiUrl}edit/videos?${queryString.stringify(params)}`, {

packages/app/public/samples/claps/empty_project.yaml

@@ -7,15 +7,18 @@
   description: An exciting new project
   synopsis: ""
   licence: ""
-  orientation: landscape
+  imageRatio: landscape
   durationInMs: 18000
   width: 1024
   height: 575
   defaultVideoModel: AnimateDiff-Lightning
-  extraPositivePrompt: []
-  screenplay: ""
+  imagePrompt: ""
+  storyPrompt: ""
+  systemPrompt: ""
   isLoop: false
   isInteractive: false
+  bpm: 120
+  frameRate: 24
 - id: e16478cf-3535-48e9-98e8-f702ec9ca348
   track: 0
   startTimeInMs: 0

packages/app/src/app/api/assistant/samples.ts

@@ -11,7 +11,7 @@ import {
       "prompt": "",
       "startTimeInMs": 767000,
       "endTimeInMs": 769000,
-      "category": "STORYBOARD"
+      "category": "IMAGE"
     },
     {
       "segmentId": 2,
@@ -25,7 +25,7 @@ import {
       "prompt": "",
       "startTimeInMs": 769000,
       "endTimeInMs": 771000,
-      "category": "STORYBOARD"
+      "category": "IMAGE"
     },
     {
       "segmentId": 4,

packages/app/src/app/api/resolve/providers/aitube/index.ts

@@ -32,7 +32,7 @@ export async function resolveSegment(
     segments: request.segments,
   })
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     const resolvedClap = await editClapStoryboards({
       clap,
       completionMode: ClapCompletionMode.PARTIAL,
@@ -40,19 +40,19 @@ export async function resolveSegment(
       token: '<TODO>',
     })
 
-    const storyboards = resolvedClap.segments.filter(
-      (s) => s.category === ClapSegmentCategory.STORYBOARD
+    const storyboardImages = resolvedClap.segments.filter(
+      (s) => s.category === ClapSegmentCategory.IMAGE
     )
 
-    const storyboard = storyboards.at(0)
+    const storyboardImage = storyboardImages.at(0)
 
-    if (!storyboard) {
-      throw new Error(`failed to generate a storyboard`)
+    if (!storyboardImage) {
+      throw new Error(`failed to generate a storyboard image`)
     }
 
     return {
       ...request.segment,
-      ...(storyboard as TimelineSegment),
+      ...(storyboardImage as TimelineSegment),
     }
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
     const resolvedClap = await editClapVideos({

packages/app/src/app/api/resolve/providers/bigmodel/index.ts

@@ -22,14 +22,14 @@ export async function resolveSegment(
     note 
     if (!request.prompts.image.positive) {
       console.error(
-        `resolveSegment: cannot resolve a storyboard with an empty prompt`
+        `resolveSegment: cannot resolve a storyboard image with an empty prompt`
       )
       return segment
     }
     */
 
     if (!request.prompts.video.image) {
-      throw new Error(`cannot generate a video without a storyboard`)
+      throw new Error(`cannot generate a video without a storyboard image`)
     }
 
     // https://bigmodel.cn/dev/api#cogvideox

packages/app/src/app/api/resolve/providers/comfy-comfydeploy/index.ts

@@ -14,7 +14,7 @@ export async function resolveSegment(
     throw new Error(`Missing API key for "ComfyDeploy"`)
   }
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     const inputFields =
       request.settings.imageGenerationWorkflow.inputFields || []
 

packages/app/src/app/api/resolve/providers/comfy-comfyicu/index.ts

@@ -15,7 +15,7 @@ export async function resolveSegment(
     throw new Error(`Missing API key for "Comfy.icu"`)
   }
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     const workflowId =
       request.settings.imageGenerationWorkflow.id.split('://').pop() || ''
 

packages/app/src/app/api/resolve/providers/comfy-huggingface/index.ts

@@ -12,7 +12,7 @@ export async function resolveSegment(
   if (!request.settings.huggingFaceApiKey) {
     throw new Error(`Missing API key for "Hugging Face"`)
   }
-  if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category !== ClapSegmentCategory.IMAGE) {
     throw new Error(
       `Clapper doesn't support ${request.segment.category} generation for provider "Comfy Hugging Face". Please open a pull request with (working code) to solve this!`
     )

packages/app/src/app/api/resolve/providers/comfy-replicate/index.ts

@@ -18,7 +18,7 @@ export async function resolveSegment(
 
   const segment: TimelineSegment = request.segment
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     const inputFields =
       request.settings.imageGenerationWorkflow.inputFields || []
 

packages/app/src/app/api/resolve/providers/comfyui/index.ts

@@ -46,13 +46,12 @@ export async function resolveSegment(
   ).init()
 
   if (
-    [ClapSegmentCategory.STORYBOARD, ClapSegmentCategory.VIDEO].includes(
+    [ClapSegmentCategory.IMAGE, ClapSegmentCategory.VIDEO].includes(
       request.segment.category
     )
   ) {
     const clapWorkflow = {
-      [ClapSegmentCategory.STORYBOARD]:
-        request.settings.imageGenerationWorkflow,
+      [ClapSegmentCategory.IMAGE]: request.settings.imageGenerationWorkflow,
       [ClapSegmentCategory.VIDEO]: request.settings.videoGenerationWorkflow,
     }[request.segment.category]
 

packages/app/src/app/api/resolve/providers/falai/index.ts

@@ -1,7 +1,7 @@
 import * as fal from '@fal-ai/serverless-client'
 import { TimelineSegment } from '@aitube/timeline'
 import { FalAiImageSize, ResolveRequest } from '@aitube/clapper-services'
-import { ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap'
+import { ClapImageRatio, ClapSegmentCategory } from '@aitube/clap'
 import {
   FalAiAudioResponse,
   FalAiImageResponse,
@@ -31,7 +31,7 @@ export async function resolveSegment(
   // for doc see:
   // https://fal.ai/models/fal-ai/fast-sdxl/api
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     model = request.settings.imageGenerationWorkflow.data || ''
 
     if (!request.prompts.image.positive) {
@@ -78,9 +78,9 @@ export async function resolveSegment(
 
     // this was the previous system
     /*
-      request.meta.orientation === ClapMediaOrientation.SQUARE
+      request.meta.orientation === ClapImageRatio.SQUARE
         ? FalAiImageSize.SQUARE_HD
-        : request.meta.orientation === ClapMediaOrientation.PORTRAIT
+        : request.meta.orientation === ClapImageRatio.PORTRAIT
           ? FalAiImageSize.PORTRAIT_16_9
           : FalAiImageSize.LANDSCAPE_16_9
     */
@@ -254,7 +254,7 @@ export async function resolveSegment(
       segment.assetUrl = result?.video?.url || ''
     } else if (model === 'fal-ai/stable-video') {
       if (!request.prompts.video.image) {
-        throw new Error(`cannot generate a video without a storyboard`)
+        throw new Error(`cannot generate a video without a storyboard image`)
       }
 
       const result = (await fal.run(model, {

packages/app/src/app/api/resolve/providers/falai/runFaceSwap.ts

@@ -17,7 +17,7 @@ export async function runFaceSwap(
 
   const segment: TimelineSegment = request.segment
 
-  if (segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (segment.category === ClapSegmentCategory.IMAGE) {
     const imageFaceswapWorkflowModel =
       request.settings.imageFaceswapWorkflow.data || ''
 

packages/app/src/app/api/resolve/providers/gradio/index.ts

@@ -9,7 +9,7 @@ export async function resolveSegment(
 ): Promise<TimelineSegment> {
   const segment = request.segment
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     segment.assetUrl = await callGradioApi({
       url: request.settings.gradioApiUrlForImage,
       inputs: request.prompts.image,

packages/app/src/app/api/resolve/providers/huggingface/index.ts

@@ -21,7 +21,7 @@ export async function resolveSegment(
     request.settings.huggingFaceApiKey
   )
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     segment.assetUrl = await generateImage(request)
   } else if (request.segment.category === ClapSegmentCategory.DIALOGUE) {
     segment.assetUrl = await generateVoice(request)

packages/app/src/app/api/resolve/providers/letzai/index.ts

@@ -16,7 +16,7 @@ export async function resolveSegment(
 
   let model = request.settings.imageGenerationWorkflow.data || ''
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     model = request.settings.imageGenerationWorkflow.data || ''
 
     if (!request.prompts.image.positive) {

packages/app/src/app/api/resolve/providers/modelslab/index.ts

@@ -12,7 +12,7 @@ export async function resolveSegment(
   if (!request.settings.modelsLabApiKey) {
     throw new Error(`Missing API key for "ModelsLab.com"`)
   }
-  if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category !== ClapSegmentCategory.IMAGE) {
     throw new Error(
       `Clapper doesn't support ${request.segment.category} generation for provider "ModelsLab". Please open a pull request with (working code) to solve this!`
     )

packages/app/src/app/api/resolve/providers/piapi/index.ts

@@ -1,6 +1,6 @@
 import { TimelineSegment } from '@aitube/timeline'
 import { ResolveRequest } from '@aitube/clapper-services'
-import { ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap'
+import { ClapImageRatio, ClapSegmentCategory } from '@aitube/clap'
 
 import { getWorkflowInputValues } from '../getWorkflowInputValues'
 import { createImage } from './midjourney/createImage'
@@ -18,7 +18,7 @@ export async function resolveSegment(
 
   let model = request.settings.imageGenerationWorkflow.data || ''
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     model = request.settings.imageGenerationWorkflow.data || ''
 
     if (!request.prompts.image.positive) {
@@ -65,7 +65,7 @@ export async function resolveSegment(
 
     if (!request.prompts.video.image) {
       console.error(
-        `resolveSegment: cannot generate video without a storyboard`
+        `resolveSegment: cannot generate video without a storyboard image`
       )
       return segment
     }
@@ -77,9 +77,9 @@ export async function resolveSegment(
     if (workflow.id === 'piapi://kling/v1/video') {
       // can only be 16:9,9:16,1:1
       const aspectRatio =
-        request.meta.orientation === ClapMediaOrientation.SQUARE
+        request.meta.orientation === ClapImageRatio.SQUARE
           ? '1:1'
-          : request.meta.orientation === ClapMediaOrientation.PORTRAIT
+          : request.meta.orientation === ClapImageRatio.PORTRAIT
             ? '9:16'
             : '16:9'
 

packages/app/src/app/api/resolve/providers/replicate/index.ts

@@ -1,6 +1,6 @@
 import Replicate from 'replicate'
 
-import { ClapMediaOrientation, ClapSegmentCategory } from '@aitube/clap'
+import { ClapImageRatio, ClapSegmentCategory } from '@aitube/clap'
 import { ResolveRequest } from '@aitube/clapper-services'
 import { TimelineSegment } from '@aitube/timeline'
 import { getWorkflowInputValues } from '../getWorkflowInputValues'
@@ -17,15 +17,15 @@ export async function resolveSegment(
 
   const segment = request.segment
 
-  if (request.segment.category == ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category == ClapSegmentCategory.IMAGE) {
     const { workflowValues } = getWorkflowInputValues(
       request.settings.imageGenerationWorkflow
     )
 
     const aspectRatio =
-      request.meta.orientation === ClapMediaOrientation.SQUARE
+      request.meta.orientation === ClapImageRatio.SQUARE
         ? '1:1'
-        : request.meta.orientation === ClapMediaOrientation.PORTRAIT
+        : request.meta.orientation === ClapImageRatio.PORTRAIT
           ? '9:16'
           : '16:9'
 

packages/app/src/app/api/resolve/providers/replicate/runFaceSwap.ts

@@ -14,7 +14,7 @@ export async function runFaceSwap(
 
   const segment: TimelineSegment = request.segment
 
-  if (segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (segment.category === ClapSegmentCategory.IMAGE) {
     const imageFaceswapWorkflowModel =
       request.settings.imageFaceswapWorkflow.data || ''
 

packages/app/src/app/api/resolve/providers/stabilityai/generateImage.ts

@@ -1,4 +1,4 @@
-import { ClapMediaOrientation } from '@aitube/clap'
+import { ClapImageRatio } from '@aitube/clap'
 
 import { ResolveRequest, StabilityAiImageSize } from '@aitube/clapper-services'
 
@@ -22,9 +22,9 @@ export async function generateImage(request: ResolveRequest): Promise<string> {
   }
 
   const aspectRatio =
-    request.meta.orientation === ClapMediaOrientation.SQUARE
+    request.meta.orientation === ClapImageRatio.SQUARE
       ? StabilityAiImageSize.SQUARE
-      : request.meta.orientation === ClapMediaOrientation.PORTRAIT
+      : request.meta.orientation === ClapImageRatio.PORTRAIT
         ? StabilityAiImageSize.PORTRAIT_9_16
         : StabilityAiImageSize.LANDSCAPE_16_9
 

packages/app/src/app/api/resolve/providers/stabilityai/index.ts

@@ -13,7 +13,7 @@ export async function resolveSegment(
 
   const segment = request.segment
 
-  if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
+  if (request.segment.category === ClapSegmentCategory.IMAGE) {
     segment.assetUrl = await generateImage(request)
   } else if (request.segment.category === ClapSegmentCategory.VIDEO) {
     segment.assetUrl = await generateVideo(request)

packages/app/src/app/embed/EmbeddedPlayer.tsx

@@ -11,7 +11,8 @@ export function EmbeddedPlayer() {
   const isPlaying = useMonitor((s) => s.isPlaying)
   const togglePlayback = useMonitor((s) => s.togglePlayback)
 
-  const meta = useTimeline((s) => s.meta)
+  const isInteractive = useTimeline((s) => s.isInteractive)
+  const isLive = useTimeline((s) => s.isLive)
 
   const [isOverlayVisible, setOverlayVisible] = useState(true)
 
@@ -64,8 +65,8 @@ export function EmbeddedPlayer() {
           >
             <div
               className={cn(`flex h-full flex-row items-center`, {
-                'bg-yellow-500/100': meta.isInteractive,
-                'bg-red-500/100': meta.isLive,
+                'bg-yellow-500/100': isInteractive,
+                'bg-red-500/100': isLive,
               })}
               style={{
                 width: '100%', // <-- TODO: compute the % of progression within the experience
@@ -90,7 +91,7 @@ export function EmbeddedPlayer() {
                 onClick={togglePlayback}
               />
               <StaticOrInteractiveTag
-                isInteractive={meta.isInteractive}
+                isInteractive={isInteractive}
                 size="md"
                 className=""
               />

packages/app/src/components/editors/FilterEditor/FilterViewer/index.tsx

@@ -14,7 +14,7 @@ import { useEffect, useState } from 'react'
 // TODO: move this to the renderer service
 // also since filters use WebGPU, I think one day we can run them in real-time
 // over the video as well (or maybe using WebGL)
-function useCurrentlyVisibleStoryboard(): string | undefined {
+function useVisibleStoryboardImages(): string | undefined {
   const { activeStoryboardSegment } = useRenderer((s) => s.bufferedSegments)
 
   // can't return something if there is nothing
@@ -25,7 +25,7 @@ function useCurrentlyVisibleStoryboard(): string | undefined {
   return activeStoryboardSegment.assetUrl
 }
 
-function useFilteredStoryboard(input?: string): string | undefined {
+function useFilteredStoryboardImages(input?: string): string | undefined {
   const current = useFilterEditor((s) => s.current)
   const runFilterPipeline = useFilterEditor((s) => s.runFilterPipeline)
   const [result, setResult] = useState('')
@@ -65,8 +65,8 @@ export function FilterViewer() {
   const undo = useFilterEditor((s) => s.undo)
   const redo = useFilterEditor((s) => s.redo)
 
-  const input = useCurrentlyVisibleStoryboard()
-  const output = useFilteredStoryboard(input)
+  const input = useVisibleStoryboardImages()
+  const output = useFilteredStoryboardImages(input)
 
   const hasBetaAccess = useUI((s) => s.hasBetaAccess)
 

packages/app/src/components/editors/ProjectEditor/index.tsx

@@ -35,7 +35,7 @@ export function ProjectEditor() {
   return (
     <FormSection label={'Project Settings'} className="p-4">
       <FormInput<string>
-        label={'title'}
+        label={'Title'}
         value={current.title || ''}
         defaultValue=""
         onChange={(title) => {
@@ -76,18 +76,32 @@ export function ProjectEditor() {
         minValue={256}
         maxValue={1024}
       />
-      {/*
-      for this one we will need some kind of draft mode
-      */}
+      <FormInput<number>
+        label={'BPM (Beats Per Minute) (WIP)'}
+        value={current.bpm || 110}
+        defaultValue={110}
+        minValue={1}
+        maxValue={500}
+      />
+      <FormInput<number>
+        label={'Frame rate (WIP)'}
+        value={current.frameRate || 24}
+        defaultValue={24}
+        minValue={1}
+        maxValue={1000}
+      />
       <FormInput<string>
-        label={'Global prompt keywords ("3D render, comical"..)'}
-        value={
-          Array.isArray(current.extraPositivePrompt)
-            ? current.extraPositivePrompt.join(', ')
-            : ''
-        }
-        onChange={(newKeywords) => {
-          // const keywords = newKeywords.split(",").map(x => x.trim())
+        label={'Global image/video prompt ("3D render, 1970 style..")'}
+        value={current.imagePrompt || ''}
+        onChange={(imagePrompt) => {
+          setCurrent({ ...current, imagePrompt })
+        }}
+      />
+      <FormInput<string>
+        label={'Global assistant prompt ("don\'t use swear words..")'}
+        value={current.systemPrompt || ''}
+        onChange={(systemPrompt) => {
+          setCurrent({ ...current, systemPrompt })
         }}
       />
       <FormInput<string>

packages/app/src/components/monitor/DynamicPlayer/StoryboardBuffer.tsx

@@ -30,7 +30,7 @@ export function StoryboardBuffer({
         className
       )}
       src={src}
-      alt="storyboard"
+      alt="storyboard image"
     />
   )
 }

packages/app/src/components/monitor/PlayerControls/index.tsx

@@ -26,7 +26,7 @@ export function PlayerControls({ className }: { className?: string }) {
   const jumpAt = useMonitor((s) => s.jumpAt)
 
   const cursorTimestampAtInMs = useTimeline((s) => s.cursorTimestampAtInMs)
-  const totalDurationInMs = useTimeline((s) => s.totalDurationInMs)
+  const durationInMs = useTimeline((s) => s.durationInMs)
 
   const handleAccelerate = () => {}
 
@@ -77,7 +77,7 @@ export function PlayerControls({ className }: { className?: string }) {
         />
       </div>
       <Counter
-        valueInMs={totalDurationInMs}
+        valueInMs={durationInMs}
         color={theme.monitorSecondaryTextColor || theme.defaultTextColor || ''}
       />
     </div>

packages/app/src/components/toolbars/editors-menu/EditorsSideMenu.tsx

@@ -28,12 +28,7 @@ export function EditorsSideMenu() {
   const theme = useTheme()
   return (
     <div
-      className="
-      w-10 md:w-11 lg:w-12 xl:w-13 flex
-      h-full
-      flex-col items-center justify-between
-      border-r
-      transition-all duration-200 ease-in-out"
+      className="flex h-full w-10 flex-col items-center justify-between border-r transition-all duration-200 ease-in-out md:w-11 lg:w-12 xl:w-13"
       style={{
         backgroundColor:
           theme.editorMenuBgColor || theme.defaultBgColor || '#eeeeee',
@@ -41,10 +36,7 @@ export function EditorsSideMenu() {
           theme.editorBorderColor || theme.defaultBorderColor || '#eeeeee',
       }}
     >
-      <div className="
-      flex h-full w-full flex-col items-center
-      transition-all duration-200 ease-in-out
-      ">
+      <div className="flex h-full w-full flex-col items-center transition-all duration-200 ease-in-out">
         <EditorsSideMenuItem view={EditorView.PROJECT} label="Project settings">
           <MdMovieEdit />
         </EditorsSideMenuItem>

packages/app/src/components/toolbars/editors-menu/EditorsSideMenuItem.tsx

@@ -54,15 +54,14 @@ export function EditorsSideMenuItem({
 
   return (
     <Tooltip delayDuration={0}>
-      <TooltipTrigger asChild disabled={!tooltipLabel}
-      className="
-      h-8 md:h-9 xl:h-11
-      w-full
-      transition-all duration-200 ease-in-out
-      ">
+      <TooltipTrigger
+        asChild
+        disabled={!tooltipLabel}
+        className="h-8 w-full transition-all duration-200 ease-in-out md:h-9 xl:h-11"
+      >
         <div
           className={cn(
-            `flex h-8 md:h-9 xl:h-11 w-full flex-col`,
+            `flex h-8 w-full flex-col md:h-9 xl:h-11`,
             `transition-all duration-150 ease-out`,
             `items-center justify-center`,
             unmanaged || isActive ? '' : `cursor-pointer`,

packages/app/src/components/toolbars/top-menu/IsBusy/index.tsx

@@ -6,7 +6,7 @@ export function IsBusy({ nbPendingTasks = 0 }: { nbPendingTasks?: number }) {
   return (
     <GoDotFill
       className={cn(
-        `-mt-[9px] ml-[1px] -mr-1.5 h-1.5 w-1.5 text-yellow-400`,
+        `-mr-1.5 -mt-[9px] ml-[1px] h-1.5 w-1.5 text-yellow-400`,
         nbPendingTasks > 0 ? 'animate-pulse opacity-100' : 'opacity-0'
       )}
     />

packages/app/src/components/toolbars/top-menu/TopMenuLogo/index.tsx

@@ -83,7 +83,7 @@ export function TopMenuLogo() {
             <BiSolidMoviePlay style={{ color }} />
           )}
           <span
-            className="hidden md:inline ld:text-md scale-[88%] pr-0.5 font-bold tracking-[-0.03em] md:pr-1 lg:pr-2"
+            className="ld:text-md hidden scale-[88%] pr-0.5 font-bold tracking-[-0.03em] md:inline md:pr-1 lg:pr-2"
             style={{ color }}
           >
             Clapper

packages/app/src/components/toolbars/top-menu/assistant/index.tsx

@@ -18,7 +18,7 @@ import { useAutocomplete } from '@/services/autocomplete/useAutocomplete'
 
 export function TopMenuAssistant() {
   const setShowSettings = useUI((s) => s.setShowSettings)
-  const storyboardsToStory = useAutocomplete((s) => s.storyboardsToStory)
+  const convertImagesToStory = useAutocomplete((s) => s.convertImagesToStory)
 
   const hasBetaAccess = useUI((s) => s.hasBetaAccess)
 
@@ -41,7 +41,7 @@ export function TopMenuAssistant() {
               <MenubarSeparator />
               <MenubarItem
                 onClick={() => {
-                  storyboardsToStory()
+                  convertImagesToStory()
                 }}
               >
                 Storyboards-to-captions (beta, client-side AI)

packages/app/src/components/ui/menubar.tsx

@@ -38,7 +38,7 @@ const MenubarTrigger = React.forwardRef<
   <MenubarPrimitive.Trigger
     ref={ref}
     className={cn(
-      'flex cursor-default select-none items-center rounded-sm px-1.5 lg:px-2 py-1.5 text-xs lg:text-sm font-normal outline-none data-[state=open]:bg-neutral-100 data-[state=open]:text-neutral-900 focus:bg-neutral-100 focus:text-neutral-900 dark:text-neutral-400 dark:data-[state=open]:bg-neutral-800 dark:data-[state=open]:text-neutral-300 dark:focus:bg-neutral-800 dark:focus:text-neutral-300',
+      'flex cursor-default select-none items-center rounded-sm px-1.5 py-1.5 text-xs font-normal outline-none data-[state=open]:bg-neutral-100 data-[state=open]:text-neutral-900 focus:bg-neutral-100 focus:text-neutral-900 dark:text-neutral-400 dark:data-[state=open]:bg-neutral-800 dark:data-[state=open]:text-neutral-300 dark:focus:bg-neutral-800 dark:focus:text-neutral-300 lg:px-2 lg:text-sm',
       className
     )}
     {...props}

packages/app/src/lib/utils/decodeOutput.ts

@@ -25,7 +25,7 @@ export async function decodeOutput(input: any): Promise<string> {
     // which is a unreasonable since a few frames quickly add up to 10 Mb,
     // we can't afford to have a 20 Gb .clap file
     //
-    // if you really want to have a pro, Hollywood-grade storyboard storage,
+    // if you really want to have a pro, Hollywood-grade storyboard image storage,
     // this isn't impossible but then you need to use either file paths or remote URL paths
     // and if you want some lossless like this, we should add a parameter to support that
     const jpegImageAsBase64 = await convertToJpeg(base64Url)

packages/app/src/lib/utils/formatSegmentForExport.ts

@@ -66,7 +66,7 @@ export function formatSegmentForExport(
 
   const isExportableToFile =
     (segment.category === ClapSegmentCategory.VIDEO ||
-      segment.category === ClapSegmentCategory.STORYBOARD ||
+      segment.category === ClapSegmentCategory.IMAGE ||
       segment.category === ClapSegmentCategory.DIALOGUE ||
       segment.category === ClapSegmentCategory.SOUND ||
       segment.category === ClapSegmentCategory.MUSIC) &&

packages/app/src/services/api/resolve.ts

@@ -13,6 +13,7 @@ import {
 } from '@aitube/clapper-services'
 import { useSettings } from '../settings'
 import {
+  ClapMeta,
   ClapSegmentCategory,
   ClapWorkflowEngine,
   ClapWorkflowProvider,
@@ -24,12 +25,14 @@ export async function resolve(
   req: Partial<ResolveRequest>
 ): Promise<TimelineSegment> {
   const { getRequestSettings }: SettingsStore = useSettings.getState()
-  const { meta }: TimelineStore = useTimeline.getState()
+  const timeline: TimelineStore = useTimeline.getState()
+
+  const meta = timeline.getClapMeta()
 
   const defaultTimelineSegment: TimelineSegment =
     await clapSegmentToTimelineSegment(
       newSegment({
-        category: ClapSegmentCategory.STORYBOARD,
+        category: ClapSegmentCategory.IMAGE,
       })
     )
 
@@ -67,6 +70,10 @@ export async function resolve(
       : [],
     mainCharacterId: req.mainCharacterId || undefined,
     mainCharacterEntity: req.mainCharacterEntity || undefined,
+
+    // jbilcke-hf: I don't think we need all of those fields
+    // for our request, especially since some are a bit large
+    // and probably slow-down all our requests eg. the story prompt, thumbnail..
     meta,
     prompts: getDefaultResolveRequestPrompts(req.prompts),
   }

packages/app/src/services/assistant/updateStoryAndScene.ts

@@ -150,7 +150,7 @@ export async function updateStoryAndScene({
       }
     } else {
       // if the LLM tries to add a new camera, we must
-      // also create corresponding video and storyboards segments as well
+      // also create corresponding video and storyboard image segments as well
       if (segment.category === ClapSegmentCategory.CAMERA) {
         segmentsToAdd.push(
           await clapSegmentToTimelineSegment(
@@ -167,7 +167,7 @@ export async function updateStoryAndScene({
             newSegment({
               ...segmentProperties,
               prompt: 'movie still',
-              category: ClapSegmentCategory.STORYBOARD,
+              category: ClapSegmentCategory.IMAGE,
               outputType: ClapOutputType.IMAGE,
             })
           )

packages/app/src/services/assistant/useAssistant.ts

@@ -187,7 +187,7 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
       bufferedSegments: { activeSegments },
     } = useRenderer.getState()
     const timeline: TimelineStore = useTimeline.getState()
-    const { meta, scenes, addSegment, entityIndex } = timeline
+    const { description, scenes, entityIndex } = timeline
 
     // note: here `settings` is not the store's state itself (with methods etc)
     // but a snapshot of the serializable state values only
@@ -269,7 +269,7 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
       fullScene: scene?.sequenceFullText || '',
       actionLine: scene?.line || '',
       entities: entityIndex,
-      projectInfo: meta.description,
+      projectInfo: description,
       history: get().history,
     }
 

packages/app/src/services/autocomplete/types.ts

@@ -12,7 +12,7 @@ export type AutocompleteControls = {
    * @param params
    * @returns
    */
-  storyboardsToStory: (params?: {
+  convertImagesToStory: (params?: {
     startTimeInMs?: number
     endTimeInMs?: number
   }) => Promise<void>

packages/app/src/services/autocomplete/useAutocomplete.ts

@@ -26,7 +26,7 @@ import { AutocompleteStore } from './types'
 export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
   ...getDefaultAutocompleteState(),
 
-  storyboardsToStory: async (
+  convertImagesToStory: async (
     params: {
       startTimeInMs?: number
       endTimeInMs?: number
@@ -46,7 +46,7 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
       : 0
     const endTimeInMs = isValidNumber(params?.endTimeInMs)
       ? params?.endTimeInMs!
-      : timeline.totalDurationInMs
+      : timeline.durationInMs
 
     const range = { startTimeInMs, endTimeInMs }
 
@@ -56,33 +56,35 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
 
       // since this is very long task, we can run it in the background
       visibility: TaskVisibility.BACKGROUND,
-      initialMessage: `Analyzing storyboards..`,
-      successMessage: `Analyzing storyboards.. 100% done`,
+      initialMessage: `Analyzing images..`,
+      successMessage: `Analyzing images.. 100% done`,
       value: 0,
     })
 
     set({ isRunning: true })
 
     try {
-      const storyboards = filterSegments<TimelineSegment>(
+      const storyboardImages = filterSegments<TimelineSegment>(
         ClapSegmentFilteringMode.ANY,
         range,
         timeline.segments,
-        ClapSegmentCategory.STORYBOARD
-      ).filter((storyboard) => storyboard.assetUrl.startsWith('data:'))
+        ClapSegmentCategory.IMAGE
+      ).filter((storyboardImage) =>
+        storyboardImage.assetUrl.startsWith('data:')
+      )
 
       let i = 0
       let progress = 0
       // to keep things light and in the background, we use an async for loop
-      for (const storyboard of storyboards) {
+      for (const storyboardImage of storyboardImages) {
         const isStillRunning = get().isRunning
         if (!isStillRunning) {
           break
         }
 
         try {
-          console.log(`analyzing storyboard:`, storyboard)
-          const frames = [storyboard.assetUrl]
+          console.log(`analyzing storyboard image:`, storyboardImage)
+          const frames = [storyboardImage.assetUrl]
           const captions = await extractCaptionsFromFrames(
             frames,
             (
@@ -93,7 +95,7 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
               // this will be counting from to 100%, for each call to extractCaptionsFromFrames()
               // so TODO @Julian: adjust this for the right calculation
               // task.setProgress({
-              //   message: `Analyzing storyboards (${progress}%)`,
+              //   message: `Analyzing storyboard images (${progress}%)`,
               //   value: progress,
               // })
             }
@@ -101,12 +103,12 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
 
           i++
 
-          const relativeProgress = i / storyboards.length
+          const relativeProgress = i / storyboardImages.length
 
           progress += relativeProgress * 100
 
           task.setProgress({
-            message: `Analyzing storyboards (${Math.round(progress)}%)`,
+            message: `Analyzing images (${Math.round(progress)}%)`,
             value: progress,
           })
 
@@ -168,8 +170,8 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
                 category: ClapSegmentCategory.CAMERA,
                 prompt: 'medium-shot',
                 label: 'medium-shot',
-                startTimeInMs: storyboard.startTimeInMs,
-                endTimeInMs: storyboard.endTimeInMs,
+                startTimeInMs: storyboardImage.startTimeInMs,
+                endTimeInMs: storyboardImage.endTimeInMs,
                 status: ClapSegmentStatus.COMPLETED,
                 track: timeline.findFreeTrack({ startTimeInMs, endTimeInMs }), // track row index
               })
@@ -187,8 +189,8 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
                   category,
                   prompt,
                   label: prompt,
-                  startTimeInMs: storyboard.startTimeInMs,
-                  endTimeInMs: storyboard.endTimeInMs,
+                  startTimeInMs: storyboardImage.startTimeInMs,
+                  endTimeInMs: storyboardImage.endTimeInMs,
                   status: ClapSegmentStatus.COMPLETED,
                   track: timeline.findFreeTrack({ startTimeInMs, endTimeInMs }), // track row index
                 })
@@ -199,13 +201,16 @@ export const useAutocomplete = create<AutocompleteStore>((set, get) => ({
 
           await timeline.addSegments({ segments })
         } catch (err) {
-          console.error(`failed to analyze a storyboard:`, err)
+          console.error(`failed to analyze a storyboard image:`, err)
         }
 
         // TODO: use a special prompt to get categorized captions
       }
     } catch (err) {
-      console.error(`storyboardsToStory(): failed to analyze storyboards:`, err)
+      console.error(
+        `convertImagesToStory(): failed to analyze storyboard images:`,
+        err
+      )
     } finally {
       task.success()
       set({ isRunning: false })

packages/app/src/services/editors/project-editor/useProjectEditor.ts

@@ -12,8 +12,9 @@ export const useProjectEditor = create<ProjectEditorStore>((set, get) => ({
   setCurrent: (current?: ClapMeta) => {
     const timeline: TimelineStore = useTimeline.getState()
     set({ current })
-    if (current && timeline?.meta) {
-      Object.assign(timeline.meta, current)
+    if (current && timeline) {
+      // note: we really need to be sure that current is valid here
+      Object.assign(timeline, current)
     }
   },
   undo: () => {},

packages/app/src/services/editors/script-editor/useScriptEditor.ts

@@ -40,13 +40,13 @@ export const useScriptEditor = create<ScriptEditorStore>((set, get) => ({
     const { highlightElements, textModel } = get()
 
     set({
-      current: clap.meta.screenplay,
-      lastPublished: clap.meta.screenplay,
+      current: clap.meta.storyPrompt,
+      lastPublished: clap.meta.storyPrompt,
     })
 
     try {
       // we need to update the model
-      textModel?.setValue(clap.meta.screenplay)
+      textModel?.setValue(clap.meta.storyPrompt)
     } catch (err) {
       // to catch the "Error: Model is disposed!"
       // this can happen if the timing isn't right,

packages/app/src/services/editors/workflow-editor/getSegmentWorkflowProviderAndEngine.ts

@@ -25,7 +25,7 @@ export function getSegmentWorkflowProviderAndEngine({
   lipsyncEngine?: ClapWorkflowEngine
 } {
   const generationWorkflow: ClapWorkflow | undefined =
-    segment.category === ClapSegmentCategory.STORYBOARD
+    segment.category === ClapSegmentCategory.IMAGE
       ? settings.imageGenerationWorkflow
       : segment.category === ClapSegmentCategory.VIDEO
         ? settings.videoGenerationWorkflow
@@ -44,7 +44,7 @@ export function getSegmentWorkflowProviderAndEngine({
     generationWorkflow?.engine || undefined
 
   const faceswapWorkflow: ClapWorkflow | undefined =
-    segment.category === ClapSegmentCategory.STORYBOARD
+    segment.category === ClapSegmentCategory.IMAGE
       ? settings.imageFaceswapWorkflow
       : segment.category === ClapSegmentCategory.VIDEO
         ? settings.videoFaceswapWorkflow

packages/app/src/services/editors/workflow-editor/workflows/comfyui/getComfyWorkflow.ts

@@ -6,7 +6,7 @@ export function getComfyWorkflow(category: ClapSegmentCategory) {
 
   let comfyWorkflow
 
-  if (category === ClapSegmentCategory.STORYBOARD) {
+  if (category === ClapSegmentCategory.IMAGE) {
     comfyWorkflow = settings.comfyClapWorkflowForImage
   } else if (category === ClapSegmentCategory.VIDEO) {
     comfyWorkflow = settings.comfyClapWorkflowForVideo

packages/app/src/services/io/ffmpegUtils.ts

@@ -380,7 +380,7 @@ export async function createFullSilentVideo(
       continue
     }
 
-    const inputFilename = `input_${index}_${UUID()}.${input.category === ClapSegmentCategory.STORYBOARD ? 'png' : 'mp4'}`
+    const inputFilename = `input_${index}_${UUID()}.${input.category === ClapSegmentCategory.IMAGE ? 'png' : 'mp4'}`
     await ffmpeg.writeFile(inputFilename, input.data)
 
     const segmentDuration = (input.endTimeInMs - input.startTimeInMs) / 1000
@@ -388,7 +388,7 @@ export async function createFullSilentVideo(
 
     let outputFilename = `output_${index}_${UUID()}.mp4`
 
-    if (input.category === ClapSegmentCategory.STORYBOARD) {
+    if (input.category === ClapSegmentCategory.IMAGE) {
       // Handle image input
       console.log(`${TAG}: Processing image input`)
       const ffmpegCommand = [

packages/app/src/services/io/formats/edl.ts

@@ -1,16 +1,5 @@
-import {
-  ClapAssetSource,
-  ClapProject,
-  ClapSegment,
-  ClapSegmentCategory,
-  UUID,
-} from '@aitube/clap'
-import {
-  TimelineSegment,
-  timelineSegmentToClapSegment,
-  TimelineStore,
-  useTimeline,
-} from '@aitube/timeline'
+import { ClapAssetSource, ClapProject } from '@aitube/clap'
+import { TimelineStore, useTimeline } from '@aitube/timeline'
 import {
   ExportableSegment,
   formatSegmentForExport,
@@ -28,12 +17,7 @@ const secondsToTimecode = (seconds: number, fps: number): string => {
 
 export async function generateEDL(): Promise<string> {
   const timeline: TimelineStore = useTimeline.getState()
-  const {
-    meta,
-    getClap,
-    totalDurationInMs,
-    segments: timelineSegments,
-  } = timeline
+  const { title, getClap, segments: timelineSegments } = timeline
 
   const clap: ClapProject | null = await getClap()
   if (!clap) {
@@ -51,7 +35,7 @@ export async function generateEDL(): Promise<string> {
     return folderName.slice(0, 7) // Limit to 7 characters for EDL compatibility
   }
 
-  let edlContent = `TITLE: ${meta.title}\nFCM: NON-DROP FRAME\n\n`
+  let edlContent = `TITLE: ${title}\nFCM: NON-DROP FRAME\n\n`
   let timelineInPoint = 0
 
   const exportableSegments: ExportableSegment[] = timelineSegments

packages/app/src/services/io/formats/fcp.ts

@@ -1,16 +1,5 @@
-import {
-  ClapAssetSource,
-  ClapProject,
-  ClapSegment,
-  ClapSegmentCategory,
-  UUID,
-} from '@aitube/clap'
-import {
-  TimelineSegment,
-  timelineSegmentToClapSegment,
-  TimelineStore,
-  useTimeline,
-} from '@aitube/timeline'
+import { ClapProject, UUID } from '@aitube/clap'
+import { TimelineStore, useTimeline } from '@aitube/timeline'
 import {
   ExportableSegment,
   formatSegmentForExport,
@@ -18,14 +7,9 @@ import {
 
 export async function generateFCP(): Promise<string> {
   const timeline: TimelineStore = useTimeline.getState()
-  const {
-    meta,
-    getClap,
-    totalDurationInMs,
-    segments: timelineSegments,
-  } = timeline
+  const { title, width, height, getClap, segments: timelineSegments } = timeline
 
-  const DEFAULT_FRAME_RATE = 30
+  const DEFAULT_FRAME_RATE = 24
 
   const formatFCPTime = (
     timeInMs: number,
@@ -41,12 +25,12 @@ export async function generateFCP(): Promise<string> {
   }
 
   const createAssetFormat = (id: string): string => {
-    return /* XML */ `<format id="${id}" name="FFVideoFormat${meta.height}p${DEFAULT_FRAME_RATE}" frameDuration="${formatFCPTime(1000 / DEFAULT_FRAME_RATE)}" width="${meta.width}" height="${meta.height}"/>`
+    return /* XML */ `<format id="${id}" name="FFVideoFormat${height}p${DEFAULT_FRAME_RATE}" frameDuration="${formatFCPTime(1000 / DEFAULT_FRAME_RATE)}" width="${width}" height="${height}"/>`
   }
 
   const resources: string[] = []
   const assetClips: string[] = []
-  const formatId = `r${meta.width}x${meta.height}`
+  const formatId = `r${width}x${height}`
 
   resources.push(createAssetFormat(formatId))
 
@@ -89,8 +73,8 @@ export async function generateFCP(): Promise<string> {
     ${resources.join('\n')}
   </resources>
   <library>
-    <event name="${meta.title}">
-      <project name="${meta.title}">
+    <event name="${title}">
+      <project name="${title}">
         <sequence format="${formatId}" tcStart="0s" tcFormat="NDF" audioLayout="stereo" audioRate="48k">
           <spine>
             ${assetClips.join('\n')}
@@ -104,14 +88,20 @@ export async function generateFCP(): Promise<string> {
 
 export async function generateFCP7XML(): Promise<string> {
   const timeline: TimelineStore = useTimeline.getState()
-  const { meta, segments: timelineSegments } = timeline
+  const {
+    title,
+    durationInMs,
+    width,
+    height,
+    segments: timelineSegments,
+  } = timeline
 
   let xmlContent = `<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE xmeml>
 <xmeml version="5">
   <sequence>
-    <name>${meta.title}</name>
-    <duration>${meta.durationInMs / 1000}</duration>
+    <name>${title}</name>
+    <duration>${durationInMs / 1000}</duration>
     <rate>
       <timebase>30</timebase>
       <ntsc>FALSE</ntsc>
@@ -120,8 +110,8 @@ export async function generateFCP7XML(): Promise<string> {
       <video>
         <format>
           <samplecharacteristics>
-            <width>${meta.width}</width>
-            <height>${meta.height}</height>
+            <width>${width}</width>
+            <height>${height}</height>
           </samplecharacteristics>
         </format>
         <track>

packages/app/src/services/io/formats/mlt.ts

@@ -8,7 +8,13 @@ import { TimelineStore, useTimeline } from '@aitube/timeline'
 
 export async function generateMLT(): Promise<string> {
   const timeline: TimelineStore = useTimeline.getState()
-  const { meta, segments: timelineSegments } = timeline
+  const {
+    title,
+    width,
+    height,
+    durationInMs,
+    segments: timelineSegments,
+  } = timeline
 
   const segments: ExportableSegment[] = timelineSegments
     .map((segment, i) => formatSegmentForExport(segment, i))
@@ -19,7 +25,7 @@ export async function generateMLT(): Promise<string> {
   )
 
   const storyboards: ExportableSegment[] = segments.filter(
-    ({ segment }) => segment.category === ClapSegmentCategory.STORYBOARD
+    ({ segment }) => segment.category === ClapSegmentCategory.IMAGE
   )
 
   const dialogues: ExportableSegment[] = segments.filter(
@@ -36,11 +42,11 @@ export async function generateMLT(): Promise<string> {
 
   // want to see some colors? install es6-string-html in your VSCode
   return /* XML */ `<?xml version="1.0" standalone="no"?>
-<mlt LC_NUMERIC="C" version="7.24.0" title="${meta.title}" producer="main_bin">
+<mlt LC_NUMERIC="C" version="7.24.0" title="${title}" producer="main_bin">
 <profile
-description="${meta.width}:${meta.height}"
-width="${meta.width}"
-height="${meta.height}"
+description="${width}:${height}"
+width="${width}"
+height="${height}"
 progressive="0"
 sample_aspect_num="1"
 sample_aspect_den="1"
@@ -58,8 +64,8 @@ colorspace="709"
 <playlist id="main_bin">
 <property name="xml_retain">1</property>
 </playlist>
-<producer id="black" in="00:00:00.000" out="${formatDuration(meta.durationInMs)}">
-<property name="length">${formatDuration(meta.durationInMs)}</property>
+<producer id="black" in="00:00:00.000" out="${formatDuration(durationInMs)}">
+<property name="length">${formatDuration(durationInMs)}</property>
 <property name="eof">pause</property>
 <property name="resource">0</property>
 <property name="aspect_ratio">1</property>
@@ -68,7 +74,7 @@ colorspace="709"
 <property name="set.test_audio">0</property>
 </producer>
 <playlist id="background">
-<entry producer="black" in="00:00:00.000" out="${formatDuration(meta.durationInMs)}" />
+<entry producer="black" in="00:00:00.000" out="${formatDuration(durationInMs)}" />
 </playlist>
 ${segments
   .map(
@@ -76,8 +82,8 @@ ${segments
 <producer
 id="${shortId}"
 in="${formatDuration(0)}"
-out="${formatDuration(meta.durationInMs)}">
-<property name="length">${formatDuration(meta.durationInMs)}</property>
+out="${formatDuration(durationInMs)}">
+<property name="length">${formatDuration(durationInMs)}</property>
 <property name="eof">pause</property>
 <property name="resource">${filePath}</property>
 <property name="ttl">1</property>
@@ -85,8 +91,8 @@ out="${formatDuration(meta.durationInMs)}">
 <property name="meta.media.progressive">1</property>
 <property name="seekable">1</property>
 <property name="format">1</property>
-<property name="meta.media.width">${meta.width}</property>
-<property name="meta.media.height">${meta.height}</property>
+<property name="meta.media.width">${width}</property>
+<property name="meta.media.height">${height}</property>
 <property name="mlt_service">qimage</property>
 <property name="creation_time">${
       segment.createdAt || new Date().toISOString()
@@ -140,8 +146,8 @@ ${storyboards
 </playlist>
 ${[...dialogues, ...sounds, ...music].map(
   ({ segment, filePath, fileName, shortId }) => /* XML */ `
-<chain id="${shortId}" out="${formatDuration(meta.durationInMs)}">
-<property name="length">${formatDuration(meta.durationInMs)}</property>
+<chain id="${shortId}" out="${formatDuration(durationInMs)}">
+<property name="length">${formatDuration(durationInMs)}</property>
 <property name="eof">pause</property>
 <property name="resource">${filePath}</property>
 <property name="mlt_service">avformat-novalidate</property>
@@ -220,7 +226,7 @@ ${music.map(
 id="tractor0"
 title="Shotcut version 24.04.28"
 in="00:00:00.000"
-out="${formatDuration(meta.durationInMs)}">
+out="${formatDuration(durationInMs)}">
 <property name="shotcut">1</property>
 <property name="shotcut:projectAudioChannels">2</property>
 <property name="shotcut:projectFolder">1</property>

packages/app/src/services/io/formats/otio.ts

@@ -49,11 +49,11 @@ interface OTIOTimeline {
 
 export async function generateOTIO(): Promise<string> {
   const timeline: TimelineStore = useTimeline.getState()
-  const { meta, segments: timelineSegments } = timeline
+  const { title, segments: timelineSegments } = timeline
 
   const otioData: OTIOTimeline = {
     OTIO_SCHEMA: 'Timeline.1',
-    name: meta.title,
+    name: title,
     global_start_time: null,
     tracks: {
       OTIO_SCHEMA: 'Stack.1',

packages/app/src/services/io/parseFileIntoSegments.ts

@@ -121,7 +121,7 @@ export async function parseFileIntoSegments({
         status: ClapSegmentStatus.COMPLETED,
         // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
         label: `${file.name}`, // a short label to name the segment (optional, can be human or LLM-defined)
-        category: ClapSegmentCategory.STORYBOARD,
+        category: ClapSegmentCategory.IMAGE,
 
         assetUrl,
         assetDurationInMs: durationInMs,
@@ -132,20 +132,20 @@ export async function parseFileIntoSegments({
         track: track ? track + 1 : undefined,
       }
 
-      const storyboard = await clapSegmentToTimelineSegment(
+      const storyboardImage = await clapSegmentToTimelineSegment(
         newSegment(partialStoryboard)
       )
 
       if (isValidNumber(track)) {
-        storyboard.track = track
+        storyboardImage.track = track
       }
 
-      storyboard.outputType = ClapOutputType.IMAGE
+      storyboardImage.outputType = ClapOutputType.IMAGE
 
       // we assume we want it to be immediately visible
-      storyboard.visibility = SegmentVisibility.VISIBLE
+      storyboardImage.visibility = SegmentVisibility.VISIBLE
 
-      newSegments.push(storyboard)
+      newSegments.push(storyboardImage)
 
       break
     }
@@ -171,6 +171,8 @@ export async function parseFileIntoSegments({
         durationInSteps,
       })
 
+      useTimeline.setState({ bpm })
+
       // TODO: use the correct drop time
       const startTimeInMs = isValidNumber(maybeStartTimeInMs)
         ? maybeStartTimeInMs!

packages/app/src/services/io/useIO.ts

@@ -3,7 +3,8 @@
 import {
   ClapAssetSource,
   ClapEntity,
-  ClapMediaOrientation,
+  ClapImageRatio,
+  ClapMeta,
   ClapOutputType,
   ClapProject,
   ClapSegment,
@@ -189,22 +190,23 @@ export const useIO = create<IOStore>((set, get) => ({
           title: projectName,
           description: `${projectName} (${fileName})`,
           synopsis: '',
-          licence:
-            "This OpenClap file is just a conversion from the original screenplay and doesn't claim any copyright or intellectual property. All rights reserved to the original intellectual property and copyright holders. Using OpenClap isn't piracy.",
+          licence: '',
 
-          orientation: ClapMediaOrientation.LANDSCAPE,
+          imageRatio: ClapImageRatio.LANDSCAPE,
           durationInMs: frames.length * durationInMs,
 
           // TODO: those should come from the Clapper user settings
 
           width: 1024,
           height: 576,
-
-          defaultVideoModel: '', // <-- we should deprecate this no?
-          extraPositivePrompt: [],
-          screenplay: '',
+          imagePrompt: '',
+          storyPrompt: '',
+          systemPrompt: '',
           isLoop: false,
           isInteractive: false,
+
+          bpm: 120,
+          frameRate: 24,
         },
       })
     )
@@ -528,10 +530,11 @@ export const useIO = create<IOStore>((set, get) => ({
     const timeline: TimelineStore = useTimeline.getState()
 
     const {
-      meta,
-      getClap,
-      totalDurationInMs,
+      width,
+      height,
+      durationInMs,
       segments: timelineSegments,
+      getClap,
     } = timeline
 
     const clap = await getClap()
@@ -584,7 +587,7 @@ export const useIO = create<IOStore>((set, get) => ({
             })
           } else if (
             filePath.startsWith('image/') ||
-            segment.category === ClapSegmentCategory.STORYBOARD
+            segment.category === ClapSegmentCategory.IMAGE
           ) {
             images.push({
               data: base64DataUriToUint8Array(segment.assetUrl),
@@ -619,9 +622,9 @@ export const useIO = create<IOStore>((set, get) => ({
     const fullVideo = await createFullVideo(
       videoInputs,
       audios,
-      meta.width,
-      meta.height,
-      totalDurationInMs,
+      width,
+      height,
+      durationInMs,
       (progress, message) => {
         task.setProgress({
           message: `Rendering video (${Math.round(progress)}%)`,
@@ -650,7 +653,7 @@ export const useIO = create<IOStore>((set, get) => ({
     try {
       const timeline: TimelineStore = useTimeline.getState()
 
-      const { meta, segments: timelineSegments } = timeline
+      const { segments: timelineSegments } = timeline
 
       const segments: ExportableSegment[] = timelineSegments
         .map((segment, i) => formatSegmentForExport(segment, i))
@@ -658,7 +661,9 @@ export const useIO = create<IOStore>((set, get) => ({
 
       let files: fflate.AsyncZippable = {}
 
-      files['screenplay.txt'] = fflate.strToU8(meta.screenplay)
+      const meta = timeline.getClapMeta()
+
+      files['story_prompt.txt'] = fflate.strToU8(meta.storyPrompt)
 
       files['meta.json'] = fflate.strToU8(JSON.stringify(meta, null, 2))
 
@@ -762,7 +767,7 @@ export const useIO = create<IOStore>((set, get) => ({
           audios,
           meta.width,
           meta.height,
-          timeline.totalDurationInMs,
+          timeline.durationInMs,
           (progress, message) => {
             task.setProgress({
               message,