Spaces:

jbilcke-hf
/

clapper

Running on CPU Upgrade

App Files Files Community

jbilcke-hf HF staff commited on Jun 11

Commit

001cba6

•

1 Parent(s): 7f4dc49

working on the assistant

Browse files

Files changed (16) hide show

package-lock.json +4 -4
package.json +1 -1
src/app/embed/README.md +7 -0
src/app/embed/embed.tsx +51 -0
src/app/embed/page.tsx +33 -0
src/app/page.tsx +3 -5
src/components/assistant/{ChatView.FINISH_ME → ChatView.tsx} +17 -10
src/components/monitor/DynamicPlayer/DynamicBuffer.tsx +41 -0
src/components/monitor/DynamicPlayer/StoryboardBuffer.tsx +29 -0
src/components/monitor/DynamicPlayer/index.tsx +63 -31
src/controllers/README.md +83 -0
src/controllers/monitor/getDefaultMonitorState.ts +1 -0
src/controllers/monitor/types.ts +4 -0
src/controllers/monitor/useMonitor.ts +32 -1
src/controllers/renderer/useRenderer.ts +17 -3
src/lib/core/constants.ts +1 -1

package-lock.json CHANGED Viewed

@@ -10,7 +10,7 @@
       "dependencies": {
         "@aitube/clap": "0.0.26",
         "@aitube/engine": "0.0.19",
-        "@aitube/timeline": "0.0.21",
         "@fal-ai/serverless-client": "^0.10.3",
         "@huggingface/hub": "^0.15.1",
         "@huggingface/inference": "^2.7.0",
@@ -114,9 +114,9 @@
       }
     },
     "node_modules/@aitube/timeline": {
-      "version": "0.0.21",
-      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.21.tgz",
-      "integrity": "sha512-FQ7YPYG7K3F4RlfW5EboenkGn6Y4a/UjlU+ZDKUBIbZSTx15QiNXFBsLcxwj58FLThuHp/90TuHog202MOI+lw==",
       "dependencies": {
         "date-fns": "^3.6.0",
         "react-virtualized-auto-sizer": "^1.0.24"

       "dependencies": {
         "@aitube/clap": "0.0.26",
         "@aitube/engine": "0.0.19",
+        "@aitube/timeline": "0.0.22",
         "@fal-ai/serverless-client": "^0.10.3",
         "@huggingface/hub": "^0.15.1",
         "@huggingface/inference": "^2.7.0",
       }
     },
     "node_modules/@aitube/timeline": {
+      "version": "0.0.22",
+      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.22.tgz",
+      "integrity": "sha512-T3/DY+c3tZgR5rwzcgptN4JzUI9VfxbvuO5/4SP6uHVccoA3SX5X2smmpzO1lLcfwHoyy+HvE+TejcKAG2oTPg==",
       "dependencies": {
         "date-fns": "^3.6.0",
         "react-virtualized-auto-sizer": "^1.0.24"

package.json CHANGED Viewed

@@ -12,7 +12,7 @@
   "dependencies": {
     "@aitube/clap": "0.0.26",
     "@aitube/engine": "0.0.19",
-    "@aitube/timeline": "0.0.21",
     "@fal-ai/serverless-client": "^0.10.3",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/inference": "^2.7.0",

   "dependencies": {
     "@aitube/clap": "0.0.26",
     "@aitube/engine": "0.0.19",
+    "@aitube/timeline": "0.0.22",
     "@fal-ai/serverless-client": "^0.10.3",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/inference": "^2.7.0",

src/app/embed/README.md ADDED Viewed

	@@ -0,0 +1,7 @@

+# /embed
+`/embed` is a simplified paged used to render a .clap project without
+much of the editing UI
+Note that for users without any rendering settings,
+this will only be able to playback pre-generated content

src/app/embed/embed.tsx ADDED Viewed

	@@ -0,0 +1,51 @@

+"use client"
+import React, { useRef } from "react"
+import { useTimeline } from "@aitube/timeline"
+import { Toaster } from "@/components/ui/sonner"
+import { cn } from "@/lib/utils"
+import { TooltipProvider } from "@/components/ui/tooltip"
+import { Monitor } from "@/components/monitor"
+import { SettingsDialog } from "@/components/settings"
+import { LoadingDialog } from "@/components/dialogs/loader/LoadingDialog"
+import { TopBar } from "@/components/toolbars/top-bar"
+export function Embed() {
+  const ref = useRef<HTMLDivElement>(null)
+  const isEmpty = useTimeline(s => s.isEmpty)
+  return (
+    <TooltipProvider>
+    <div
+      ref={ref}
+      className={cn(`
+      dark
+      select-none
+      fixed
+      flex flex-col
+      w-screen h-screen
+      overflow-hidden
+      items-center justify-center
+      font-light
+      text-stone-900/90 dark:text-stone-100/90
+      `)}
+      style={{
+        backgroundImage: "repeating-radial-gradient( circle at 0 0, transparent 0, #000000 7px ), repeating-linear-gradient( #37353455, #373534 )"
+      }}
+      >
+      <TopBar />
+      <div className={cn(
+        `flex flex-row flex-grow w-full overflow-hidden`,
+        isEmpty ? "opacity-0" : "opacity-100"
+      )}>
+        <Monitor />
+      </div>
+      <SettingsDialog />
+      <LoadingDialog />
+      <Toaster />
+    </div>
+    </TooltipProvider>
+  );
+}

src/app/embed/page.tsx ADDED Viewed

	@@ -0,0 +1,33 @@

+"use client"
+import { useEffect, useState } from "react"
+import Head from "next/head"
+import Script from "next/script"
+import { Embed } from "./embed"
+// https://nextjs.org/docs/pages/building-your-application/optimizing/fonts
+export default function EmbedPage() {
+  const [isLoaded, setLoaded] = useState(false)
+  useEffect(() => { setLoaded(true) }, [])
+  return (
+    <>
+      <Head>
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossOrigin="anonymous" />
+        <link rel="preconnect" href="https://fonts.googleapis.com" crossOrigin="anonymous" />
+        <meta name="viewport" content="width=device-width, initial-scale=0.86, maximum-scale=5.0, minimum-scale=0.86" />
+      </Head>
+      <Script id="gtm">{`(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+})(window,document,'script','dataLayer','GTM-WD55Z2KN');`}</Script>
+      <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WD55Z2KN"
+height="0" width="0" style={{ display: "none", visibility: "hidden" }}></iframe></noscript>
+      <main>
+        {isLoaded && <Embed />}
+      </main>
+    </>
+  )
+}

src/app/page.tsx CHANGED Viewed

@@ -4,13 +4,11 @@ import { useEffect, useState } from "react"
 import Head from "next/head"
 import Script from "next/script"
-import { cn } from "@/lib/utils/cn"
 import { Main } from "./main"
 // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts
-export default function Page() {
   const [isLoaded, setLoaded] = useState(false)
   useEffect(() => { setLoaded(true) }, [])
   return (
@@ -24,8 +22,8 @@ export default function Page() {
 new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
 j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
-})(window,document,'script','dataLayer','GTM-K98T8ZFZ');`}</Script>
-      <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-K98T8ZFZ"
 height="0" width="0" style={{ display: "none", visibility: "hidden" }}></iframe></noscript>
       <main>
         {isLoaded && <Main />}

 import Head from "next/head"
 import Script from "next/script"
 import { Main } from "./main"
 // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts
+export default function MainPage() {
   const [isLoaded, setLoaded] = useState(false)
   useEffect(() => { setLoaded(true) }, [])
   return (
 new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
 j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+})(window,document,'script','dataLayer','GTM-WD55Z2KN');`}</Script>
+      <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WD55Z2KN"
 height="0" width="0" style={{ display: "none", visibility: "hidden" }}></iframe></noscript>
       <main>
         {isLoaded && <Main />}

src/components/assistant/{ChatView.FINISH_ME → ChatView.tsx} RENAMED Viewed

@@ -1,14 +1,15 @@
 "use client"
 import { useState, useTransition } from "react"
-import { ClapOutputType, ClapProject, ClapScene, ClapSegment, ClapSegmentCategory, ClapSegmentFilteringMode, filterSegments, newSegment } from "@aitube/clap"
-import { Input } from "@/components/ui/input"
-import { queryAssistant } from "@/app/api/assistant/providers/openai/askAssistant"
-import { useSettingsa } from "@/controllers/settings"
 import { DEFAULT_DURATION_IN_MS_PER_STEP, findFreeTrack, useTimeline } from "@aitube/timeline"
 import { useAssistant } from "@/controllers/assistant/useAssistant"
 export function ChatView() {
   const [_isPending, startTransition] = useTransition()
@@ -20,9 +21,9 @@ export function ChatView() {
   */
   const [draft, setDraft] = useState("")
-  const runCommand = useAssistant((state) => state.runCommand)
-  const history = useAssistant((state) => state.history)
-  const addEventToHistory = useAssistant((state) => state.addEventToHistory)
   /*
   const updateSegment = useApp((state) => state.updateSegment)
@@ -142,7 +143,8 @@ export function ChatView() {
         })
         console.log("Creating new existing segment:", newSeg)
-        addSegment(newSeg)
         addEventToHistory({
           senderId: "assistant",
@@ -157,11 +159,16 @@ export function ChatView() {
           label: prompt,
         })
         updateSegment({
           ...match,
           prompt,
           label: prompt,
         })
         addEventToHistory({
           senderId: "assistant",

 "use client"
 import { useState, useTransition } from "react"
+import { ClapOutputType, ClapProject, ClapSegment, ClapSegmentCategory, newSegment } from "@aitube/clap"
 import { DEFAULT_DURATION_IN_MS_PER_STEP, findFreeTrack, useTimeline } from "@aitube/timeline"
 import { useAssistant } from "@/controllers/assistant/useAssistant"
+import { queryAssistant } from "@/app/api/assistant/providers/openai/askAssistant"
+import { useSettings } from "@/controllers/settings"
+import { ChatBubble } from "./ChatBubble"
+import { Input } from "../ui/input"
 export function ChatView() {
   const [_isPending, startTransition] = useTransition()
   */
   const [draft, setDraft] = useState("")
+  const runCommand = useAssistant((s) => s.runCommand)
+  const history = useAssistant((s) => s.history)
+  const addEventToHistory = useAssistant((s) => s.addEventToHistory)
   /*
   const updateSegment = useApp((state) => state.updateSegment)
         })
         console.log("Creating new existing segment:", newSeg)
+        console.log(`TODO Julian: add the segment!!`)
+        // addSegment(newSeg)
         addEventToHistory({
           senderId: "assistant",
           label: prompt,
         })
+        console.log(`TODO Julian: update the segment!!`)
+        // addSegment(newSeg)
+        /*
         updateSegment({
           ...match,
           prompt,
           label: prompt,
         })
+          */
         addEventToHistory({
           senderId: "assistant",

src/components/monitor/DynamicPlayer/DynamicBuffer.tsx ADDED Viewed

	@@ -0,0 +1,41 @@

+"use client"
+import { ClapOutputType, ClapSegment } from "@aitube/clap"
+import { cn } from "@/lib/utils"
+import { VideoClipBuffer } from "./VideoClipBuffer"
+import { StoryboardBuffer } from "./StoryboardBuffer"
+export const DynamicBuffer = ({
+  segment,
+  isPlaying = false,
+  isVisible = false,
+}: {
+  segment?: ClapSegment
+  isPlaying?: boolean
+  isVisible?: boolean
+}): JSX.Element | null => {
+  const src = `${segment?.assetUrl || ""}`
+  if (!src) { return null }
+  const outputType = segment!.outputType
+  const className = cn(isVisible ? `opacity-100` : `opacity-0`)
+  return (
+   <>
+     {outputType === ClapOutputType.VIDEO
+       ? <VideoClipBuffer
+        src={src}
+        isPlaying={isPlaying}
+        className={className}
+      />
+      : <StoryboardBuffer
+        src={src}
+        className={className}
+      />}
+    </>
+  )
+}

src/components/monitor/DynamicPlayer/StoryboardBuffer.tsx ADDED Viewed

	@@ -0,0 +1,29 @@

+import { cn } from "@/lib/utils"
+export function StoryboardBuffer({
+  src,
+  className,
+}: {
+  src?: string;
+  className?: string;
+}) {
+  if (!src) { return null }
+  return (
+    <img
+      className={cn(
+        `absolute`,
+        `h-full w-full rounded-md overflow-hidden`,
+        // iseally we could only use the ease-out and duration-150
+        // to avoid a weird fade to grey,
+        // but the ease out also depends on which video is on top of each other,
+        // in term of z-index, so we should also intervert this
+        `transition-all duration-100 ease-out`,
+        className
+      )}
+      src={src}
+    />
+  )
+}

src/components/monitor/DynamicPlayer/index.tsx CHANGED Viewed

@@ -8,9 +8,10 @@ import { useTimeline } from "@aitube/timeline"
 import { useRequestAnimationFrame } from "@/lib/hooks"
 import { MonitoringMode } from "@/controllers/monitor/types"
-import { VideoClipBuffer } from "./VideoClipBuffer"
 import { useRenderLoop } from "@/controllers/renderer/useRenderLoop"
 import { useRenderer } from "@/controllers/renderer/useRenderer"
 export const DynamicPlayer = ({
   className,
@@ -25,20 +26,43 @@ export const DynamicPlayer = ({
   // this should only be called once and at only one place in the project!
   useRenderLoop()
-  const { activeVideoSegment, upcomingVideoSegment } = useRenderer(s => s.bufferedSegments)
-  const currentVideoUrl = activeVideoSegment?.assetUrl || ""
-  // the upcoming video we want to preload (note: we just want to preload it, not display it just yet)
-  const preloadVideoUrl = upcomingVideoSegment?.assetUrl || ""
-  const [buffer1Value, setBuffer1Value] = useState("")
-  const [buffer2Value, setBuffer2Value] = useState("")
   const [activeBufferNumber, setActiveBufferNumber] = useState(1)
   const timeoutRef = useRef<NodeJS.Timeout>()
-  const fadeDurationInMs = 300
   useEffect(() => {
     setMonitoringMode(MonitoringMode.DYNAMIC)
@@ -56,57 +80,65 @@ export const DynamicPlayer = ({
   })
   useEffect(() => {
     // trivial case: we are at the initial state
-    if (!buffer1Value && !buffer2Value) {
-      setBuffer1Value(currentVideoUrl)
-      setBuffer2Value(preloadVideoUrl)
       setActiveBufferNumber(1)
     }
-  }, [buffer1Value, currentVideoUrl, preloadVideoUrl])
-  // console.log("cursorInSteps:", cursorInSteps)
   useEffect(() => {
-    /*
-    console.log("ATTENTION: something changed among those: ", {
-      currentVideoUrl, preloadVideoUrl
-    })
-    */
     clearTimeout(timeoutRef.current)
     const newActiveBufferNumber = activeBufferNumber === 1 ? 2 : 1
-    // console.log(`our pre-loaded video should already be available in buffer ${newActiveBufferNumber}`)
     setActiveBufferNumber(newActiveBufferNumber)
     timeoutRef.current = setTimeout(() => {
       // by now one buffer should be visible, and the other should be hidden
       // so let's update the invisible one
       if (newActiveBufferNumber === 2) {
-        setBuffer1Value(preloadVideoUrl)
       } else {
-        setBuffer2Value(preloadVideoUrl)
       }
     }, fadeDurationInMs + 200) // let's add some security in here
     return () => {
       clearTimeout(timeoutRef.current)
     }
-  }, [currentVideoUrl, preloadVideoUrl])
   return (
     <div className={cn(`@container flex flex-col flex-grow w-full`, className)}>
-      <VideoClipBuffer
-        src={buffer1Value}
         isPlaying={isPlaying}
-        className={cn(activeBufferNumber === 1 ? `opacity-100` : `opacity-0`)}
       />
-      <VideoClipBuffer
-        src={buffer2Value}
         isPlaying={isPlaying}
-        className={cn(activeBufferNumber === 2 ? `opacity-100` : `opacity-0`)}
       />
     </div>
   )

 import { useRequestAnimationFrame } from "@/lib/hooks"
 import { MonitoringMode } from "@/controllers/monitor/types"
 import { useRenderLoop } from "@/controllers/renderer/useRenderLoop"
 import { useRenderer } from "@/controllers/renderer/useRenderer"
+import { ClapSegment } from "@aitube/clap"
+import { DynamicBuffer } from "./DynamicBuffer"
 export const DynamicPlayer = ({
   className,
   // this should only be called once and at only one place in the project!
   useRenderLoop()
+  const {
+    activeVideoSegment,
+    upcomingVideoSegment,
+    activeStoryboardSegment,
+    upcomingStoryboardSegment
+  } = useRenderer(s => s.bufferedSegments)
+  console.log(`DynamicPlayer:`, {
+    activeVideoSegment,
+    upcomingVideoSegment,
+    activeStoryboardSegment,
+    upcomingStoryboardSegment
+  })
+  const currentSegment =
+    activeVideoSegment?.assetUrl
+    ? activeVideoSegment
+    : activeStoryboardSegment?.assetUrl
+    ? activeStoryboardSegment
+    : undefined
+  // the upcoming asset we want to preload (note: we just want to preload it, not display it just yet)
+  const preloadSegment =
+    upcomingVideoSegment?.assetUrl
+    ? upcomingVideoSegment
+    : upcomingStoryboardSegment?.assetUrl
+    ? upcomingStoryboardSegment
+    : undefined
+  const [dataUriBuffer1, setDataUriBuffer1] = useState<ClapSegment | undefined>()
+  const [dataUriBuffer2, setDataUriBuffer2] = useState<ClapSegment | undefined>()
   const [activeBufferNumber, setActiveBufferNumber] = useState(1)
   const timeoutRef = useRef<NodeJS.Timeout>()
+  const fadeDurationInMs = 250
   useEffect(() => {
     setMonitoringMode(MonitoringMode.DYNAMIC)
   })
+  // performance optimization:
+  // we only look at the first part since it might be huge
+  // for assets, using a smaller header lookup like 256 or even 512 doesn't seem to be enough
+  const currentSegmentKey = `${currentSegment?.assetUrl || ""}`.slice(0, 1024)
+  const preloadSegmentKey = `${preloadSegment?.assetUrl || ""}`.slice(0, 1024)
+  const dataUriBuffer1Key = `${dataUriBuffer1?.assetUrl || ""}`.slice(0, 1024)
+  const dataUriBuffer2Key = `${dataUriBuffer2?.assetUrl || ""}`.slice(0, 1024)
   useEffect(() => {
     // trivial case: we are at the initial state
+    if (!dataUriBuffer1 && !dataUriBuffer2) {
+      setDataUriBuffer1(currentSegment)
+      setDataUriBuffer2(preloadSegment)
       setActiveBufferNumber(1)
     }
+  }, [
+    dataUriBuffer1Key,
+    dataUriBuffer2Key,
+    currentSegmentKey,
+    preloadSegmentKey
+  ])
   useEffect(() => {
     clearTimeout(timeoutRef.current)
     const newActiveBufferNumber = activeBufferNumber === 1 ? 2 : 1
     setActiveBufferNumber(newActiveBufferNumber)
     timeoutRef.current = setTimeout(() => {
       // by now one buffer should be visible, and the other should be hidden
       // so let's update the invisible one
       if (newActiveBufferNumber === 2) {
+        setDataUriBuffer1(preloadSegment)
       } else {
+        setDataUriBuffer2(preloadSegment)
       }
     }, fadeDurationInMs + 200) // let's add some security in here
     return () => {
       clearTimeout(timeoutRef.current)
     }
+}, [
+  currentSegmentKey,
+  preloadSegmentKey
+])
   return (
     <div className={cn(`@container flex flex-col flex-grow w-full`, className)}>
+      <DynamicBuffer
+        segment={dataUriBuffer1}
         isPlaying={isPlaying}
+        isVisible={activeBufferNumber === 1}
       />
+      <DynamicBuffer
+        segment={dataUriBuffer2}
         isPlaying={isPlaying}
+        isVisible={activeBufferNumber === 2}
       />
     </div>
   )

src/controllers/README.md ADDED Viewed

	@@ -0,0 +1,83 @@

+# /controllers
+Each controller represents a core features of Clapper.
+They each have their own state (managed using Zustand, so it can be used in and out of a React component), and should be considered like tiny independent apps (each manage one important thing), although they deeply relate to each other.
+For instance multiple controllers might need to tap into the Settings controller to pull parameters or default values.
+## Assistant
+The chatbot assistant.
+Usage:
+```typescript
+useAssistant()
+```
+## Audio
+Audio management, including playing back audio files for dialogues, sound effects and music.
+Usage:
+```typescript
+useAudio()
+```
+## IO
+Input/output management, to import and export various file formats.
+Usage:
+```typescript
+useIO()
+```
+## Monitor
+The video monitor is the big video display component used to preview an existing (pre-generated) full video, or preview a work in progress project composed of many tiny video clips.
+The monitor is responsible for handling the play, pause, seek etc..
+functions.
+Usage:
+```typescript
+useMonitor()
+```
+## Renderer
+This is the engine that can generate a video stream on the fly from a sequence of tiny video and audio clips.
+Usage:
+```typescript
+useRenderer()
+```
+## Settings
+Responsible for organizing user preferences, which are serialized
+and persisted into the browser local storage, that way no login or account management is necessary.
+Usage:
+```typescript
+useSettings()
+```
+## UI
+The user interface controller, through which you can show/hidden elements of the interface.
+Usage:
+```typescript
+useUI()
+```

src/controllers/monitor/getDefaultMonitorState.ts CHANGED Viewed

@@ -2,6 +2,7 @@ import { MonitoringMode, MonitorState } from "./types"
 export function getDefaultMonitorState(): MonitorState {
   const state: MonitorState = {
     mode: MonitoringMode.NONE,
     lastTimelineUpdateAtInMs: 0,
     isPlaying: false,

 export function getDefaultMonitorState(): MonitorState {
   const state: MonitorState = {
+    shortcutsAreBound: false,
     mode: MonitoringMode.NONE,
     lastTimelineUpdateAtInMs: 0,
     isPlaying: false,

src/controllers/monitor/types.ts CHANGED Viewed

@@ -5,6 +5,7 @@ export enum MonitoringMode {
 }
 export type MonitorState = {
   mode: MonitoringMode
   lastTimelineUpdateAtInMs: number
   isPlaying: boolean
@@ -12,6 +13,9 @@ export type MonitorState = {
 }
 export type MonitorControls = {
   setMonitoringMode: (mode: MonitoringMode) => void
   setStaticVideoRef: (staticVideoRef: HTMLVideoElement) => void

 }
 export type MonitorState = {
+  shortcutsAreBound: boolean
   mode: MonitoringMode
   lastTimelineUpdateAtInMs: number
   isPlaying: boolean
 }
 export type MonitorControls = {
+  bindShortcuts: () => void
   setMonitoringMode: (mode: MonitoringMode) => void
   setStaticVideoRef: (staticVideoRef: HTMLVideoElement) => void

src/controllers/monitor/useMonitor.ts CHANGED Viewed

@@ -11,7 +11,32 @@ import { getDefaultMonitorState } from "./getDefaultMonitorState"
 export const useMonitor = create<MonitorStore>((set, get) => ({
   ...getDefaultMonitorState(),
   setMonitoringMode: (mode: MonitoringMode) => {
     set({ mode })
   },
@@ -104,4 +129,10 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
     set({ lastTimelineUpdateAtInMs })
   },
-}))

 export const useMonitor = create<MonitorStore>((set, get) => ({
   ...getDefaultMonitorState(),
+  bindShortcuts: () => {
+    if (get().shortcutsAreBound) { return }
+    document.addEventListener("keydown", (event) => {
+      const element = event.target as unknown as HTMLElement
+      if (event.code === "Space" &&
+        // those exception are important, otherwise we won't be able to add spaces
+        // in the search boxes, edit fields, or even the script editor
+        element.nodeName !== "INPUT" &&
+        element.nodeName !== "TEXTAREA") {
+        console.log("[SHORTCUT DETECTED] User pressed space key outside a text input: toggling video playback")
+        // prevent the default behavior, which is strange (automatic scroll to the buttom)
+        // https://www.jankollars.com/posts/preventing-space-scrolling/
+        event.preventDefault()
+        get().togglePlayback()
+      }
+    })
+    set({
+      shortcutsAreBound: true
+    })
+  },
   setMonitoringMode: (mode: MonitoringMode) => {
     set({ mode })
   },
     set({ lastTimelineUpdateAtInMs })
   },
+}))
+setTimeout(() => {
+  if (typeof document !== "undefined") {
+    useMonitor.getState().bindShortcuts()
+  }
+}, 0)

src/controllers/renderer/useRenderer.ts CHANGED Viewed

@@ -46,8 +46,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
     const segments = clapSegments as RuntimeSegment[]
     const results: BufferedSegments = getDefaultBufferedSegments()
-    // console.log("useRenderer: computeBufferedSegments() called")
     // we could use a temporal index to keep things efficient here
     // thiere is this relatively recent algorithm, the IB+ Tree,
@@ -60,7 +59,6 @@ export const useRenderer = create<RendererStore>((set, get) => ({
       if (inActiveShot) {
         const isActiveVideo = segment.category === ClapSegmentCategory.VIDEO && segment.assetUrl
-        // const isActiveStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
         if (isActiveVideo) {
           results.activeSegments.push(segment)
           results.activeVideoSegment = segment
@@ -79,6 +77,14 @@ export const useRenderer = create<RendererStore>((set, get) => ({
           results.activeAudioSegments.push(segment)
           results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
         }
       }
       const inUpcomingShot =
@@ -107,6 +113,14 @@ export const useRenderer = create<RendererStore>((set, get) => ({
           results.upcomingAudioSegments.push(segment)
           results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
         }
       }
     }

     const segments = clapSegments as RuntimeSegment[]
     const results: BufferedSegments = getDefaultBufferedSegments()
     // we could use a temporal index to keep things efficient here
     // thiere is this relatively recent algorithm, the IB+ Tree,
       if (inActiveShot) {
         const isActiveVideo = segment.category === ClapSegmentCategory.VIDEO && segment.assetUrl
         if (isActiveVideo) {
           results.activeSegments.push(segment)
           results.activeVideoSegment = segment
           results.activeAudioSegments.push(segment)
           results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
         }
+        const isActiveStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
+        if (isActiveStoryboard) {
+          results.activeSegments.push(segment)
+          results.activeStoryboardSegment = segment
+          results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
+        }
       }
       const inUpcomingShot =
           results.upcomingAudioSegments.push(segment)
           results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
         }
+        const isUpcomingStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
+        if (isUpcomingStoryboard) {
+          results.upcomingSegments.push(segment)
+          results.upcomingStoryboardSegment = segment
+          results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
+        }
       }
     }

src/lib/core/constants.ts CHANGED Viewed

@@ -4,7 +4,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 export const APP_NAME = "Clapper AI"
-export const APP_REVISION = "r2024-06-11"
 export const APP_DOMAIN = "Clapper.app"
 export const APP_LINK = "https://clapper.app"

 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 export const APP_NAME = "Clapper AI"
+export const APP_REVISION = "r20240611-2256"
 export const APP_DOMAIN = "Clapper.app"
 export const APP_LINK = "https://clapper.app"