fix bug in the .env

.env

@@ -70,7 +70,7 @@ RENDERING_HF_INFERENCE_API_FILE_TYPE="image/png"
 
 # An experimental RENDERING engine (sorry it is not very documented yet, so you can use one of the other engines)
 RENDERING_VIDEOCHAIN_API_URL="http://localhost:7860"
-you decide
+
 RENDERING_OPENAI_API_BASE_URL="https://api.openai.com/v1"
 RENDERING_OPENAI_API_MODEL="dall-e-3"
 

src/app/engine/render.ts

@@ -371,6 +371,9 @@ export async function newRender({
           prompt,
           // negativePrompt, unused for now
 
+          // for a future version of the comic factory
+          identityImage: "",
+
           nbFrames,
 
           nbSteps: nbInferenceSteps, // 20 = fast, 30 = better, 50 = best

src/app/interface/bottom-bar/bottom-bar.tsx

@@ -13,28 +13,33 @@ import { SettingsDialog } from "../settings-dialog"
 import { useLocalStorage } from "usehooks-ts"
 import { localStorageKeys } from "../settings-dialog/localStorageKeys"
 import { defaultSettings } from "../settings-dialog/defaultSettings"
+import { getParam } from "@/lib/getParam"
 
 function BottomBar() {
   // deprecated, as HTML-to-bitmap didn't work that well for us
-  // const page = useStore(state => state.page)
-  // const download = useStore(state => state.download)
-  // const pageToImage = useStore(state => state.pageToImage)
+  // const page = useStore(s => s.page)
+  // const download = useStore(s => s.download)
+  // const pageToImage = useStore(s => s.pageToImage)
 
-  const isGeneratingStory = useStore(state => state.isGeneratingStory)
-  const prompt = useStore(state => state.prompt)
-  const panelGenerationStatus = useStore(state => state.panelGenerationStatus)
+  const isGeneratingStory = useStore(s => s.isGeneratingStory)
+  const prompt = useStore(s => s.prompt)
+  const panelGenerationStatus = useStore(s => s.panelGenerationStatus)
 
-  const preset = useStore(state => state.preset)
+  const preset = useStore(s => s.preset)
+  
+  const canSeeBetaFeatures = getParam<boolean>("beta", false)
 
   const allStatus = Object.values(panelGenerationStatus)
   const remainingImages = allStatus.reduce((acc, s) => (acc + (s ? 1 : 0)), 0)
 
-  const upscaleQueue = useStore(state => state.upscaleQueue)
-  const renderedScenes = useStore(state => state.renderedScenes)
-  const removeFromUpscaleQueue = useStore(state => state.removeFromUpscaleQueue)
-  const setRendered = useStore(state => state.setRendered)
+  const upscaleQueue = useStore(s => s.upscaleQueue)
+  const renderedScenes = useStore(s => s.renderedScenes)
+  const removeFromUpscaleQueue = useStore(s => s.removeFromUpscaleQueue)
+  const setRendered = useStore(s => s.setRendered)
   const [isUpscaling, setUpscaling] = useState(false)
 
+  const downloadClap = useStore(s => s.downloadClap)
+
   const [hasGeneratedAtLeastOnce, setHasGeneratedAtLeastOnce] = useLocalStorage<boolean>(
     localStorageKeys.hasGeneratedAtLeastOnce,
     defaultSettings.hasGeneratedAtLeastOnce
@@ -147,6 +152,17 @@ function BottomBar() {
            </Button>
         </div>
           */}
+          {canSeeBetaFeatures ? <Button
+            onClick={downloadClap}
+            disabled={!prompt?.length}
+          >
+            <span className="hidden md:inline">{
+            remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels ⌛` : `Save .clap`
+            }</span>
+            <span className="inline md:hidden">{
+              remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `Save .clap`
+            }</span>
+        </Button> : null}
           <Button
             onClick={handlePrint}
             disabled={!prompt?.length}

src/app/main.tsx

@@ -222,14 +222,17 @@ export default function Main() {
           // update the frontend
           // console.log("updating the frontend..")
           setCaptions(ref.current.newCaptions)
-          setPanels(ref.current.newPanelsPrompts)    
-
+          setPanels(ref.current.newPanelsPrompts)
           setGeneratingStory(false)
+
+          // TODO generate the clap here
+          
         } catch (err) {
           console.log("main.tsx: LLM generation failed:", err)
           setGeneratingStory(false)
           break
         }
+
         if (currentPanel > (currentNbPanels / 2)) {
           console.log("main.tsx: we are halfway there, hold tight!")
           // setWaitABitMore(true)

src/app/store/index.ts

@@ -1,7 +1,7 @@
 "use client"
 
 import { create } from "zustand"
-import html2canvas from "html2canvas"
+import { ClapProject, newClap, newSegment, serializeClap } from "@aitube/clap"
 
 import { FontName } from "@/lib/fonts"
 import { Preset, PresetName, defaultPreset, getPreset, getRandomPreset } from "@/app/engine/presets"
@@ -13,6 +13,7 @@ export const useStore = create<{
   prompt: string
   font: FontName
   preset: Preset
+  currentClap?: ClapProject
   currentNbPanelsPerPage: number
   maxNbPanelsPerPage: number
   currentNbPages: number
@@ -70,6 +71,10 @@ export const useStore = create<{
   // setPage: (page: HTMLDivElement) => void
 
   generate: (prompt: string, presetName: PresetName, layoutName: LayoutName) => void
+
+  computeClap: () => Promise<ClapProject>
+
+  downloadClap: () => Promise<void>
 }>((set, get) => ({
   prompt:
     (getParam("stylePrompt", "") || getParam("storyPrompt", ""))
@@ -78,6 +83,7 @@ export const useStore = create<{
   font: "actionman",
   preset: getPreset(getParam("preset", defaultPreset)),
 
+  currentClap: undefined,
   currentNbPanelsPerPage: 4,
   maxNbPanelsPerPage: 4,
   currentNbPages: 1,
@@ -399,5 +405,114 @@ export const useStore = create<{
       layout: layouts[0],
       layouts,
     })
+  },
+  computeClap: async (): Promise<ClapProject> => {
+    const {
+      currentNbPanels,
+      prompt,
+      panels,
+      renderedScenes,
+      captions
+    } = get()
+
+    const defaultSegmentDurationInMs = 7000
+
+    let currentElapsedTimeInMs = 0
+
+
+    const clap: ClapProject = newClap({
+      meta: {
+        title: "Untitled", // we don't need a title actually
+        description: prompt,
+        prompt: prompt,
+        synopsis: "",
+        licence: "",
+        orientation: "landscape",
+        width: 512,
+        height: 288,
+        isInteractive: false,
+        isLoop: false,
+        durationInMs: panels.length * defaultSegmentDurationInMs,
+        defaultVideoModel: "SDXL",
+      }
+    })
+
+    for (let i = 0; i < panels.length; i++) {
+
+      const panel = panels[i]
+      const caption = panels[i]
+      const renderedScene = renderedScenes[`${i}`]
+
+      clap.segments.push(newSegment({
+        track: 1,
+        startTimeInMs: currentElapsedTimeInMs,
+        assetDurationInMs: defaultSegmentDurationInMs,
+        category: "storyboard",
+        prompt: panel,
+        outputType: "image"
+      }))
+  
+      clap.segments.push(newSegment({
+        track: 2,
+        startTimeInMs: currentElapsedTimeInMs,
+        assetDurationInMs: defaultSegmentDurationInMs,
+        category: "interface",
+        prompt: caption,
+        // assetUrl: `data:text/plain;base64,${btoa(title)}`,
+        assetUrl: caption,
+        outputType: "text"
+      }))
+  
+      clap.segments.push(newSegment({
+        track: 3,
+        startTimeInMs: currentElapsedTimeInMs,
+        assetDurationInMs: defaultSegmentDurationInMs,
+        category: "dialogue",
+        prompt: caption,
+        outputType: "audio"
+      }))
+  
+      // the presence of a camera is mandatory
+      clap.segments.push(newSegment({
+        track: 4,
+        startTimeInMs: currentElapsedTimeInMs,
+        assetDurationInMs: defaultSegmentDurationInMs,
+        category: "camera",
+        prompt: "video",
+        outputType: "text"
+      }))
+  
+      currentElapsedTimeInMs += defaultSegmentDurationInMs
+    }
+
+    set({ currentClap: clap })
+
+    return clap
+  },
+
+  downloadClap: async () => {
+    const { computeClap } = get()
+
+    const currentClap = await computeClap()
+
+    if (!currentClap) { throw new Error(`cannot save a clap.. if there is no clap`) }
+
+    const currentClapBlob: Blob = await serializeClap(currentClap)
+
+    // Create an object URL for the compressed clap blob
+    const objectUrl = URL.createObjectURL(currentClapBlob)
+  
+    // Create an anchor element and force browser download
+    const anchor = document.createElement("a")
+    anchor.href = objectUrl
+
+    anchor.download = "my_ai_comic.clap"
+
+    document.body.appendChild(anchor) // Append to the body (could be removed once clicked)
+    anchor.click() // Trigger the download
+  
+    // Cleanup: revoke the object URL and remove the anchor element
+    URL.revokeObjectURL(objectUrl)
+    document.body.removeChild(anchor)
   }
 }))

src/types.ts

@@ -15,12 +15,8 @@ export interface RenderRequest {
   // actionnables are names of things like "chest", "key", "tree", "chair" etc
   actionnables: string[]
 
-  // note: this is the number of frames for Zeroscope,
-  // which is currently configured to only output 3 seconds, so:
-  // nbFrames=8 -> 1 sec
-  // nbFrames=16 -> 2 sec
-  // nbFrames=24 -> 3 sec
-  nbFrames: number // min: 1, max: 24
+  nbFrames: number
+  nbFPS: number
 
   nbSteps: number // min: 1, max: 50
 
@@ -52,6 +48,8 @@ export interface RenderRequest {
   wait: boolean // wait until the job is completed
 
   analyze: boolean // analyze the image to generate a caption (optional)
+
+  identityImage: string // reference image for the main entity
 }
 
 export interface ImageSegment {