fixed upscaling

.env

@@ -1,6 +1,6 @@
-NEXT_PUBLIC_BASE_URL=https://jbilcke-hf-fishtank.hf.space
+NEXT_PUBLIC_BASE_URL=https://jbilcke-hf-ai-comic-factory.hf.space
 # NEXT_PUBLIC_RENDERING_ENGINE_API=https://hysts-zeroscope-v2.hf.space
 RENDERING_ENGINE_API=https://jbilcke-hf-videochain-api.hf.space
-#VC_SECRET_ACCESS_TOKEN=<SECRET>
+#VC_SECRET_ACCESS_TOKEN=<secret>
 #HF_API_TOKEN=<SECRET>
 #HF_INFERENCE_ENDPOINT_URL=<SECRET>

next.config.js

@@ -4,6 +4,7 @@ const nextConfig = {
   
   experimental: {
     serverActions: true,
+    serverActionsBodySizeLimit: '8mb',
   },
 }
 

src/app/engine/presets.ts

@@ -123,13 +123,12 @@ export const presets: Record<string, Preset> = {
     font: "actionman",
     llmPrompt: "american comic",
     imagePrompt: (prompt: string) => [
-      `american comic about ${prompt}`,
-      "single panel",
-      "modern american comic",
-      "comicbook style",
-      "2010s",
-      "digital print",
-      "color comicbook",
+      `modern american comic about ${prompt}`,
+      //"single panel",
+      "digital color comicbook style",
+      // "2010s",
+      // "digital print",
+      // "color comicbook",
       // "color drawing"
     ],
     negativePrompt: () => [
@@ -183,13 +182,12 @@ export const presets: Record<string, Preset> = {
     font: "actionman",
     llmPrompt: "american comic",
     imagePrompt: (prompt: string) => [
-      `american comic about ${prompt}`,
-      "single panel",
-      "american comic",
-      "comicbook style",
+      `vintage american color comic about ${prompt}`,
+      // "single panel",
+     //  "comicbook style",
       "1950",
       "50s",
-      "color comicbook",
+      // "color comicbook",
       // "color drawing"
     ],
     negativePrompt: () => [
@@ -244,15 +242,12 @@ export const presets: Record<string, Preset> = {
     font: "actionman",
     llmPrompt: "new pulp science fiction",
     imagePrompt: (prompt: string) => [
-      `color comic panel`,
+      `vintage color pulp comic panel`,
       `${prompt}`,
       "40s",
       "1940",
-      "vintage comic",
-      "pulp magazine",
-      "pulp science fiction",
       "vintage science fiction",
-      "single panel",
+      // "single panel",
       // "comic album"
     ],
     negativePrompt: () => [
@@ -311,9 +306,9 @@ export const presets: Record<string, Preset> = {
       "tintin style",
       "french comic panel",
       "franco-belgian style",
-      "color panel",
-      "bande dessinée",
-      "single panel",
+     //  "color panel",
+     //  "bande dessinée",
+      // "single panel",
       // "comic album"
     ],
     negativePrompt: () => [
@@ -343,8 +338,8 @@ export const presets: Record<string, Preset> = {
       "franco-belgian style",
       "bande dessinée",
       "single panel",
-      "comical",
-      "comic album",
+      // "comical",
+      // "comic album",
       // "color drawing"
     ],
     negativePrompt: () => [

src/app/engine/render.ts

@@ -19,7 +19,7 @@ export async function newRender({
   width: number
   height: number
 }) {
-  console.log(`newRender(${prompt})`)
+  // console.log(`newRender(${prompt})`)
   if (!prompt) {
     console.error(`cannot call the rendering API without a prompt, aborting..`)
     throw new Error(`cannot call the rendering API without a prompt, aborting..`)
@@ -37,7 +37,7 @@ export async function newRender({
 
 
   try {
-    console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
+    // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
 
     const res = await fetch(`${apiUrl}/render`, {
       method: "POST",
@@ -99,7 +99,7 @@ export async function getRender(renderId: string) {
 
   let defaulResult: RenderedScene = {
     renderId: "",
-    status: "error",
+    status: "pending",
     assetUrl: "",
     alt: "",
     maskUrl: "",
@@ -114,7 +114,7 @@ export async function getRender(renderId: string) {
       headers: {
         Accept: "application/json",
         "Content-Type": "application/json",
-        // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
+        Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
       },
       cache: 'no-store',
     // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
@@ -134,6 +134,62 @@ export async function getRender(renderId: string) {
     const response = (await res.json()) as RenderedScene
     // console.log("response:", response)
     return response
+  } catch (err) {
+    console.error(err)
+    defaulResult.status = "error"
+    defaulResult.error = `${err}`
+    // Gorgon.clear(cacheKey)
+    return defaulResult
+  }
+
+  // }, cacheDurationInSec * 1000)
+}
+
+export async function upscaleImage(image: string): Promise<{
+  assetUrl: string
+  error: string
+}> {
+  if (!image) {
+    console.error(`cannot call the rendering API without an image, aborting..`)
+    throw new Error(`cannot call the rendering API without an image, aborting..`)
+  }
+
+  let defaulResult = {
+    assetUrl: "",
+    error: "failed to fetch the data",
+  }
+
+  try {
+    // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
+    const res = await fetch(`${apiUrl}/upscale`, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
+      },
+      cache: 'no-store',
+      body: JSON.stringify({ image, factor: 3 })
+    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+    // next: { revalidate: 1 }
+    })
+
+    // console.log("res:", res)
+    // The return value is *not* serialized
+    // You can return Date, Map, Set, etc.
+    
+    // Recommendation: handle errors
+    if (res.status !== 200) {
+      // This will activate the closest `error.js` Error Boundary
+      throw new Error('Failed to fetch data')
+    }
+    
+    const response = (await res.json()) as {
+      assetUrl: string
+      error: string
+    }
+    // console.log("response:", response)
+    return response
   } catch (err) {
     console.error(err)
     // Gorgon.clear(cacheKey)

src/app/interface/bottom-bar/index.tsx

@@ -5,6 +5,9 @@ import { base64ToFile } from "@/lib/base64ToFile"
 import { uploadToHuggingFace } from "@/lib/uploadToHuggingFace"
 import { cn } from "@/lib/utils"
 import { About } from "../about"
+import { startTransition, useState } from "react"
+import { upscaleImage } from "@/app/engine/render"
+import { sleep } from "@/lib/sleep"
 
 export function BottomBar() {
   const download = useStore(state => state.download)
@@ -17,15 +20,42 @@ export function BottomBar() {
 
   const allStatus = Object.values(panelGenerationStatus)
   const remainingImages = allStatus.reduce((acc, s) => (acc + (s ? 1 : 0)), 0)
-  
-  /*
+
+  const upscaleQueue = useStore(state => state.upscaleQueue)
+  const renderedScenes = useStore(state => state.renderedScenes)
+  const removeFromUpscaleQueue = useStore(state => state.removeFromUpscaleQueue)
+  const setRendered = useStore(state => state.setRendered)
+  const [isUpscaling, setUpscaling] = useState(false)
+
   const handleUpscale = () => {
+    setUpscaling(true)
     startTransition(() => {
-      we are blocked here, because we don't know the render id
-      also  we will have to store a state about the 
+      const fn = async () => {
+        for (let [panelId, renderedScene] of Object.entries(upscaleQueue)) {
+          try {
+            console.log(`upscaling panel ${panelId} (${renderedScene.renderId})`)
+            const result = await upscaleImage(renderedScene.assetUrl)
+            await sleep(1000)
+            if (result.assetUrl) {
+              console.log(`upscale successful, removing ${panelId} (${renderedScene.renderId}) from upscale queue`)
+              setRendered(panelId, {
+                ...renderedScene,
+                assetUrl: result.assetUrl
+              })
+              removeFromUpscaleQueue(panelId)
+            }
+
+          } catch (err) {
+            console.error(`failed to upscale: ${err}`)
+          }
+        }
+        
+        setUpscaling(false)
+      }
+
+      fn()
     })
   }
-  */
 
   const handleShare = async () => {
     const dataUrl = await pageToImage()
@@ -83,16 +113,6 @@ ${uploadUrl
       )}>
         <About />
       </div>
-      {/*
-      <div>
-        <Button
-          onClick={handleUpscale}
-          disabled={!prompt?.length && remainingImages}
-        >
-          Upscale
-        </Button>
-      </div>
-      */}
       <div className={cn(
       `flex flex-row`,
       `animation-all duration-300 ease-in-out`,
@@ -100,6 +120,16 @@ ${uploadUrl
       `space-x-3`,
       `scale-[0.9]`
     )}>
+      <div>
+        <Button
+          onClick={handleUpscale}
+          disabled={!prompt?.length || remainingImages > 0 || !Object.values(upscaleQueue).length}
+        >
+          {isUpscaling
+            ? `${allStatus.length - Object.values(upscaleQueue).length}/${allStatus.length} ⌛`
+            : "Upscale"}
+        </Button>
+      </div>
         <div>
           <Button
             onClick={handlePrint}
@@ -114,10 +144,10 @@ ${uploadUrl
             disabled={!prompt?.length}
           >
             <span className="hidden md:inline">{
-            remainingImages ? `${allStatus.length - remainingImages}/4 panels ⌛` : `Save`
+            remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} panels ⌛` : `Save`
             }</span>
             <span className="inline md:hidden">{
-              remainingImages ? `${allStatus.length - remainingImages}/4 ⌛` : `Save`
+              remainingImages ? `${allStatus.length - remainingImages}/${allStatus.length} ⌛` : `Save`
             }</span>
            </Button>
         </div>

src/app/interface/panel/index.tsx

@@ -26,9 +26,12 @@ export function Panel({
   width?: number
   height?: number
  }) {
+  const panelId = `${panel}`
+
   const ref = useRef<HTMLImageElement>(null)
   const font = useStore(state => state.font)
   const preset = useStore(state => state.preset)
+
   const setGeneratingImages = useStore(state => state.setGeneratingImages)
 
   const [imageWithText, setImageWithText] = useState("")
@@ -41,13 +44,18 @@ export function Panel({
   const zoomLevel = useStore(state => state.zoomLevel)
   const showCaptions = useStore(state => state.showCaptions)
 
-  // const setCaption = useStore(state => state.setCaption)
-  // const captions = useStore(state => state.captions)
-  // const caption = captions[panel] || ""
+  const addToUpscaleQueue = useStore(state => state.addToUpscaleQueue)
 
   const [_isPending, startTransition] = useTransition()
-  const [rendered, setRendered] = useState<RenderedScene>(getInitialRenderedScene())
+  const renderedScenes = useStore(state => state.renderedScenes)
+  const setRendered = useStore(state => state.setRendered)
+
+  const rendered = renderedScenes[panel] || getInitialRenderedScene()
+
+  // keep a ref in sync
   const renderedRef = useRef<RenderedScene>()
+  const renderedKey = JSON.stringify(rendered)
+  useEffect(() => { renderedRef.current = rendered }, [renderedKey])
 
   const timeoutRef = useRef<any>(null)
 
@@ -60,41 +68,40 @@ export function Panel({
     if (!prompt?.length) { return }
 
     // important: update the status, and clear the scene
-    setGeneratingImages(panel, true)
+    setGeneratingImages(panelId, true)
 
     // just to empty it
-    setRendered(getInitialRenderedScene())
+    setRendered(panelId, getInitialRenderedScene())
 
     setTimeout(() => {
       startTransition(async () => {
 
-      console.log(`Loading panel ${panel}..`)
+      // console.log(`Loading panel ${panel}..`)
     
       let newRendered: RenderedScene
       try {
         newRendered = await newRender({ prompt, width, height })
       } catch (err) {
-        console.log("Failed to load the panel! Don't worry, we are retrying..")
+        // "Failed to load the panel! Don't worry, we are retrying..")
         newRendered = await newRender({ prompt, width, height })
       }
 
       if (newRendered) {
         // console.log("newRendered:", newRendered)
-        setRendered(renderedRef.current = newRendered)
-        // addRenderedScene(newRendered)
+        setRendered(panelId, newRendered)
 
         // but we are still loading!
       } else {
-        setRendered(renderedRef.current = {
+        setRendered(panelId, {
           renderId: "",
-          status: "error",
+          status: "pending",
           assetUrl: "",
           alt: "",
           maskUrl: "",
-          error: "failed to fetch the data",
+          error: "",
           segments: []
         })
-        setGeneratingImages(panel, false)
+        setGeneratingImages(panelId, false)
         return
       }
     })
@@ -111,15 +118,15 @@ export function Panel({
         return
       }
       try {
-        setGeneratingImages(panel, true)
+        setGeneratingImages(panelId, true)
         // console.log(`Checking job status API for job ${renderedRef.current?.renderId}`)
         const newRendered = await getRender(renderedRef.current.renderId)
         // console.log("got a response!", newRendered)
 
         if (JSON.stringify(renderedRef.current) !== JSON.stringify(newRendered)) {
-          console.log("updated panel:", newRendered)
-          setRendered(renderedRef.current = newRendered)
-          setGeneratingImages(panel, true)
+          // console.log("updated panel:", newRendered)
+          setRendered(panelId, renderedRef.current = newRendered)
+          setGeneratingImages(panelId, true)
         }
         // console.log("status:", newRendered.status)
 
@@ -128,17 +135,18 @@ export function Panel({
           timeoutRef.current = setTimeout(checkStatus, delay)
         } else if (newRendered.status === "error" || 
         (newRendered.status === "completed" && !newRendered.assetUrl?.length)) {
-          console.log(`panel got an error and/or an empty asset url :/ "${newRendered.error}", but let's try to recover..`)
+          // console.log(`panel got an error and/or an empty asset url :/ "${newRendered.error}", but let's try to recover..`)
           try {
             const newAttempt = await newRender({ prompt, width, height })
-            setRendered(renderedRef.current = newAttempt)
+            setRendered(panelId, newAttempt)
           } catch (err) {
-            console.error("yeah sorry, something is wrong.. aborting")
-            setGeneratingImages(panel, false)
+            console.error("yeah sorry, something is wrong.. aborting", err)
+            setGeneratingImages(panelId, false)
           }
         } else {
           console.log("panel finished!")
-          setGeneratingImages(panel, false)
+          setGeneratingImages(panelId, false)
+          addToUpscaleQueue(panelId, newRendered)
         }
       } catch (err) {
         console.error(err)
@@ -251,7 +259,6 @@ export function Panel({
         zoomLevel > 40 ? `border-b-[0.5px] md:border-b-[1px]` :
         `border-transparent md:border-b-[0.5px]`,
         `print:border-b-[1.5px]`,
-        showCaptions ? `block` : `hidden`,
         `truncate`,
 
         zoomLevel > 200 ? `p-4 md:p-8` :
@@ -271,8 +278,11 @@ export function Panel({
         zoomLevel > 120 ? `text-3xs md:text-xl` :
         zoomLevel > 100 ? `text-4xs md:text-lg` :
         zoomLevel > 90 ? `text-5xs md:text-sm` :
-        zoomLevel > 40 ? `hidden md:block md:text-xs` :
-        `hidden md:block md:text-2xs`,
+        zoomLevel > 40 ? `md:text-xs` : `md:text-2xs`,
+
+        showCaptions ? (
+          zoomLevel > 90 ? `block` : `hidden md:block`
+        ) : `hidden`,
       )}
       >{caption || ""}
         </div>

src/app/main.tsx

@@ -30,7 +30,6 @@ export default function Main() {
 
   const [waitABitMore, setWaitABitMore] = useState(false)
 
-
   // react to prompt changes
   useEffect(() => {
     if (!prompt) { return }
@@ -42,27 +41,29 @@ export default function Main() {
       try {
 
         const llmResponse = await getStory({ preset, prompt })
-        console.log("response:", llmResponse)
+        console.log("LLM responded:", llmResponse)
 
         // we have to limit the size of the prompt, otherwise the rest of the style won't be followed
 
         let limitedPrompt = prompt.slice(0, 77)
-        console.log("had to cut the length og the prompt to:", limitedPrompt)
+        if (limitedPrompt.length !== prompt.length) {
+          console.log("Sorry folks, the prompt was cut to:", limitedPrompt)
+        }
 
         const panelPromptPrefix = preset.imagePrompt(limitedPrompt).join(", ")
-        console.log("panel prompt prefix:", panelPromptPrefix)
-    
+
         const nbPanels = 4
         const newPanels: string[] = []
         const newCaptions: string[] = []
         setWaitABitMore(true)
-
+        console.log("Panel prompts for SDXL:")
         for (let p = 0; p < nbPanels; p++) {
           newCaptions.push(llmResponse[p]?.caption || "...")
-          const newPanel = [panelPromptPrefix, llmResponse[p]?.instructions || ""]
-          newPanels.push(newPanel.map(chunk => chunk).join(", "))
+          const newPanel = [panelPromptPrefix, llmResponse[p]?.instructions || ""].map(chunk => chunk).join(", ")
+          newPanels.push(newPanel)
+          console.log(newPanel)
         }
-        console.log("newPanels:", newPanels)
+   
         setCaptions(newCaptions)
         setPanels(newPanels)
       } catch (err) {

src/app/queries/getStory.ts

@@ -54,7 +54,7 @@ export const getStory = async ({
     }
   }
 
-  console.log("Raw response from LLM:", result)
+  // console.log("Raw response from LLM:", result)
   const tmp = cleanJson(result)
   
   let llmResponse: LLMResponse = []
@@ -63,6 +63,7 @@ export const getStory = async ({
     llmResponse = dirtyLLMJsonParser(tmp)
   } catch (err) {
     console.log(`failed to read LLM response: ${err}`)
+    console.log(`original response was:`, result)
 
       // in case of failure here, it might be because the LLM hallucinated a completely different response,
       // such as markdown. There is no real solution.. but we can try a fallback:

src/app/store/index.ts

@@ -15,7 +15,9 @@ export const useStore = create<{
   nbFrames: number
   panels: string[]
   captions: string[]
+  upscaleQueue: Record<string, RenderedScene>
   showCaptions: boolean
+  renderedScenes: Record<string, RenderedScene>
   layout: LayoutName
   layouts: LayoutName[]
   zoomLevel: number
@@ -24,6 +26,9 @@ export const useStore = create<{
   panelGenerationStatus: Record<number, boolean>
   isGeneratingText: boolean
   atLeastOnePanelIsBusy: boolean
+  setRendered: (panelId: string, renderedScene: RenderedScene) => void
+  addToUpscaleQueue: (panelId: string, renderedScene: RenderedScene) => void
+  removeFromUpscaleQueue: (panelId: string) => void
   setPrompt: (prompt: string) => void
   setFont: (font: FontName) => void
   setPreset: (preset: Preset) => void
@@ -35,7 +40,7 @@ export const useStore = create<{
   setZoomLevel: (zoomLevel: number) => void
   setPage: (page: HTMLDivElement) => void
   setGeneratingStory: (isGeneratingStory: boolean) => void
-  setGeneratingImages: (panelId: number, value: boolean) => void
+  setGeneratingImages: (panelId: string, value: boolean) => void
   setGeneratingText: (isGeneratingText: boolean) => void
   pageToImage: () => Promise<string>
   download: () => Promise<void>
@@ -47,6 +52,8 @@ export const useStore = create<{
   nbFrames: 1,
   panels: [],
   captions: [],
+  upscaleQueue: {} as Record<string, RenderedScene>,
+  renderedScenes: {} as Record<string, RenderedScene>,
   showCaptions: false,
   layout: defaultLayout,
   layouts: [defaultLayout, defaultLayout],
@@ -56,6 +63,31 @@ export const useStore = create<{
   panelGenerationStatus: {},
   isGeneratingText: false,
   atLeastOnePanelIsBusy: false,
+  setRendered: (panelId: string, renderedScene: RenderedScene) => {
+    const { renderedScenes } = get()
+    set({
+      renderedScenes: {
+        ...renderedScenes,
+        [panelId]: renderedScene
+      }
+    })
+  },
+  addToUpscaleQueue: (panelId: string, renderedScene: RenderedScene) => {
+    const { upscaleQueue } = get()
+    set({
+      upscaleQueue: {
+        ...upscaleQueue,
+        [panelId]: renderedScene
+      },
+    })
+  },
+  removeFromUpscaleQueue: (panelId: string) => {
+    const upscaleQueue = { ...get().upscaleQueue }
+    delete upscaleQueue[panelId]
+    set({
+      upscaleQueue,
+    })
+  },
   setPrompt: (prompt: string) => {
     const existingPrompt = get().prompt
     if (prompt === existingPrompt) { return }
@@ -112,9 +144,8 @@ export const useStore = create<{
     set({ page })
   },
   setGeneratingStory: (isGeneratingStory: boolean) => set({ isGeneratingStory }),
-  setGeneratingImages: (panelId: number, value: boolean) => {
-
-    const panelGenerationStatus: Record<number, boolean> = {
+  setGeneratingImages: (panelId: string, value: boolean) => {
+    const panelGenerationStatus: Record<string, boolean> = {
       ...get().panelGenerationStatus,
       [panelId]: value
     }

src/lib/sleep.ts

@@ -0,0 +1,6 @@
+export const sleep = async (durationInMs: number) =>
+  new Promise((resolve) => {
+    setTimeout(() => {
+      resolve(true)
+    }, durationInMs)
+  })