exhumation of this old project 🧟

.env

@@ -1,4 +1,6 @@
 
+HF_API_KEY=""
+
 # ----------- CENSORSHIP -------
 # Due to abuse by users, I've had to add a censorship/fingerprinting mechanism
 ENABLE_CENSORSHIP="false"

.nvmrc

@@ -1 +1 @@
-v20.9.0
+v20.15.1

next.config.js

@@ -1,11 +1,6 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: 'standalone',
-  
-  experimental: {
-    serverActions: true,
-    serverActionsBodySizeLimit: '8mb',
-  },
 }
 
 module.exports = nextConfig

package.json

@@ -9,7 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@gradio/client": "1.2.1",
+    "@gradio/client": "1.5.0",
+    "@huggingface/inference": "^2.8.0",
     "@photo-sphere-viewer/core": "5.1.7",
     "@photo-sphere-viewer/markers-plugin": "5.1.7",
     "@photo-sphere-viewer/video-plugin": "5.1.7",
@@ -31,46 +32,39 @@
     "@radix-ui/react-switch": "1.0.3",
     "@radix-ui/react-toast": "1.1.4",
     "@radix-ui/react-tooltip": "1.0.6",
-    "@react-pdf/renderer": "3.1.12",
     "@types/node": "20.4.2",
     "@types/react": "18.2.15",
     "@types/react-dom": "18.2.7",
     "@types/uuid": "9.0.2",
-    "autoprefixer": "10.4.14",
-    "class-variance-authority": "0.6.1",
-    "clsx": "2.0.0",
-    "cmdk": "0.2.0",
+    "autoprefixer": "10.4.20",
+    "class-variance-authority": "0.7.0",
+    "clsx": "2.1.1",
+    "cmdk": "^1.0.0",
     "cookies-next": "2.1.2",
-    "date-fns": "2.30.0",
-    "eslint": "8.45.0",
-    "eslint-config-next": "13.4.10",
-    "html2canvas": "1.4.1",
-    "lucide-react": "0.260.0",
-    "next": "13.4.10",
+    "eslint": "^8.57.0",
+    "eslint-config-next": "^14.2.5",
+    "lucide-react": "0.427.0",
+    "next": "14.2.5",
     "photo-sphere-viewer-lensflare-plugin": "1.1.1",
     "pick": "0.0.1",
-    "postcss": "8.4.26",
+    "postcss": "8.4.41",
     "react": "18.2.0",
     "react-circular-progressbar": "2.1.0",
     "react-dom": "18.2.0",
     "react-photo-sphere-viewer": "3.3.5-psv5.1.4",
-    "react-virtualized-auto-sizer": "1.0.20",
-    "sbd": "1.0.19",
-    "sharp": "0.32.5",
-    "styled-components": "6.0.7",
-    "tailwind-merge": "1.13.2",
-    "tailwindcss": "3.3.3",
-    "tailwindcss-animate": "1.0.6",
-    "tesseract.js": "4.1.2",
-    "ts-node": "10.9.1",
-    "typescript": "5.1.6",
+    "sharp": "0.33.4",
+    "styled-components": "6.1.12",
+    "tailwind-merge": "2.5.2",
+    "tailwindcss": "3.4.9",
+    "tailwindcss-animate": "1.0.7",
+    "ts-node": "^10.9.2",
+    "typescript": "5.5.4",
     "usehooks-ts": ".9.1",
     "uuid": "9.0.0",
-    "zustand": "4.4.1"
+    "zustand": "4.5.4"
   },
   "devDependencies": {
     "@types/qs": "6.9.7",
-    "@types/react-virtualized": "9.21.22",
-    "@types/sbd": "1.0.3"
+    "@types/react-virtualized": "9.21.22"
   }
 }

src/app/engine/getPanoramaFlux.ts

@@ -0,0 +1,58 @@
+"use server"
+
+import { HfInference, HfInferenceEndpoint } from '@huggingface/inference'
+
+import { filterOutBadWords } from "./censorship"
+
+export async function getPanoramaFlux({
+  prompt,
+}: {
+  prompt: string
+}): Promise<string> {
+  if (!prompt) {
+    console.error(`cannot call the rendering API without a prompt, aborting..`)
+    throw new Error(`cannot call the rendering API without a prompt, aborting..`)
+  }
+
+  prompt = [
+    `hdri view`,
+    `highly detailed`,
+    `intricate details`,
+    filterOutBadWords(prompt)
+  ].join(', ')
+
+
+  console.log(`calling API with prompt: ${prompt}`)
+    
+  const hf: HfInferenceEndpoint = new HfInference(
+    `${process.env.HF_API_KEY}`
+  )
+    
+  const blob: Blob = await hf.textToImage({
+    model: "<put a 360° flux model here>",
+    inputs: prompt,
+    parameters: {
+      height: 1024,
+      width: 2048,
+  
+      // this triggers the following exception:
+      // Error: __call__() got an unexpected keyword argument 'negative_prompt'
+      // negative_prompt: request.prompts.image.negative || '',
+  
+      /**
+       * The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference.
+       */
+      // num_inference_steps?: number;
+      /**
+       * Guidance scale: Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, usually at the expense of lower image quality.
+       */
+      // guidance_scale?: number;
+    },
+  })
+  
+  // console.log('output from Hugging Face Inference API:', blob)
+  
+  const buffer = Buffer.from(await blob.arrayBuffer())
+  
+  return `data:${blob.type || 'image/jpeg'};base64,${buffer.toString('base64')}`
+}

src/app/engine/getPanoramaSDXL.ts

@@ -0,0 +1,20 @@
+"use server"
+
+import { Client } from "@gradio/client"
+
+export async function getPanoramaSDXL({
+  prompt
+}: {
+  prompt: string
+}): Promise<string> {
+  const app = await Client.connect("jbilcke-hf/panorama-api")
+
+  const result = await app.predict("/predict", [
+    process.env.MICRO_SERVICE_SECRET_TOKEN || "",
+    prompt,
+    undefined
+  ])
+  console.log(`result:`, result)
+
+  return ""
+}

src/app/engine/render.ts

@@ -1,245 +0,0 @@
-"use server"
-
-
-import { RenderRequest, RenderedScene } from "@/types"
-import { generateSeed } from "@/lib/generateSeed"
-import { sleep } from "@/lib/sleep"
-import { filterOutBadWords } from "./censorship"
-
-export async function newRender({
-  prompt,
-  clearCache,
-}: {
-  prompt: string
-  clearCache: boolean
-}) {
-  if (!prompt) {
-    console.error(`cannot call the rendering API without a prompt, aborting..`)
-    throw new Error(`cannot call the rendering API without a prompt, aborting..`)
-  }
-
-  prompt = [
-    `hdri view`,
-    `highly detailed`,
-    `intricate details`,
-    filterOutBadWords(prompt)
-  ].join(', ')
- 
-  // return await Gorgon.get(cacheKey, async () => {
-
-  let defaulResult: RenderedScene = {
-    renderId: "",
-    status: "error",
-    assetUrl: "",
-    alt: prompt || "",
-    maskUrl: "",
-    error: "failed to fetch the data",
-    segments: []
-   }
-
-  try {
-    console.log(`calling Gradio space with prompt: ${prompt}`)
-
-    const request = {
-      prompt,
-      nbFrames: 1, // when nbFrames is 1, we will only generate static images
-      nbSteps: 35, // 20 = fast, 30 = better, 50 = best
-      actionnables: [],
-      segmentation: "disabled", // one day we will remove this param, to make it automatic
-      width: 1024,
-      height: 768,
-
-      // on VideoQuest we use an aggressive setting: 4X upscaling
-      // this generates images that can be slow to load, but that's
-      // not too much of an issue since we use async loading
-      upscalingFactor: 1,
-
-      // note that we never disable the cache completely for VideoQuest
-      // that's because in the feedbacks people prefer speed to avoid frustration
-      cache: clearCache ? "renew" : "use",
-
-    } as Partial<RenderRequest>
-
-    console.table(request)
-
-    if (renderingEngine === "REPLICATE") {
-      if (!replicateToken) {
-        throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
-      }
-      if (!replicateModel) {
-        throw new Error(`you need to configure your REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
-      }
-      if (!replicateModelVersion) {
-        throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
-      }
-      const replicate = new Replicate({ auth: replicateToken })
-
-      // console.log("Calling replicate..")
-      const seed = generateSeed()
-      const prediction = await replicate.predictions.create({
-        version: replicateModelVersion,
-        input: { prompt, seed }
-      })
-      
-      // console.log("prediction:", prediction)
-
-      // no need to reply straight away: good things take time
-      // also our friends at Replicate won't like it if we spam them with requests
-      await sleep(12000)
-
-      return {
-        renderId: prediction.id,
-        status: "pending",
-        assetUrl: "", 
-        alt: prompt,
-        error: prediction.error,
-        maskUrl: "",
-        segments: []
-      } as RenderedScene
-    } else {
-
-      const res = await fetch(`${apiUrl}/render`, {
-        method: "POST",
-        headers: {
-          Accept: "application/json",
-          "Content-Type": "application/json",
-          // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
-        },
-        body: JSON.stringify(request),
-        cache: 'no-store',
-      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-      // next: { revalidate: 1 }
-      })
-
-      // console.log("res:", res)
-      // The return value is *not* serialized
-      // You can return Date, Map, Set, etc.
-      
-      // Recommendation: handle errors
-      if (res.status !== 200) {
-        // This will activate the closest `error.js` Error Boundary
-        throw new Error('Failed to fetch data')
-      }
-      
-      const response = (await res.json()) as RenderedScene
-      // console.log("response:", response)
-      return response
-    }
-  } catch (err) {
-    console.log("request failed:", err)
-    console.error(err)
-    // Gorgon.clear(cacheKey)
-    return defaulResult
-  }
-
-  // }, cacheDurationInSec * 1000)
-}
-
-export async function getRender(renderId: string) {
-  if (!renderId) {
-    console.error(`cannot call the rendering API without a renderId, aborting..`)
-    throw new Error(`cannot call the rendering API without a renderId, aborting..`)
-  }
-
-  let defaulResult: RenderedScene = {
-    renderId: "",
-    status: "pending",
-    assetUrl: "",
-    alt: "",
-    maskUrl: "",
-    error: "",
-    segments: []
-  }
-
-  try {
-
-
-    if (renderingEngine === "REPLICATE") {
-      if (!replicateToken) {
-        throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
-      }
-      if (!replicateModel) {
-        throw new Error(`you need to configure your REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
-      }
-
-      // const replicate = new Replicate({ auth: replicateToken })
-
-      // console.log("Calling replicate..")
-      // const prediction = await replicate.predictions.get(renderId)
-      // console.log("Prediction:", prediction)
-
-       // console.log(`calling GET https://api.replicate.com/v1/predictions/${renderId}`)
-       const res = await fetch(`https://api.replicate.com/v1/predictions/${renderId}`, {
-        method: "GET",
-        headers: {
-          // Accept: "application/json",
-          // "Content-Type": "application/json",
-          Authorization: `Token ${replicateToken}`,
-        },
-        cache: 'no-store',
-      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-      // next: { revalidate: 1 }
-      })
-    
-      // console.log("res:", res)
-      // The return value is *not* serialized
-      // You can return Date, Map, Set, etc.
-      
-      // Recommendation: handle errors
-      if (res.status !== 200) {
-        // This will activate the closest `error.js` Error Boundary
-        throw new Error('Failed to fetch data')
-      }
-      
-      const response = (await res.json()) as any
-      // console.log("response:", response)
-
-      return  {
-        renderId,
-        status: response?.error ? "error" : response?.status === "succeeded" ?  "completed" : "pending",
-        assetUrl: `${response?.output || ""}`,
-        alt: `${response?.input?.prompt || ""}`,
-        error: `${response?.error || ""}`,
-        maskUrl: "",
-        segments: []
-      } as RenderedScene
-    } else {
-
-      // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
-      const res = await fetch(`${apiUrl}/render/${renderId}`, {
-        method: "GET",
-        headers: {
-          Accept: "application/json",
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
-        },
-        cache: 'no-store',
-      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-      // next: { revalidate: 1 }
-      })
-
-      // console.log("res:", res)
-      // The return value is *not* serialized
-      // You can return Date, Map, Set, etc.
-      
-      // Recommendation: handle errors
-      if (res.status !== 200) {
-        // This will activate the closest `error.js` Error Boundary
-        throw new Error('Failed to fetch data')
-      }
-      
-      const response = (await res.json()) as RenderedScene
-      // console.log("response:", response)
-
-      return response
-    }
-  } catch (err) {
-    console.error(err)
-    defaulResult.status = "error"
-    defaulResult.error = `${err}`
-    // Gorgon.clear(cacheKey)
-    return defaulResult
-  }
-
-  // }, cacheDurationInSec * 1000)
-}

src/app/engine/upscaleImage.ts

@@ -0,0 +1,77 @@
+"use server"
+
+import { Client } from "@gradio/client"
+
+export async function upscalePanorama({
+  image,
+  prompt
+}: {
+  image: string
+  prompt: string
+}): Promise<string> {
+  const app = await Client.connect("jbilcke-hf/clarity-upscaler-api")
+
+ // const dataUri = await fetch(`data:image/jpeg;base64,${base64Data}`)
+  const dataUri = await fetch(image)
+
+  const imageBlob = await dataUri.blob()
+
+  const result = await app.predict("/predict", {
+    "Secret Token": process.env.MICRO_SERVICE_SECRET_TOKEN || "",
+
+    // convert the base64 image to blob
+    "Image": imageBlob,
+
+    Prompt: `360° HDRI panorama photo, ${prompt}`,
+
+    "Negative Prompt": "blurry, cropped",
+
+    "Scalue Factor": 2,
+  
+    "Dynamic": 6,
+
+    "Creativity": 0.35,
+
+    "Resemblance": 0.6,
+
+    "tiling_width": 112,
+
+    "tiling_height": 144,
+
+   // epicrealism_naturalSinRC1VAE.safetensors [84d76a0328]', 'juggernaut_reborn.safetensors [338b85bc4f]', 'flat2DAnimerge_v45Sharp.safetensors'],
+    "sd_model": "juggernaut_reborn.safetensors [338b85bc4f]",
+    "scheduler": "DPM++ 3M SDE Karras",
+  })
+  /*
+
+  inputs.append(gr.Slider(
+      label="Num Inference Steps", info='''Number of denoising steps''', value=18,
+      minimum=1, maximum=100, step=1,
+  ))
+  
+  inputs.append(gr.Number(
+      label="Seed", info='''Random seed. Leave blank to randomize the seed''', value=1337
+  ))
+  
+  inputs.append(gr.Checkbox(
+      label="Downscaling", info='''Downscale the image before upscaling. Can improve quality and speed for images with high resolution but lower quality''', value=False
+  ))
+  
+  inputs.append(gr.Number(
+      label="Downscaling Resolution", info='''Downscaling resolution''', value=768
+  ))
+  
+  inputs.append(gr.Textbox(
+      label="Lora Links", info='''Link to a lora file you want to use in your upscaling. Multiple links possible, seperated by comma'''
+  ))
+  
+  inputs.append(gr.Textbox(
+      label="Custom Sd Model", info='''Link to a custom safetensors checkpoint file you want to use in your upscaling. Will overwrite sd_model checkpoint.'''
+  ))
+  
+  ])
+  */
+  console.log(`result:`, result)
+
+  return ""
+}

src/app/firehose/{page.tsx → page.txt}

@@ -1,6 +1,6 @@
 "use client"
 
-import { useEffect, useState, useTransition } from "react"
+import { Suspense, useEffect, useState, useTransition } from "react"
 
 import { Post } from "@/types"
 import { cn } from "@/lib/utils"
@@ -13,7 +13,7 @@ import { Delete } from "./delete"
 import Link from "next/link"
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
 
-export default function FirehosePage() {
+function PageContent() {
   const searchParams = useSearchParams()
   const [_isPending, startTransition] = useTransition()
   const [posts, setPosts] = useState<Post[]>([])
@@ -35,12 +35,7 @@ export default function FirehosePage() {
   }
 
   return (
-    <TooltipProvider delayDuration={100}>
-      <div className={cn(
-        `light fixed w-full h-full flex flex-col items-center bg-slate-300 text-slate-800`,
-        ``,
-        actionman.className
-        )}>
+    <>
         <div className="w-full flex flex-col items-center overflow-y-scroll">
           <div className="flex flex-col space-y-2 pt-18 mb-6">
             <h1 className="text-4xl md:text-6xl lg:text-[70px] xl:text-[100px] text-cyan-700">🌐 Text-to-panorama</h1>
@@ -96,6 +91,20 @@ export default function FirehosePage() {
           </div>
         </div>
         <Delete post={toDelete} moderationKey={moderationKey} onDelete={handleOnDelete} />
+    </>
+  )
+}
+
+
+export default function FirehosePage() {
+  return (
+    <TooltipProvider delayDuration={100}>
+      <div className={cn(
+        `light fixed w-full h-full flex flex-col items-center bg-slate-300 text-slate-800`,
+        ``,
+        actionman.className
+        )}>
+        <Suspense><PageContent /></Suspense>
       </div>
     </TooltipProvider>
   )

src/app/generate/page.tsx

@@ -1,6 +1,6 @@
 "use client"
 
-import { useEffect, useRef, useTransition } from "react"
+import { Suspense, useEffect, useTransition } from "react"
 
 import { cn } from "@/lib/utils"
 import { TopMenu } from "../interface/top-menu"
@@ -9,31 +9,22 @@ import { fonts } from "@/lib/fonts"
 import { useStore } from "../store"
 import { BottomBar } from "../interface/bottom-bar"
 import { SphericalImage } from "../interface/spherical-image"
-import { getRender, newRender } from "../engine/render"
-import { RenderedScene } from "@/types"
-import { getPost, postToCommunity } from "../engine/community"
+import { getPanoramaSDXL } from "../engine/getPanoramaSDXL"
+import { getPost } from "../engine/community"
 import { useSearchParams } from "next/navigation"
 
-export default function GeneratePage() {
+function PageContent() {
   const searchParams = useSearchParams()
   const [_isPending, startTransition] = useTransition()
   const postId = (searchParams.get("postId") as string) || ""
 
   const prompt = useStore(s => s.prompt)
   const setPrompt = useStore(s => s.setPrompt)
-  const setRendered = useStore(s => s.setRendered)
-  const renderedScene = useStore(s => s.renderedScene)
+  const assetUrl = useStore(s => s.assetUrl)
+  const setAssetUrl = useStore(s => s.setAssetUrl)
   const isLoading = useStore(s => s.isLoading)
   const setLoading = useStore(s => s.setLoading)
 
-  // keep a ref in sync
-  const renderedRef = useRef<RenderedScene>()
-  const renderedKey = JSON.stringify(renderedScene)
-  useEffect(() => { renderedRef.current = renderedScene }, [renderedKey])
-  
-  const timeoutRef = useRef<any>(null)
-  
-  const delay = 3000
 
   // react to prompt changes
   useEffect(() => {
@@ -44,89 +35,21 @@ export default function GeneratePage() {
     // if (isLoading) { return }
 
     startTransition(async () => {
- 
       try {
-        const rendered = await newRender({ prompt, clearCache: true })
-        setRendered(rendered)
-      } catch (err) {
-        console.error(err)
-      } finally {
-      }
-    })
-  }, [prompt]) // important: we need to react to preset changes too
-
-
-  const checkStatus = () => {
-    startTransition(async () => {
-      clearTimeout(timeoutRef.current)
-
-      if (renderedRef.current?.status === "completed") {
-        console.log("rendering job is already completed")
-        return
-      }
-
-      if (!renderedRef.current?.renderId || renderedRef.current?.status !== "pending") {
-        timeoutRef.current = setTimeout(checkStatus, delay)
-        return
-      }
-
-      try {
-        // console.log(`Checking job status API for job ${renderedRef.current?.renderId}`)
-        const newRendered = await getRender(renderedRef.current.renderId)
-        if (!newRendered) {
-          throw new Error(`getRender failed`)
-        }
-        // console.log("got a response!", newRendered)
-
-        if (JSON.stringify(renderedRef.current) !== JSON.stringify(newRendered)) {
-          // console.log("updated panel:", newRendered)
-          setRendered(renderedRef.current = newRendered)
-        }
-        // console.log("status:", newRendered.status)
-
-        if (newRendered.status === "pending") {
-          console.log("job not finished")
-          timeoutRef.current = setTimeout(checkStatus, delay)
-        } else if (newRendered.status === "error" || 
-        (newRendered.status === "completed" && !newRendered.assetUrl?.length)) {
-          console.log(`panorama got an error and/or an empty asset url :/ "${newRendered.error}", but let's try to recover..`)
+        const assetUrl = await getPanoramaSDXL({ prompt })
+        if (assetUrl) {
+          setAssetUrl(assetUrl)
           setLoading(false)
         } else {
-          console.log("panorama finished:", newRendered)
-          /*
-          let's disable the community for now
-          
-          try {
-            await postToCommunity({
-              prompt,
-              model: "jbilcke-hf/sdxl-panorama",
-              assetUrl: newRendered.assetUrl,
-            })
-          } catch (err) {
-            console.log("failed to post to community, but it's no big deal")
-          }
-          */
-          setRendered(newRendered)
-          setLoading(false)
+          console.log(`panorama got an error and/or an empty asset url`)
         }
       } catch (err) {
         console.error(err)
-        timeoutRef.current = setTimeout(checkStatus, delay)
+      } finally {
+        setLoading(false)
       }
     })
-  }
- 
-  useEffect(() => {
-    // console.log("starting timeout")
-    clearTimeout(timeoutRef.current)
-    
-    // normally it should reply in < 1sec, but we could also use an interval
-    timeoutRef.current = setTimeout(checkStatus, delay)
-
-    return () => {
-      clearTimeout(timeoutRef.current)
-    }
-  }, [prompt])
+  }, [prompt]) // important: we need to react to preset changes too
 
   useEffect(() => {
     if (!postId) {
@@ -144,15 +67,7 @@ export default function GeneratePage() {
         // because we are set the app to "is loading"
         // setPrompt(post.prompt)
 
-        setRendered({
-          renderId: postId,
-          status: "completed",
-          assetUrl: post.assetUrl, 
-          alt: post.prompt,
-          error: "",
-          maskUrl: "",
-          segments: []
-        })
+        setAssetUrl(post.assetUrl)
         setLoading(false)
       } catch (err) {
         console.error("failed to get post: ", err)
@@ -162,19 +77,16 @@ export default function GeneratePage() {
   }, [postId])
 
   return (
-    <div className="">
-      <TopMenu />
+    <>
       <div className={cn(
         `fixed inset-0 w-screen h-screen overflow-y-scroll`,
         fonts.actionman.className
       )}>
-        {renderedScene.assetUrl ? <SphericalImage
-          rendered={renderedScene}
-          onEvent={(() => {}) as any}
+        {assetUrl ? <SphericalImage
+          assetUrl={assetUrl}
           debug={true}
         /> : null}
       </div>
-      <BottomBar />
       <div className={cn(
         `print:hidden`,
         `z-20 fixed inset-0`,
@@ -193,6 +105,17 @@ export default function GeneratePage() {
           {isLoading ? 'Generating metaverse location in the latent space..' : ''}
         </div>
       </div>
+    </>
+  )
+}
+
+
+export default function GeneratePage() {
+  return (
+    <div className="">
+      <Suspense><TopMenu /></Suspense>
+      <Suspense><PageContent /></Suspense>
+      <BottomBar />
     </div>
   )
 }

src/app/interface/display/index.tsx

@@ -1,10 +1,9 @@
-import { RenderedScene } from "@/types"
 
-export function Display ({ rendered }: { rendered: RenderedScene }) {
+export function Display ({ assetUrl }: { assetUrl: string }) {
   return (
     <>
       <img
-        src={rendered.assetUrl || undefined}
+        src={assetUrl || undefined}
         className="fixed w-screen top-0 left-0 right-0"
       />
     </>

src/app/interface/spherical-image/index.tsx

@@ -1,33 +1,27 @@
 "use client"
 
 import { useEffect, useRef, useState } from "react"
-import { PanoramaPosition, PluginConstructor, Point, Position, SphericalPosition, Viewer } from "@photo-sphere-viewer/core"
-import { LensflarePlugin, ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
-
-import { MouseEventHandler, RenderedScene } from "@/types"
+import { PluginConstructor, Viewer } from "@photo-sphere-viewer/core"
+import { ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
 
 import { useImageDimension } from "@/lib/useImageDimension"
-import { lightSourceNames } from "@/lib/lightSourceNames"
 
 type PhotoSpherePlugin = (PluginConstructor | [PluginConstructor, any])
 
 export function SphericalImage({
-  rendered,
-  onEvent,
+  assetUrl,
   className,
   debug,
 }: {
-  rendered: RenderedScene
-  onEvent: MouseEventHandler
+  assetUrl: string
   className?: string
   debug?: boolean
 }) {
 
 
-  const imageDimension = useImageDimension(rendered.assetUrl)
-  const maskDimension = useImageDimension(rendered.maskUrl)
+  const imageDimension = useImageDimension(assetUrl)
 
-  const sceneConfig = JSON.stringify({ rendered, debug, imageDimension, maskDimension })
+  const sceneConfig = JSON.stringify({ assetUrl, debug, imageDimension })
   const [lastSceneConfig, setLastSceneConfig] = useState<string>(sceneConfig)
   const rootContainerRef = useRef<HTMLDivElement>(null)
   const viewerContainerRef = useRef<HTMLElement>()
@@ -39,7 +33,7 @@ export function SphericalImage({
   const options = {
     defaultZoomLvl,
     fisheye: false, // ..no!
-    overlay: rendered.maskUrl || undefined,
+    overlay:  undefined,
     overlayOpacity: debug ? 0.5 : 0,
     /*
     panoData: {
@@ -56,47 +50,6 @@ export function SphericalImage({
     */
   }
 
-
-  const cacheRef = useRef("")
-  useEffect(() => {
-    const listener = (e: DragEvent) => {
-      if (!rootContainerRef.current) { return }
-
-      // TODO: check if we are currently dragging an object
-      // if yes, then we should check if clientX and clientY are matching the 
-      const boundingRect = rootContainerRef.current.getBoundingClientRect()
-
-      // abort if we are not currently dragging over our display area
-      if (e.clientX < boundingRect.left) { return }
-      if (e.clientX > (boundingRect.left + boundingRect.width)) { return }
-      if (e.clientY < boundingRect.top) { return }
-      if (e.clientY > (boundingRect.top + boundingRect.height)) { return }
-
-      const containerX = e.clientX - boundingRect.left
-      const containerY = e.clientY - boundingRect.top
-    
-      const relativeX = containerX / boundingRect.width
-      const relativeY = containerY / boundingRect.height
-
-      const key = `${relativeX},${relativeY}`
-
-      // to avoid use
-      if (cacheRef.current === key) {
-        return
-      }
-      // console.log(`DRAG: calling onEvent("hover", ${relativeX}, ${relativeY})`)
-
-      cacheRef.current = key
-      onEvent("hover", relativeX, relativeY)
-    }
-
-    document.addEventListener('drag', listener)
-
-    return () => {
-      document.removeEventListener('drag', listener)
-    }
-  }, [onEvent])
-
   useEffect(() => {
     const task = async () => {
       // console.log("SphericalImage: useEffect")
@@ -114,50 +67,7 @@ export function SphericalImage({
           ...options,
         }
 
-        const lensflares: { id: string; position: SphericalPosition; type: number }[] = []
-          
-        if (maskDimension.width && imageDimension.width) {
-
-          // console.log("rendered.segments:", rendered.segments)
-   
-          rendered.segments
-            .filter(segment => lightSourceNames.includes(segment.label))
-            .forEach(light => {
-            // console.log("light detected", light)
-            const [x1, y1, x2, y2] = light.box
-            const [centerX, centerY] = [(x1 + x2) / 2, (y1 + y2) / 2]
-            // console.log("center:", { centerX, centerY })
-            const [relativeX, relativeY] = [centerX / maskDimension.width, centerY/ maskDimension.height]
-            // console.log("relative:", { relativeX, relativeY})
-
-            const panoramaPosition: PanoramaPosition = {
-              textureX: relativeX * imageDimension.width,
-              textureY: relativeY * imageDimension.height
-            }
-            // console.log("panoramaPosition:", panoramaPosition)
-
-            const position = viewer.dataHelper.textureCoordsToSphericalCoords(panoramaPosition)
-            // console.log("sphericalPosition:", position)
-            if ( // make sure coordinates are valid
-              !isNaN(position.pitch) && isFinite(position.pitch) &&
-              !isNaN(position.yaw) && isFinite(position.yaw)) {
-              lensflares.push({
-                id: `flare_${lensflares.length}`,
-                position,
-                type: 0,
-              })
-            }
-          })
-        }
-
-        // console.log("lensflares:", lensflares)
-        const lensFlarePlugin = viewer.getPlugin<LensflarePlugin>("lensflare")
-        lensFlarePlugin.setLensflares(lensflares)
-
-        // console.log("SphericalImage: calling setOptions")
-        // console.log("SphericalImage: changing the panorama to: " + rendered.assetUrl.slice(0, 120))
-      
-        await viewer.setPanorama(rendered.assetUrl, {
+        await viewer.setPanorama(assetUrl, {
           ...newOptions,
           showLoader: false,
         })
@@ -173,86 +83,24 @@ export function SphericalImage({
       }
     }
     task()
-  }, [sceneConfig, rendered.assetUrl, viewerRef.current, maskDimension.width, imageDimension])
-
-  const handleEvent = async (event: React.MouseEvent<HTMLDivElement, MouseEvent>, isClick: boolean) => {
-    const rootContainer = rootContainerRef.current
-    const viewer = viewerRef.current
-    const viewerContainer = viewerContainerRef.current
-
-    /*
-    if (isClick) console.log(`handleEvent(${isClick})`, {
-      " imageDimension.width": imageDimension.width,
-      "rendered.maskUrl": rendered.maskUrl
-    })
-    */
-  
-    if (!viewer || !rootContainer || !viewerContainer || !imageDimension.width || !rendered.maskUrl) {
-      return
-    }
-
-    const containerRect = viewerContainer.getBoundingClientRect()
-    // if (isClick) console.log("containerRect:", containerRect)
-
-    const containerY = event.clientY - containerRect.top
-    // console.log("containerY:", containerY)
-    
-    const position: Position = viewer.getPosition()
-
-    const viewerPosition: Point = viewer.dataHelper.sphericalCoordsToViewerCoords(position)
-    // if (isClick) console.log("viewerPosition:", viewerPosition)
+  }, [sceneConfig, assetUrl, viewerRef.current, imageDimension])
 
-    // we want to ignore events that are happening in the toolbar
-    // note that we will probably hide this toolbar at some point,
-    // to implement our own UI
-    if (isClick && containerY > (containerRect.height - 40)) {
-      // console.log("we are in the toolbar.. ignoring the click")
-      return
-    }
-
-    const panoramaPosition: PanoramaPosition = viewer.dataHelper.sphericalCoordsToTextureCoords(position)
-
-    if (typeof panoramaPosition.textureX !== "number" || typeof panoramaPosition.textureY !== "number") {
-      return
-    }
-    
-    const relativeX = panoramaPosition.textureX / imageDimension.width
-    const relativeY = panoramaPosition.textureY / imageDimension.height
 
-    onEvent(isClick ? "click" : "hover", relativeX, relativeY)
-  }
-
-  if (!rendered.assetUrl) {
+  if (!assetUrl) {
     return null
   }
 
   return (
     <div
       ref={rootContainerRef}
-      onMouseMove={(event) => {
-        handleEvent(event, false)
-        setMouseMoved(true)
-      }}
-      onMouseUp={(event) => {
-        if (!mouseMoved) {
-          handleEvent(event, true)
-        }
-        setMouseMoved(false)
-      }}
-      onMouseDown={() => {
-        setMouseMoved(false)
-      }}
       >
       <ReactPhotoSphereViewer
-        src={rendered.assetUrl}
+        src={assetUrl}
         container=""
         containerClass={className}
         
         height="100vh"
         width="100%"
-        
-        // to access a plugin we must use viewer.getPlugin()
-        plugins={[[LensflarePlugin, { lensflares: [] }]]}
 
         {...options}
 
@@ -263,24 +111,7 @@ export function SphericalImage({
         }}
     
         onReady={(instance) => {
-          viewerRef.current = instance
           viewerContainerRef.current = instance.container
-
-          /*
-          const markersPlugs = instance.getPlugin(MarkersPlugin);
-          if (!markersPlugs)
-            return;
-          markersPlugs.addMarker({
-            id: "imageLayer2",
-            imageLayer: "drone.png",
-            size: { width: 220, height: 220 },
-            position: { yaw: '130.5deg', pitch: '-0.1deg' },
-            tooltip: "Image embedded in the scene"
-          });
-          markersPlugs.addEventListener("select-marker", () => {
-            console.log("asd");
-          });
-          */
         }}
 
       />

src/app/interface/top-menu/index.tsx

@@ -11,7 +11,7 @@ export function TopMenu() {
   const prompt = useStore(s => s.prompt)
 
   const setPrompt = useStore(s => s.setPrompt)
-  const setRendered = useStore(s => s.setRendered)
+  const setAssetUrl = useStore(s => s.setAssetUrl)
 
   const isLoading = useStore(s => s.isLoading)
   const setLoading = useStore(s => s.setLoading)
@@ -26,15 +26,7 @@ export function TopMenu() {
     const promptChanged = draftPrompt.trim() !== prompt.trim()
     if (!isLoading && (promptChanged)) {
       // important: we reset!
-      setRendered({
-        renderId: "",
-        status: "pending",
-        assetUrl: "",
-        alt: "",
-        error: "",
-        maskUrl: "",
-        segments: []
-      })
+      setAssetUrl("")
       setPrompt(draftPrompt)
     }
   }

src/app/page.tsx

@@ -1,10 +1,12 @@
 "use server"
 
 import Head from "next/head"
+import { Suspense } from "react"
 
 // import Firehose from "./firehose/page"
 import Generate from "./generate/page"
 
+
 // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts 
 
 export default async function Page() {
@@ -18,7 +20,7 @@ export default async function Page() {
       <main className={
         `light bg-zinc-50 text-stone-900
         `}>
-        <Generate />
+        <Suspense><Generate /></Suspense>
       </main>
     </>
   )

src/app/store/index.ts

@@ -2,33 +2,23 @@
 
 import { create } from "zustand"
 
-import { RenderedScene } from "@/types"
-
 export const useStore = create<{
   prompt: string
-  renderedScene: RenderedScene
+  assetUrl: string
   isLoading: boolean
   setLoading: (isLoading: boolean) => void
-  setRendered: (renderedScene: RenderedScene) => void
+  setAssetUrl: (assetUrl: string) => void
   setPrompt: (prompt: string) => void
 }>((set, get) => ({
   prompt: "",
-  renderedScene: {
-    renderId: "",
-    status: "pending",
-    assetUrl: "",
-    alt: "",
-    error: "",
-    maskUrl: "",
-    segments: []
-  },
+  assetUrl: "",
   isLoading: false,
   setLoading: (isLoading: boolean) => {
     set({ isLoading })
   },
-  setRendered: (renderedScene: RenderedScene) => {
+  setAssetUrl: (assetUrl: string) => {
     set({
-      renderedScene
+      assetUrl
     })
   },
   setPrompt: (prompt: string) => {

src/lib/getInitialRenderedScene.ts

@@ -1,11 +0,0 @@
-import { RenderedScene } from "@/types"
-
-export const getInitialRenderedScene = (): RenderedScene => ({
-  renderId: "",
-  status: "pending",
-  assetUrl: "", 
-  alt: "",
-  error: "",
-  maskUrl: "",
-  segments: []
-})

src/types.ts

@@ -1,92 +1,4 @@
-export type ProjectionMode = 'cartesian' | 'spherical'
 
-export type MouseEventType = "hover" | "click"
-
-export type MouseEventHandler = (type: MouseEventType, x: number, y: number) => Promise<void>
-
-export type CacheMode = "use" | "renew" | "ignore"
-
-export interface RenderRequest {
-  prompt: string
-
-  // whether to use video segmentation
-  // disabled (default)
-  // firstframe: we only analyze the first frame
-  // allframes: we analyze all the frames
-  segmentation: 'disabled' | 'firstframe' | 'allframes'
-
-  // segmentation will only be executed if we have a non-empty list of actionnables
-  // actionnables are names of things like "chest", "key", "tree", "chair" etc
-  actionnables: string[]
-
-  // note: this is the number of frames for Zeroscope,
-  // which is currently configured to only output 3 seconds, so:
-  // nbFrames=8 -> 1 sec
-  // nbFrames=16 -> 2 sec
-  // nbFrames=24 -> 3 sec
-  nbFrames: number // min: 1, max: 24
-
-  nbSteps: number // min: 1, max: 50
-
-  seed: number
-
-  width: number // fixed at 1024 for now
-  height: number // fixed at 512 for now
-
-  // upscaling factor
-  // 0: no upscaling
-  // 1: no upscaling
-  // 2: 2x larger
-  // 3: 3x larger
-  // 4x: 4x larger, up to 4096x4096 (warning: a PNG of this size can be 50 Mb!)
-  upscalingFactor: number
-
-  projection: ProjectionMode
-
-  cache: CacheMode
-
-  wait: boolean // wait until the job is completed
-
-  analyze: boolean // analyze the image to generate a caption (optional)
-}
-
-export interface ImageSegment {
-  id: number
-  box: number[]
-  color: number[]
-  label: string
-  score: number 
-}
-
-export type RenderedSceneStatus =
-  | "pending"
-  | "completed"
-  | "error"
-
-export interface RenderedScene {
-  renderId: string
-  status: RenderedSceneStatus
-  assetUrl: string 
-  alt: string
-  error: string
-  maskUrl: string
-  segments: ImageSegment[]
-}
-
-export interface ImageAnalysisRequest {
-  image: string // in base64
-  prompt: string
-}
-
-export interface ImageAnalysisResponse {
-  result: string
-  error?: string
-}
-
-export type RenderingEngine =
-  | "VIDEOCHAIN"
-  | "OPENAI"
-  | "REPLICATE"
 
 export type PostVisibility =
   | "featured" // featured by admins