work in progress

src/app/agents/{ant.ts → city.ts}

@@ -2,40 +2,62 @@ import { pick } from "./pick"
 import { Agent, Scene } from "./types"
 
 const actions = [
-  "working on lavae",
-  "slicing leaves",
-  "attacking a beetle",
-  "foraging",
-  "cutting a sugar cube",
-  "collecting sugar",
-  "collecting aphids"
+  "busy pedestrians",
+  "busy traffic",
+  "typical street life",
+  "skyscrapper being constructed",
+  "a building is on fire",
 ]
 
 const positions = [
-  "on a leave",
-  "on a tree branch",
-  "on sand",
-  "on the ground"
+  "city center with skyscrappers",
+  "city center with a hospital",
+  "market area",
+  "residential area with small houses",
+  "residential area and houses with pools",
+  "industrial area with a smoking factory",
+  "beachfront area with villas",
+  "theme park with one big rollercoaster"
+]
+
+const lights = [
+  "during the day",
+  // "during the night",
+]
+
+const actionnables = [
+  "building",
+  "road",
+  "car",
+  "tower",
+  "tree",
+  "river",
+  "sea"
 ]
 
 export const agent: Agent = {
-  title: "Ant",
-  type: "ant",
+  title: "City",
+  type: "city",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
+    const light = pick(lights)
 
     const prompt = [
-      `close-up shot of a couple of ants`,
+      `static isometrical view of 3D rendered city`,
       action,
       position,
+      light,
+      `isometric`,
+      `game`,
       `high res`,
-      `documentary`,
     ].join(", ")
 
     return {
       action,
       position,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/{fox.ts → dungeon.ts}

@@ -2,42 +2,67 @@ import { pick } from "./pick"
 import { Agent, Scene } from "./types"
 
 const actions = [
-  "waiting",
-  "jumping",
-  "eating a mouse",
-  "looking at camera",
-  "touch a rock",
-  "touching grass",
-  "drinking from a water hole"
+  "not moving",
+  "walking in",
+  "looking up",
+  "looking down",
+  "looking left",
+  "looking right",
+  "looking around"
 ]
 
 const positions = [
-  "in the forest",
-  "in a plain",
-  "in front of a fox hole",
-  "in front of a bush"
+  "corridor with a beautiful wooden door at the end, wooden floor and stone walls",
+  "a beautiful wooden door",
+  "beautiful room with stone walls and wooden floor",
+  "large ball room with stone pillars, stone floor and red carpet",
+  "a cosy room with a fireplace, stone walls and wooden floor",
+  "a fireplace with stone walls",
+  "a cold dungeon with stone walls",
+  "a damp medieval jail cell with stone walls and wooden floor"
 ]
 
+const lights = [
+  "lit through windows",
+  "lit through wall-mounted torchs"
+  // "poorly lit"
+]
+
+const actionnables = [
+  "floor",
+  "fireplace",
+  "door",
+  "window",
+  "chair",
+  "table",
+  "torch"
+]
 
 export const agent: Agent = {
-  title: "Fox",
-  type: "fox",
+  title: "Dungeon",
+  type: "dungeon",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
+    const light = pick(lights)
 
     const prompt = [
-      `medium shot of a fox`,
+      `first-person footage`,
       action,
       position,
-      `high res`,
+      light,
+      `medieval`,
+      `photography`,
       `documentary`,
+      `high res`,
     ].join(", ")
 
     return {
       action,
       position,
+      light,
+      actionnables,
       prompt
     }
   }
-}
+}

src/app/agents/index.ts

@@ -1,11 +1,11 @@
 import { Agent, AgentType } from "./types"
 
-import { agent as ant } from "./ant"
-import { agent as fish } from "./fish"
-import { agent as fox } from "./fox"
+import { agent as pirates } from "./pirates"
+import { agent as city } from "./city"
+import { agent as dungeon } from "./dungeon"
 
-export const agents = { ant, fish, fox }
+export const agents = { pirates, city, dungeon }
 
-export const defaultAgent: AgentType = "fox"
+export const defaultAgent: AgentType = "pirates"
 
 export const getAgent = (type?: AgentType) => agents[type || defaultAgent] || agents[defaultAgent]

src/app/agents/{fish.ts → pirates.ts}

@@ -21,24 +21,51 @@ const positions = [
   "hiding in the coral"
 ]
 
+const lights = [
+  "during the day",
+]
+
+const actionnables = [
+  "chest",
+  // "door",
+  // "window",
+  // "sail",
+  // "capstan",
+  // "ship's wheel",
+  // "hat",
+  // "barrel",
+  // "cannon",
+  // "rope",
+  // "bucket",
+  "parrot",
+  // "wooden leg"
+]
+
 export const agent: Agent = {
-  title: "Fish",
-  type: "fish",
+  title: "Pirates",
+  type: "pirates",
   simulate: (): Scene => {
     const action = pick(actions)
     const position = pick(positions)
+    const light = pick(lights)
 
+    // this prompt is beautiful:
+    // screenshot from an adventure videogame, inside the hold of a pirate ship, with a pirate chest in the center, at sunset, beautiful, award winning, unreal engine, intricate details
     const prompt = [
-      `medium shot of a clownfish`,
-      action,
-      position,
-      `in front of yellow coral`,
-      `high res underwater footage`,
+      `screenshot from an adventure videogame`,
+      `inside the hold of a pirate ship`,
+      `with a pirate chest in the center`,
+      `a parrot`,
+      `and a wooden leg`,
+      `at sunset`,
+      `unreal engine`,
     ].join(", ")
 
     return {
       action,
       position,
+      light,
+      actionnables,
       prompt
     }
   }

src/app/agents/types.ts

@@ -1,8 +1,10 @@
-export type AgentType = 'ant' | 'fish' | 'fox'
+export type AgentType = 'pirates' | 'city' | 'dungeon'
 
 export interface Scene {
   action: string
   position: string
+  light: string
+  actionnables: string[]
   prompt: string
 }
 

src/app/layout.tsx

@@ -5,8 +5,8 @@ import { Inter } from 'next/font/google'
 const inter = Inter({ subsets: ['latin'] })
 
 export const metadata: Metadata = {
-  title: 'VideoQuest 🎞🛡',
-  description: 'VideoQuest',
+  title: 'Generative AI Adventure',
+  description: 'Generative AI Adventure',
 }
 
 export default function RootLayout({

src/app/main.tsx

@@ -2,7 +2,7 @@
 
 import { useEffect, useRef, useState, useTransition } from "react"
 
-import { VideoPlayer } from "@/components/business/video-player"
+import { ImageRenderer } from "@/components/business/image-renderer"
 
 import {
   Select,
@@ -15,20 +15,25 @@ import {
 import { render } from "./render"
 import { Agent, AgentType, Scene } from "./agents/types"
 import { agents, defaultAgent, getAgent } from "./agents"
+import { ImageSegment, RenderedScene } from "./types"
 
 export default function Main() {
-  const [url, setUrl] = useState<string>()
   const [isPending, startTransition] = useTransition()
-  const [agent, setAgent] = useState<Agent>()
   const [scene, setScene] = useState<Scene>()
+  const [rendered, setRendered] = useState<RenderedScene>({
+    assetUrl: "", 
+    error: "",
+    maskBase64: "",
+    segments:[]
+  })
   const ref = useRef<AgentType>(defaultAgent)
    
   useEffect(() => {
     
     const updateView = async () => {
-      console.log(`update view..`)
+      // console.log(`update view..`)
 
-      startTransition(async () => {
+      await startTransition(async () => {
 
         // console.log(`getting agent..`)
         const type = ref?.current
@@ -38,19 +43,27 @@ export default function Main() {
         const scene = agent.simulate()
 
         // console.log(`rendering scene..`)
-        const newUrl = await render(scene.prompt)
+        const newRendered = await render(
+          scene.prompt,
+          scene.actionnables.slice(0, 5) // too many can slow us down it seems
+        )
 
         if (type !== ref?.current) {
-          console.log("agent type changed while we were rendering")
+          console.log("agent type changed! reloading scene")
           setTimeout(() => { updateView() }, 0)
           return
         } 
 
-        // console.log(`newUrl: ${newUrl}`)
-        setUrl(newUrl)
-        setAgent(agent)
-        setScene(scene)
-        setTimeout(() => { updateView()}, 2000)
+
+        if (newRendered.assetUrl) {
+          setRendered(newRendered)
+          // console.log(`got a new url: ${newUrl}`)
+          setScene(scene)
+          setTimeout(() => { updateView()}, 1000)
+        } else {
+          // console.log(`going to wait a bit more: ${newUrl}`)
+          setTimeout(() => { updateView()}, newRendered.error ? 6000 : 3000)
+        }
       })
     }
 
@@ -62,12 +75,17 @@ export default function Main() {
     <div className="flex flex-col w-full pt-4">
       <div className="flex flex-col space-y-3 px-2">
         <div className="flex flex-row items-center space-x-3">
-          <label className="flex">Agent model:</label>
+          <label className="flex">Select a game:</label>
           <Select
             defaultValue={defaultAgent}
             onValueChange={(value) => {
               ref.current = value as AgentType
-              setUrl("")
+              setRendered({
+                assetUrl: "", 
+                error: "",
+                maskBase64: "",
+                segments:[]
+              })
             }}>
             <SelectTrigger className="w-[180px]">
               <SelectValue placeholder="Type" />
@@ -79,12 +97,21 @@ export default function Main() {
             </SelectContent>
           </Select>
         </div>
-        {(url && scene) ? <div>
+        <p>Note: changing the model might take up to 1 minute</p>
+          
+        {(scene) ? <div>
           <p>Action: {scene.action}</p>
           <p>Position: {scene.position}</p>
+          <p>Light: {scene.light}</p>
         </div> : null}
+        <div className="flex flex-col">
+        {rendered.segments.map((segment, i) => 
+          <div key={i}>
+            {segment.label} ({segment.score})
+          </div>)}
+        </div>
       </div>
-      <VideoPlayer url={url} />
+      <ImageRenderer rendered={rendered} />
     </div>
   )
 }

src/app/render.ts

@@ -1,10 +1,19 @@
 "use server"
 
+import { RenderedScene } from "./types"
+
 // note: there is no / at the end in the variable
 // so we have to add it ourselves if needed
 const apiUrl = process.env.RENDERING_ENGINE_API
 
-export async function render(prompt: string) {
+export async function render(prompt: string, actionnables: string[] = []) {
+  let defaulResult: RenderedScene = {
+    assetUrl: "",
+    maskBase64: "",
+    error: "",
+    segments: []
+  }
+
   try {
     console.log(`calling ${apiUrl}/render with prompt: ${prompt}`)
     const res = await fetch(`${apiUrl}/render`, {
@@ -14,12 +23,20 @@ export async function render(prompt: string) {
         "Content-Type": "application/json",
         // Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`,
       },
-      body: JSON.stringify({ prompt }),
+      body: JSON.stringify({
+        prompt,
+        // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
+        nbFrames: 1, // when nbFrames is 1, we will only generate static images
+        nbSteps: 20,
+        actionnables,
+        segmentation: "firstframe", // one day we will remove this param, to make it automatic
+      }),
        cache: 'no-store',
      // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
      // next: { revalidate: 1 }
     })
 
+    // console.log("res:", res)
     // The return value is *not* serialized
     // You can return Date, Map, Set, etc.
     
@@ -29,11 +46,11 @@ export async function render(prompt: string) {
       throw new Error('Failed to fetch data')
     }
     
-    const { url } = await res.json()
-
-    return url
+    const response = (await res.json()) as RenderedScene
+    // console.log("response:", response)
+    return response
   } catch (err) {
     console.error(err)
-    return ""
+    return defaulResult
   }
 }

src/app/types.ts

@@ -0,0 +1,40 @@
+
+export interface RenderRequest {
+  prompt: string
+
+  // whether to use video segmentation
+  // disabled (default)
+  // firstframe: we only analyze the first frame
+  // allframes: we analyze all the frames
+  segmentation: 'disabled' | 'firstframe' | 'allframes'
+
+  // segmentation will only be executed if we have a non-empty list of actionnables
+  // actionnables are names of things like "chest", "key", "tree", "chair" etc
+  actionnables: string[]
+
+  // note: this is the number of frames for Zeroscope,
+  // which is currently configured to only output 3 seconds, so:
+  // nbFrames=8 -> 1 sec
+  // nbFrames=16 -> 2 sec
+  // nbFrames=24 -> 3 sec
+  nbFrames: number // min: 1, max: 24
+
+  nbSteps: number // min: 1, max: 50
+
+  seed: number
+}
+
+export interface ImageSegment {
+  id: number
+  box: number[]
+  color: number[]
+  label: string
+  score: number 
+}
+
+export interface RenderedScene {
+  assetUrl: string 
+  error: string
+  maskBase64: string
+  segments: ImageSegment[]
+}

src/components/business/image-renderer.tsx

@@ -0,0 +1,88 @@
+import { useRef } from "react"
+
+import { ImageSegment, RenderedScene } from "@/app/types"
+
+export const ImageRenderer = ({
+  assetUrl = "",
+  maskBase64 = "",
+  segments = []
+}: RenderedScene) => {
+  const imgRef = useRef<HTMLImageElement | null>(null)
+  const canvas = document.createElement('canvas')
+  const context = canvas.getContext('2d')
+  
+  const getPixelColor = (x: number, y: number) => {
+    console.log("getting pixel color")
+    if (!context) {
+      throw new Error("Unable to get context from canvas")
+    }
+
+    const imgData = context.getImageData(x, y, 1, 1).data
+
+    return `[${imgData[0]},${imgData[1]},${imgData[2]},${imgData[3]/255}]`
+  }
+
+  const handleMouseDown = (event: React.MouseEvent) => {
+    const boundingRect = imgRef.current!.getBoundingClientRect()
+  
+    const x = event.clientX - boundingRect.left
+    const y = event.clientY - boundingRect.top
+  
+    if (maskBase64) {
+      const image = new Image()
+
+      image.onload = function () {
+        if (context) {
+          context.drawImage(image, 0, 0)
+
+          const clickedColor = getPixelColor(x, y) as any
+          
+          let closestSegment: ImageSegment | null = null
+
+          let minDistance = Infinity
+          
+          segments.forEach(segment => {
+            const segmentColor = segment.color.slice(0,3) // get the RGB part only
+            
+            const distance = Math.sqrt(
+              Math.pow(clickedColor[0] - segmentColor[0], 2) +
+              Math.pow(clickedColor[1] - segmentColor[1], 2) +
+              Math.pow(clickedColor[2] - segmentColor[2], 2)
+            )
+            
+            if(distance < minDistance) {
+              minDistance = distance;
+              closestSegment = segment;
+            }
+          })
+  
+          console.log("closestSegment:", closestSegment)
+          console.log(closestSegment) // Here is the closest matching segment
+        }
+      }
+
+      image.src = maskBase64
+      // image.src = "data:image/png;base64," + maskBase64;
+    } else {
+      console.log("No mask available, aborting..")
+    }
+  }
+
+
+  if (!assetUrl) {
+    return <div className="flex w-full h-screen items-center justify-center text-center">
+      <div>Rendering first frame.. (might take around 30s)</div>
+    </div>
+  }
+
+  return (
+    <div className="w-full py-8 px-2">
+      <img
+        src={assetUrl}
+        ref={imgRef}
+        className="w-full rounded-md overflow-hidden"
+        onMouseDown={handleMouseDown}
+      />
+    </div>
+  )
+}

src/components/business/{video-player.tsx → video-renderer.tsx}

@@ -1,10 +1,10 @@
 "use client"
 
-export const VideoPlayer = ({ url }: { url?: string }) => {
+export const VideoRenderer = ({ url }: { url?: string }) => {
 
   if (!url) {
     return <div className="flex w-full h-screen items-center justify-center text-center">
-      <div>Generating first frames.. (might take around 30s)</div>
+      <div>Rendering first frames.. (might take around 30s)</div>
     </div>
   }