things are taking shape

package-lock.json

@@ -8,6 +8,7 @@
       "name": "video-quest",
       "version": "0.0.0",
       "dependencies": {
+        "@huggingface/agents": "^0.0.4",
         "@huggingface/inference": "^2.6.1",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.3",
@@ -184,6 +185,17 @@
         "react-dom": ">=16.8.0"
       }
     },
+    "node_modules/@huggingface/agents": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/@huggingface/agents/-/agents-0.0.4.tgz",
+      "integrity": "sha512-jjsiMEba2dLLYUkMnX0g+bddM/9De0tk73RAiXL0uE117qdDOlUK/D+tI1krDfxpn9J+Qu9w7BqmufIMZFhV4Q==",
+      "dependencies": {
+        "@huggingface/inference": "^2.6.1"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@huggingface/inference": {
       "version": "2.6.1",
       "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.6.1.tgz",

package.json

@@ -9,6 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@huggingface/agents": "^0.0.4",
     "@huggingface/inference": "^2.6.1",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.3",

src/app/agents/index.ts

@@ -1,11 +0,0 @@
-import { Agent, AgentType } from "./types"
-
-import { agent as pirates } from "./pirates"
-import { agent as city } from "./city"
-import { agent as dungeon } from "./dungeon"
-
-export const agents = { pirates, city, dungeon }
-
-export const defaultAgent: AgentType = "pirates"
-
-export const getAgent = (type?: AgentType) => agents[type || defaultAgent] || agents[defaultAgent]

src/app/agents/server.ts

@@ -1,10 +0,0 @@
-"use server"
-
-import { HfInference } from "@huggingface/inference"
-
-const hfi = new HfInference(process.env.HF_API_TOKEN)
-const hf = hfi.endpoint(`${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`)
-
-export async function decideNextSteps(userAction: string) {
-  return ""
-}

src/app/agents/types.ts

@@ -1,15 +0,0 @@
-export type AgentType = 'pirates' | 'city' | 'dungeon'
-
-export interface Scene {
-  action: string
-  position: string
-  light: string
-  actionnables: string[]
-  prompt: string
-}
-
-export interface Agent {
-  title: string
-  type: AgentType
-  simulate: () => Scene
-}

src/app/{agents → games}/city.ts

@@ -1,5 +1,4 @@
-import { pick } from "./pick"
-import { Agent, Scene } from "./types"
+import { Game } from "./types"
 
 const actions = [
   "busy pedestrians",
@@ -25,40 +24,33 @@ const lights = [
   // "during the night",
 ]
 
-const actionnables = [
+const initialSituation = [
+  `over the city town center`,
+  `at noon`,
+].join(", ")
+
+const initialActionnables = [
   "building",
   "road",
   "car",
   "tower",
   "tree",
   "river",
-  "sea"
+  "sea",
+  "house"
 ]
 
-export const agent: Agent = {
+export const game: Game = {
   title: "City",
   type: "city",
-  simulate: (): Scene => {
-    const action = pick(actions)
-    const position = pick(positions)
-    const light = pick(lights)
-
-    const prompt = [
-      `static isometrical view of 3D rendered city`,
-      action,
-      position,
-      light,
-      `isometric`,
-      `game`,
-      `high res`,
-    ].join(", ")
-
-    return {
-      action,
-      position,
-      light,
-      actionnables,
-      prompt
-    }
-  }
-}
+  initialSituation,
+  initialActionnables,
+  getScenePrompt: (situation?: string) => [
+    `isometrical bird view of 3D rendered city`,
+    situation || initialSituation,
+    `game screenshot`,
+    `isometric`,
+    `unreal engine`,
+    `high res`,
+  ]
+}

src/app/{agents → games}/dungeon.ts

@@ -1,5 +1,4 @@
-import { pick } from "./pick"
-import { Agent, Scene } from "./types"
+import { Game, Scene } from "./types"
 
 const actions = [
   "not moving",
@@ -24,45 +23,37 @@ const positions = [
 
 const lights = [
   "lit through windows",
-  "lit through wall-mounted torchs"
+  "lit through wall-mounted torches"
   // "poorly lit"
 ]
 
-const actionnables = [
-  "floor",
-  "fireplace",
+const initialSituation = [
+  `inside a beautiful room with stone walls and wooden floor`,
+  `a fireplace on the wall and a metal chest in the center with a large lock`,
+].join(", ")
+
+const initialActionnables = [
+ //  "floor",
+  // "fireplace",
   "door",
-  "window",
+ // "window",
   "chair",
+  "chest",
+  "key",
   "table",
-  "torch"
+  // torch"
 ]
 
-export const agent: Agent = {
+export const game: Game = {
   title: "Dungeon",
   type: "dungeon",
-  simulate: (): Scene => {
-    const action = pick(actions)
-    const position = pick(positions)
-    const light = pick(lights)
-
-    const prompt = [
-      `first-person footage`,
-      action,
-      position,
-      light,
-      `medieval`,
-      `photography`,
-      `documentary`,
-      `high res`,
-    ].join(", ")
-
-    return {
-      action,
-      position,
-      light,
-      actionnables,
-      prompt
-    }
-  }
-}
+  initialSituation,
+  initialActionnables,
+  getScenePrompt: (situation?: string) => [
+    `screenshot from an adventure videogame`,
+        // `first-person footage`,
+    situation || initialSituation,
+    `medieval`,
+    `unreal engine`,
+  ].join(", ")
+}

src/app/games/index.ts

@@ -0,0 +1,11 @@
+import { GameType } from "./types"
+
+import { game as pirates } from "./pirates"
+import { game as city } from "./city"
+import { game as dungeon } from "./dungeon"
+
+export const games = { pirates, city, dungeon }
+
+export const defaultGame: GameType = "pirates"
+
+export const getGame = (type?: GameType) => games[type || defaultGame] || games[defaultGame]

src/app/{agents → games}/pirates.ts

@@ -1,5 +1,4 @@
-import { pick } from "./pick"
-import { Agent, Scene } from "./types"
+import { Game } from "./types"
 
 const actions = [
   "idling",
@@ -25,8 +24,9 @@ const lights = [
   "during the day",
 ]
 
-const actionnables = [
+const initialActionnables = [
   "chest",
+  "box",
   // "door",
   // "window",
   // "sail",
@@ -34,41 +34,33 @@ const actionnables = [
   // "ship's wheel",
   // "hat",
   // "barrel",
-  "cannon",
+  // "cannon",
   // "rope",
   // "bucket",
-  "skull",
+ // "skull",
   "parrot",
   "lock",
-  "ship",
+  // "ship",
   // "wooden leg"
 ]
 
-export const agent: Agent = {
+const initialSituation = [
+  `inside the hold of a pirate ship`,
+  `a pirate chest in the center with a large lock`,
+  `a parrot on top of it`,
+  `at sunset`,
+].join(", ")
+
+export const game: Game = {
   title: "Pirates",
   type: "pirates",
-  simulate: (): Scene => {
-    const action = pick(actions)
-    const position = pick(positions)
-    const light = pick(lights)
-
+  initialSituation,
+  initialActionnables,
+  getScenePrompt: (situation?: string) => [
     // this prompt is beautiful:
     // screenshot from an adventure videogame, inside the hold of a pirate ship, with a pirate chest in the center, at sunset, beautiful, award winning, unreal engine, intricate details
-    const prompt = [
       `screenshot from an adventure videogame`,
-      `inside the hold of a pirate ship`,
-      `a pirate chest in the center with a large lock`,
-      `a parrot on top of it`,
-      `at sunset`,
+      situation || initialSituation,
       `unreal engine`,
-    ].join(", ")
-
-    return {
-      action,
-      position,
-      light,
-      actionnables,
-      prompt
-    }
-  }
+    ],
 }

src/app/games/types.ts

@@ -0,0 +1,14 @@
+export type GameType = 'pirates' | 'city' | 'dungeon'
+
+export interface Scene {
+  actionnables: string[]
+  prompt: string
+}
+
+export interface Game {
+  title: string
+  type: GameType
+  initialSituation: string
+  initialActionnables: string[]
+  getScenePrompt: (situation?: string) => string | string[]
+}

src/app/main.tsx

@@ -13,40 +13,51 @@ import {
 } from "@/components/ui/select"
 
 import { render } from "./render"
-import { AgentType, Scene } from "./agents/types"
-import { agents, defaultAgent, getAgent } from "./agents"
+
 import { RenderedScene } from "./types"
+import { predict } from "./predict"
+import { GameType } from "./games/types"
+import { defaultGame, games, getGame } from "./games"
 
 export default function Main() {
   const [isPending, startTransition] = useTransition()
-  const [scene, setScene] = useState<Scene>()
   const [rendered, setRendered] = useState<RenderedScene>({
     assetUrl: "", 
     error: "",
     maskBase64: "",
     segments:[]
   })
-  const ref = useRef<AgentType>(defaultAgent)
-   
-
-  const loadNextScene = async () => {
+  const ref = useRef<GameType>(defaultGame)
+  const [situation, setSituation] = useState("")
+  const [scene, setScene] = useState("")
+  const [dialogue, setDialogue] = useState("")
+  const [hoveredActionnable, setHoveredActionnable] = useState("")
+  const [isLoading, setLoading] = useState(true)
+
+  const loadNextScene = async (nextSituation?: string, nextActionnables?: string[]) => {
     // console.log(`update view..`)
+    setLoading(true)
 
     await startTransition(async () => {
 
       // console.log(`getting agent..`)
+      // note: we use a ref so that it can be changed in the background
       const type = ref?.current
-      const agent = getAgent(type)
-
-      // console.log(`asking agent to determine things..`)
-      const scene = agent.simulate()
+      const game = getGame(type)
 
       // console.log(`rendering scene..`)
       const newRendered = await render(
-        scene.prompt,
-        scene.actionnables.slice(0, 5) // too many can slow us down it seems
+        // SCENE PROMPT
+        [...game.getScenePrompt(nextSituation)].join(", "),
+
+        // ACTIONNABLES
+        (Array.isArray(nextActionnables) && nextActionnables.length
+          ? nextActionnables
+          : game.initialActionnables
+        ).slice(0, 6) // too many can slow us down it seems
       )
 
+      // detect if something changed in the background
       if (type !== ref?.current) {
         console.log("agent type changed! reloading scene")
         setTimeout(() => { loadNextScene() }, 0)
@@ -54,9 +65,11 @@ export default function Main() {
       } 
 
       if (newRendered.assetUrl) {
-        setRendered(newRendered)
-        // console.log(`got a new url: ${newUrl}`)
+        // console.log(`got a new url: ${newRendered.assetUrl}`)
         setScene(scene)
+
+        setRendered(newRendered)
+        setLoading(false)
       }
     })
   }
@@ -65,15 +78,108 @@ export default function Main() {
     loadNextScene()
   }, [])
 
-  const handleUserAction = (action: string) => {
-    console.log("user action:", action)
-
+  const handleUserAction = async (actionnable: string) => {
+    console.log("user actionnable:", actionnable)
+ 
     // TODO: ask Llama2 what to do about it
     // we need a frame and some actionnables,
     // perhaps even some music or sound effects
 
-    console.log("we don't know what to do, so we just load the next frame!")
-    loadNextScene()
+    await startTransition(async () => {
+ 
+      setLoading(true)
+
+      const game = getGame(ref.current)
+      const initialPrompt = [...game.getScenePrompt()].join(", ")
+
+      const currentPrompt = situation
+        ? [...game.getScenePrompt(situation)].join(", ")
+        : initialPrompt
+
+      try {
+        const basePrompt = [
+          `QUESTION: You are the AI game master of a role video game.`,
+          initialPrompt !== currentPrompt ? `The initial scene of the game was this: "${initialPrompt}".` : '',
+          `The player is currently in this scene: "${currentPrompt}".`,
+          `The player has just clicked on "${actionnable}".`
+        ]
+
+        console.log("ask the LLM to invent next steps..")
+
+        const rawSituation = await predict([
+          ...basePrompt,
+          `Please describe the new scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc. You must include important objects, that the user can click on (eg. characters, doors, vehicles, useful objects).\nANSWER:`
+        ].join(" "))
+
+        console.log(`rawSituation: `, rawSituation)
+
+        if (!rawSituation) {
+          throw new Error("failed to generate the situation")
+        }
+        const newSituation = `${rawSituation.split("QUESTION:")[0] || ""}`
+        if (!newSituation) {
+          throw new Error("failed to parse the situation")
+        }
+
+        console.log(`newSituation: `, newSituation)
+
+        const rawActionnables = await predict([
+          ...basePrompt,
+          `Here are the 4 most important objects visible in this scene, that the user can click on. The list is in JSON (list of strings). You must list basic name of things (eg. "parrot", "chest", "spaceship", "glass", "door", "person", "window", "light", "knob", "button" etc..) \nJSON = [`
+        ].join(" "))
+        console.log(`rawActionnables: `, rawActionnables)
+
+ 
+        if (!rawActionnables) {
+          throw new Error("failed to generate the actionnables")
+        }
+
+        let newActionnables = []
+        try {
+          newActionnables = (JSON.parse(
+            `[${rawActionnables.split("]")[0] || ""}]`
+          ) as string[]).map(item =>
+            // clean the words to remove any punctuation
+            item.replace(/\W/g, '').trim()
+          )
+
+          if (!newActionnables.length) {
+            throw new Error("no actionnables")
+          }
+        } catch (err) {
+          throw new Error("failed to parse the actionnables")
+        }
+ 
+        console.log(`newActionnables: `, newActionnables)
+
+ 
+        const rawDialogue = await predict([
+          ...basePrompt,
+          `As a game master, what should you say next? (Only reply with 2 sentences, please).\nANSWER:`
+        ].join(" "))
+        console.log(`rawDialogue: `, rawDialogue)
+
+        if (!rawDialogue) {
+          throw new Error("failed to generate the dialogue")
+        }
+        const newDialogue = `${rawDialogue.split("QUESTION:")[0] || ""}`
+        if (!newDialogue) {
+            throw new Error("failed to parse the dialogue")
+        }
+        console.log(`newDialogue: `, newDialogue)
+
+
+        setDialogue(newDialogue)
+        setSituation(newSituation)
+
+        console.log("loading next scene..")
+        await loadNextScene(newSituation, newActionnables)
+
+        // todo we could also use useEffect
+      } catch (err) {
+       console.error(err)
+      }
+    })
   }
 
   return (
@@ -82,9 +188,9 @@ export default function Main() {
         <div className="flex flex-row items-center space-x-3">
           <label className="flex">Select a story:</label>
           <Select
-            defaultValue={defaultAgent}
+            defaultValue={defaultGame}
             onValueChange={(value) => {
-              ref.current = value as AgentType
+              ref.current = value as GameType
               setRendered({
                 assetUrl: "", 
                 error: "",
@@ -96,27 +202,30 @@ export default function Main() {
               <SelectValue placeholder="Type" />
             </SelectTrigger>
             <SelectContent>
-              {Object.entries(agents).map(([key, agent]) =>
-              <SelectItem key={key} value={key}>{agent.title}</SelectItem>
+              {Object.entries(games).map(([key, game]) =>
+              <SelectItem key={key} value={key}>{game.title}</SelectItem>
               )}
             </SelectContent>
           </Select>
         </div>
-        <p>Note: it takes about 1 minute to generate a new game panel</p>
-          
-        {(scene) ? <div>
-          <p>Action: {scene.action}</p>
-          <p>Position: {scene.position}</p>
-          <p>Light: {scene.light}</p>
-        </div> : null}
-        <div className="flex flex-col">
-        {rendered.segments.map((segment, i) => 
-          <div key={i}>
-            {segment.label} ({segment.score})
+        <p className="text-xl">The server is blowing up! Loading a panel may take a few minutes.</p>
+        <p className="text-xl">{dialogue}</p>
+        <div className="flex flex-row">
+          <div className="text-xl mr-2">🔎 Possible items:</div>
+          {rendered.segments.map((segment, i) => 
+          <div key={i} className="flex flex-row text-xl mr-2">
+            <div className="">{segment.label}</div>
+            {i < (rendered.segments.length - 1) ? <div>,</div> : null}
           </div>)}
         </div>
+        <p className="text-xl font-normal">You may be looking at.. <span className="font-bold">{hoveredActionnable || "nothing"}</span></p>
       </div>
-      <ImageRenderer rendered={rendered} onUserAction={handleUserAction} />
+      <ImageRenderer
+        rendered={rendered}
+        onUserAction={handleUserAction}
+        onUserHover={setHoveredActionnable}
+        isLoading={isLoading}
+      />
     </div>
   )
 }

src/app/predict.ts

@@ -0,0 +1,46 @@
+"use server"
+
+import { HfInference } from "@huggingface/inference"
+
+const hfi = new HfInference(process.env.HF_API_TOKEN)
+const hf = hfi.endpoint(`${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`)
+
+export async function predict(inputs: string) {
+
+  let instructions = ""
+  try {
+    for await (const output of hf.textGenerationStream({
+      inputs,
+      parameters: {
+        do_sample: true,
+
+        // hard limit for max_new_tokens is 1512
+        max_new_tokens: 200, // 1150,
+        return_full_text: false,
+      }
+    })) {
+      instructions += output.token.text
+      process.stdout.write(output.token.text)
+      if (
+        instructions.includes("</s>") || 
+        instructions.includes("<s>") ||
+        instructions.includes("<|end|>") ||
+        instructions.includes("<|assistant|>")
+      ) {
+        break
+      }
+    }
+  } catch (err) {
+    console.error(`error during generation: ${err}`)
+  }
+
+  // need to do some cleanup of the garbage the LLM might have gave us
+  return (
+    instructions
+    .replaceAll("<|end|>", "")
+    .replaceAll("<s>", "")
+    .replaceAll("</s>", "")
+    .replaceAll("<|assistant|>", "")
+    .replaceAll('""', '"')
+  )
+}

src/components/business/image-renderer.tsx

@@ -9,9 +9,13 @@ export const ImageRenderer = ({
     segments = []
   },
   onUserAction,
+  onUserHover,
+  isLoading = false,
 }: {
   rendered: RenderedScene
-  onUserAction: (action: string) => void
+  onUserAction: (actionnable: string) => void
+  onUserHover: (actionnable: string) => void
+  isLoading?: boolean
 }) => {
   const imgRef = useRef<HTMLImageElement | null>(null)
   const canvasRef = useRef<HTMLCanvasElement | null>(null)
@@ -20,7 +24,7 @@ export const ImageRenderer = ({
 
   useEffect(() => {
     if (maskBase64) {
-      console.log("maskBase64:", maskBase64)
+      // console.log("maskBase64:", maskBase64)
       const img = new Image();
       img.onload = function () {
         canvasRef.current = document.createElement('canvas');
@@ -31,7 +35,7 @@ export const ImageRenderer = ({
       }
       img.src = "data:image/png;base64," + maskBase64;
     } else {
-      console.log("error, no maskBase64 detected!")
+      // console.log("error, no maskBase64 detected!")
     }
   }, [maskBase64]);
 
@@ -82,15 +86,21 @@ export const ImageRenderer = ({
       if(distance < minDistance) {
         minDistance = distance;
         closestSegment = segment;
+        console.log(`${distance} -> ${segment.label}: score = ${segment.score}`)
       }
     });
 
     return closestSegment;
   }
 
-  const handleMouseEvent = (event: React.MouseEvent, isClickEvent: boolean = false) => {
+  const handleMouseEvent = async (event: React.MouseEvent, isClickEvent: boolean = false) => {
     if (!contextRef.current) return; // Return early if mask image has not been loaded yet
   
+    if (isLoading) {
+      // we ignore all user interactions
+      return false
+    }
+
     const boundingRect = imgRef.current!.getBoundingClientRect();
     const x = event.clientX - boundingRect.left;
     const y = event.clientY - boundingRect.top;
@@ -98,22 +108,28 @@ export const ImageRenderer = ({
     const newSegment = getSegmentAt(x, y)
 
     if (actionnable !== newSegment.label) {
+      if (newSegment.label) {
+        console.log(`User is hovering "${newSegment.label}"`);
+      } else {
+        console.log(`Nothing in the area`);
+      }
       setActionnable(newSegment.label)
     }
 
-    if (!newSegment.label) { return }
-
-    console.log("actionnable: ", actionnable)
-
     if (isClickEvent) {
-      console.log("User clicked on " + actionnable);
-      // onUserAction(actionnable);
+      if (!newSegment.label) {
+        return
+      }
+      console.log("User clicked on " + newSegment.label)
+      onUserAction(actionnable)
+    } else {
+      onUserHover(actionnable)
     }
   };
 
   if (!assetUrl) {
     return <div className="flex w-full h-screen items-center justify-center text-center">
-      <div>Rendering first frame.. (might take around 30s)</div>
+      <div>Generating a new panel..</div>
     </div>
   }
 
@@ -150,17 +166,22 @@ export const ImageRenderer = ({
         */
 
   return (
-    <div className="w-full py-8 px-2">
+    <div className={[
+      "w-full py-8 px-2",
+      isLoading ? "animate-pulse" : ""
+    ].join(" ")
+    }>
       <div className="relative w-full">
         <img
-          src={"data:image/png;base64," + maskBase64}
+          src={assetUrl}
+          // src={"data:image/png;base64," + maskBase64}
           ref={imgRef}
           width="1024px"
           height="512px"
           className={
             [
-              "absolute top-0 left-0 opacity-30",
-              actionnable ? "cursor-pointer" : ""
+              "absolute top-0 left-0",
+              actionnable && !isLoading ? "cursor-pointer" : ""
             ].join(" ")
           }
           onMouseDown={(event) => handleMouseEvent(event, true)}