fix for video segmentation

src/app/games/enchanters.ts

@@ -14,6 +14,8 @@ const initialActionnables = [
   "lake",
   "roof",
   "boat",
+  "mountain",
+  "sky"
 ]
 
 export const game: Game = {

src/app/games/flamenco.ts

@@ -0,0 +1,41 @@
+import { macondo } from "@/lib/fonts"
+import { Game } from "./types"
+
+const initialSituation = [
+  `beautiful view of an art deco building in new york`,
+  `looking up`,
+  `entrance desk`,
+  `pigeon character`,
+  `day of the dead makeup`,
+  `artdeco bridge`,
+].join(", ")
+
+const initialActionnables = [
+  "pigeon",
+  "face",
+  "person",
+  "building",
+  "light",
+  "decoration",
+  "box",
+  "desk",
+  "gate",
+  "door"
+]
+
+export const game: Game = {
+  title: "Sad Flamenco",
+  type: "flamenco",
+  engine: "spherical_image",
+  className: macondo.className,
+  initialSituation,
+  initialActionnables,
+  getScenePrompt: (situation?: string) => [
+    `photo of an artdeco scene`,
+    `grimfandango screenshot`,
+    `unreal engine`,
+    `1920 mexico`,
+    situation || initialSituation,
+  ]
+}
+

src/app/games/index.ts

@@ -6,9 +6,11 @@ import { game as dungeon } from "./dungeon"
 import { game as doom } from "./doom"
 import { game as vernian } from "./vernian"
 import { game as enchanters } from "./enchanters"
+import { game as flamenco } from "./flamenco"
+import { game as pharaoh } from "./pharaoh"
 
-export const games = { pirates, city, dungeon, doom, vernian, enchanters}
+export const games = { pirates, city, dungeon, doom, vernian, enchanters, flamenco, pharaoh}
 
-export const defaultGame: GameType = "enchanters"
+export const defaultGame: GameType = "flamenco"
 
 export const getGame = (type?: GameType) => games[type || defaultGame] || games[defaultGame]

src/app/games/pharaoh.ts

@@ -0,0 +1,35 @@
+import { macondo } from "@/lib/fonts"
+import { Game } from "./types"
+
+const initialSituation = [
+  `looking at a beautiful pyramid, ancient egypt, during golden hour, surrounded by sand dunes, near the Nile`,
+].join(", ")
+
+const initialActionnables = [
+  "pyramid",
+  "person",
+  "rocks",
+  "dune",
+  "sceptre",
+  "tree",
+  "river",
+  "boat",
+  "crocodile"
+]
+
+export const game: Game = {
+  title: "Pharaoh",
+  type: "pharaoh",
+  engine: "spherical_image",
+  className: macondo.className,
+  initialSituation,
+  initialActionnables,
+  getScenePrompt: (situation?: string) => [
+    `Screenshot from a videogame`,
+    `unreal engine`,
+    `ancient egypt`, 
+    `first person`,
+    situation || initialSituation,
+  ]
+}
+

src/app/games/types.ts

@@ -1,6 +1,6 @@
 import { EngineType } from "../engines"
 
-export type GameType = "pirates" | "city" | "dungeon" | "doom" | "vernian" | "enchanters"
+export type GameType = "pirates" | "city" | "dungeon" | "doom" | "vernian" | "enchanters" | "pharaoh" | "flamenco"
 
 export interface Scene {
   actionnables: string[]

src/app/main.tsx

@@ -35,7 +35,7 @@ const getInitialRenderedScene = (): RenderedScene => ({
 export default function Main() {
   const [isPending, startTransition] = useTransition()
   const [rendered, setRendered] = useState<RenderedScene>(getInitialRenderedScene())
-  const renderedRef = useRef<RenderedScene>()
+  const historyRef = useRef<RenderedScene[]>([])
   const router = useRouter()
   const pathname = usePathname()
   const searchParams = useSearchParams()
@@ -50,7 +50,7 @@ export default function Main() {
   const debug = (searchParams.get('debug') === "true")
 
   const [situation, setSituation] = useState("")
-  const [scene, setScene] = useState("")
+
   const [dialogue, setDialogue] = useState("")
   const [hoveredActionnable, setHoveredActionnable] = useState("")
 
@@ -59,14 +59,13 @@ export default function Main() {
 
   const loopRef = useRef<any>(null)
 
-
   const loadNextScene = async (nextSituation?: string, nextActionnables?: string[]) => {
     
     await startTransition(async () => {
       console.log("Rendering a scene for " + game.type)
 
       // console.log(`rendering scene..`)
-      renderedRef.current = await newRender({
+      const newRendered = await newRender({
         engine,
 
         // SCENE PROMPT
@@ -79,7 +78,7 @@ export default function Main() {
         ).slice(0, 10) // too many can slow us down it seems
       })
 
-      console.log("got the first version of our scene!", renderedRef.current)
+      console.log("got the first version of our scene!", newRendered)
 
       // detect if type game type changed while we were busy
       if (game?.type !== gameRef?.current) {
@@ -87,16 +86,17 @@ export default function Main() {
         return
       }
 
-      setScene(scene)
-
-      setRendered(renderedRef.current)
+      historyRef.current.unshift(newRendered)
+      setRendered(newRendered)
     })
   }
 
   const checkRenderedLoop = async () => {
     // console.log("checkRenderedLoop! rendered:", renderedRef.current)
     clearTimeout(loopRef.current)
-    if (!renderedRef.current?.renderId || renderedRef.current?.status !== "pending") {
+  
+    if (!historyRef.current[0]?.renderId || historyRef.current[0]?.status !== "pending") {
+
       // console.log("let's try again in a moments")
       loopRef.current = setTimeout(() => checkRenderedLoop(), 200)
       return
@@ -106,13 +106,13 @@ export default function Main() {
     await startTransition(async () => {
       // console.log(`getting latest updated scene..`)
       try {
-        if (!renderedRef.current?.renderId) {
+        if (!historyRef.current[0]?.renderId) {
           throw new Error(`missing renderId`)
         }
 
       
         // console.log(`calling getRender(${renderedRef.current.renderId})`)
-        const newRendered = await getRender(renderedRef.current.renderId)
+        const newRendered = await getRender(historyRef.current[0]?.renderId)
         // console.log(`got latest updated scene:`, renderedRef.current)
 
         // detect if type game type changed while we were busy
@@ -122,13 +122,13 @@ export default function Main() {
         }
 
     
-        const before = JSON.stringify(renderedRef.current)
+        const before = JSON.stringify(historyRef.current[0])
         const after = JSON.stringify(newRendered)
 
         if (after !== before) {
           console.log("updating scene..")
-          renderedRef.current = newRendered
-          setRendered(renderedRef.current)
+          historyRef.current[0] = newRendered
+          setRendered(historyRef.current[0])
 
           if (newRendered.status === "completed") {
             setBusy(busyRef.current = false)
@@ -283,7 +283,7 @@ export default function Main() {
       </div>
 
       <div className={[
-        "flex flex-col w-full pt-4 space-y-3 px-2 text-gray-50 dark:text-gray-50",
+        "flex flex-col w-full pt-4 space-y-3 text-gray-50 dark:text-gray-50",
         getGame(gameRef.current).className // apply the game theme
       ].join(" ")}>
         <div className="flex flex-row">
@@ -299,7 +299,7 @@ export default function Main() {
             {i < (clickables.length - 1) ? <div>,</div> : null}
           </div>)}
         </div>
-        <div className="text-xl p-3 rounded-xl backdrop-blur-sm bg-white/30">
+        <div className="text-xl p-4 rounded-xl backdrop-blur-sm bg-white/30">
           You are looking at: <span className="font-bold">{hoveredActionnable || "nothing"}</span>
         </div>
         <Renderer
@@ -311,7 +311,7 @@ export default function Main() {
           engine={engine}
           debug={debug}
         />
-        <div className="text-xl rounded-xl backdrop-blur-sm bg-white/30">{dialogue}</div>
+        <div className="text-xl rounded-xl backdrop-blur-sm bg-white/30 p-4">{dialogue}</div>
       </div>
     </div>
   )

src/app/queries/getActionnables.ts

@@ -77,8 +77,22 @@ export const getActionnables = async ({
         throw new Error("no actionnables")
       }
     } catch (err) {
-      console.log("failed to repair and recover a valid JSON!")
-      throw new Error("failed to parse the actionnables")
+      console.log("failed to repair and recover a valid JSON! Using a generic fallback..")
+
+      return [
+        "door",
+        "rock",
+        "window",
+        "table",
+        "ground",
+        "sky",
+        "box",
+        "tree",
+        "wall",
+        "floor"
+      ]
+      
+      // throw new Error("failed to parse the actionnables")
     }
   }
 

src/app/queries/getDialogue.ts

@@ -41,6 +41,7 @@ Here is the original scene in which the user was located at first, which will in
         basePrompt,
         `You must imagine a funny response to speak in reaction to what the player did, like in some old point and click video games.`,
         `Please limit yourself to only a 1 or 2 sentences, please.`,
+        `Also please don't say things like "Well, well, well", it is annoying.`
       ].filter(item => item).join("\n")
     },
     {

src/app/render.ts

@@ -54,7 +54,7 @@ export async function newRender({
         prompt,
         // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
         nbFrames, // when nbFrames is 1, we will only generate static images
-        nbSteps: isForVideo ? 20 : 30, // 20 = fast, 30 = better, 50 = best
+        nbSteps: isForVideo ? 25 : 35, // 20 = fast, 30 = better, 50 = best
         actionnables,
         segmentation: "firstframe", // one day we will remove this param, to make it automatic
         width: isForVideo ? 576 : 1024,
@@ -72,7 +72,7 @@ export async function newRender({
           prompt,
           // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation
           nbFrames, // when nbFrames is 1, we will only generate static images
-          nbSteps: isForVideo ? 12 : 30, // 20 = fast, 30 = better, 50 = best
+          nbSteps: isForVideo ? 25 : 35, // 20 = fast, 30 = better, 50 = best
           actionnables,
           segmentation: "firstframe", // one day we will remove this param, to make it automatic
           width: isForVideo ? 576 : 1024,

src/components/business/cartesian-video.tsx

@@ -44,11 +44,15 @@ export const CartesianVideo = forwardRef(({
       muted
       autoPlay
       loop
+      width="1024px"
+      height="512px"
       className="absolute"
     />
     {debug && <img
       src={rendered.maskUrl || undefined}
       className="absolute opacity-50"
+      width="1024px"
+      height="512px"
     />}
   </div>
   )