add "movie director assistant" LLM step

TODO.md

@@ -0,0 +1,6 @@
+
+
+to  allow multiple videos to be processed a the same time:
+
+[ ] yield from the loop at each step
+[ ] random processing of videos

package-lock.json

@@ -14,15 +14,19 @@
         "@types/express": "^4.17.17",
         "@types/ffmpeg-concat": "^1.1.2",
         "@types/uuid": "^9.0.2",
+        "eventsource-parser": "^1.0.0",
         "express": "^4.18.2",
         "ffmpeg-concat": "^1.3.0",
         "fluent-ffmpeg": "^2.1.2",
         "fs-extra": "^11.1.1",
+        "gpt-tokens": "^1.1.1",
         "node-fetch": "^3.3.1",
+        "openai": "^3.3.0",
         "puppeteer": "^20.8.0",
         "temp-dir": "^3.0.0",
         "ts-node": "^10.9.1",
-        "uuid": "^9.0.0"
+        "uuid": "^9.0.0",
+        "yaml": "^2.3.1"
       }
     },
     "node_modules/@babel/code-frame": {
@@ -686,6 +690,14 @@
       "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz",
       "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg=="
     },
+    "node_modules/axios": {
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dependencies": {
+        "follow-redirects": "^1.14.8"
+      }
+    },
     "node_modules/b4a": {
       "version": "1.6.4",
       "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz",
@@ -1222,6 +1234,11 @@
         "ms": "2.0.0"
       }
     },
+    "node_modules/decimal.js": {
+      "version": "10.4.3",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
+      "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA=="
+    },
     "node_modules/decompress-response": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
@@ -1479,6 +1496,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/eventsource-parser": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.0.0.tgz",
+      "integrity": "sha512-9jgfSCa3dmEme2ES3mPByGXfgZ87VbP97tng1G2nWwWx6bV2nYxm2AWCrbQjXToSe+yYlqaZNtxffR9IeQr95g==",
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
     "node_modules/execa": {
       "version": "0.10.0",
       "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz",
@@ -1772,6 +1797,25 @@
         "node": ">=0.8.0"
       }
     },
+    "node_modules/follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/foreground-child": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
@@ -2218,6 +2262,16 @@
         "through2": "^0.6.3"
       }
     },
+    "node_modules/gpt-tokens": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/gpt-tokens/-/gpt-tokens-1.1.1.tgz",
+      "integrity": "sha512-fB1u0ZH7PywF9FByfWCqn6Hpp3so/pFUmk3AiV4QlOskr57LK8Ds3YJOjdemWKRGJQ+2pT9ikt++Eb+/et9gTQ==",
+      "dependencies": {
+        "decimal.js": "^10.4.3",
+        "js-tiktoken": "^1.0.7",
+        "openai": "^3.3.0"
+      }
+    },
     "node_modules/graceful-fs": {
       "version": "4.2.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -2609,6 +2663,14 @@
       "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz",
       "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg=="
     },
+    "node_modules/js-tiktoken": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.7.tgz",
+      "integrity": "sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==",
+      "dependencies": {
+        "base64-js": "^1.5.1"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -3349,6 +3411,28 @@
         "wrappy": "1"
       }
     },
+    "node_modules/openai": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz",
+      "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==",
+      "dependencies": {
+        "axios": "^0.26.0",
+        "form-data": "^4.0.0"
+      }
+    },
+    "node_modules/openai/node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/optionator": {
       "version": "0.8.3",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
@@ -5059,6 +5143,14 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
     },
+    "node_modules/yaml": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz",
+      "integrity": "sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
     "node_modules/yargs": {
       "version": "17.7.1",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz",

package.json

@@ -21,14 +21,18 @@
     "@types/express": "^4.17.17",
     "@types/ffmpeg-concat": "^1.1.2",
     "@types/uuid": "^9.0.2",
+    "eventsource-parser": "^1.0.0",
     "express": "^4.18.2",
     "ffmpeg-concat": "^1.3.0",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^11.1.1",
+    "gpt-tokens": "^1.1.1",
     "node-fetch": "^3.3.1",
+    "openai": "^3.3.0",
     "puppeteer": "^20.8.0",
     "temp-dir": "^3.0.0",
     "ts-node": "^10.9.1",
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "yaml": "^2.3.1"
   }
 }

src/config.mts

@@ -2,9 +2,9 @@ import path from "node:path"
 
 export const storagePath = `${process.env.VC_STORAGE_PATH || './sandbox'}`
 
-export const tasksDirPath = path.join(storagePath, "tasks")
-export const pendingTasksDirFilePath = path.join(tasksDirPath, "pending")
-export const completedTasksDirFilePath =  path.join(tasksDirPath, "completed")
+export const metadataDirPath = path.join(storagePath, "metadata")
+export const pendingMetadataDirFilePath = path.join(metadataDirPath, "pending")
+export const completedMetadataDirFilePath =  path.join(metadataDirPath, "completed")
 
 export const filesDirPath = path.join(storagePath, "files")
 export const pendingFilesDirFilePath = path.join(filesDirPath, "pending")

src/data/all_words.json

@@ -4269,7 +4269,7 @@
     "tank",
     "tap",
     "target",
-    "task",
+    "video",
     "taste",
     "tax",
     "tea",

src/data/good_words.json

@@ -4259,7 +4259,7 @@
     "tank",
     "tap",
     "target",
-    "task",
+    "video",
     "taste",
     "tax",
     "tea",

src/index.mts

@@ -4,17 +4,21 @@ import path from "node:path"
 import { validate as uuidValidate } from "uuid"
 import express from "express"
 
-import { VideoTask, VideoTaskRequest } from "./types.mts"
+import { Video, VideoStatus, VideoAPIRequest } from "./types.mts"
 import { parseVideoRequest } from "./utils/parseVideoRequest.mts"
-import { savePendingTask } from "./scheduler/savePendingTask.mts"
-import { getTask } from "./scheduler/getTask.mts"
+import { savePendingVideo } from "./scheduler/savePendingVideo.mts"
+import { getVideo } from "./scheduler/getVideo.mts"
 import { main } from "./main.mts"
 import { completedFilesDirFilePath } from "./config.mts"
-import { deleteTask } from "./scheduler/deleteTask.mts"
-import { getPendingTasks } from "./scheduler/getPendingTasks.mts"
+import { markVideoAsToDelete } from "./scheduler/markVideoAsToDelete.mts"
+import { markVideoAsToAbort } from "./scheduler/markVideoAsToAbort.mts"
+import { markVideoAsToPause } from "./scheduler/markVideoAsToPause.mts"
+import { markVideoAsPending } from "./scheduler/markVideoAsPending.mts"
+import { getPendingVideos } from "./scheduler/getPendingVideos.mts"
 import { hasValidAuthorization } from "./utils/hasValidAuthorization.mts"
-import { getAllTasksForOwner } from "./scheduler/getAllTasksForOwner.mts"
+import { getAllVideosForOwner } from "./scheduler/getAllVideosForOwner.mts"
 import { initFolders } from "./initFolders.mts"
+import { sortVideosByYoungestFirst } from "./utils/sortVideosByYoungestFirst.mts"
 
 initFolders()
 // to disable all processing (eg. to debug)
@@ -26,8 +30,8 @@ const port = 7860
 
 app.use(express.json())
 
-app.post("/", async (req, res) => {
-  const request = req.body as VideoTaskRequest
+app.post("/:ownerId", async (req, res) => {
+  const request = req.body as VideoAPIRequest
 
   if (!hasValidAuthorization(req.headers)) {
     console.log("Invalid authorization")
@@ -37,68 +41,45 @@ app.post("/", async (req, res) => {
     return
   }
   
-  let task: VideoTask = null
+  const ownerId = req.params.ownerId
 
-  console.log(`creating task from request..`)
-  console.log(`request: `, JSON.stringify(request))
-  try {
-    task = await parseVideoRequest(request)
-  } catch (err) {
-    console.error(`failed to create task: ${task} (${err})`)
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
     res.status(400)
-    res.write(JSON.stringify({ error: "query seems to be malformed" }))
+    res.write(JSON.stringify({ error: `invalid owner id` }))
     res.end()
     return
   }
 
-  console.log(`saving task ${task.id}`)
+  let video: Video = null
+
+  console.log(`creating video from request..`)
+  console.log(`request: `, JSON.stringify(request))
   try {
-    await savePendingTask(task)
-    res.status(200)
-    res.write(JSON.stringify(task))
-    res.end()
+    video = await parseVideoRequest(ownerId, request)
   } catch (err) {
-    console.error(err)
-    res.status(500)
-    res.write(JSON.stringify({ error: "couldn't save the task" }))
-    res.end()
-  }
-})
-
-// only get the tasks for a specific owner
-app.get("/owner/:ownerId", async (req, res) => {
-  if (!hasValidAuthorization(req.headers)) {
-    console.log("Invalid authorization")
-    res.status(401)
-    res.write(JSON.stringify({ error: "invalid token" }))
-    res.end()
-    return
-  }
-
-  const ownerId = req.params.ownerId
-
-  if (!uuidValidate(ownerId)) {
-    console.error("invalid owner id")
+    console.error(`failed to create video: ${video} (${err})`)
     res.status(400)
-    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.write(JSON.stringify({ error: "query seems to be malformed" }))
     res.end()
     return
   }
 
+  console.log(`saving video ${video.id}`)
   try {
-    const tasks = await getAllTasksForOwner(ownerId)
+    await savePendingVideo(video)
     res.status(200)
-    res.write(JSON.stringify(tasks, null, 2))
+    res.write(JSON.stringify(video))
     res.end()
   } catch (err) {
     console.error(err)
     res.status(500)
-    res.write(JSON.stringify({ error: `couldn't get the tasks for owner ${ownerId}` }))
+    res.write(JSON.stringify({ error: "couldn't save the video" }))
     res.end()
   }
 })
 
-app.get("/download/:id\.mp4", async (req, res) => {
+app.get("/:ownerId/:videoId\.mp4", async (req, res) => {
     
   /*
   for simplicity, let's skip auth when fetching videos
@@ -113,7 +94,7 @@ app.get("/download/:id\.mp4", async (req, res) => {
   }
   */
 
-  const [ownerId, videoId] = `${req.params.id}`.split("_")
+  const ownerId = req.params.ownerId
 
   if (!uuidValidate(ownerId)) {
     console.error("invalid owner id")
@@ -123,6 +104,7 @@ app.get("/download/:id\.mp4", async (req, res) => {
     return
   }
 
+  const videoId = req.params.videoId
 
   if (!uuidValidate(videoId)) {
     console.error("invalid video id")
@@ -132,9 +114,9 @@ app.get("/download/:id\.mp4", async (req, res) => {
     return
   }
 
-  let task: VideoTask = null
+  let video: Video = null
   try {
-    task = await getTask(ownerId, videoId)
+    video = await getVideo(ownerId, videoId)
     console.log(`returning video ${videoId} to owner ${ownerId}`)
   } catch (err) {
     res.status(404)
@@ -143,7 +125,7 @@ app.get("/download/:id\.mp4", async (req, res) => {
     return
   }
 
-  const completedFilePath = path.join(completedFilesDirFilePath, task.fileName)
+  const completedFilePath = path.join(completedFilesDirFilePath, video.fileName)
 
   // note: we DON'T want to use the pending file path, as there may be operations on it
   // (ie. a process might be busy writing stuff to it)
@@ -177,8 +159,9 @@ app.get("/download/:id\.mp4", async (req, res) => {
   }
 })
 
-// get all pending tasks
-app.get("/", async (req, res) => {
+// get metadata (json)
+app.get("/:ownerId/:videoId", async (req, res) => {
+    
   if (!hasValidAuthorization(req.headers)) {
     console.log("Invalid authorization")
     res.status(401)
@@ -187,20 +170,106 @@ app.get("/", async (req, res) => {
     return
   }
 
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+  const videoId = req.params.videoId
+
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
+    res.end()
+    return
+  }
+
   try {
-    const tasks = await getPendingTasks()
+    const video = await getVideo(ownerId, videoId)
     res.status(200)
-    res.write(JSON.stringify(tasks, null, 2))
+    res.write(JSON.stringify(video))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(404)
+    res.write(JSON.stringify({ error: "couldn't find this video" }))
+    res.end()
+  }
+})
+
+// only get the videos for a specific owner
+app.get("/:ownerId", async (req, res) => {
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
+
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error(`invalid owner d ${ownerId}`)
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id ${ownerId}` }))
+    res.end()
+    return
+  }
+
+  try {
+    const videos = await getAllVideosForOwner(ownerId)
+    sortVideosByYoungestFirst(videos)
+    res.status(200)
+    res.write(JSON.stringify(videos, null, 2))
+    res.end()
+  } catch (err) {
+    console.error(err)
+    res.status(500)
+    res.write(JSON.stringify({ error: `couldn't get the videos for owner ${ownerId}` }))
+    res.end()
+  }
+})
+
+// get all pending videos - this is for admin usage only
+app.get("/", async (req, res) => {
+  if (!hasValidAuthorization(req.headers)) {
+    // this is what users will see in the space - but no need to show something scary
+    console.log("Invalid authorization")
+    res.status(200)
+    res.write(`<html><head></head><body>
+This space is the REST API used by VideoChain UI:<br/>
+<a href="https://jbilcke-hf-videochain-ui.hf.space" target="_blank">https://jbilcke-hf-videochain-ui.hf.space</a>
+    </body></html>`)
+    res.end()
+    // res.status(401)
+    // res.write(JSON.stringify({ error: "invalid token" }))
+    // res.end()
+    return
+  }
+
+  try {
+    const videos = await getPendingVideos()
+    res.status(200)
+    res.write(JSON.stringify(videos, null, 2))
     res.end()
   } catch (err) {
     console.error(err)
     res.status(500)
-    res.write(JSON.stringify({ error: "couldn't get the tasks" }))
+    res.write(JSON.stringify({ error: "couldn't get the videos" }))
     res.end()
   }
 })
 
-app.get("/:id", async (req, res) => {
+
+// edit a video
+app.patch("/:ownerId/:videoId", async (req, res) => {
     
   if (!hasValidAuthorization(req.headers)) {
     console.log("Invalid authorization")
@@ -210,38 +279,113 @@ app.get("/:id", async (req, res) => {
     return
   }
 
-  const [ownerId, videoId] = `${req.params.id}`.split("_")
+  const ownerId = req.params.ownerId
 
   if (!uuidValidate(ownerId)) {
-    console.error("invalid owner id")
+    console.error(`invalid owner id ${ownerId}`)
     res.status(400)
-    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.write(JSON.stringify({ error: `invalid owner id ${ownerId}` }))
     res.end()
     return
   }
 
+  const videoId = req.params.videoId
 
   if (!uuidValidate(videoId)) {
-    console.error("invalid video id")
+    console.error(`invalid video id ${videoId}`)
     res.status(400)
-    res.write(JSON.stringify({ error: `invalid video id` }))
+    res.write(JSON.stringify({ error: `invalid video id ${videoId}` }))
     res.end()
     return
   }
 
+  let status: VideoStatus = "unknown"
   try {
-    const task = await getTask(ownerId, videoId)
-    res.status(200)
-    res.write(JSON.stringify(task))
-    res.end()
+    const request = req.body as { status: VideoStatus }
+    if (['pending', 'abort', 'delete', 'pause'].includes(request.status)) {
+      status = request.status
+    } else {
+      throw new Error(`invalid video status "${request.status}"`)
+    }
   } catch (err) {
-    console.error(err)
-    res.status(404)
-    res.write(JSON.stringify({ error: "couldn't find this task" }))
+    console.error(`invalid parameter (${err})`)
+    res.status(401)
+    res.write(JSON.stringify({ error: `invalid parameter (${err})` }))
     res.end()
+    return
+  }
+
+  switch (status) {
+    case 'delete': 
+      try {
+        await markVideoAsToDelete(ownerId, videoId)
+        console.log(`deleting video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to delete video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to delete video ${videoId}` }))
+        res.end()
+      }
+      break
+
+    case 'abort': 
+      try {
+        await markVideoAsToAbort(ownerId, videoId)
+        console.log(`aborted video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to abort video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to abort video ${videoId}` }))
+        res.end()
+      }
+      break
+
+    case 'pause': 
+      try {
+        await markVideoAsToPause(ownerId, videoId)
+        console.log(`paused video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to pause video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to pause video ${videoId}` }))
+        res.end()
+      }
+      break
+    
+    case 'pending': 
+      try {
+        await markVideoAsPending(ownerId, videoId)
+        console.log(`unpausing video ${videoId}`)
+        res.status(200)
+        res.write(JSON.stringify({ success: true }))
+        res.end()
+      } catch (err) {
+        console.error(`failed to unpause video ${videoId} (${err})`)
+        res.status(500)
+        res.write(JSON.stringify({ error: `failed to unpause video ${videoId}` }))
+        res.end()
+      }
+      break
+
+    default:
+      console.log(`unsupported status ${status}`)
+      res.status(401)
+      res.write(JSON.stringify({ error: `unsupported status ${status}` }))
+      res.end()
   }
 })
 
+// delete a video - this is legacy, we should use other functions instead
+/*
 app.delete("/:id", async (req, res) => {
     
   if (!hasValidAuthorization(req.headers)) {
@@ -270,27 +414,32 @@ app.delete("/:id", async (req, res) => {
     return
   }
 
-  let task: VideoTask = null
+  // ecurity note: we always check the existence if the video first
+  // that's because we are going to delete all the associated files with a glob,
+  // so we must be sure the id is not a system path or something ^^
+  let video: Video = null
   try {
-    task = await getTask(ownerId, videoId)
+    video = await getVideo(ownerId, videoId)
   } catch (err) {
     console.error(err)
     res.status(404)
-    res.write(JSON.stringify({ error: "couldn't find this task" }))
+    res.write(JSON.stringify({ error: "couldn't find this video" }))
     res.end()
+    return
   }
 
   try {
-    await deleteTask(task)
+    await markVideoAsToDelete(ownerId, videoId)
     res.status(200)
     res.write(JSON.stringify({ success: true }))
     res.end()
   } catch (err) {
     console.error(err)
     res.status(500)
-    res.write(JSON.stringify({ success: false, error: "failed to delete the task" }))
+    res.write(JSON.stringify({ success: false, error: "failed to delete the video" }))
     res.end()
   }
 })
+*/
 
 app.listen(port, () => { console.log(`Open http://localhost:${port}`) })

src/initFolders.mts

@@ -1,11 +1,18 @@
-import { tasksDirPath, pendingTasksDirFilePath, completedTasksDirFilePath, filesDirPath, pendingFilesDirFilePath, completedFilesDirFilePath } from "./config.mts"
+import {
+  metadataDirPath,
+  pendingMetadataDirFilePath,
+  completedMetadataDirFilePath,
+  filesDirPath,
+  pendingFilesDirFilePath,
+  completedFilesDirFilePath
+} from "./config.mts"
 import { createDirIfNeeded } from "./utils/createDirIfNeeded.mts"
 
 export const initFolders = () => {
   console.log(`initializing folders..`)
-  createDirIfNeeded(tasksDirPath)
-  createDirIfNeeded(pendingTasksDirFilePath)
-  createDirIfNeeded(completedTasksDirFilePath)
+  createDirIfNeeded(metadataDirPath)
+  createDirIfNeeded(pendingMetadataDirFilePath)
+  createDirIfNeeded(completedMetadataDirFilePath)
   createDirIfNeeded(filesDirPath)
   createDirIfNeeded(pendingFilesDirFilePath)
   createDirIfNeeded(completedFilesDirFilePath)

src/llm/enrichVideoSpecsUsingLLM.mts

@@ -0,0 +1,79 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+import { Video, VideoAPIRequest } from "../types.mts"
+import { generateYAML } from "./openai/generateYAML.mts"
+import { HallucinatedVideoRequest, OpenAIErrorResponse } from "./types.mts"
+import { getQueryChatMessages } from "../preproduction/prompts.mts"
+import { getValidNumber } from "../utils/getValidNumber.mts"
+
+
+export const enrichVideoSpecsUsingLLM = async (video: Video): Promise<Video> => {
+
+  const messages: ChatCompletionRequestMessage[] = getQueryChatMessages(video.videoPrompt)
+  
+  const defaultValue = {} as unknown as HallucinatedVideoRequest
+
+  // console.log("enrichVideoSpecsUsingLLM: messages = ", messages)
+
+  let hallucinatedVideo: HallucinatedVideoRequest
+
+
+  const referenceShot = video.shots[0]
+  video.shots = []
+  // console.log("referenceShot:", referenceShot)
+  
+  try {
+    hallucinatedVideo = await generateYAML<HallucinatedVideoRequest>(
+      messages,
+      defaultValue
+    )
+    console.log("enrichVideoSpecsUsingLLM: hallucinatedVideo = ", hallucinatedVideo)
+  } catch (err) {
+
+    let error: OpenAIErrorResponse = err?.response?.data?.error as unknown as OpenAIErrorResponse
+    if (!error) {
+      error = { message: `${err || ""}` } as unknown as OpenAIErrorResponse
+    }
+    
+    console.error(JSON.stringify(error, null, 2))
+    throw new Error(`failed to call the LLM: ${error.message}`)
+  }
+
+  // const video = JSON.parse(JSON.stringify(referenceVideo)) as Video
+
+  // TODO here we should make some verifications and perhaps even some conversions
+  // betwen the LLM response and the actual format used in a videoRequest
+  video.backgroundAudioPrompt = hallucinatedVideo.backgroundAudioPrompt || video.backgroundAudioPrompt
+  video.foregroundAudioPrompt = hallucinatedVideo.foregroundAudioPrompt || video.foregroundAudioPrompt
+  video.actorPrompt = hallucinatedVideo.actorPrompt || video.actorPrompt
+  video.actorVoicePrompt = hallucinatedVideo.actorVoicePrompt || video.actorVoicePrompt
+
+  video.noise = typeof hallucinatedVideo.noise !== "undefined"
+    ? (`${hallucinatedVideo.noise || ""}`.toLowerCase() === "true")
+    : video.noise
+  
+  video.noiseAmount = typeof hallucinatedVideo.noiseAmount !== "undefined"
+    ? getValidNumber(hallucinatedVideo.noiseAmount, 0, 10, 2)
+    : video.noiseAmount
+
+  video.outroDurationMs = typeof hallucinatedVideo.outroDurationMs !== "undefined"
+    ? getValidNumber(hallucinatedVideo.outroDurationMs, 0, 3000, 500)
+    : video.outroDurationMs
+
+  const hallucinatedShots = Array.isArray(hallucinatedVideo.shots) ? hallucinatedVideo.shots : []
+
+
+  for (const hallucinatedShot of hallucinatedShots) {
+    const shot = JSON.parse(JSON.stringify(referenceShot))
+    shot.shotPrompt = hallucinatedShot.shotPrompt || shot.shotPrompt
+    shot.environmentPrompt = hallucinatedShot.environmentPrompt || shot.environmentPrompt
+    shot.photographyPrompt = hallucinatedShot.photographyPrompt || shot.photographyPrompt
+    shot.actionPrompt = hallucinatedShot.actionPrompt || shot.actionPrompt
+    shot.foregroundAudioPrompt = hallucinatedShot.foregroundAudioPrompt || shot.foregroundAudioPrompt
+    video.shots.push(shot)
+  }
+
+  console.log("enrichVideoSpecsUsingLLM: video = ", video)
+
+  return video
+}

src/llm/openai/createChatCompletion.mts

@@ -0,0 +1,105 @@
+import { ChatCompletionRequestMessage } from "openai"
+import { GPTTokens } from "gpt-tokens"
+
+import { openai } from "./openai.mts"
+import { runModerationCheck } from "./runModerationCheck.mts"
+import { getUserContent } from "./getUserContent.mts"
+import { getTextPrompt } from "./getTextPrompt.mts"
+
+export const createChatCompletion = async (
+  messages: ChatCompletionRequestMessage[],
+  model = "gpt-4"
+): Promise<string> => {
+  // this is the part added by the user, and the one we need to check against the moderation API
+  const userContent = getUserContent(messages)
+
+  const check = await runModerationCheck(userContent)
+
+  if (check.flagged) {
+    console.error("Thoughtcrime: content flagged by the AI police", {
+      userContent,
+      moderationResult: check,
+    })
+    return "Thoughtcrime: content flagged by the AI police"
+  }
+
+  const rawPrompt = getTextPrompt(messages)
+
+
+  // for doc: https://www.npmjs.com/package/gpt-tokens
+  const usageInfo = new GPTTokens({
+    // Plus enjoy a 25% cost reduction for input tokens on GPT-3.5 Turbo (0.0015 per 1K input tokens)
+    plus    : false,
+    model   : "gpt-4",
+    messages: messages as any,
+  })
+
+  console.table({
+    "Tokens prompt": usageInfo.promptUsedTokens,
+    "Tokens completion": usageInfo.completionUsedTokens,
+    "Tokens total": usageInfo.usedTokens,
+  })
+
+  // Price USD:  0.000298
+  console.log("Price USD: ", usageInfo.usedUSD)
+
+  // const tokenLimit = 4000
+
+  const maxTokens = 4000 - usageInfo.promptUsedTokens
+
+  console.log("maxTokens:", maxTokens)
+  /*
+  console.log("settings:", {
+    tokenLimit,
+    promptLength: rawPrompt.length,
+    promptTokenLengh: rawPrompt.length / 1.9,
+    maxTokens
+  })
+
+  console.log("createChatCompletion(): raw prompt length:", rawPrompt.length)
+  console.log(
+    `createChatCompletion(): requesting ${maxTokens} of the ${tokenLimit} tokens availables`
+  )
+  */
+
+  console.log("query:", {
+    model,
+    // messages,
+    user: "Anonymous User",
+    temperature: 0.7,
+    max_tokens: maxTokens,
+    // stop: preset.stop?.length ? preset.stop : undefined,
+  })
+
+  const response = await openai.createChatCompletion({
+    model,
+    messages,
+    // TODO use the Hugging Face Login username here
+    user: "Anonymous User",
+    temperature: 0.7,
+
+    // 30 tokens is about 120 characters
+    // we don't want more, as it will take longer to respond
+    max_tokens: maxTokens,
+    // stop: preset.stop?.length ? preset.stop : undefined,
+  })
+
+  const { choices } = response.data
+
+  if (!choices.length) {
+    console.log("createChatCompletion(): no choice found in the LLM response..")
+    return ""
+  }
+  const firstChoice = choices[0]
+
+  if (firstChoice?.message?.role !== "assistant") {
+    console.log(
+      "createChatCompletion(): something went wrong, the model imagined the user response?!"
+    )
+    return ""
+  }
+
+  console.log("createChatCompletion(): response", firstChoice.message.content)
+
+  return firstChoice.message.content || ""
+}

src/llm/openai/createChatCompletionStream.mts

@@ -0,0 +1,66 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+import { openai } from "./openai.mts"
+import { streamCompletion } from "./stream.mts"
+import { getTextPrompt } from "./getTextPrompt.mts"
+
+export const createChatCompletionStream = async (
+  messages: ChatCompletionRequestMessage[],
+  model: string,
+  onMessage: (message: string) => Promise<void>,
+  onEnd = () => Promise<void>
+) => {
+  try {
+    const rawPrompt = getTextPrompt(messages)
+
+    const tokenLimit = 4096 // 8000
+
+    const maxTokens = Math.round(tokenLimit - rawPrompt.length / 1.9)
+
+    const completion = await openai.createCompletion({
+      model,
+      prompt: messages,
+      temperature: 0.7,
+      max_tokens: Math.min(30, maxTokens),
+      stream: true,
+    })
+
+    for await (const message of streamCompletion(completion as any)) {
+      try {
+        const parsed = JSON.parse(message)
+        const { text } = parsed.choices[0]
+
+        try {
+          await onMessage(text)
+        } catch (err) {
+          console.error(
+            'Could not process stream message (callback failed)',
+            message,
+            err
+          )
+        }
+      } catch (error) {
+        console.error('Could not JSON parse stream message', message, error)
+      }
+    }
+    try {
+      await onEnd()
+    } catch (err) {
+      console.error('onEnd callback failed', err)
+    }
+  } catch (error: any) {
+    if (error.code) {
+      try {
+        const parsed = JSON.parse(error.body)
+        console.error('An error occurred during OpenAI request: ', parsed)
+      } catch (error) {
+        console.error(
+          'An error occurred during OpenAI request (invalid json): ',
+          error
+        )
+      }
+    } else {
+      console.error('An error occurred during OpenAI request', error)
+    }
+  }
+}

src/llm/openai/generateYAML.mts

@@ -0,0 +1,42 @@
+import { ChatCompletionRequestMessage } from "openai"
+import { parse } from "yaml"
+
+import { createChatCompletion } from "./createChatCompletion.mts"
+
+export const generateYAML = async <T,>(messages: ChatCompletionRequestMessage[] = [], defaultValue?: T): Promise<T> => {
+
+  const defaultResult = defaultValue || ({} as T)
+
+  if (!messages.length) {
+    return defaultResult
+  }
+
+  const output = await createChatCompletion(messages)
+
+  let raw = ""
+
+  // cleanup any remains of the markdown response
+  raw = output.split("```")[0]
+
+  // remove any remaining `
+  const input = raw.replaceAll("`", "")
+
+  try {
+    const obj = parse(input) as T
+
+    if (obj === null || typeof obj === undefined) {
+      throw new Error("couldn't parse YAML")
+    }
+
+    return obj
+  } catch (err) {
+    // just in case, we also try JSON!
+    const obj = JSON.parse(input) as T
+
+    if (obj === null || typeof obj === undefined) {
+      throw new Error("couldn't parse JSON")
+    }
+
+    return obj
+  }
+}

src/llm/openai/getTextPrompt.mts

@@ -0,0 +1,4 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+export const getTextPrompt = (prompt: ChatCompletionRequestMessage[]) =>
+  prompt.reduce((acc, item) => acc.concat(item.content), "") || ""

src/llm/openai/getUserContent.mts

@@ -0,0 +1,7 @@
+import { ChatCompletionRequestMessage } from "openai"
+
+export const getUserContent = (prompt: ChatCompletionRequestMessage[]) =>
+  prompt
+    .filter((message) => message.role === "user")
+    .map((message) => message.content)
+    .join("\n")

src/llm/openai/openai.mts

@@ -0,0 +1,7 @@
+import { Configuration, OpenAIApi } from "openai"
+
+export const openai = new OpenAIApi(
+  new Configuration({
+    apiKey: process.env.VC_OPENAI_API_KEY
+  })
+)

src/llm/openai/runModerationCheck.mts

@@ -0,0 +1,30 @@
+import {
+  CreateModerationResponseResultsInnerCategories,
+  CreateModerationResponseResultsInnerCategoryScores,
+} from "openai"
+
+import { openai } from "./openai.mts"
+
+export const runModerationCheck = async (
+  input = ''
+): Promise<{
+  categories?: CreateModerationResponseResultsInnerCategories
+  category_scores?: CreateModerationResponseResultsInnerCategoryScores
+  flagged: boolean
+}> => {
+  if (!input || !input.length) {
+    console.log(`skipping moderation check as input length is too shot`)
+    return {
+      flagged: false,
+    }
+  }
+
+  const response = await openai.createModeration({ input })
+  const { results } = response.data
+
+  if (!results.length) {
+    throw new Error(`failed to call the moderation endpoint`)
+  }
+
+  return results[0]
+}

src/llm/openai/stream.mts

@@ -0,0 +1,35 @@
+import { Readable } from "node:stream"
+
+async function* chunksToLines(
+  chunksAsync: AsyncIterable<Buffer>
+): AsyncIterable<string> {
+  let previous = ""
+  for await (const chunk of chunksAsync) {
+    const bufferChunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+    previous += bufferChunk
+    let eolIndex
+    while ((eolIndex = previous.indexOf("\n")) >= 0) {
+      // line includes the EOL
+      const line = previous.slice(0, eolIndex + 1).trimEnd()
+      if (line === "data: [DONE]") break
+      if (line.startsWith("data: ")) yield line
+      previous = previous.slice(eolIndex + 1)
+    }
+  }
+}
+
+async function* linesToMessages(
+  linesAsync: AsyncIterable<string>
+): AsyncIterable<string> {
+  for await (const line of linesAsync) {
+    const message = line.substring("data :".length)
+
+    yield message
+  }
+}
+
+export async function* streamCompletion(
+  stream: Readable
+): AsyncGenerator<string, void, undefined> {
+  yield* linesToMessages(chunksToLines(stream))
+}

src/llm/types.mts

@@ -0,0 +1,25 @@
+// note: this has to exactly match what is in the prompt, in ../preproduction/prompts.mts
+export interface HallucinatedVideoRequest {
+  backgroundAudioPrompt: string; // describe the background audio (crowd, birds, wind, sea etc..)
+  foregroundAudioPrompt: string; // describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
+  actorPrompt: string; // describe the physical look of the main actor visible in the shot (man, woman, old, young, hair, glasses, clothes etc)
+  actorVoicePrompt: string; // describe the main actor voice (man, woman, old, young, amused, annoyed.. etc)
+  noise: boolean; // whether to apply movie noise or not
+  noiseAmount: number; // (integer) the amount of ffmpeg noise (film grain) to apply. 0 is none, 10 is a lot
+  outroDurationMs: number; // in milliseconds. An outro generally only lasts between 0 and 3000 (3s)
+  
+  shots: Array<{
+    shotPrompt: string; // describe the main elements of a shot, in excruciating details. You must include ALL those parameters: characters, shot story, what is happening. How they look, the textures, the expressions, their clothes. The color, materials and style of clothes. 
+    environmentPrompt: string; // describe the environment, in excruciating details. You must include ALL those parameters: Lights, atmosphere and weather (misty, dust, clear, rain, snow..). Time of the day and hour of the day. Furnitures, their shape, style, era. The materials used for each object. The global time period, time of the day, era. Explain if anything is moving in the backgroung.
+    photographyPrompt: string; // describe the photography, in excruciating details. You must include ALL those parameters: Camera angle, position and movement. Type of shot and angle. Lighting. Mood. Settings. Tint of the lights. Position of the sun or moon. Shadows and their direction. Camera shutter speed, blur, bokeh, aperture.
+    actionPrompt: string; // describe the dynamics of a shot, in excruciating details. You must include ALL those parameters: What is happening, who and what is moving. Which entity are in movements. What are the directions, starting and ending position. At which speed entities or objects are moving. Is there motion blur, slow motion, timelapse etc.
+    foregroundAudioPrompt: string; // describe the sounds in a concise way (eg. ringing bells, underwater sound and whistling dolphin, cat mewong etc),
+  }>
+}
+
+export interface OpenAIErrorResponse {
+  message: string
+  type: string
+  param: any
+  code: any
+}

src/main.mts

@@ -1,22 +1,26 @@
 import { initFolders } from "./initFolders.mts"
-import { getPendingTasks } from "./scheduler/getPendingTasks.mts"
-import { processTask } from "./scheduler/processTask.mts"
+import { getPendingVideos } from "./scheduler/getPendingVideos.mts"
+import { processVideo } from "./scheduler/processVideo.mts"
+import { sortPendingVideosByLeastCompletedFirst } from "./utils/sortPendingVideosByLeastCompletedFirst.mts"
 
 export const main = async () => {
 
-  const tasks = await getPendingTasks()
-  if (!tasks.length) {
+  const videos = await getPendingVideos()
+  if (!videos.length) {
     setTimeout(() => {
       main()
     }, 500)
     return
   }
 
-  console.log(`there are ${tasks.length} pending tasks`)
-  for (const task of tasks) {
-    await processTask(task)
+  console.log(`there are ${videos.length} pending videos`)
+
+  sortPendingVideosByLeastCompletedFirst(videos)
+
+  for (const video of videos) {
+    await processVideo(video)
   }
-  console.log(`processed ${tasks.length} tasks`)
+  console.log(`processed ${videos.length} videos`)
 
   setTimeout(() => {
     main()

src/preproduction/mocks.mts

@@ -0,0 +1,36 @@
+import { Video, VideoShot } from "../types.mts"
+
+export const mockShots: VideoShot[] = [
+  {
+    "shotPrompt": "In the extreme wide shot, a flock of ducks is converging on the Central Park, coming from multiple directions. Their feathers are glossy and clean, casting off varying degrees of green, brown and white",
+    "environmentPrompt": "Central Park at sunrise, the park looks slightly misty, the sky is tinged with shades of pink and orange as the day breaks. There's dew on the grass, and the leaves on trees are rustling in the light breeze",
+    "photographyPrompt": "Eye-level shot with a slight tilt in the camera, capturing the panorama of the park. There's natural lighting, sun just rising. The camera zooms out to capture the ducks entering the park. Shutter speed is slow to capture the movement of ducks",
+    "actionPrompt": "Large groups of ducks waddle into the park from various directions, some fly in groups, landing on the pond with small splashes. Movement is slow, slightly sped up to depict the invasion",
+    "foregroundAudioPrompt": "A symphony of soft quacking and rustling feathers",
+  },
+  {
+    "shotPrompt": "In the medium shot, a group of ducks are by the pond, pecking at the ground and frolicking in the water. One male mallard is particularly captivating with its emerald green head and healthy body",
+    "environmentPrompt": "It's a sunny spring day in Central Park. The pond is surrounded by lush, green vegetation and dappled with sunlight filtering through the leaves",
+    "photographyPrompt": "Low angle shot near the water level, the camera moves in a crane shot to capture ducks in action, and the camera's aperture is partially open. Natural sunlight creates playful shadows",
+    "actionPrompt": "Ducks are pecking at the ground, dabbling at the water's edge and frolicking in the pond. The camera tracks a particularly majestic mallard navigating through the pond",
+    "foregroundAudioPrompt": "Sounds of ducks quacking and splashing in the water"
+  },
+  {
+    "shotPrompt": "Close-up shot of a mother duck with ducklings following her in a line on the grass and into the water",
+    "environmentPrompt": "Central Park, by one of the smaller ponds, surrounded by green trees. Sun is high up giving off warm, radiant light",
+    "photographyPrompt": "High angle shot, focusing on the line of ducklings following their mother. The camera follows the ducklings. The setting is bright and clear with sun illuminating the ducklings",
+    "actionPrompt": "Mother duck is leading her ducklings from the grass into the water, the ducklings obediently follow, creating a neat line. The whole scene feels peaceful",
+    "foregroundAudioPrompt": "Ducklings' high pitched chirping, soft lapping of water at the edge of the pond"
+  }
+] as any
+
+export const mock: Video = {
+  "backgroundAudioPrompt": "City ambience mixed with the rustling leaves and the chirping birds in the park",
+  "foregroundAudioPrompt": "Rustling feathers, soft quacking, flapping wings, occasional splash in the pond",
+  "actorPrompt": "Main actors are ducks - a variety of breeds, mostly mallards: males with glossy green heads and females in mottled brown; all plump, medium-sized waterfowl",
+  "actorVoicePrompt": "Soft, low pitched quacking of adult ducks and higher pitched chirping of ducklings",
+  "noise": true,
+  "noiseAmount": 2,
+  "outroDurationMs": 1500,
+  "shots": mockShots
+} as any

src/preproduction/prompts.mts

@@ -0,0 +1,119 @@
+
+
+// types of movie shots: https://www.nfi.edu/types-of-film-shots/
+
+import { ChatCompletionRequestMessage } from "openai"
+
+export const getQueryChatMessages = (sceneDescription: string): ChatCompletionRequestMessage[] => {
+  return [
+  {
+    role: "system",
+    name: "moviemaking_rules",
+    content: `# Context
+You are an AI Movie Director Assistant, and you need to help generating input requests (movie "specs") for an automated movie generation API.
+The format expected by the API must be in YAML. The TypeScript schema for this YAML file is:
+\`\`\`typescript
+{
+  backgroundAudioPrompt: string; // describe the background audio (crowd, birds, wind, sea etc..)
+  foregroundAudioPrompt: string; // describe the foreground audio (cars revving, footsteps, objects breaking, explosion etc)
+  actorPrompt: string; // describe the physical look of the main actor visible in the shot (man, woman, old, young, hair, glasses, clothes etc)
+  actorVoicePrompt: string; // describe the main actor voice (man, woman, old, young, amused, annoyed.. etc)
+  noise: boolean; // whether to apply movie noise or not
+  noiseAmount: number; // (integer) the amount of noise (film grain) to apply. This is mapped from the FFmpeg filter (0 is none, 10 is already a lot)
+  outroDurationMs: number; // in milliseconds. An outro generally only lasts between 0 and 3000 (3s)
+  shots: Array<{
+    shotPrompt: string; // describe the main elements of a shot, in excruciating details. You must include ALL those parameters: characters, shot story, what is happening. How they look, the textures, the expressions, their clothes. The color, materials and style of clothes. 
+    environmentPrompt: string; // describe the environment, in excruciating details. You must include ALL those parameters: Lights, atmosphere and weather (misty, dust, clear, rain, snow..). Time of the day and hour of the day. Furnitures, their shape, style, era. The materials used for each object. The global time period, time of the day, era. Explain if anything is moving in the backgroung.
+    photographyPrompt: string; // describe the photography, in excruciating details. You must include ALL those parameters: Camera angle, position and movement. Type of shot and angle. Lighting. Mood. Settings. Tint of the lights. Position of the sun or moon. Shadows and their direction. Camera shutter speed, blur, bokeh, aperture.
+    actionPrompt: string; // describe the dynamics of a shot, in excruciating details. You must include ALL those parameters: What is happening, who and what is moving. Which entity are in movements. What are the directions, starting and ending position. At which speed entities or objects are moving. Is there motion blur, slow motion, timelapse etc.
+    foregroundAudioPrompt: string; // describe the sounds in a concise way (eg. ringing bells, underwater sound and whistling dolphin, cat mewong etc),
+  }>
+}
+\`\`\`
+# Guidelines
+You will generate 3 shots by default, unless more or less are specified.
+Is it crucial to repeat the elements consituting a sequence of multiple shots verbatim from one shot to another.
+For instance, you will have to repeat exactly what a character or background look like, how they are dressed etc.
+This will ensure consistency from one scene to another.
+## Creating a movie
+Here are some guidelines regarding film-making:
+- The distance your subject is to the camera impacts how the audience feels about them.
+- Subject will appear largest in a close-up or choker shot and smallest in a wide or long shot.
+- Camera movement is a technique for changing the relationship between the subject and the camera frame, controlling the delivery of the narrative. It helps to give additional meaning to what’s happening on the screen.
+- Do not hesitate to combine types of shots with camera movement shots and camera position (angle) shots.
+## Shots
+Single shot: where the shot only captures one subject.
+Two shot: which has only two characters.
+Three shot: when three characters are in the frame.
+Point-of-view shot (POV): shows the scene from the point of view of one of the characters, makes the audience feel that they are there seeing what the character is seeing.
+Over-the-shoulder shot (OTS): shows the subject from behind the shoulder of another character.
+Over-the-hip (OTH) shot, in which the camera is placed on the hip of one character and the focus is on the subject.
+Reverse angle shot: which is approximately 180 degrees opposite the previous shot.
+Reaction shot: which shows the character’s reaction to the previous shot.
+Weather shot: where the subject of the filming is the weather.
+Extreme wide shot/extreme long shot: used to show the subject and the entire area of the environment they are in.
+Wide shot/long shot: used to focus on the subject while still showing the scene the subject is in.
+Medium shot: shows the subject from the knees up, and is often referred to as the 3/4 shot.
+Medium close-up shot: The subject fills the frame. It is somewhere between a medium close-up and a close-up.
+Close-up shot: shows emotions and detailed reactions, with the subject filling the entire frame.
+Choker shot: shows the subject’s face from just above the eyebrows to just below the mouth and is between a close-up and an extreme close-up.
+Extreme close-up shot: shows the detail of an object, such as one a character is handling, or a person, such as just their eyes or moving lips.
+Full shot: similar to a wide shot except that it focuses on the character in the frame, showing them from head to toe.
+Cowboy shot: similar to the medium shot except that the character is shown from the hips or waist up.
+Establishing shot: a long shot at the beginning of a scene that shows objects, buildings, and other elements of a setting from a distance to establish where the next sequence of events takes place.
+## Camera angles
+Eye-level shot: This is when the camera is placed at the same height as the eyes of the characters.
+Low angle shot: This shot frames the subject from a low height, often used to emphasize differences in power between characters.
+Aerial shot/helicopter shot: Taken from way up high, this shot is usually from a drone or helicopter to establish the expanse of the surrounding landscape.
+High angle shot: This is when the subject is framed with the camera looking down at them.
+Birds-eye-view shot/overhead shot: This is a shot taken from way above the subject, usually including a significant amount of the surrounding environment to create a sense of scale or movement.
+Shoulder-level shot: This is where the camera is approximately the same height as the character’s shoulders.
+Hip-level shot: The camera is approximately at the height of the character’s hips.
+Knee-level shot: The camera is approximately at the same level as the character’s knees.
+Ground-level shot: When the height of the camera is at ground level with the character, this shot captures what’s happening on the ground the character is standing on.
+Dutch-angle/tilt shot: This is where the camera is tilted to the side.
+Cut-in shot: This type of shot cuts into the action on the screen to offer a different view of something happening in this main scene.
+Cutaway shot: As a shot that cuts away from the main action on the screen, it’s used to focus on secondary action and add more information for greater understanding for the audience.
+Master shot: A long shot that captures most or all of the action happening in a scene.
+Deep focus: A shot that keeps everything on the screen in sharp focus, including the foreground, background, and middle ground.
+Locked-down shot: With this shot, the camera is fixed in one position and the action continues off-screen.
+## Camera movements
+Zoom Shot: involves changing the focal length of the lens to zoom in or out during filming.
+Pan shot:  involves moving the camera from side to side to show something to the audience or help them better follow the sequence of events.
+Tilt shot: similar to a pan shot, except moving the camera up and down.
+Dolly shot: the camera is attached to a dolly that moves on tracks and can possibly move up and down.
+Truck shot: you move the entire camera on a fixed point and the motion goes from side to side.
+Pedestal shot: the entire camera is moved vertically, not just the angle of view, and is often combined with panning and/or tilting.
+Static/fixed shot: where there is no camera movement, and the shot emphasizes the movement of the subject in the environment.
+Arc shot: where the camera moves in an arc pattern around the subject to give the audience a better perspective of their surroundings.
+Crab shot: a less-common version of tracking a subject where the dolly the camera is on goes sideways.
+Dolly zoom shot: the position of the camera and focal length are changed simultaneously.
+Whip pan shot/swish pan shot: used to create a blur as you pan from one shot to the next.
+Tracking shot: the camera follows the subject, either from behind or at their side, moving with them.
+Whip tilt shot: used to create a blur panning from one shot to the next vertically.
+Bridging shot: denotes a shift in place or time.
+## Focus
+Focus pull: focus the lens to keep the subject within an acceptable focus range.
+Rack focus: focus is more aggressively shifted from subject A to subject B.
+Tilt-shift: parts of the image are in focus while other parts are out of focus.
+Deep focus: both the subject and the environment are in focus.
+Shallow focus: subject is crisp and in focus while the background is out of focus.
+## Camera angles
+High-angle
+Low-angle
+Over-the-shoulder
+Bird’s eye
+Dutch angle/tilt`
+  },
+  {
+    role: "user",
+    name: "movie_director",
+    content: `# Task
+Please generate the movie spec YAML based on the following description:
+${sceneDescription}.
+# YAML
+\`\`\`
+`
+  },
+]
+}

src/production/generateShot.mts

@@ -135,7 +135,7 @@ export const generateShot = async ({
     const interpolationSteps = 3
     const interpolatedFramesPerSecond = 24
     await interpolateVideo(
-      task,
+      video,
       interpolationSteps,
       interpolatedFramesPerSecond
     )
@@ -194,7 +194,7 @@ export const generateShot = async ({
       audioFileName = foregroundAudioFileName
     }
 
-    await addAudioToVideo(task, audioFileName)
+    await addAudioToVideo(video, audioFileName)
   }
 
   console.log("returning result to user..")

src/scheduler/deleteTask.mts

@@ -1,33 +0,0 @@
-
-import { existsSync, promises as fs } from "node:fs"
-import path from "node:path"
-
-import tmpDir from "temp-dir"
-
-import { VideoTask } from "../types.mts"
-import { completedTasksDirFilePath, completedFilesDirFilePath, pendingTasksDirFilePath, pendingFilesDirFilePath } from "../config.mts"
-import { deleteFileIfExists } from "../utils/deleteFileIfExists.mts"
-
-
-export const deleteTask = async (task: VideoTask) => {
-  const taskFileName = `${task.ownerId}_${task.id}.json`
-  const videoFileName = task.fileName
-  
-  // .mp4 files
-  const tmpFilePath = path.join(tmpDir, videoFileName)
-  const pendingVideoPath = path.join(pendingFilesDirFilePath, videoFileName)
-  const completedVideoPath = path.join(completedFilesDirFilePath, videoFileName)
-
-  // .json files
-  const pendingTaskPath = path.join(pendingTasksDirFilePath, taskFileName)
-  const completedTaskPath = path.join(completedTasksDirFilePath, taskFileName)
-
-  await deleteFileIfExists(tmpFilePath)
-  await deleteFileIfExists(pendingVideoPath)
-  await deleteFileIfExists(completedVideoPath)
-  await deleteFileIfExists(pendingTaskPath)
-  await deleteFileIfExists(completedTaskPath)
-
-  // TODO: we didn't delete any audio file!
-  console.log(`note: we didn't delete any audio file!`)
-}

src/scheduler/deleteVideo.mts

@@ -0,0 +1,28 @@
+import tmpDir from "temp-dir"
+import { validate as uuidValidate } from "uuid"
+
+import { completedMetadataDirFilePath, completedFilesDirFilePath, pendingMetadataDirFilePath, pendingFilesDirFilePath } from "../config.mts"
+import { deleteFilesWithName } from "../utils/deleteAllFilesWith.mts"
+
+
+// note: we make sure ownerId and videoId are *VALID*
+// otherwise an attacker could try to delete important files!
+export const deleteVideo = async (ownerId: string, videoId?: string) => {
+  if (!uuidValidate(ownerId)) {
+    throw new Error(`fatal error: ownerId ${ownerId} is invalid!`)
+  }
+
+  if (videoId && !uuidValidate(videoId)) {
+    throw new Error(`fatal error: videoId ${videoId} is invalid!`)
+  }
+  const id = videoId ? `${ownerId}_${videoId}` : ownerId
+
+  // this should delete everything, including audio files
+  // however we still have some temporary files with a name that is unique:
+  // we should probably rename those
+  await deleteFilesWithName(tmpDir, id)
+  await deleteFilesWithName(completedMetadataDirFilePath, id)
+  await deleteFilesWithName(completedFilesDirFilePath, id)
+  await deleteFilesWithName(pendingMetadataDirFilePath, id)
+  await deleteFilesWithName(pendingFilesDirFilePath, id)
+}

src/scheduler/getAllTasksForOwner.mts

@@ -1,9 +0,0 @@
-import { VideoTask } from "../types.mts"
-import { getCompletedTasks } from "./getCompletedTasks.mts"
-import { getPendingTasks } from "./getPendingTasks.mts"
-
-export const getAllTasksForOwner = async (ownerId: string): Promise<VideoTask[]> => {
-  const pendingTasks = await getPendingTasks(ownerId)
-  const completedTasks = await getCompletedTasks(ownerId)
-  return [...pendingTasks, ...completedTasks]
-}

src/scheduler/getAllVideosForOwner.mts

@@ -0,0 +1,9 @@
+import { Video } from "../types.mts"
+import { getCompletedVideos } from "./getCompletedVideos.mts"
+import { getPendingVideos } from "./getPendingVideos.mts"
+
+export const getAllVideosForOwner = async (ownerId: string): Promise<Video[]> => {
+  const pendingVideos = await getPendingVideos(ownerId)
+  const completedVideos = await getCompletedVideos(ownerId)
+  return [...pendingVideos, ...completedVideos]
+}

src/scheduler/getCompletedTasks.mts

@@ -1,9 +0,0 @@
-import { VideoTask } from "../types.mts"
-import { completedTasksDirFilePath } from "../config.mts"
-import { readTasks } from "./readTasks.mts"
-
-export const getCompletedTasks = async (ownerId?: string): Promise<VideoTask[]> => {
-  const completedTasks = await readTasks(completedTasksDirFilePath, ownerId)
-
-  return completedTasks
-}

src/scheduler/getCompletedVideos.mts

@@ -0,0 +1,9 @@
+import { Video } from "../types.mts"
+import { completedMetadataDirFilePath } from "../config.mts"
+import { readVideoMetadataFiles } from "./readVideoMetadataFiles.mts"
+
+export const getCompletedVideos = async (ownerId?: string): Promise<Video[]> => {
+  const completedVideos = await readVideoMetadataFiles(completedMetadataDirFilePath, ownerId)
+
+  return completedVideos
+}

src/scheduler/getPendingTasks.mts

@@ -1,9 +0,0 @@
-import { VideoTask } from "../types.mts"
-import { pendingTasksDirFilePath } from "../config.mts"
-import { readTasks } from "./readTasks.mts"
-
-export const getPendingTasks = async (ownerId?: string): Promise<VideoTask[]> => {
-  const pendingTasks = await readTasks(pendingTasksDirFilePath, ownerId)
-
-  return pendingTasks
-}

src/scheduler/getPendingVideos.mts

@@ -0,0 +1,9 @@
+import { Video } from "../types.mts"
+import { pendingMetadataDirFilePath } from "../config.mts"
+import { readVideoMetadataFiles } from "./readVideoMetadataFiles.mts"
+
+export const getPendingVideos = async (ownerId?: string): Promise<Video[]> => {
+  const pendingVideos = await readVideoMetadataFiles(pendingMetadataDirFilePath, ownerId)
+
+  return pendingVideos
+}

src/scheduler/getTask.mts

@@ -1,23 +0,0 @@
-import path from "node:path"
-
-import { completedTasksDirFilePath, pendingTasksDirFilePath } from "../config.mts"
-import { readTask } from "./readTask.mts"
-
-export const getTask = async (ownerId: string, videoId: string) => {
-  const taskFileName = `${ownerId}_${videoId}.json`
-
-  const completedTaskFilePath = path.join(completedTasksDirFilePath, taskFileName)
-  const pendingTaskFilePath = path.join(pendingTasksDirFilePath, taskFileName)
-
-  try {
-    const completedTask = await readTask(completedTaskFilePath)
-    return completedTask
-  } catch (err) {
-    try {
-      const pendingTask = await readTask(pendingTaskFilePath)
-      return pendingTask
-    } catch (err) {
-      throw new Error(`couldn't find video task ${videoId} for owner ${ownerId}`)
-    }
-  }
-}

src/scheduler/getVideo.mts

@@ -0,0 +1,23 @@
+import path from "node:path"
+
+import { completedMetadataDirFilePath, pendingMetadataDirFilePath } from "../config.mts"
+import { readVideoMetadataFile } from "./readVideoMetadataFile.mts"
+
+export const getVideo = async (ownerId: string, videoId: string) => {
+  const videoFileName = `${ownerId}_${videoId}.json`
+
+  const completedVideoMetadataFilePath = path.join(completedMetadataDirFilePath, videoFileName)
+  const pendingVideoMetadataFilePath = path.join(pendingMetadataDirFilePath, videoFileName)
+
+  try {
+    const completedVideo = await readVideoMetadataFile(completedVideoMetadataFilePath)
+    return completedVideo
+  } catch (err) {
+    try {
+      const pendingVideo= await readVideoMetadataFile(pendingVideoMetadataFilePath)
+      return pendingVideo
+    } catch (err) {
+      throw new Error(`couldn't find video ${videoId} for owner ${ownerId}`)
+    }
+  }
+}

src/scheduler/getVideoStatus.mts

@@ -0,0 +1,13 @@
+import { Video, VideoStatus } from "../types.mts"
+
+import { getVideo } from "./getVideo.mts"
+
+export const getVideoStatus = async (video: Video): Promise<VideoStatus> => {
+  try {
+    const { status } = await getVideo(video.ownerId, video.id)
+    return status
+  } catch (err) {
+    console.log(`failed to get the video status.. weird`)
+  }
+  return "unknown"
+}

src/scheduler/markVideoAsPending.mts

@@ -0,0 +1,25 @@
+import { updatePendingVideo } from "./updatePendingVideo.mts"
+import { getVideo } from "./getVideo.mts"
+
+export const markVideoAsPending = async (ownerId: string, videoId: string) => {
+  try {
+    const video = await getVideo(ownerId, videoId)
+    if (video.status === "abort" ) {
+      // actually, if we wanted to, we could ressurect it..
+      console.log(`cannot mark video as pending: video ${videoId} is aborted`)
+    } else if (video.status === "completed") {
+      console.log(`video ${videoId} is already completed`)
+    } else if (video.status === "delete") {
+      console.log(`cannot mark video as pending: video ${videoId} is marked for deletion`)
+    } else if (video.status === "pending") {
+      console.log(`video ${videoId} is already pending`)
+    } {
+      video.status = "pending"
+      await updatePendingVideo(video)
+      return true
+    }
+  } catch (err) {
+    console.error(`failed to mark video as pending ${videoId}`)
+  }
+  return false
+}

src/scheduler/markVideoAsToAbort.mts

@@ -0,0 +1,22 @@
+import { updatePendingVideo } from "./updatePendingVideo.mts"
+import { getVideo } from "./getVideo.mts"
+
+export const markVideoAsToAbort = async (ownerId: string, videoId: string) => {
+  try {
+    const video = await getVideo(ownerId, videoId)
+    if (video.status === "abort" ) {
+      console.log(`video ${videoId} is already aborted`)
+    } else if (video.status === "delete") {
+      console.log(`cannot abort: video ${videoId} is marked for deletion`)
+    }  else if (video.status === "completed") {
+      console.log(`cannot abort: video ${videoId} is completed`)
+    } {
+      video.status = "abort"
+      await updatePendingVideo(video)
+      return true
+    }
+  } catch (err) {
+    console.error(`failed to abort video ${videoId}`)
+  }
+  return false
+}

src/scheduler/markVideoAsToDelete.mts

@@ -0,0 +1,23 @@
+import { updatePendingVideo } from "./updatePendingVideo.mts"
+import { getVideo } from "./getVideo.mts"
+import { deleteVideo } from "./deleteVideo.mts"
+
+export const markVideoAsToDelete = async (ownerId: string, videoId: string) => {
+  try {
+    const video = await getVideo(ownerId, videoId)
+    if (video.status === "delete" ) {
+      console.log(`video ${videoId} is already marked for deletion`)
+    } else if (video.status === "completed" ) {
+      console.log(`video ${videoId} is completed: we can delete immediately`)
+      await deleteVideo(ownerId, videoId)
+      return true
+    } else {
+      video.status = "delete"
+      await updatePendingVideo(video)
+      return true
+    }
+  } catch (err) {
+    console.error(`failed to delete video ${videoId}`)
+  }
+  return false
+}

src/scheduler/markVideoAsToPause.mts

@@ -0,0 +1,24 @@
+import { updatePendingVideo } from "./updatePendingVideo.mts"
+import { getVideo } from "./getVideo.mts"
+
+export const markVideoAsToPause = async (ownerId: string, videoId: string) => {
+  try {
+    const video = await getVideo(ownerId, videoId)
+    if (video.status === "abort" ) {
+      console.log(`cannot pause: video ${videoId} is being aborted`)
+    } else if (video.status === "completed") {
+      console.log(`cannot pause: video ${videoId} is completed`)
+    } else if (video.status === "delete") {
+      console.log(`cannot pause: video ${videoId} is marked for deletion`)
+    } else if (video.status === "pause") {
+      console.log(`video ${videoId} is already paused`)
+    } {
+      video.status = "pause"
+      await updatePendingVideo(video)
+      return true
+    }
+  } catch (err) {
+    console.error(`failed to mark video as paused ${videoId}`)
+  }
+  return false
+}

src/scheduler/{processTask.mts → processVideo.mts}

@@ -1,44 +1,49 @@
 import { v4 as uuidv4 } from "uuid"
 
-import { saveCompletedTask } from "./saveCompletedTask.mts"
-import { savePendingTask } from "./savePendingTask.mts"
-import { updatePendingTask } from "./updatePendingTask.mts"
-import { VideoShot, VideoTask } from "../types.mts"
-import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
+import { Video } from "../types.mts"
+
 import { generateVideo } from "../production/generateVideo.mts"
-import { copyVideoFromTmpToPending } from "../utils/copyVideoFromTmpToPending.mts"
-import { copyVideoFromTmpToCompleted } from "../utils/copyVideoFromTmpToCompleted.mts"
 import { upscaleVideo } from "../production/upscaleVideo.mts"
 import { interpolateVideo } from "../production/interpolateVideo.mts"
 import { postInterpolation } from "../production/postInterpolation.mts"
-import { moveVideoFromPendingToCompleted } from "../utils/moveVideoFromPendingToCompleted.mts"
 import { assembleShots } from "../production/assembleShots.mts"
-import { copyVideoFromPendingToCompleted } from "../utils/copyVideoFromPendingToCompleted.mts"
 import { generateAudio } from "../production/generateAudio.mts"
-import { mergeAudio } from "../production/mergeAudio.mts"
 import { addAudioToVideo } from "../production/addAudioToVideo.mts"
 
-export const processTask = async (task: VideoTask) => {
-  console.log(`processing video task ${task.id}`)
+import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
+import { copyVideoFromTmpToPending } from "../utils/copyVideoFromTmpToPending.mts"
+import { copyVideoFromPendingToCompleted } from "../utils/copyVideoFromPendingToCompleted.mts"
 
-  // something isn't right, the task is already completed
-  if (task.completed) {
-    console.log(`video task ${task.id} is already completed`)
-    await saveCompletedTask(task)
-    return
-  }
+import { saveAndCheckIfNeedToStop } from "./saveAndCheckIfNeedToStop.mts"
+import { enrichVideoSpecsUsingLLM } from "../llm/enrichVideoSpecsUsingLLM.mts"
+
+export const processVideo = async (video: Video) => {
 
-  // always count 1 more step, for the final assembly
+  // just a an additional precaution, for consistency and robustness
+  if (["pause", "completed", "abort", "delete"].includes(video.status)) { return }
 
-  let nbTotalSteps = 1
+  console.log(`processing video video ${video.id}`)
 
-  for (const shot of task.shots) {
+  // always count 2 more steps: 1 for the LLM, 1 for the final assembly
+
+  let nbTotalSteps = 2
+
+  for (const shot of video.shots) {
     nbTotalSteps += shot.nbTotalSteps
   }
 
   let nbCompletedSteps = 0
 
-  for (const shot of task.shots) {
+  if (!video.hasGeneratedSpecs) {
+    await enrichVideoSpecsUsingLLM(video)
+
+    nbCompletedSteps++
+    video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+
+    if (await saveAndCheckIfNeedToStop(video)) { return }
+  }
+
+  for (const shot of video.shots) {
     nbCompletedSteps += shot.nbCompletedSteps
 
     // skip shots completed previously
@@ -56,6 +61,8 @@ export const processTask = async (task: VideoTask) => {
     // const nbFramesForBaseModel = Math.min(3, Math.max(1, Math.round(duration))) * 8
     const nbFramesForBaseModel = 24
 
+    if (await saveAndCheckIfNeedToStop(video)) { return }
+
     if (!shot.hasGeneratedPreview) {
       console.log("generating a preview of the final result..")
       let generatedPreviewVideoUrl = ""
@@ -73,21 +80,20 @@ export const processTask = async (task: VideoTask) => {
 
         await copyVideoFromTmpToPending(shot.fileName)
 
-        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+        await copyVideoFromPendingToCompleted(shot.fileName, video.fileName)
 
         shot.hasGeneratedPreview = true
         shot.nbCompletedSteps++
         nbCompletedSteps++
         shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
-
-        await updatePendingTask(task)
+        video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
 
+        if (await saveAndCheckIfNeedToStop(video)) { return }
       } catch (err) {
         console.error(`failed to generate preview for shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
-        task.error = `failed to generate preview for shot ${shot.id} (will try again later)`
-        await updatePendingTask(task)
+        video.error = `failed to generate preview for shot ${shot.id} (will try again later)`
+        if (await saveAndCheckIfNeedToStop(video)) { return }
         break
       }
 
@@ -117,16 +123,17 @@ export const processTask = async (task: VideoTask) => {
         shot.nbCompletedSteps++
         nbCompletedSteps++
         shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+        video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
 
-        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+        await copyVideoFromPendingToCompleted(shot.fileName, video.fileName)
 
-        await updatePendingTask(task)
+        if (await saveAndCheckIfNeedToStop(video)) { return }
       } catch (err) {
         console.error(`failed to generate shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
-        task.error = `failed to generate shot ${shot.id} (will try again later)`
-        await updatePendingTask(task)
+        video.error = `failed to generate shot ${shot.id} (will try again later)`
+        if (await saveAndCheckIfNeedToStop(video)) { return }
+
         break
       }
 
@@ -141,16 +148,18 @@ export const processTask = async (task: VideoTask) => {
         shot.nbCompletedSteps++
         nbCompletedSteps++
         shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+        video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
 
-        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+        await copyVideoFromPendingToCompleted(shot.fileName, video.fileName)
+
+        if (await saveAndCheckIfNeedToStop(video)) { return }
 
-        await updatePendingTask(task)
       } catch (err) {
         console.error(`failed to upscale shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
-        task.error = `failed to upscale shot ${shot.id} (will try again later)`
-        await updatePendingTask(task)
+        video.error = `failed to upscale shot ${shot.id} (will try again later)`
+        if (await saveAndCheckIfNeedToStop(video)) { return }
+
         break
       }
     }
@@ -182,17 +191,17 @@ export const processTask = async (task: VideoTask) => {
         shot.nbCompletedSteps++
         nbCompletedSteps++
         shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+        video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
 
-        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+        await copyVideoFromPendingToCompleted(shot.fileName, video.fileName)
 
-        await updatePendingTask(task)
+        if (await saveAndCheckIfNeedToStop(video)) { return }
 
       } catch (err) {
         console.error(`failed to interpolate shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
-        task.error = `failed to interpolate shot ${shot.id} (will try again later)`
-        await updatePendingTask(task)
+        video.error = `failed to interpolate shot ${shot.id} (will try again later)`
+        if (await saveAndCheckIfNeedToStop(video)) { return }
         break
       }
     }
@@ -215,22 +224,22 @@ export const processTask = async (task: VideoTask) => {
         shot.nbCompletedSteps++
         nbCompletedSteps++
         shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+        video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
 
-        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+        await copyVideoFromPendingToCompleted(shot.fileName, video.fileName)
 
-        await updatePendingTask(task)
+        if (await saveAndCheckIfNeedToStop(video)) { return }
       } catch (err) {
         console.error(`failed to post-process shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
-        task.error = `failed to post-process shot ${shot.id} (will try again later)`
-        await updatePendingTask(task)
+        video.error = `failed to post-process shot ${shot.id} (will try again later)`
+        if (await saveAndCheckIfNeedToStop(video)) { return }
         break
       }
     }
 
 
-    let foregroundAudioFileName = `${task.ownerId}_${task.id}_${shot.id}_${uuidv4()}.m4a`
+    let foregroundAudioFileName = `${video.ownerId}_${video.id}_${shot.id}_${uuidv4()}.m4a`
 
     if (!shot.hasGeneratedForegroundAudio) {
       if (shot.foregroundAudioPrompt) {
@@ -243,19 +252,19 @@ export const processTask = async (task: VideoTask) => {
           shot.nbCompletedSteps++
           nbCompletedSteps++
           shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-          task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+          video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
 
           await addAudioToVideo(shot.fileName, foregroundAudioFileName)
 
-          await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+          await copyVideoFromPendingToCompleted(shot.fileName, video.fileName)
 
-          await updatePendingTask(task)
+          if (await saveAndCheckIfNeedToStop(video)) { return }
 
         } catch (err) {
           console.error(`failed to generate foreground audio for ${shot.id} (${err})`)
           // something is wrong, let's put the whole thing back into the queue
-          task.error = `failed to generate foreground audio ${shot.id} (will try again later)`
-          await updatePendingTask(task)
+          video.error = `failed to generate foreground audio ${shot.id} (will try again later)`
+          if (await saveAndCheckIfNeedToStop(video)) { return }
           break
         }
       } else {
@@ -263,8 +272,8 @@ export const processTask = async (task: VideoTask) => {
         shot.nbCompletedSteps++
         nbCompletedSteps++
         shot.progressPercent = Math.round((shot.nbCompletedSteps / shot.nbTotalSteps) * 100)
-        task.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
-        await updatePendingTask(task)
+        video.progressPercent = Math.round((nbCompletedSteps / nbTotalSteps) * 100)
+        if (await saveAndCheckIfNeedToStop(video)) { return }
       }
     }
 
@@ -272,56 +281,54 @@ export const processTask = async (task: VideoTask) => {
     shot.completedAt = new Date().toISOString()
     shot.progressPercent = 100
 
-    task.nbCompletedShots++
+    video.nbCompletedShots++
 
-    await updatePendingTask(task)
+    if (await saveAndCheckIfNeedToStop(video)) { return }
   }
 
   console.log(`end of the loop:`)
-  console.log(`nb completed shots: ${task.nbCompletedShots}`)
-  console.log(`len of the shot array: ${task.shots.length}`)
+  console.log(`nb completed shots: ${video.nbCompletedShots}`)
+  console.log(`len of the shot array: ${video.shots.length}`)
   
-  if (task.nbCompletedShots === task.shots.length) {
-    console.log(`we have completed the whole video sequence!`)
-    console.log(`assembling the video..`)
+  // now time to check the end game
 
-    if (task.shots.length === 1) {
+  if (video.nbCompletedShots === video.shots.length) {
+    console.log(`we have generated each individual shot!`)
+    console.log(`assembling the fonal..`)
+
+    if (!video.hasAssembledVideo) {
+
+    if (video.shots.length === 1) {
       console.log(`we only have one shot, so this gonna be easy`)
-      task.hasAssembledVideo = true
+      video.hasAssembledVideo = true
 
       // the single shot (so, the first) becomes the final movie
-      await moveVideoFromPendingToCompleted(task.shots[0].fileName, task.fileName)
-
-      await updatePendingTask(task)
-    }
+      await copyVideoFromPendingToCompleted(video.shots[0].fileName, video.fileName)
 
-    if (!task.hasAssembledVideo) {
-      console.log(`assembling the ${task.shots.length} shots together (might take a while)`)
-      try {
-        await assembleShots(task.shots, task.fileName)
-        console.log(`finished assembling the ${task.shots.length} shots together!`)
+      if (await saveAndCheckIfNeedToStop(video)) { return }
+    } else {
+        console.log(`assembling ${video.shots.length} shots together, might take a while`)
+        try {
+          await assembleShots(video.shots, video.fileName)
+          console.log(`finished assembling the ${video.shots.length} shots together!`)
 
-        await moveVideoFromPendingToCompleted(task.fileName)
+          await copyVideoFromPendingToCompleted(video.fileName)
 
-        task.hasAssembledVideo = true
+          video.hasAssembledVideo = true
 
-        await updatePendingTask(task)
-      } catch (err) {
-        console.error(`failed to assemble the shots together (${err})`)
-        // something is wrong, let's put the whole thing back into the queue
-        task.error = `failed to assemble the shots together (will try again later)`
-        await updatePendingTask(task)
-        return
+          if (await saveAndCheckIfNeedToStop(video)) { return }
+        } catch (err) {
+          console.error(`failed to assemble the shots together (${err})`)
+          // something is wrong, let's put the whole thing back into the queue
+          video.error = `failed to assemble the shots together (will try again later)`
+          if (await saveAndCheckIfNeedToStop(video)) { return }
+          return
+        }
       }
     }
 
     nbCompletedSteps++
-    task.progressPercent = 100
-    task.completed = true
-    task.completedAt = new Date().toISOString()
-    await updatePendingTask(task)
-
-    console.log(`moving task to completed tasks..`)
-    await saveCompletedTask(task)
+    video.completed = true
+    if (await saveAndCheckIfNeedToStop(video)) { return }
   }
 }

src/scheduler/readTask.mts

@@ -1,11 +0,0 @@
-import { promises as fs } from "node:fs"
-
-import { VideoTask } from "../types.mts"
-
-export const readTask = async (taskFilePath: string): Promise<VideoTask> => {
-  const task = JSON.parse(
-    await fs.readFile(taskFilePath, 'utf8')
-  ) as VideoTask
-
-  return task
-}

src/scheduler/readTasks.mts

@@ -1,45 +0,0 @@
-import path from "node:path"
-import { promises as fs } from "node:fs"
-
-import { VideoTask } from "../types.mts"
-import { readTask } from "./readTask.mts"
-
-
-export const readTasks = async (taskDirFilePath: string, ownerId?: string): Promise<VideoTask[]> => {
-
-  let tasksFiles: string[] = []
-  try {
-    const filesInDir = await fs.readdir(taskDirFilePath)
-    // console.log("filesInDir:", filesInDir)
-
-    // we only keep valid files (in UUID.json format)
-    tasksFiles = filesInDir.filter(fileName =>
-      fileName.match(/[a-z0-9\-_]\.json/i) && (ownerId ? fileName.includes(ownerId): true)
-    )
-
-    // console.log("tasksfiles:", tasksFiles)
-  } catch (err) {
-    console.log(`failed to read tasks: ${err}`)
-  }
-
-  const tasks: VideoTask[] = []
-
-  for (const taskFileName of tasksFiles) {
-    // console.log("taskFileName:", taskFileName)
-    const taskFilePath = path.join(taskDirFilePath, taskFileName)
-    try {
-      const task = await readTask(taskFilePath)
-      tasks.push(task)
-    } catch (parsingErr) {
-      console.log(`failed to read ${taskFileName}: ${parsingErr}`)
-      console.log(`deleting corrupted file ${taskFileName}`)
-      try {
-        await fs.unlink(taskFilePath)
-      } catch (unlinkErr) {
-        console.log(`failed to unlink ${taskFileName}: ${unlinkErr}`)
-      }
-    }
-  }
-
-  return tasks
-}

src/scheduler/readVideoMetadataFile.mts

@@ -0,0 +1,11 @@
+import { promises as fs } from "node:fs"
+
+import { Video } from "../types.mts"
+
+export const readVideoMetadataFile = async (videoMetadataFilePath: string): Promise<Video> => {
+  const video = JSON.parse(
+    await fs.readFile(videoMetadataFilePath, 'utf8')
+  ) as Video
+
+  return video
+}

src/scheduler/readVideoMetadataFiles.mts

@@ -0,0 +1,44 @@
+import path from "node:path"
+import { promises as fs } from "node:fs"
+
+import { Video } from "../types.mts"
+import { readVideoMetadataFile } from "./readVideoMetadataFile.mts"
+
+export const readVideoMetadataFiles = async (videoMetadataDirFilePath: string, ownerId?: string): Promise<Video[]> => {
+
+  let videosMetadataFiles: string[] = []
+  try {
+    const filesInDir = await fs.readdir(videoMetadataDirFilePath)
+    // console.log("filesInDir:", filesInDir)
+
+    // we only keep valid files (in UUID.json format)
+    videosMetadataFiles = filesInDir.filter(fileName =>
+      fileName.match(/[a-z0-9\-_]\.json/i) && (ownerId ? fileName.includes(ownerId): true)
+    )
+
+    // console.log("videosfiles:", videosFiles)
+  } catch (err) {
+    console.log(`failed to read videos: ${err}`)
+  }
+
+  const videos: Video[] = []
+
+  for (const videoMetadataFileName of videosMetadataFiles) {
+    // console.log("videoFileName:", videoFileName)
+    const videoMetadataFilePath = path.join(videoMetadataDirFilePath, videoMetadataFileName)
+    try {
+      const videoMetadata = await readVideoMetadataFile(videoMetadataFilePath)
+      videos.push(videoMetadata)
+    } catch (parsingErr) {
+      console.log(`failed to read ${videoMetadataFileName}: ${parsingErr}`)
+      console.log(`deleting corrupted file ${videoMetadataFileName}`)
+      try {
+        await fs.unlink(videoMetadataFilePath)
+      } catch (unlinkErr) {
+        console.log(`failed to unlink ${videoMetadataFileName}: ${unlinkErr}`)
+      }
+    }
+  }
+
+  return videos
+}

src/scheduler/saveAndCheckIfNeedToStop.mts

@@ -0,0 +1,68 @@
+import { Video } from "../types.mts"
+import { deleteVideo } from "./deleteVideo.mts"
+import { getVideoStatus } from "./getVideoStatus.mts"
+import { saveCompletedVideo } from "./saveCompletedVideo.mts"
+import { updatePendingVideo } from "./updatePendingVideo.mts"
+
+export const saveAndCheckIfNeedToStop = async (video: Video): Promise<boolean> => {
+  
+  const status = await getVideoStatus(video)
+  const isToDelete =  status === "delete"
+  const isToAbort = status === "abort"
+  const isToPause = status === "pause"
+
+  // well, normally no other process is supported to mark a video as "completed"
+  // while we are busy processing it
+  // but maybe in the future, we can afford to waste procesing power to do the "who goes faster"..?
+  // const isCompleted = status === "completed"
+  
+  const mustStop = isToAbort || isToPause || isToDelete
+
+  // deletion is the most priority check, as we just need to ignore all the rest
+  if (isToDelete) {
+    await deleteVideo(video.ownerId, video.id)
+    return mustStop
+  }
+
+  // then we give priority to the pending video: maybe it is done?
+  if (video.completed) {
+    console.log(`video ${video.id} is completed!`)
+    video.progressPercent = 100
+    video.completedAt = new Date().toISOString()
+    video.status = "completed"
+    await updatePendingVideo(video)
+    await saveCompletedVideo(video)
+    return mustStop
+  }
+
+
+  if (isToPause) {
+    console.log(`we've been requested to pause the video`)
+    video.status = "pause"
+    await updatePendingVideo(video)
+    return mustStop
+  }
+  
+  if (isToAbort) {
+    console.log(`we've been requested to cancel the video`)
+    
+    // we are not going to update the percentage, because we want to keep the
+    // info that we aborted mid-course
+    video.completed = true
+
+    // watch what we do here: we mark the video as completed
+    // that's because "abort" is a temporary status
+    video.status = "completed"
+
+    video.completedAt = new Date().toISOString()
+    await updatePendingVideo(video)
+    await saveCompletedVideo(video)
+
+    return mustStop
+  }
+
+  await updatePendingVideo(video)
+  
+  // tell the loop if it should be aborted or not
+  return mustStop
+}

src/scheduler/saveCompletedTask.mts

@@ -1,12 +0,0 @@
-import path from "path"
-
-import { VideoTask } from "../types.mts"
-import { completedTasksDirFilePath, pendingTasksDirFilePath } from "../config.mts"
-import { moveFile } from "../utils/moveFile.mts"
-
-export const saveCompletedTask = async (task: VideoTask) => {
-  const fileName = `${task.ownerId}_${task.id}.json`
-  const pendingFilePath = path.join(pendingTasksDirFilePath, fileName)
-  const completedFilePath = path.join(completedTasksDirFilePath, fileName)
-  await moveFile(pendingFilePath, completedFilePath)
-}

src/scheduler/saveCompletedVideo.mts

@@ -0,0 +1,12 @@
+import path from "path"
+
+import { Video } from "../types.mts"
+import { completedMetadataDirFilePath, pendingMetadataDirFilePath } from "../config.mts"
+import { moveFile } from "../utils/moveFile.mts"
+
+export const saveCompletedVideo = async (video: Video) => {
+  const metadataFileName = `${video.ownerId}_${video.id}.json`
+  const pendingMetadataFilePath = path.join(pendingMetadataDirFilePath, metadataFileName)
+  const completedMetadataFilePath = path.join(completedMetadataDirFilePath, metadataFileName)
+  await moveFile(pendingMetadataFilePath, completedMetadataFilePath)
+}

src/scheduler/savePendingTask.mts

@@ -1,11 +0,0 @@
-import { promises as fs } from "node:fs"
-import path from "path"
-
-import { VideoTask } from "../types.mts"
-import { pendingTasksDirFilePath } from "../config.mts"
-
-export const savePendingTask = async (task: VideoTask) => {
-  const fileName = `${task.ownerId}_${task.id}.json`
-  const filePath = path.join(pendingTasksDirFilePath, fileName)
-  await fs.writeFile(filePath, JSON.stringify(task, null, 2), "utf8")
-}

src/scheduler/savePendingVideo.mts

@@ -0,0 +1,11 @@
+import { promises as fs } from "node:fs"
+import path from "path"
+
+import { Video } from "../types.mts"
+import { pendingMetadataDirFilePath } from "../config.mts"
+
+export const savePendingVideo = async (video: Video) => {
+  const fileName = `${video.ownerId}_${video.id}.json`
+  const filePath = path.join(pendingMetadataDirFilePath, fileName)
+  await fs.writeFile(filePath, JSON.stringify(video, null, 2), "utf8")
+}

src/scheduler/{updatePendingTask.mts → updatePendingVideo.mts}

@@ -1,16 +1,16 @@
 import { promises as fs } from "node:fs"
 import path from "path"
 
-import { VideoTask } from "../types.mts"
-import { pendingTasksDirFilePath } from "../config.mts"
+import { Video } from "../types.mts"
+import { pendingMetadataDirFilePath } from "../config.mts"
 
-export const updatePendingTask = async (task: VideoTask) => {
+export const updatePendingVideo = async (video: Video) => {
   try {
-    const fileName = `${task.ownerId}_${task.id}.json`
-    const filePath = path.join(pendingTasksDirFilePath, fileName)
-    await fs.writeFile(filePath, JSON.stringify(task, null, 2), "utf8")
+    const fileName = `${video.ownerId}_${video.id}.json`
+    const filePath = path.join(pendingMetadataDirFilePath, fileName)
+    await fs.writeFile(filePath, JSON.stringify(video, null, 2), "utf8")
   } catch (err) {
-    console.error(`Failed to update the task. Probably an issue with the serialized object or the file system: ${err}`)
+    console.error(`Failed to update the video. Probably an issue with the serialized object or the file system: ${err}`)
     // we do not forward the exception, there is no need
     // we will just try again the job later (even if it means losing a bit of data)
   }

src/tests/checkStatus.mts

@@ -9,6 +9,6 @@ const response = await fetch(`${server}/${videoId}`, {
 });
 
 console.log('response:', response)
-const task = await response.json()
+const video = await response.json()
 
-console.log("task:", JSON.stringify(task, null, 2))
+console.log("video:", JSON.stringify(video, null, 2))

src/tests/submitVideo.mts

@@ -18,6 +18,6 @@ const response = await fetch(`${server}/`, {
 
 
 console.log('response:', response)
-const task = await response.json()
+const video = await response.json()
 
-console.log("task:", JSON.stringify(task, null, 2))
+console.log("video:", JSON.stringify(video, null, 2))