study of potential ComfyUI integration

package-lock.json

@@ -1,15 +1,16 @@
 {
   "name": "@jbilcke-hf/clap-viewer",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@jbilcke-hf/clap-viewer",
-      "version": "0.0.1",
+      "version": "0.0.2",
       "dependencies": {
-        "@aitube/clap": "0.0.23",
-        "@aitube/timeline": "0.0.6",
+        "@aitube/clap": "0.0.24",
+        "@aitube/engine": "0.0.15",
+        "@aitube/timeline": "0.0.7",
         "@huggingface/hub": "^0.15.0",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.4",
@@ -45,7 +46,7 @@
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.0",
         "cmdk": "^0.2.1",
-        "eslint": "8.56.0",
+        "eslint": "8.57.0",
         "eslint-config-next": "14.1.0",
         "lucide-react": "^0.334.0",
         "next": "^14.2.3",
@@ -60,6 +61,7 @@
         "react-hook-consent": "^3.5.3",
         "react-icons": "^5.2.1",
         "react-reflex": "^4.2.6",
+        "replicate": "^0.30.1",
         "sharp": "^0.33.4",
         "sonner": "^1.4.41",
         "tailwind-merge": "^2.3.0",
@@ -73,30 +75,37 @@
         "uuid": "^9.0.1",
         "yaml": "^2.4.2",
         "zustand": "^4.5.2"
-      },
-      "devDependencies": {}
+      }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.23",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.23.tgz",
-      "integrity": "sha512-+XOsWUgEax8yhmCBuWDFETh6mJTT5gYBNf8rChP55qX6le8bFxSBOdLirEdQ+SLnOLwRNthgCF4tWcT3GOLMVw==",
+      "version": "0.0.24",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.24.tgz",
+      "integrity": "sha512-iMmxyPmVh1LWNUlWKK2rRnCSZ/CLmBeb1ADNr5pj0dFI51NNezipYXjZv9dBOb5y3FKBkL+jbEcy7kQFgKHhpw==",
       "dependencies": {
         "pure-uuid": "^1.8.1"
       },
       "peerDependencies": {
-        "yaml": "^2.4.2"
+        "yaml": "^2.4.3"
+      }
+    },
+    "node_modules/@aitube/engine": {
+      "version": "0.0.15",
+      "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.15.tgz",
+      "integrity": "sha512-SxlTGOLHNPh93PY/hCXuNgCUWV7Uky9Q4mBKS4uA7+gReeYBT//x/sAWqIvV8MEj2dkx1Br3iTse1/rcVay49w==",
+      "peerDependencies": {
+        "@aitube/clap": "0.0.24"
       }
     },
     "node_modules/@aitube/timeline": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.6.tgz",
-      "integrity": "sha512-0R8JgaylnY4sJL2WhE4Yx6U2+R7G9ifaUmTCt1XpcmB3dLHUicNH4i8Wgmbndnvr7qR7QzICm2SWI5sgiVAT4g==",
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.7.tgz",
+      "integrity": "sha512-eQbp2fcrazuW6prKfGAfcEVNvHgO+ZVgh5aHJXSQfCDty8dLw54MlvbsZw9RXntxMjGu3OzqgC6PCSAZldebFQ==",
       "dependencies": {
         "date-fns": "^3.6.0",
         "react-virtualized-auto-sizer": "^1.0.24"
       },
       "peerDependencies": {
-        "@aitube/clap": "0.0.23",
+        "@aitube/clap": "0.0.24",
         "@radix-ui/react-slider": "^1.1.2",
         "@react-spring/three": "^9.7.3",
         "@react-spring/types": "^9.7.3",
@@ -631,9 +640,9 @@
       }
     },
     "node_modules/@eslint/js": {
-      "version": "8.56.0",
-      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.56.0.tgz",
-      "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==",
+      "version": "8.57.0",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz",
+      "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==",
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
@@ -3092,6 +3101,18 @@
         "react": ">= 16.8.0"
       }
     },
+    "node_modules/abort-controller": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
+      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+      "optional": true,
+      "dependencies": {
+        "event-target-shim": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=6.5"
+      }
+    },
     "node_modules/acorn": {
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@@ -3325,15 +3346,18 @@
       }
     },
     "node_modules/array.prototype.tosorted": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.3.tgz",
-      "integrity": "sha512-/DdH4TiTmOKzyQbp/eadcCVexiCb36xJg7HshYOYJnNZFDj33GEv0P7GxsynpShhq4OLYJzbGcBDkLsDt7MnNg==",
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.4.tgz",
+      "integrity": "sha512-p6Fx8B7b7ZhL/gmUsAy0D15WhvDccw3mnGNbZpi3pmeJdxtWsj2jEaI4Y6oo3XiHfzuSgPwKc04MYt6KgvC/wA==",
       "dependencies": {
-        "call-bind": "^1.0.5",
+        "call-bind": "^1.0.7",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.22.3",
-        "es-errors": "^1.1.0",
+        "es-abstract": "^1.23.3",
+        "es-errors": "^1.3.0",
         "es-shim-unscopables": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
       }
     },
     "node_modules/arraybuffer.prototype.slice": {
@@ -3627,9 +3651,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001626",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001626.tgz",
-      "integrity": "sha512-JRW7kAH8PFJzoPCJhLSHgDgKg5348hsQ68aqb+slnzuB5QFERv846oA/mRChmlLAOdEDeOkRn3ynb1gSFnjt3w==",
+      "version": "1.0.30001627",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001627.tgz",
+      "integrity": "sha512-4zgNiB8nTyV/tHhwZrFs88ryjls/lHiqFhrxCW4qSTeuRByBVnPYpDInchOIySWknznucaf31Z4KYqjfbrecVw==",
       "funding": [
         {
           "type": "opencollective",
@@ -4614,15 +4638,15 @@
       }
     },
     "node_modules/eslint": {
-      "version": "8.56.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.56.0.tgz",
-      "integrity": "sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==",
+      "version": "8.57.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz",
+      "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==",
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.6.1",
         "@eslint/eslintrc": "^2.1.4",
-        "@eslint/js": "8.56.0",
-        "@humanwhocodes/config-array": "^0.11.13",
+        "@eslint/js": "8.57.0",
+        "@humanwhocodes/config-array": "^0.11.14",
         "@humanwhocodes/module-importer": "^1.0.1",
         "@nodelib/fs.walk": "^1.2.8",
         "@ungap/structured-clone": "^1.2.0",
@@ -5049,6 +5073,24 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/event-target-shim": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
+      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
+      "optional": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "optional": true,
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -6353,9 +6395,9 @@
       }
     },
     "node_modules/meshline": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/meshline/-/meshline-3.2.1.tgz",
-      "integrity": "sha512-/Zrhq1sbQCtqSsrud0hN/U8wOdKKOcxCWefEowZRHsosIcy1p87+2PlWSNO4s9zOoT/zjrQR8YikXYao8XCqVQ==",
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/meshline/-/meshline-3.3.1.tgz",
+      "integrity": "sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ==",
       "peerDependencies": {
         "three": ">=0.137"
       }
@@ -7126,6 +7168,15 @@
         "url": "https://github.com/prettier/prettier?sponsor=1"
       }
     },
+    "node_modules/process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
+      "optional": true,
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
     "node_modules/promise-worker-transferable": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/promise-worker-transferable/-/promise-worker-transferable-1.0.4.tgz",
@@ -7464,6 +7515,22 @@
         "pify": "^2.3.0"
       }
     },
+    "node_modules/readable-stream": {
+      "version": "4.5.2",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
+      "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
+      "optional": true,
+      "dependencies": {
+        "abort-controller": "^3.0.0",
+        "buffer": "^6.0.3",
+        "events": "^3.3.0",
+        "process": "^0.11.10",
+        "string_decoder": "^1.3.0"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      }
+    },
     "node_modules/readdirp": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -7517,6 +7584,20 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/replicate": {
+      "version": "0.30.1",
+      "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.30.1.tgz",
+      "integrity": "sha512-9JZPia6p7UCNfye2kddb2AHcTf3ngyckzq4y5FZkuhhF06AuuOHKN1q6NRSp9iAch7Htmlhm+kRGc+EQbzY8Kw==",
+      "engines": {
+        "git": ">=2.11.0",
+        "node": ">=18.0.0",
+        "npm": ">=7.19.0",
+        "yarn": ">=1.7.0"
+      },
+      "optionalDependencies": {
+        "readable-stream": ">=4.0.0"
+      }
+    },
     "node_modules/require-from-string": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
@@ -7665,6 +7746,26 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "optional": true
+    },
     "node_modules/safe-regex-test": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz",
@@ -7914,6 +8015,15 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/string_decoder": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+      "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+      "optional": true,
+      "dependencies": {
+        "safe-buffer": "~5.2.0"
+      }
+    },
     "node_modules/string-width": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",

package.json

@@ -10,8 +10,9 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.23",
-    "@aitube/timeline": "0.0.6",
+    "@aitube/clap": "0.0.24",
+    "@aitube/engine": "0.0.15",
+    "@aitube/timeline": "0.0.7",
     "@huggingface/hub": "^0.15.0",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
@@ -47,7 +48,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "cmdk": "^0.2.1",
-    "eslint": "8.56.0",
+    "eslint": "8.57.0",
     "eslint-config-next": "14.1.0",
     "lucide-react": "^0.334.0",
     "next": "^14.2.3",
@@ -62,6 +63,7 @@
     "react-hook-consent": "^3.5.3",
     "react-icons": "^5.2.1",
     "react-reflex": "^4.2.6",
+    "replicate": "^0.30.1",
     "sharp": "^0.33.4",
     "sonner": "^1.4.41",
     "tailwind-merge": "^2.3.0",
@@ -75,6 +77,5 @@
     "uuid": "^9.0.1",
     "yaml": "^2.4.2",
     "zustand": "^4.5.2"
-  },
-  "devDependencies": {}
+  }
 }

src/components/interface/settings/constants.tsx

@@ -0,0 +1,13 @@
+import { Settings } from "@/types"
+
+const version = `CLAP_CONFIG_V0_`
+
+export const localStorageKeys: Record<keyof Settings, string> = {
+  comfyVendor: `${version}COMFY_VENDOR`,
+  comfyApiKey: `${version}COMFY_API_KEY`,
+
+  storyboardGenerationStrategy: `${version}STORYBOARD_GENERATION_STRATEGY`,
+  videoGenerationStrategy: `${version}VIDEO_GENERATION_STRATEEGY`,
+
+  maxNbAssetsToGenerateInParallel: `${version}MAX_NB_ASSETS_TO_GENERATE_IN_PARQLLEL`,
+}

src/lib/core/getDefaultSettings.ts

@@ -0,0 +1,13 @@
+import { AssetGenerationStrategy, ComfyVendor, Settings } from "@/types"
+
+export function getDefaultSettings(): Settings {
+  return {
+    comfyVendor: ComfyVendor.CUSTOM,
+    comfyApiKey: "",
+
+    storyboardGenerationStrategy: AssetGenerationStrategy.ON_DEMAND,
+    videoGenerationStrategy: AssetGenerationStrategy.ON_DEMAND,
+
+    maxNbAssetsToGenerateInParallel: 1,
+  }
+  }

src/lib/core/index.ts

@@ -0,0 +1,2 @@
+export { getFinalVideo } from "./getFinalVideo"
+export { removeFinalVideos } from "./removeFinalVideos"

src/server/comfy/formatStoryboardWorkflow.ts

@@ -0,0 +1,23 @@
+import { getVideoPrompt } from "@aitube/engine"
+
+import { ComfyNode, RenderRequest } from "@/types"
+
+export function formatStoryboardWorkflow(request: RenderRequest) {
+  // parse the node array from the ComfyUI workflow
+  const nodes = Object.values(JSON.parse(request.comfyWorkflow)) as ComfyNode[]
+
+  const storyboardPrompt = getVideoPrompt(
+    request.segments,
+    request.entities
+  )
+
+  for (const node of nodes) {
+    if (typeof node.inputs.text === "string") {
+      if (node._meta.title.includes("Prompt")) {
+        node.inputs.text = storyboardPrompt
+      }
+    }
+  }
+
+  return JSON.stringify(nodes)
+}

src/server/comfy/huggingface.ts

@@ -0,0 +1,15 @@
+"use server"
+
+
+export async function run({
+  apiKey,
+  workflow
+}: {
+  apiKey: string
+  workflow: string
+}): Promise<string> {
+
+  // TODO: call Hugging Face
+   
+  throw new Error(`Hugging Face isn't supported yet`)
+}

src/server/comfy/index.ts

@@ -0,0 +1,21 @@
+"use server"
+
+import { RenderRequest } from "@/types"
+
+import { run as runWithReplicate } from "./replicate"
+import { run as runWithHuggingFace } from "./huggingface"
+import { formatStoryboardWorkflow } from "./formatStoryboardWorkflow"
+
+export async function run(request: RenderRequest): Promise<string> {
+
+  const workflow = formatStoryboardWorkflow(request)
+
+  // TODO support Hugging Face as well
+  // const await runWithHuggingFace({
+  const result = await runWithReplicate({
+    apiKey: request.comfyApiKey,
+    workflow,
+  })
+
+  return result
+}

src/server/comfy/replicate.ts

@@ -0,0 +1,26 @@
+"use server"
+
+import Replicate from 'replicate'
+
+export async function run({
+  apiKey,
+  workflow
+}: {
+  apiKey: string
+  workflow: string
+}): Promise<string> {
+
+  const replicate = new Replicate({ auth: apiKey })
+
+  const cogId = "fofr/any-comfyui-workflow:74f12621dc9f9b7cdca50d03941b8ddb3a368d7f5a1bb16fb7e1b87f05d96bf5"
+
+  const output = await replicate.run(cogId, {
+    input: {
+      workflow_json: workflow
+    }
+  })
+
+  console.log(`response:`, output)
+
+  return ""
+}

src/types.ts

@@ -1,3 +1,5 @@
+import { ClapEntity, ClapSegment } from "@aitube/clap"
+
 export type TaskStatus =
   | "idle"
   | "generating"
@@ -21,3 +23,85 @@ export type GenerationStage =
   | "videos"
   | "final"
   | "idle"
+
+export enum ComfyVendor {
+  CUSTOM = "CUSTOM",
+  HUGGINGFACE = "HUGGINGFACE",
+  REPLICATE = "REPLICATE",
+}
+
+export enum AssetGenerationStrategy {
+
+  // generate assets when the user asks for it explicitely
+  ON_DEMAND = "ON_DEMAND_",
+
+  // generate missing assets currently visible on screen, and that's all
+  ON_SCREEN_ONLY = "ON_SCREEN_ONLY",
+
+  // generate missing assets visible on screen first,
+  // then pre-generate surrouding assets (a bit before and after)
+  ON_SCREEN_THEN_SURROUNDING = "ON_SCREEN_THEN_SURROUNDING",
+
+  // generate missing assets visible on screen first,
+  // then pre-generate *ALL* the remaining project's assets
+  //
+  // this is hardcore! only GPU-rich people shoud use this one!
+  ON_SCREEN_THEN_ALL = "ON_SCREEN_THEN_ALL",
+}
+
+// TODO: rename the *Model fields to better indicate if this is a LLM or RENDER mdoel
+export type Settings = {
+  comfyVendor: ComfyVendor
+  comfyApiKey: string
+
+  storyboardGenerationStrategy: AssetGenerationStrategy
+  videoGenerationStrategy: AssetGenerationStrategy
+
+  maxNbAssetsToGenerateInParallel: number
+}
+
+export type RenderRequest = {
+  comfyVendor: ComfyVendor
+
+  // secret vendor api key to use (provided by the user)
+  comfyApiKey: string
+
+  // URL to use for the vendor
+  // vendorUrl: string
+
+  // ComfyUI workflow to use
+  comfyWorkflow: string
+
+  // the slice to use for rendering
+  segments: ClapSegment[]
+
+  entities: Record<string, ClapEntity>
+}
+
+export type ComfyParameter =
+  | string
+  | number
+  | boolean
+  | Array< string | number | boolean>
+
+export type ComfyNode = {
+  inputs: Record<string, ComfyParameter>
+  /*
+  inputs: {
+    "seed": number
+    "steps": number
+    "cfg": number
+    "sampler_name":string
+    "scheduler": string
+    "denoise": number
+    "model": any[]
+    "positive": any[]
+    "negative": any[]
+    "latent_image": any[]
+  }
+    */
+  class_type: string
+  _meta: {
+    title: string
+  }
+}