preparation work for the video-to-clap project

next.config.js

@@ -15,6 +15,15 @@ const nextConfig = {
     //   The image may be corrupted or an unsupported format.
     unoptimized: true,
   },
+  // workaround for transformers.js issues
+  webpack: (config) => {
+    config.resolve.alias = {
+        ...config.resolve.alias,
+        "sharp$": false,
+        "onnxruntime-node$": false,
+    }
+    return config;
+},
   async headers() {
     return [
       {

package-lock.json

@@ -57,6 +57,7 @@
         "@types/dom-speech-recognition": "^0.0.4",
         "@upstash/ratelimit": "^1.1.3",
         "@upstash/redis": "^1.31.1",
+        "@xenova/transformers": "github:xenova/transformers.js#v3",
         "autoprefixer": "10.4.19",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.1",
@@ -68,6 +69,7 @@
         "fs-extra": "^11.2.0",
         "is-hotkey": "^0.2.0",
         "lucide-react": "^0.396.0",
+        "mediainfo.js": "^0.3.2",
         "mlt-xml": "^2.0.2",
         "monaco-editor": "^0.50.0",
         "next": "^14.2.5",
@@ -2253,6 +2255,14 @@
       "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.2.tgz",
       "integrity": "sha512-vlwUJsj/QJcR/oLXvV+JBKheaVk9pqfAPYiS136cjHEDTeTW5/+ePpM6uKOc56oxqwrUjh5T0JylHJU8vyqr1A=="
     },
+    "node_modules/@huggingface/jinja": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.2.2.tgz",
+      "integrity": "sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@huggingface/tasks": {
       "version": "0.10.22",
       "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.22.tgz",
@@ -2864,6 +2874,18 @@
         "url": "https://github.com/chalk/strip-ansi?sponsor=1"
       }
     },
+    "node_modules/@isaacs/fs-minipass": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
+      "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+      "optional": true,
+      "dependencies": {
+        "minipass": "^7.0.4"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/@jridgewell/gen-mapping": {
       "version": "0.3.5",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
@@ -3406,6 +3428,60 @@
         "url": "https://opencollective.com/preact"
       }
     },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
+    },
     "node_modules/@radix-ui/number": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.0.tgz",
@@ -6176,6 +6252,19 @@
         "url": "https://opencollective.com/vitest"
       }
     },
+    "node_modules/@xenova/transformers": {
+      "version": "3.0.0-alpha.0",
+      "resolved": "git+ssh://git@github.com/xenova/transformers.js.git#1b4d2428225ef8f63be94bfa38a0d7fd81ac7c0c",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@huggingface/jinja": "^0.2.2",
+        "onnxruntime-web": "^1.18.0",
+        "sharp": "^0.33.2"
+      },
+      "optionalDependencies": {
+        "onnxruntime-node": "^1.18.0"
+      }
+    },
     "node_modules/abort-controller": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
@@ -6991,6 +7080,15 @@
         "node": ">= 6"
       }
     },
+    "node_modules/chownr": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
+      "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+      "optional": true,
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/class-variance-authority": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.0.tgz",
@@ -9045,6 +9143,11 @@
         "node": "^10.12.0 || >=12.0.0"
       }
     },
+    "node_modules/flatbuffers": {
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz",
+      "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ=="
+    },
     "node_modules/flatted": {
       "version": "3.3.1",
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
@@ -9592,6 +9695,11 @@
         "node": ">=12.0.0"
       }
     },
+    "node_modules/guid-typescript": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
+      "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="
+    },
     "node_modules/has-bigints": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
@@ -10772,6 +10880,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/long": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
+      "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q=="
+    },
     "node_modules/loose-envify": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
@@ -10851,6 +10964,20 @@
         "is-buffer": "~1.1.6"
       }
     },
+    "node_modules/mediainfo.js": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/mediainfo.js/-/mediainfo.js-0.3.2.tgz",
+      "integrity": "sha512-SC8z72ESV1z2lq2zfheoo9zgcoqjeBn0mzq6MhIn3aqlkh3RV84FwOiMtYA0HWY7mi1igM89Jcll4r2sk/yyZA==",
+      "dependencies": {
+        "yargs": "^17.7.2"
+      },
+      "bin": {
+        "mediainfo.js": "dist/esm/cli.js"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/merge-stream": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@@ -10946,6 +11073,52 @@
         "node": ">=16 || 14 >=14.17"
       }
     },
+    "node_modules/minizlib": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz",
+      "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==",
+      "optional": true,
+      "dependencies": {
+        "minipass": "^7.0.4",
+        "rimraf": "^5.0.5"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/minizlib/node_modules/rimraf": {
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.9.tgz",
+      "integrity": "sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==",
+      "optional": true,
+      "dependencies": {
+        "glob": "^10.3.7"
+      },
+      "bin": {
+        "rimraf": "dist/esm/bin.mjs"
+      },
+      "engines": {
+        "node": "14 >=14.20 || 16 >=16.20 || >=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/mkdirp": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
+      "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
+      "optional": true,
+      "bin": {
+        "mkdirp": "dist/cjs/src/bin.js"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/ml-array-mean": {
       "version": "1.1.6",
       "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz",
@@ -11540,6 +11713,40 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/onnxruntime-common": {
+      "version": "1.18.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.18.0.tgz",
+      "integrity": "sha512-lufrSzX6QdKrktAELG5x5VkBpapbCeS3dQwrXbN0eD9rHvU0yAWl7Ztju9FvgAKWvwd/teEKJNj3OwM6eTZh3Q=="
+    },
+    "node_modules/onnxruntime-node": {
+      "version": "1.18.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.18.0.tgz",
+      "integrity": "sha512-iTnFcxKpmywCatx8ov4GTbECe3tJk2Bp1OA2mWRJde78q+7tpPYBhKMnwhlaoKy9oKQcy4UoEuuhoy2PSD13ww==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "win32",
+        "darwin",
+        "linux"
+      ],
+      "dependencies": {
+        "onnxruntime-common": "1.18.0",
+        "tar": "^7.0.1"
+      }
+    },
+    "node_modules/onnxruntime-web": {
+      "version": "1.18.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.18.0.tgz",
+      "integrity": "sha512-o1UKj4ABIj1gmG7ae0RKJ3/GT+3yoF0RRpfDfeoe0huzRW4FDRLfbkDETmdFAvnJEXuYDE0YT+hhkia0352StQ==",
+      "dependencies": {
+        "flatbuffers": "^1.12.0",
+        "guid-typescript": "^1.0.9",
+        "long": "^5.2.3",
+        "onnxruntime-common": "1.18.0",
+        "platform": "^1.3.6",
+        "protobufjs": "^7.2.4"
+      }
+    },
     "node_modules/openai": {
       "version": "4.52.7",
       "resolved": "https://registry.npmjs.org/openai/-/openai-4.52.7.tgz",
@@ -11868,6 +12075,11 @@
         "node": ">= 6"
       }
     },
+    "node_modules/platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="
+    },
     "node_modules/playwright": {
       "version": "1.45.2",
       "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.45.2.tgz",
@@ -12230,6 +12442,29 @@
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
       "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
     },
+    "node_modules/protobufjs": {
+      "version": "7.3.2",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.3.2.tgz",
+      "integrity": "sha512-RXyHaACeqXeqAKGLDl68rQKbmObRsTIn4TYVUUug1KfS47YWCo5MacGITEryugIgZqORCvJWEk4l449POg5Txg==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/psl": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
@@ -13704,6 +13939,32 @@
         "node": ">=6"
       }
     },
+    "node_modules/tar": {
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz",
+      "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==",
+      "optional": true,
+      "dependencies": {
+        "@isaacs/fs-minipass": "^4.0.0",
+        "chownr": "^3.0.0",
+        "minipass": "^7.1.2",
+        "minizlib": "^3.0.1",
+        "mkdirp": "^3.0.1",
+        "yallist": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/tar/node_modules/yallist": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
+      "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+      "optional": true,
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",

package.json

@@ -68,6 +68,7 @@
     "@types/dom-speech-recognition": "^0.0.4",
     "@upstash/ratelimit": "^1.1.3",
     "@upstash/redis": "^1.31.1",
+    "@xenova/transformers": "github:xenova/transformers.js#v3",
     "autoprefixer": "10.4.19",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
@@ -79,6 +80,7 @@
     "fs-extra": "^11.2.0",
     "is-hotkey": "^0.2.0",
     "lucide-react": "^0.396.0",
+    "mediainfo.js": "^0.3.2",
     "mlt-xml": "^2.0.2",
     "monaco-editor": "^0.50.0",
     "next": "^14.2.5",

public/workers/captioning.worker.js

@@ -0,0 +1,54 @@
+importScripts('https://cdn.jsdelivr.net/npm/@xenova/transformers@3.0.0');
+
+const { AutoProcessor, AutoTokenizer, Moondream1ForConditionalGeneration, RawImage } = transformers;
+
+let processor;
+let tokenizer;
+let model;
+
+async function initializeModel() {
+  const model_id = 'Xenova/moondream2';
+  processor = await AutoProcessor.from_pretrained(model_id);
+  tokenizer = await AutoTokenizer.from_pretrained(model_id);
+  model = await Moondream1ForConditionalGeneration.from_pretrained(model_id, {
+    dtype: {
+      embed_tokens: 'fp16',
+      vision_encoder: 'fp16',
+      decoder_model_merged: 'q4',
+    },
+    device: 'webgpu',
+  });
+}
+
+async function captionImage(imageDataUrl) {
+  if (!processor || !tokenizer || !model) {
+    await initializeModel();
+  }
+
+  const prompt = 'Describe this image.';
+  const text = `<image>\n\nQuestion: ${prompt}\n\nAnswer:`;
+  const text_inputs = tokenizer(text);
+
+  const image = await RawImage.fromURL(imageDataUrl);
+  const vision_inputs = await processor(image);
+
+  const output = await model.generate({
+    ...text_inputs,
+    ...vision_inputs,
+    do_sample: false,
+    max_new_tokens: 64,
+  });
+
+  const decoded = tokenizer.batch_decode(output, { skip_special_tokens: true });
+  return decoded[0].trim();
+}
+
+self.addEventListener('message', async (event) => {
+  const { imageDataUrl } = event.data;
+  try {
+    const caption = await captionImage(imageDataUrl);
+    self.postMessage({ caption });
+  } catch (error) {
+    self.postMessage({ error: error.message });
+  }
+});

src/services/io/extractCaptionFromFrame.ts

@@ -0,0 +1,68 @@
+import {
+  AutoProcessor,
+  AutoTokenizer,
+  Moondream1ForConditionalGeneration,
+  RawImage,
+} from '@xenova/transformers'
+
+export async function extractCaptionFromFrame(
+  imageInBase64DataUri: string
+): Promise<string> {
+  if (!(navigator as any).gpu) {
+    throw new Error(`Please enable WebGPU to analyze video frames:
+    
+1. You need a modern browser such as Google Chrome 113+, Microsoft Edge 113+, Safari 18 (macOS 15), Firefox Nightly
+
+2. You need to enable WebGPU (depends on your browser, see below)
+
+2.1 For Chrome: Perform the following operations in the Chrome / Microsoft Edge address bar
+The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features). Linux experimental support also requires launching the browser with --enable-features=Vulkan.
+
+2.2 For Safari 18 (macOS 15): WebGPU is enabled by default
+
+2.3 For Firefox Nightly: Type about:config in the address bar and set 'dom.webgpu.enabled" to true
+`)
+  }
+
+  // Load processor, tokenizer and model
+  const model_id = 'Xenova/moondream2'
+  const processor = await AutoProcessor.from_pretrained(model_id)
+  const tokenizer = await AutoTokenizer.from_pretrained(model_id)
+  const model = await Moondream1ForConditionalGeneration.from_pretrained(
+    model_id,
+    {
+      dtype: {
+        embed_tokens: 'fp16', // or 'fp32'
+        vision_encoder: 'fp16', // or 'q8'
+        decoder_model_merged: 'q4', // or 'q4f16' or 'q8'
+      },
+      device: 'webgpu',
+    }
+  )
+
+  // Prepare text inputs
+  const prompt = 'Describe this image.'
+  const text = `<image>\n\nQuestion: ${prompt}\n\nAnswer:`
+  const text_inputs = tokenizer(text)
+
+  // Prepare vision inputs
+  const image = await RawImage.fromURL(imageInBase64DataUri)
+  const vision_inputs = await processor(image)
+
+  // Generate response
+  const output = await model.generate({
+    ...text_inputs,
+    ...vision_inputs,
+    do_sample: false,
+    max_new_tokens: 177,
+  })
+  const decoded = tokenizer.batch_decode(output, { skip_special_tokens: false })
+  console.log(decoded)
+  // [
+  //     '<|endoftext|><image>\n\n' +
+  //     'Question: Describe this image.\n\n' +
+  //     'Answer: A hand is holding a white book titled "The Little Book of Deep Learning" against a backdrop of a balcony with a railing and a view of a building and trees.<|endoftext|>'
+  // ]
+
+  return `${decoded[0] || ''}`
+}

src/services/io/extractFramesFromVideo.ts

@@ -0,0 +1,141 @@
+import { FFmpeg } from '@ffmpeg/ffmpeg'
+import { toBlobURL } from '@ffmpeg/util'
+import mediaInfoFactory, {
+  Track,
+  GeneralTrack,
+  VideoTrack,
+  AudioTrack,
+  TextTrack,
+  ImageTrack,
+  MenuTrack,
+  OtherTrack,
+} from 'mediainfo.js'
+
+interface FrameExtractorOptions {
+  format: 'png' | 'jpg'
+  maxWidth: number
+  maxHeight: number
+  sceneSamplingRate: number // Percentage of additional frames between scene changes (0-100)
+  onProgress?: (progress: number) => void // Callback function for progress updates
+}
+
+async function extractFramesFromVideo(
+  videoBlob: Blob,
+  options: FrameExtractorOptions
+): Promise<string[]> {
+  // Initialize MediaInfo
+  const mediaInfo = await mediaInfoFactory({ format: 'object' })
+
+  // Get video duration using MediaInfo
+  const getSize = () => videoBlob.size
+  const readChunk = (chunkSize: number, offset: number) =>
+    new Promise<Uint8Array>((resolve, reject) => {
+      const reader = new FileReader()
+      reader.onload = (event) => {
+        if (event.target?.result instanceof ArrayBuffer) {
+          resolve(new Uint8Array(event.target.result))
+        } else {
+          reject(new Error('Failed to read chunk'))
+        }
+      }
+      reader.onerror = (error) => reject(error)
+      reader.readAsArrayBuffer(videoBlob.slice(offset, offset + chunkSize))
+    })
+
+  const result = await mediaInfo.analyzeData(getSize, readChunk)
+
+  let duration: number = 0
+
+  for (const track of result.media?.track || []) {
+    ///  '@type': "General" | "Video" | "Audio" | "Text" | "Image" | "Menu" | "Other"
+    let maybeDuration: number = 0
+    if (track['@type'] === 'Audio') {
+      const audioTrack = track as AudioTrack
+      maybeDuration = audioTrack.Duration || 0
+    } else if (track['@type'] === 'Video') {
+      const videoTrack = track as VideoTrack
+      maybeDuration = videoTrack.Duration || 0
+    }
+    if (
+      typeof maybeDuration === 'number' &&
+      isFinite(maybeDuration) &&
+      !isNaN(maybeDuration)
+    ) {
+      duration = maybeDuration
+    }
+  }
+
+  if (!duration) {
+    throw new Error('Could not determine video duration (or it is length 0)')
+  }
+
+  // Initialize FFmpeg
+  const ffmpeg = new FFmpeg()
+  const baseURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd'
+
+  await ffmpeg.load({
+    coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, 'text/javascript'),
+    wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, 'application/wasm'),
+  })
+
+  // Write video file to FFmpeg's file system
+  const videoUint8Array = new Uint8Array(await videoBlob.arrayBuffer())
+  await ffmpeg.writeFile('input.mp4', videoUint8Array)
+
+  // Prepare FFmpeg command
+  const sceneFilter = `select='gt(scene,0.4)'`
+  const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'`
+  const scaleFilter = `scale=iw*min(${options.maxWidth}/iw\,${options.maxHeight}/ih):ih*min(${options.maxWidth}/iw\,${options.maxHeight}/ih)`
+
+  let lastProgress = 0
+  ffmpeg.on('log', ({ message }) => {
+    const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/)
+    if (timeMatch) {
+      const [, hours, minutes, seconds] = timeMatch
+      const currentTime =
+        parseInt(hours) * 3600 + parseInt(minutes) * 60 + parseFloat(seconds)
+      const progress = Math.min(100, Math.round((currentTime / duration) * 100))
+      if (progress > lastProgress) {
+        lastProgress = progress
+        options.onProgress?.(progress)
+      }
+    }
+  })
+
+  await ffmpeg.exec([
+    '-i',
+    'input.mp4',
+    '-vf',
+    `${sceneFilter},${additionalFramesFilter},${scaleFilter}`,
+    '-vsync',
+    '0',
+    '-q:v',
+    '2',
+    `frames_%03d.${options.format}`,
+  ])
+
+  // Read generated frames
+  const files = await ffmpeg.listDir('/')
+  const frameFiles = files.filter(
+    (file) =>
+      file.name.startsWith('frames_') &&
+      file.name.endsWith(`.${options.format}`)
+  )
+
+  const frames: string[] = []
+  for (let i = 0; i < frameFiles.length; i++) {
+    const file = frameFiles[i]
+    const frameData = await ffmpeg.readFile(file.name)
+    const base64Frame = btoa(
+      String.fromCharCode.apply(null, frameData as unknown as number[])
+    )
+    frames.push(`data:image/${options.format};base64,${base64Frame}`)
+
+    // Update progress for frame processing (from 90% to 100%)
+    options.onProgress?.(90 + Math.round(((i + 1) / frameFiles.length) * 10))
+  }
+
+  return frames
+}
+
+export default extractFramesFromVideo

src/services/io/fix-xenova-transformers.d.ts

@@ -0,0 +1 @@
+declare module '@xenova/transformers'

src/services/io/imageCaptioning.ts

@@ -0,0 +1,49 @@
+let worker: Worker | null = null
+
+function initializeWorker() {
+  if (typeof window !== 'undefined' && !worker) {
+    worker = new Worker('/captioning.worker.js')
+  }
+}
+
+export async function captionImages(
+  imageDataUrls: string[]
+): Promise<string[]> {
+  initializeWorker()
+
+  if (!worker) {
+    throw new Error(
+      'Worker could not be initialized. Are you running in a browser environment?'
+    )
+  }
+
+  const captions: string[] = []
+
+  for (const imageDataUrl of imageDataUrls) {
+    const caption = await new Promise<string>((resolve, reject) => {
+      const messageHandler = (event: MessageEvent) => {
+        worker!.removeEventListener('message', messageHandler)
+        if (event.data.error) {
+          reject(new Error(event.data.error))
+        } else {
+          resolve(event.data.caption)
+        }
+      }
+
+      worker!.addEventListener('message', messageHandler)
+      worker!.postMessage({ imageDataUrl })
+    })
+
+    captions.push(caption)
+  }
+
+  return captions
+}
+
+// Optionally, you can provide a cleanup function
+export function terminateWorker() {
+  if (worker) {
+    worker.terminate()
+    worker = null
+  }
+}