Spaces:

pdufour
/

Qwen2VL_TransformersJS_Demo

Running

App Files Files Community

Paul Dufour commited on Nov 28, 2024

Commit

ee4ce56

•

1 Parent(s): 246f1f5

Update demo

Browse files

Files changed (4) hide show

Makefile +2 -0
index.html +30 -25
index.js +30 -12
style.css +88 -0

Makefile ADDED Viewed

	@@ -0,0 +1,2 @@


1	+ run:
2	+ static-web-server -p 3000 --cors-allow-origins "" --cors-allow-headers "" --cors-expose-headers "*" -d . -e false -z

index.html CHANGED Viewed

@@ -1,13 +1,11 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="UTF-8" />
     <link rel="stylesheet" href="style.css" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Vision Demo</title>
-   <script type="importmap">
       {
         "imports": {
           "@huggingface/transformers": "https://esm.run/@huggingface/transformers@3.1.0"
@@ -15,32 +13,39 @@
       }
     </script>
 </head>
 <body>
     <h1>Vision Demo</h1>
     <div style="text-align: center">Everything runs directly in your browser (no server required)</div>
     <form id="form">
-      <label id="status">Loading model...</label>
-      <label id="container" for="upload">
-          <div id="thumb">
-            <svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
-                <path fill="#000" d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z"></path>
-            </svg>
-            Click to upload image
-            <label id="example">(or try example)</label>
-          </div>
-          <div>
-              <input type="text" placeholder="Add your prompt and hit enter" disabled>
-          </div>
-      </label>
-      <input id="upload" type="file" accept="image/*" disabled />
-      <div id="llm-output"></div>
     </form>
-    <script src="index.js" type="module">
-      document.addEventListener('DOMContentLoaded', async () => {
-        await import('./index.js');
-      });
-    </script>
 </body>
 </html>

 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="UTF-8" />
     <link rel="stylesheet" href="style.css" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Vision Demo</title>
+    <script type="importmap">
       {
         "imports": {
           "@huggingface/transformers": "https://esm.run/@huggingface/transformers@3.1.0"
       }
     </script>
 </head>
 <body>
     <h1>Vision Demo</h1>
     <div style="text-align: center">Everything runs directly in your browser (no server required)</div>
     <form id="form">
+        <div>
+            <select id="dtype-select">
+                <option value=""></option>
+                <option value="fp32">fp32</option>
+                <option value="fp16">fp16</option>
+                <option value="q8">q8</option>
+                <option value="int8">int8</option>
+                <option value="uint8">uint8</option>
+                <option value="q4">q4</option>
+                <option value="bnb4">bnb4</option>
+                <option value="q4f16">q4f16</option>
+            </select>
+            <button id="load-model" type="button">Load Model</button>
+        </div>
+        <label id="container" for="upload">
+            <div id="thumb">
+                <svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+                    <path fill="#000" d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z"></path>
+                </svg>
+                Click to upload image
+                <label id="example">(or try example)</label>
+            </div>
+            <div>
+                <input type="text" placeholder="Add your prompt and hit enter" disabled>
+            </div>
+        </label>
+        <input id="upload" type="file" accept="image/*" disabled />
+        <div id="llm-output"></div>
     </form>
+    <script src="index.js" type="module"></script>
 </body>
 </html>

index.js CHANGED Viewed

@@ -1,6 +1,5 @@
 import { AutoProcessor, Qwen2VLForConditionalGeneration, RawImage } from "@huggingface/transformers";
 const EXAMPLE_URL = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg";
 const exampleButton = document.getElementById('example');
@@ -10,6 +9,9 @@ const thumb = document.getElementById('thumb');
 const uploadInput = document.getElementById('upload');
 const form = document.getElementById('form');
 const output = document.getElementById('llm-output');
 let currentImage = '';
 let currentQuery = '';
@@ -18,15 +20,24 @@ let processor;
 let model;
 async function initializeSessions() {
-  status.textContent = 'Loading model...';
   container.classList.add('disabled');
   processor = await AutoProcessor.from_pretrained(model_id);
-  model = await Qwen2VLForConditionalGeneration.from_pretrained(model_id, { dtype: 'q4f16', device: 'webgpu' });
-  status.textContent = 'Ready';
-  status.classList.add('ready');
   uploadInput.disabled = false;
   promptInput.disabled = false;
   container.classList.remove('disabled');
@@ -34,25 +45,25 @@ async function initializeSessions() {
 async function handleQuery(imageUrl, query) {
   try {
-    status.textContent = 'Analyzing...';
     const result = await imageTextToText(imageUrl, query, (out) => {
       console.log({ out });
       output.textContent = out;
     });
   } catch (err) {
-    status.textContent = 'Error processing request';
     console.error(err);
   }
 }
-export async function imageTextToText(
   imagePath,
   query,
   cb,
 ) {
   const image = await (await RawImage.read(imagePath)).resize(448, 448);
   const conversation = [
     {
@@ -92,7 +103,11 @@ async function updatePreview(url) {
   thumb.innerHTML = '';
 }
-await initializeSessions();
 // UI Event Handlers
 exampleButton.addEventListener('click', (e) => {
@@ -121,7 +136,10 @@ form.addEventListener('submit', (e) => {
   e.preventDefault();
   if (!currentImage || !currentQuery) {
-    status.textContent = 'Please select an image and type a prompt';
   } else {
     promptInput.disabled = true;
     uploadInput.disabled = true;

 import { AutoProcessor, Qwen2VLForConditionalGeneration, RawImage } from "@huggingface/transformers";
 const EXAMPLE_URL = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg";
 const exampleButton = document.getElementById('example');
 const uploadInput = document.getElementById('upload');
 const form = document.getElementById('form');
 const output = document.getElementById('llm-output');
+const dtypeSelect = document.getElementById('dtype-select');
+const loadModelButton = document.getElementById('load-model');
+const container = document.getElementById('container');
 let currentImage = '';
 let currentQuery = '';
 let model;
 async function initializeSessions() {
+  loadModelButton.textContent = 'Loading Model...';
+  loadModelButton.classList.add('loading');
   container.classList.add('disabled');
   processor = await AutoProcessor.from_pretrained(model_id);
+  const dtype = dtypeSelect.value;
+  const options = { device: 'webgpu', };
+  if (dtype) {
+    options.dtype = dtype;
+  }
+  model = await Qwen2VLForConditionalGeneration.from_pretrained(model_id, options);
+  loadModelButton.textContent = 'Model Ready';
+  loadModelButton.classList.remove('loading');
+  loadModelButton.classList.add('ready');
+  dtypeSelect.disabled = true;
   uploadInput.disabled = false;
   promptInput.disabled = false;
   container.classList.remove('disabled');
 async function handleQuery(imageUrl, query) {
   try {
+    loadModelButton.textContent = 'Processing...';
     const result = await imageTextToText(imageUrl, query, (out) => {
       console.log({ out });
       output.textContent = out;
     });
+    loadModelButton.textContent = 'Model Ready';
   } catch (err) {
+    loadModelButton.textContent = 'Error';
     console.error(err);
   }
 }
+async function imageTextToText(
   imagePath,
   query,
   cb,
 ) {
   const image = await (await RawImage.read(imagePath)).resize(448, 448);
   const conversation = [
     {
   thumb.innerHTML = '';
 }
+loadModelButton.addEventListener('click', async () => {
+  dtypeSelect.disabled = true;
+  loadModelButton.disabled = true;
+  await initializeSessions();
+});
 // UI Event Handlers
 exampleButton.addEventListener('click', (e) => {
   e.preventDefault();
   if (!currentImage || !currentQuery) {
+    loadModelButton.textContent = 'Please select an image and type a prompt';
+    setTimeout(() => {
+      loadModelButton.textContent = 'Model Ready';
+    }, 2000);
   } else {
     promptInput.disabled = true;
     uploadInput.disabled = true;

style.css CHANGED Viewed

@@ -148,3 +148,91 @@ input[type="text"]:focus {
 #container.disabled #thumb svg {
   opacity: 0.4;
 }

 #container.disabled #thumb svg {
   opacity: 0.4;
 }
+#dtype-select {
+  width: 100%;
+  margin: 10px 0 0;
+  padding: 8px 12px;
+  font-size: 14px;
+  border: 1px solid #ddd;
+  border-radius: 6px;
+  background-color: #fff;
+  cursor: pointer;
+  margin-bottom: 10px;
+  outline: none;
+  transition: border-color 0.2s, box-shadow 0.2s;
+}
+#dtype-select:hover {
+  border-color: #bbb;
+}
+#dtype-select:focus {
+  border-color: #2196F3;
+  box-shadow: 0 0 0 2px rgba(33, 150, 243, 0.1);
+}
+#dtype-select:disabled {
+  background-color: #f5f5f5;
+  cursor: not-allowed;
+  opacity: 0.7;
+}
+#load-model {
+  width: 100%;
+  padding: 12px 20px;
+  font-size: 14px;
+  font-weight: 600;
+  background-color: #2196F3;
+  color: white;
+  border: none;
+  border-radius: 6px;
+  cursor: pointer;
+  transition: all 0.2s ease;
+  box-shadow: 0 2px 4px rgba(33, 150, 243, 0.2);
+  text-transform: uppercase;
+  letter-spacing: 0.5px;
+  margin: 0 0 15px;
+  position: relative;
+}
+#load-model:hover:not(:disabled) {
+  background-color: #1976D2;
+  box-shadow: 0 4px 8px rgba(33, 150, 243, 0.3);
+  transform: translateY(-1px);
+}
+#load-model:active:not(:disabled) {
+  transform: translateY(1px);
+  box-shadow: 0 1px 2px rgba(33, 150, 243, 0.2);
+}
+#load-model.loading {
+  color: transparent;
+  cursor: wait;
+  background-color: #90CAF9;
+}
+#load-model.loading::after {
+  content: "";
+  position: absolute;
+  width: 16px;
+  height: 16px;
+  top: 50%;
+  left: 50%;
+  margin-left: -8px;
+  margin-top: -8px;
+  border: 2px solid rgba(255, 255, 255, 0.3);
+  border-radius: 50%;
+  border-top-color: white;
+  animation: spin 1s linear infinite;
+}
+#load-model.ready {
+  background-color: #4CAF50;
+  cursor: default;
+}
+@keyframes spin {
+  to { transform: rotate(360deg); }
+}