Spaces:

Nymbo
/

GemmaOnDevice

Running

App Files Files Community

thejagstudio commited on May 10

Commit

bf8ef86

•

1 Parent(s): caf9f43

Upload 3 files

Browse files

Files changed (3) hide show

gemma-2b-it-gpu-int4.bin +3 -0
index.html +81 -29
index.js +60 -79

gemma-2b-it-gpu-int4.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:ef44d548e44a2a6f313c3f3e94a48e1de786871ad95f4cd81bfb35372032cdbd
+size 1354301440

index.html CHANGED Viewed

@@ -1,29 +1,81 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8" />
-    <link rel="stylesheet" href="style.css" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Transformers.js - Object Detection</title>
-</head>
-<body>
-    <h1>Object Detection w/ 🤗 Transformers.js</h1>
-    <label id="container" for="upload">
-        <svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
-            <path fill="#000"
-                d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z">
-            </path>
-        </svg>
-        Click to upload image
-        <label id="example">(or try example)</label>
-    </label>
-    <label id="status">Loading model...</label>
-    <input id="upload" type="file" accept="image/*" />
-    <script src="index.js" type="module"></script>
-</body>
-</html>

+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>On-device LLM Inference</title>
+        <style>
+        body {
+            font-family: Arial, sans-serif;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: center;
+            margin: 0;
+            padding-top: 20px; /* Added padding at the top */
+            height: auto; /* Changed to auto for dynamic content sizing */
+            min-height: 100vh; /* Ensure it covers at least the full viewport height */
+            background-color: #f0f0f0;
+        }
+        .container {
+            width: 80%;
+            max-width: 640px; /* Adjusted for better control over max width */
+            text-align: center;
+        }
+        h1 {
+            color: #333;
+            margin-bottom: 20px;
+        }
+        textarea {
+            width: 100%; /* Full width of the container */
+            height: 200px; /* Height adjusted for paragraphs */
+            margin: 10px 0;
+            padding: 15px; /* More padding for better readability */
+            border: 1px solid #ccc;
+            border-radius: 8px;
+            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+            font-family: inherit;
+            font-size: 16px;
+            resize: vertical; /* Users can adjust the vertical size */
+        }
+        input[type="button"] {
+            padding: 10px 20px;
+            font-size: 16px;
+            border: none;
+            border-radius: 5px;
+            background-color: #007bff;
+            color: white;
+            cursor: pointer;
+            transition: background-color 0.2s;
+        }
+        input[type="button"]:hover {
+            background-color: #0056b3;
+        }
+        input[type="button"]:disabled {
+            background-color: #ccc;
+            cursor: not-allowed;
+        }
+        </style>
+    </head>
+    <body>
+        <div class="container">
+            <h1>On-device LLM Inference</h1>
+            <label for="input">Input:</label>
+            <br>
+            <textarea id="input"></textarea>
+            <br>
+            <input
+                type="button"
+                id="submit"
+                value="Get Response"
+                disabled
+            >
+            <br>
+            <br>
+            <label for="output">Result:</label>
+            <br>
+            <textarea id="output"></textarea>
+        </div>
+        <script type="module" src="index.js"></script>
+    </body>
+</html>

index.js CHANGED Viewed

@@ -1,79 +1,60 @@
-import { pipeline, env } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.10.1';
-// Since we will download the model from the Hugging Face Hub, we can skip the local model check
-env.allowLocalModels = false;
-// Reference the elements that we will need
-const status = document.getElementById('status');
-const fileUpload = document.getElementById('upload');
-const imageContainer = document.getElementById('container');
-const example = document.getElementById('example');
-const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/city-streets.jpg';
-// Create a new object detection pipeline
-status.textContent = 'Loading model...';
-const detector = await pipeline('object-detection', 'Xenova/detr-resnet-50');
-status.textContent = 'Ready';
-example.addEventListener('click', (e) => {
-    e.preventDefault();
-    detect(EXAMPLE_URL);
-});
-fileUpload.addEventListener('change', function (e) {
-    const file = e.target.files[0];
-    if (!file) {
-        return;
-    }
-    const reader = new FileReader();
-    // Set up a callback when the file is loaded
-    reader.onload = e2 => detect(e2.target.result);
-    reader.readAsDataURL(file);
-});
-// Detect objects in the image
-async function detect(img) {
-    imageContainer.innerHTML = '';
-    imageContainer.style.backgroundImage = `url(${img})`;
-    status.textContent = 'Analysing...';
-    const output = await detector(img, {
-        threshold: 0.5,
-        percentage: true,
-    });
-    status.textContent = '';
-    output.forEach(renderBox);
-}
-// Render a bounding box and label on the image
-function renderBox({ box, label }) {
-    const { xmax, xmin, ymax, ymin } = box;
-    // Generate a random color for the box
-    const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0);
-    // Draw the box
-    const boxElement = document.createElement('div');
-    boxElement.className = 'bounding-box';
-    Object.assign(boxElement.style, {
-        borderColor: color,
-        left: 100 * xmin + '%',
-        top: 100 * ymin + '%',
-        width: 100 * (xmax - xmin) + '%',
-        height: 100 * (ymax - ymin) + '%',
-    })
-    // Draw label
-    const labelElement = document.createElement('span');
-    labelElement.textContent = label;
-    labelElement.className = 'bounding-box-label';
-    labelElement.style.backgroundColor = color;
-    boxElement.appendChild(labelElement);
-    imageContainer.appendChild(boxElement);
-}

+import { FilesetResolver, LlmInference } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-genai";
+const input = document.getElementById("input");
+const output = document.getElementById("output");
+const submit = document.getElementById("submit");
+const modelFileName = "gemma-2b-it-gpu-int4.bin";
+/**
+ * Display newly generated partial results to the output text box.
+ */
+function displayPartialResults(partialResults, complete) {
+    output.textContent += partialResults;
+    if (complete) {
+        if (!output.textContent) {
+            output.textContent = "Result is empty";
+        }
+        submit.disabled = false;
+    }
+}
+/**
+ * Main function to run LLM Inference.
+ */
+async function runDemo() {
+    const genaiFileset = await FilesetResolver.forGenAiTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-genai/wasm");
+    let llmInference;
+    submit.onclick = () => {
+        output.textContent = "";
+        submit.disabled = true;
+        llmInference.generateResponse(input.value, displayPartialResults);
+    };
+    submit.value = "Loading the model...";
+    LlmInference.createFromOptions(genaiFileset, {
+        baseOptions: { modelAssetPath: modelFileName },
+        // maxTokens: 512,  // The maximum number of tokens (input tokens + output
+        //                  // tokens) the model handles.
+        // randomSeed: 1,   // The random seed used during text generation.
+        // topK: 1,  // The number of tokens the model considers at each step of
+        //           // generation. Limits predictions to the top k most-probable
+        //           // tokens. Setting randomSeed is required for this to make
+        //           // effects.
+        // temperature:
+        //     1.0,  // The amount of randomness introduced during generation.
+        //           // Setting randomSeed is required for this to make effects.
+    })
+        .then((llm) => {
+            llmInference = llm;
+            submit.disabled = false;
+            submit.value = "Get Response";
+        })
+        .catch(() => {
+            alert("Failed to initialize the task.");
+        });
+}
+runDemo();