Spaces:
Running
Running
| <html> | |
| <head> | |
| <meta content="text/html;charset=utf-8" http-equiv="Content-Type" /> | |
| <title>Candle Llama.c Rust/WASM</title> | |
| </head> | |
| <body></body> | |
| </html> | |
| <html> | |
| <head> | |
| <meta charset="UTF-8" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | |
| <style> | |
| @import url("https://fonts.googleapis.com/css2?family=Source+Code+Pro:wght@200;300;400&family=Source+Sans+3:wght@100;200;300;400;500;600;700;800;900&display=swap"); | |
| html, | |
| body { | |
| font-family: "Source Sans 3", sans-serif; | |
| } | |
| code, | |
| output, | |
| select, | |
| pre { | |
| font-family: "Source Code Pro", monospace; | |
| } | |
| </style> | |
| <script src="https://cdn.tailwindcss.com"></script> | |
| <script type="module"> | |
| // base url for audio examples | |
| const MODELS_BASE_URL = | |
| "https://huggingface.co/karpathy/tinyllamas/resolve/main"; | |
| // models base url | |
| const MODELS = { | |
| stories15M: { | |
| url: "stories15M.bin", | |
| seq_len: 256, | |
| }, | |
| stories42M: { | |
| url: "stories42M.bin", | |
| seq_len: 1024, | |
| }, | |
| stories110M: { | |
| url: "stories110M.bin", | |
| seq_len: 1024, | |
| }, | |
| }; | |
| const llamaWorker = new Worker("./llama2cWorker.js", { | |
| type: "module", | |
| }); | |
| async function generateSequence(controller) { | |
| const getValue = (id) => document.querySelector(`#${id}`).value; | |
| const modelID = getValue("model"); | |
| const model = MODELS[modelID]; | |
| const weightsURL = `${MODELS_BASE_URL}/${model.url}`; | |
| const prompt = getValue("prompt"); | |
| const temperature = getValue("temperature"); | |
| const topP = getValue("top-p"); | |
| const repeatPenalty = getValue("repeat_penalty"); | |
| const seed = getValue("seed"); | |
| const maxSeqLen = getValue("max-seq"); | |
| function updateStatus(data) { | |
| const outStatus = document.querySelector("#output-status"); | |
| const outGen = document.querySelector("#output-generation"); | |
| const outCounter = document.querySelector("#output-counter"); | |
| switch (data.status) { | |
| case "loading": | |
| outStatus.hidden = false; | |
| outStatus.textContent = data.message; | |
| outGen.hidden = true; | |
| outCounter.hidden = true; | |
| break; | |
| case "generating": | |
| const { message, prompt, sentence, tokensSec, totalTime } = data; | |
| outStatus.hidden = true; | |
| outCounter.hidden = false; | |
| outGen.hidden = false; | |
| outGen.innerHTML = `<span class="font-semibold">${prompt}</span>${sentence.replace( | |
| /\<s\>|\<\/s\>/g, | |
| "" | |
| )}`; | |
| outCounter.innerHTML = `${(totalTime / 1000).toFixed( | |
| 2 | |
| )}s (${tokensSec.toFixed(2)} tok/s)`; | |
| break; | |
| case "complete": | |
| outStatus.hidden = true; | |
| outGen.hidden = false; | |
| break; | |
| } | |
| } | |
| return new Promise((resolve, reject) => { | |
| llamaWorker.postMessage({ | |
| weightsURL, | |
| modelID, | |
| tokenizerURL: "tokenizer.json", | |
| prompt, | |
| temp: temperature, | |
| top_p: topP, | |
| repeatPenalty, | |
| seed: BigInt(seed), | |
| maxSeqLen, | |
| command: "start", | |
| }); | |
| const handleAbort = () => { | |
| llamaWorker.postMessage({ command: "abort" }); | |
| }; | |
| const handleMessage = (event) => { | |
| const { status, error, message, prompt, sentence } = event.data; | |
| if (status) updateStatus(event.data); | |
| if (error) { | |
| llamaWorker.removeEventListener("message", handleMessage); | |
| reject(new Error(error)); | |
| } | |
| if (status === "aborted") { | |
| llamaWorker.removeEventListener("message", handleMessage); | |
| resolve(event.data); | |
| } | |
| if (status === "complete") { | |
| llamaWorker.removeEventListener("message", handleMessage); | |
| resolve(event.data); | |
| } | |
| }; | |
| controller.signal.addEventListener("abort", handleAbort); | |
| llamaWorker.addEventListener("message", handleMessage); | |
| }); | |
| } | |
| const form = document.querySelector("#form"); | |
| const prompt = document.querySelector("#prompt"); | |
| const clearBtn = document.querySelector("#clear-btn"); | |
| const runBtn = document.querySelector("#run"); | |
| const modelSelect = document.querySelector("#model"); | |
| let runController = new AbortController(); | |
| let isRunning = false; | |
| modelSelect.addEventListener("change", (e) => { | |
| const model = MODELS[e.target.value]; | |
| document.querySelector("#max-seq").max = model.seq_len; | |
| document.querySelector("#max-seq").nextElementSibling.value = | |
| model.seq_len; | |
| }); | |
| form.addEventListener("submit", async (e) => { | |
| e.preventDefault(); | |
| if (isRunning) { | |
| stopRunning(); | |
| } else { | |
| startRunning(); | |
| await generateSequence(runController); | |
| stopRunning(); | |
| } | |
| }); | |
| function startRunning() { | |
| isRunning = true; | |
| runBtn.textContent = "Stop"; | |
| } | |
| function stopRunning() { | |
| runController.abort(); | |
| runController = new AbortController(); | |
| runBtn.textContent = "Run"; | |
| isRunning = false; | |
| } | |
| clearBtn.addEventListener("click", (e) => { | |
| e.preventDefault(); | |
| prompt.value = ""; | |
| clearBtn.classList.add("invisible"); | |
| runBtn.disabled = true; | |
| stopRunning(); | |
| }); | |
| prompt.addEventListener("input", (e) => { | |
| runBtn.disabled = false; | |
| if (e.target.value.length > 0) { | |
| clearBtn.classList.remove("invisible"); | |
| } else { | |
| clearBtn.classList.add("invisible"); | |
| } | |
| }); | |
| </script> | |
| </head> | |
| <body class="container max-w-4xl mx-auto p-4 text-gray-800"> | |
| <main class="grid grid-cols-1 gap-8 relative"> | |
| <span class="absolute text-5xl -ml-[1em]"> 🕯️ </span> | |
| <div> | |
| <h1 class="text-5xl font-bold">Candle Llama2.c</h1> | |
| <h2 class="text-2xl font-bold">Rust/WASM Demo</h2> | |
| <p class="max-w-lg"> | |
| <a | |
| href="https://github.com/karpathy/llama2.c" | |
| target="_blank" | |
| class="underline hover:text-blue-500 hover:no-underline" | |
| target="_blank" | |
| >Llama2.c</a | |
| > | |
| is Andrey Karpathy's C implementation of the Llama 2 LLM model in C. | |
| This demo uses | |
| <a | |
| href="https://github.com/huggingface/candle/" | |
| target="_blank" | |
| class="underline hover:text-blue-500 hover:no-underline" | |
| >Candle | |
| </a> | |
| to run Llama2.c in the browser using rust/wasm. | |
| </p> | |
| </div> | |
| <div> | |
| <label for="model" class="font-medium">Models Options: </label> | |
| <select | |
| id="model" | |
| class="border-2 border-gray-500 rounded-md font-light"> | |
| <option value="stories15M" selected>stories 15M (60.8 MB)</option> | |
| <option value="stories42M">stories 42M (167 MB)</option> | |
| <option value="stories110M">stories 110M (438 MB)</option> | |
| </select> | |
| </div> | |
| <form | |
| id="form" | |
| class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center"> | |
| <input type="submit" hidden /> | |
| <input | |
| type="text" | |
| id="prompt" | |
| class="font-light w-full px-3 py-2 mx-1 resize-none outline-none" | |
| placeholder="Add your prompt here..." | |
| value="Once upon a time" /> | |
| <button id="clear-btn"> | |
| <svg | |
| fill="none" | |
| xmlns="http://www.w3.org/2000/svg" | |
| width="40" | |
| viewBox="0 0 70 40"> | |
| <path opacity=".5" d="M39 .2v40.2" stroke="#1F2937" /> | |
| <path | |
| d="M1.5 11.5 19 29.1m0-17.6L1.5 29.1" | |
| opacity=".5" | |
| stroke="#1F2937" | |
| stroke-width="2" /> | |
| </svg> | |
| </button> | |
| <button | |
| id="run" | |
| class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-2 w-16 rounded disabled:bg-gray-300 disabled:cursor-not-allowed"> | |
| Run | |
| </button> | |
| </form> | |
| <details> | |
| <summary class="font-medium cursor-pointer">Advanced Options</summary> | |
| <div class="grid grid-cols-3 max-w-md items-center gap-3 py-3"> | |
| <label class="text-sm font-medium" for="max-seq" | |
| >Maximum length | |
| </label> | |
| <input | |
| type="range" | |
| id="max-seq" | |
| name="max-seq" | |
| min="1" | |
| max="256" | |
| step="1" | |
| value="200" | |
| oninput="this.nextElementSibling.value = Number(this.value)" /> | |
| <output | |
| class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 200</output | |
| > | |
| <label class="text-sm font-medium" for="temperature" | |
| >Temperature</label | |
| > | |
| <input | |
| type="range" | |
| id="temperature" | |
| name="temperature" | |
| min="0" | |
| max="2" | |
| step="0.01" | |
| value="0.40" | |
| oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" /> | |
| <output | |
| class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 0.40</output | |
| > | |
| <label class="text-sm font-medium" for="top-p">Top-p</label> | |
| <input | |
| type="range" | |
| id="top-p" | |
| name="top-p" | |
| min="0" | |
| max="1" | |
| step="0.01" | |
| value="1.00" | |
| oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" /> | |
| <output | |
| class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md"> | |
| 1.00</output | |
| > | |
| <label class="text-sm font-medium" for="repeat_penalty" | |
| >Repeat Penalty</label | |
| > | |
| <input | |
| type="range" | |
| id="repeat_penalty" | |
| name="repeat_penalty" | |
| min="1" | |
| max="2" | |
| step="0.01" | |
| value="1.10" | |
| oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" /> | |
| <output | |
| class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md" | |
| >1.10</output | |
| > | |
| <label class="text-sm font-medium" for="seed">Seed</label> | |
| <input | |
| type="number" | |
| id="seed" | |
| name="seed" | |
| value="299792458" | |
| class="font-light border border-gray-700 text-right rounded-md p-2" /> | |
| <button | |
| id="run" | |
| onclick="document.querySelector('#seed').value = BigInt(Math.floor(Math.random() * 2**64-1))" | |
| class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-1 w-[50px] rounded disabled:bg-gray-300 disabled:cursor-not-allowed text-sm"> | |
| Rand | |
| </button> | |
| </div> | |
| </details> | |
| <div> | |
| <h3 class="font-medium">Generation:</h3> | |
| <div | |
| class="min-h-[250px] bg-slate-100 text-gray-500 p-4 rounded-md flex flex-col gap-2"> | |
| <div | |
| id="output-counter" | |
| hidden | |
| class="ml-auto font-semibold grid-rows-1 text-sm"></div> | |
| <p hidden id="output-generation" class="grid-rows-2"></p> | |
| <span id="output-status" class="m-auto font-light" | |
| >No output yet</span | |
| > | |
| </div> | |
| </div> | |
| </main> | |
| </body> | |
| </html> | |