Spaces:
Running
Running
| <html> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <script src="https://cdn.tailwindcss.com"></script> | |
| <script src="https://cdn.jsdelivr.net/pyodide/v0.19.1/full/pyodide.js"></script> | |
| </head> | |
| <body> | |
| <div class="container mx-auto px-4"> | |
| <h1 class="text-3xl mt-6 font-bold"> | |
| 🚚 Pyodide demo | |
| </h1> | |
| <h2 class="text-gray">Python implementation of GPT-2 Tokenizer running inside your browser</h2> | |
| <div class="text-sm text-gray-800">Open your browser console to see Pyodide output</div> | |
| <div class="text-xs text-green-800 mt-4 js-init">Initialization: ...</div> | |
| <div class="flex gap-6 mt-10"> | |
| <div class="flex-1 bg-gray-50 p-4 rounded-xl border border-gray-200/60"> | |
| <input type="text" placeholder="Enter your sentence…" value="This text is transformed into tokens" class="w-full py-3 px-6" /> | |
| <div class="flex gap-4 my-4"> | |
| <button class="js-clear bg-gray-50 flex-1 p-3 rounded font-semibold focus:outline-none">Clear</button> | |
| <button class="js-submt bg-indigo-200 flex-1 p-3 rounded font-semibold focus:outline-none">Submit</button> | |
| </div> | |
| </div> | |
| <div class="flex-1 bg-gray-50 p-4 rounded-xl border border-gray-200/60"> | |
| <textarea placeholder="Output" class="w-full py-3 px-6 font-mono"></textarea> | |
| </div> | |
| </div> | |
| <h4 class="text-xs mt-10 mb-1">Python code being run:</h4> | |
| <pre class="js-code text-gray-500 text-xs bg-gray-50 p-4 rounded-xl border border-gray-200/60"></pre> | |
| </div> | |
| <script type="text/javascript"> | |
| const divInit = document.querySelector(".js-init"); | |
| const btnClear = document.querySelector(".js-clear"); | |
| const btnSubmt = document.querySelector(".js-submt"); | |
| const inputField = document.querySelector("input[type=text]"); | |
| const URL_VOCAB = "https://huggingface.co/gpt2/resolve/main/vocab.json"; | |
| const URL_MERGES = "https://huggingface.co/gpt2/resolve/main/merges.txt"; | |
| (async function main() { | |
| const c = console; | |
| const vocab = await (await fetch(URL_VOCAB)).text(); | |
| divInit.innerHTML += `<br> Downloaded vocab from ${URL_VOCAB}`; | |
| const merges = await (await fetch(URL_MERGES)).text(); | |
| divInit.innerHTML += `<br> Downloaded merges from ${URL_MERGES}`; | |
| const py_code = await (await fetch("./encoder.py")).text(); | |
| document.querySelector(".js-code").textContent = py_code; | |
| divInit.innerHTML += `<br> Downloaded python code from present repo`; | |
| const pyodide = await loadPyodide({ | |
| indexURL : "https://cdn.jsdelivr.net/pyodide/v0.19.1/full/" | |
| }); | |
| divInit.innerHTML += `<br> Initialized Pyodide`; | |
| await pyodide.loadPackagesFromImports(py_code); | |
| pyodide.runPython(py_code); | |
| pyodide.globals.set("vocab", vocab); | |
| pyodide.globals.set("merges", merges); | |
| pyodide.runPython(`encoder = get_encoder_from_strings(vocab, merges)`); | |
| divInit.innerHTML += `<br> Initialized tokenizer`; | |
| const compute = () => { | |
| const inputVal = inputField.value; | |
| const out = Array.from(pyodide.runPython(`encoder.encode(${JSON.stringify(inputVal)})`)); | |
| /// ^ array of ints | |
| document.querySelector("textarea").value = out.join(" "); | |
| }; | |
| btnSubmt.addEventListener("click", compute); | |
| inputField.addEventListener("input", compute); | |
| btnSubmt.click(); | |
| btnClear.addEventListener("click", () => { | |
| inputField.value = ""; | |
| compute(); | |
| }); | |
| inputField.focus(); | |
| inputField.selectionStart = inputField.selectionEnd = inputField.value.length; | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |