Hugging Face's logo

Spaces:
pyodide-demo
/
gpt2-tokenizer
Running

App Files Community
julien-c HF staff commited on
Commit
669a951
•
1 Parent(s): 7349b7e

Complete demo

Browse files
Files changed (1) hide show
  1. index.html +55 -6
index.html CHANGED
@@ -1,26 +1,60 @@
1
  <!DOCTYPE html>
2
  <html>
3
  <head>
 
 
 
4
  <script src="https://cdn.jsdelivr.net/pyodide/v0.19.1/full/pyodide.js"></script>
5
  </head>
6
  <body>
7
- Pyodide test page <br>
8
- Open your browser console to see Pyodide output
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  <script type="text/javascript">
 
 
 
 
10
  const URL_VOCAB = "https://huggingface.co/gpt2/resolve/main/vocab.json";
11
  const URL_MERGES = "https://huggingface.co/gpt2/resolve/main/merges.txt";
12
 
13
 
14
  (async function main() {
 
 
15
  const vocab = await (await fetch(URL_VOCAB)).text();
 
16
  const merges = await (await fetch(URL_MERGES)).text();
17
-
18
  const py_code = await (await fetch("./encoder.py")).text();
19
- const c = console;
 
20
 
21
  const pyodide = await loadPyodide({
22
  indexURL : "https://cdn.jsdelivr.net/pyodide/v0.19.1/full/"
23
  });
 
24
 
25
  await pyodide.loadPackagesFromImports(py_code);
26
  pyodide.runPython(py_code);
@@ -28,8 +62,23 @@
28
  pyodide.globals.set("vocab", vocab);
29
  pyodide.globals.set("merges", merges);
30
  pyodide.runPython(`encoder = get_encoder_from_strings(vocab, merges)`);
31
- const out = pyodide.runPython(`encoder.encode(${JSON.stringify("Hello my name is")})`);
32
- c.log(Array.from(out));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  })();
34
  </script>
35
  </body>
 
1
  <!DOCTYPE html>
2
  <html>
3
  <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <script src="https://cdn.tailwindcss.com"></script>
7
  <script src="https://cdn.jsdelivr.net/pyodide/v0.19.1/full/pyodide.js"></script>
8
  </head>
9
  <body>
10
+ <div class="container mx-auto px-4">
11
+ <h1 class="text-3xl font-bold">
12
+ 🚚 Pyodide demo
13
+ </h1>
14
+ <h2 class="text-gray">Python implementation of GPT-2 Tokenizer running inside your browser</h2>
15
+ <div class="text-sm text-gray-800">Open your browser console to see Pyodide output</div>
16
+ <div class="text-xs text-green-800 mt-4 js-init">Initialization: ...</div>
17
+ <div class="flex gap-6 mt-10">
18
+ <div class="flex-1 bg-gray-50 p-4 rounded-xl border border-gray-200/60">
19
+ <input type="text" placeholder="Enter your sentence…" value="This text is transformed into tokens" class="w-full py-3 px-6" />
20
+
21
+ <div class="flex gap-4 my-4">
22
+ <button class="js-clear bg-gray-50 flex-1 p-3 rounded font-semibold focus:outline-none">Clear</button>
23
+ <button class="js-submt bg-indigo-200 flex-1 p-3 rounded font-semibold focus:outline-none">Submit</button>
24
+ </div>
25
+ </div>
26
+ <div class="flex-1 bg-gray-50 p-4 rounded-xl border border-gray-200/60">
27
+ <textarea placeholder="Output" class="w-full py-3 px-6 font-mono"></textarea>
28
+ </div>
29
+ </div>
30
+
31
+ <h4 class="text-xs mt-10 mb-1">Python code being run:</h4>
32
+ <pre class="js-code text-gray-500 text-xs bg-gray-50 p-4 rounded-xl border border-gray-200/60"></pre>
33
+ </div>
34
  <script type="text/javascript">
35
+ const divInit = document.querySelector(".js-init");
36
+ const btnClear = document.querySelector(".js-clear");
37
+ const btnSubmt = document.querySelector(".js-submt");
38
+ const inputField = document.querySelector("input[type=text]");
39
  const URL_VOCAB = "https://huggingface.co/gpt2/resolve/main/vocab.json";
40
  const URL_MERGES = "https://huggingface.co/gpt2/resolve/main/merges.txt";
41
 
42
 
43
  (async function main() {
44
+ const c = console;
45
+
46
  const vocab = await (await fetch(URL_VOCAB)).text();
47
+ divInit.innerHTML += `<br> Downloaded vocab from ${URL_VOCAB}`;
48
  const merges = await (await fetch(URL_MERGES)).text();
49
+ divInit.innerHTML += `<br> Downloaded merges from ${URL_MERGES}`;
50
  const py_code = await (await fetch("./encoder.py")).text();
51
+ document.querySelector(".js-code").textContent = py_code;
52
+ divInit.innerHTML += `<br> Downloaded python code from present repo`;
53
 
54
  const pyodide = await loadPyodide({
55
  indexURL : "https://cdn.jsdelivr.net/pyodide/v0.19.1/full/"
56
  });
57
+ divInit.innerHTML += `<br> Initialized Pyodide`;
58
 
59
  await pyodide.loadPackagesFromImports(py_code);
60
  pyodide.runPython(py_code);
 
62
  pyodide.globals.set("vocab", vocab);
63
  pyodide.globals.set("merges", merges);
64
  pyodide.runPython(`encoder = get_encoder_from_strings(vocab, merges)`);
65
+ divInit.innerHTML += `<br> Initialized tokenizer`;
66
+
67
+ const compute = () => {
68
+ const inputVal = inputField.value;
69
+ const out = Array.from(pyodide.runPython(`encoder.encode(${JSON.stringify(inputVal)})`));
70
+ /// ^ array of ints
71
+ document.querySelector("textarea").value = out.join(" ");
72
+ };
73
+ btnSubmt.addEventListener("click", compute);
74
+ inputField.addEventListener("input", compute);
75
+ btnSubmt.click();
76
+ btnClear.addEventListener("click", () => {
77
+ inputField.value = "";
78
+ compute();
79
+ });
80
+ inputField.focus();
81
+ inputField.selectionStart = inputField.selectionEnd = inputField.value.length;
82
  })();
83
  </script>
84
  </body>