Working prototype

README.md

@@ -1,8 +1,8 @@
 ---
-title: Gpt2 Tokenizer
-emoji: 🌖
-colorFrom: gray
-colorTo: gray
+title: Pyodide GPT-2 Tokenizer
+emoji: 🐍
+colorFrom: green
+colorTo: green
 sdk: static
 pinned: false
 ---

encoder.py

@@ -114,4 +114,12 @@ def get_encoder(model_name, models_dir):
     return Encoder(
         encoder=encoder,
         bpe_merges=bpe_merges,
-    )
+    )
+
+def get_encoder_from_strings(vocab, bpe_data):
+    encoder = json.loads(vocab)
+    bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]]
+    return Encoder(
+        encoder=encoder,
+        bpe_merges=bpe_merges,
+    )

index.html

@@ -7,15 +7,29 @@
 		Pyodide test page <br>
 		Open your browser console to see Pyodide output
 		<script type="text/javascript">
-			(async function main(){
+			const URL_VOCAB = "https://huggingface.co/gpt2/resolve/main/vocab.json";
+			const URL_MERGES = "https://huggingface.co/gpt2/resolve/main/merges.txt";
+			
+			
+			(async function main() {
+				const vocab = await (await fetch(URL_VOCAB)).text();
+				const merges = await (await fetch(URL_MERGES)).text();
+				
+				const py_code = await (await fetch("./encoder.py")).text();
+				const c = console;
+				
 				const pyodide = await loadPyodide({
 					indexURL : "https://cdn.jsdelivr.net/pyodide/v0.19.1/full/"
 				});
-				console.log(pyodide.runPython(`
-						import sys
-						sys.version
-				`));
-				console.log(pyodide.runPython("print(1 + 2)"));
+				
+				await pyodide.loadPackagesFromImports(py_code);
+				pyodide.runPython(py_code);
+				
+				pyodide.globals.set("vocab", vocab);
+				pyodide.globals.set("merges", merges);
+				pyodide.runPython(`encoder = get_encoder_from_strings(vocab, merges)`);
+				const out = pyodide.runPython(`encoder.encode(${JSON.stringify("Hello my name is")})`);
+				c.log(Array.from(out));
 			})();
 		</script>
 	</body>