fix

app.py

@@ -48,84 +48,108 @@ selected_vocab = st.sidebar.radio("", list(vocab_list.keys()))
 repo_id = vocab_list[selected_vocab]
 tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
 
+def escape_byte(b):
+    ret = ""
+    if b == 0x00: 
+        ret = "<!!!!VOCABVIEWER ASCII 0x00 NUL!!!!>"
+    elif b == 0x01: 
+        ret = "<!!!!VOCABVIEWER ASCII 0x01 SOH!!!!>"
+    elif b == 0x02: 
+        ret = "<!!!!VOCABVIEWER ASCII 0x02 STX!!!!>"
+    elif b == 0x03: 
+        ret = "<!!!!VOCABVIEWER ASCII 0x03 ETX!!!!>"
+    elif b == 0x04: 
+        ret = "<!!!!VOCABVIEWER ASCII 0x04 EOT!!!!>"
+    elif b == 0x05:
+        ret = "<!!!!VOCABVIEWER ASCII 0x05 ENQ!!!!>"
+    elif b == 0x06:
+        ret = "<!!!!VOCABVIEWER ASCII 0x06 ACK!!!!>"
+    elif b == 0x07:
+        ret = "<!!!!VOCABVIEWER ASCII 0x07 BEL!!!!>"
+    elif b == 0x08:
+        ret = "<!!!!VOCABVIEWER ASCII 0x08 BS!!!!>"
+    elif b == 0x0a:
+        ret = "<!!!!VOCABVIEWER ASCII 0x0a LF!!!!>"
+    elif b == 0x0b:
+        ret = "<!!!!VOCABVIEWER ASCII 0x0b VT!!!!>"
+    elif b == 0x0c:
+        ret = "<!!!!VOCABVIEWER ASCII 0x0c FF!!!!>"
+    elif b == 0x0d:
+        ret = "<!!!!VOCABVIEWER ASCII 0x0d CR!!!!>"
+    elif b == 0x0e:
+        ret = "<!!!!VOCABVIEWER ASCII 0x0e SO!!!!>"
+    elif b == 0x0f:
+        ret = "<!!!!VOCABVIEWER ASCII 0x0f SI!!!!>"
+    elif b == 0x10:
+        ret = "<!!!!VOCABVIEWER ASCII 0x10 DLE!!!!>"
+    elif b == 0x11:
+        ret = "<!!!!VOCABVIEWER ASCII 0x11 DC1!!!!>"
+    elif b == 0x12:
+        ret = "<!!!!VOCABVIEWER ASCII 0x12 DC2!!!!>"
+    elif b == 0x13:
+        ret = "<!!!!VOCABVIEWER ASCII 0x13 DC3!!!!>"
+    elif b == 0x14:
+        ret = "<!!!!VOCABVIEWER ASCII 0x14 DC4!!!!>"
+    elif b == 0x15:
+        ret = "<!!!!VOCABVIEWER ASCII 0x15 NAK!!!!>"
+    elif b == 0x16:
+        ret = "<!!!!VOCABVIEWER ASCII 0x16 SYN!!!!>"
+    elif b == 0x17:
+        ret = "<!!!!VOCABVIEWER ASCII 0x17 ETB!!!!>"
+    elif b == 0x18:
+        ret = "<!!!!VOCABVIEWER ASCII 0x18 CAN!!!!>"
+    elif b == 0x19:
+        ret = "<!!!!VOCABVIEWER ASCII 0x19 EM!!!!>"
+    elif b == 0x1a:
+        ret = "<!!!!VOCABVIEWER ASCII 0x1a SUB!!!!>"
+    elif b == 0x1b:
+        ret = "<!!!!VOCABVIEWER ASCII 0x1b ESC!!!!>"
+    elif b == 0x1c:
+        ret = "<!!!!VOCABVIEWER ASCII 0x1c FS!!!!>"
+    elif b == 0x1d:
+        ret = "<!!!!VOCABVIEWER ASCII 0x1d GS!!!!>"
+    elif b == 0x1e:
+        ret = "<!!!!VOCABVIEWER ASCII 0x1e RS!!!!>"
+    elif b == 0x1f:
+        ret = "<!!!!VOCABVIEWER ASCII 0x1f US!!!!>"
+    elif b == 0x20:
+        # ret = "<!!!!VOCABVIEWER ASCII 0x20 SPC!!!!>"
+        ret = " "
+    elif b == 0x7f:
+        ret = "<!!!!VOCABVIEWER ASCII 0x7f DEL!!!!>"
+
+    return ret
+
 def escape_decode(token):
     ret = token
     tok_enc = token.encode()
     if len(tok_enc) == 1:
         b = tok_enc[0]
-
-        if b == 0x00: 
-            ret = "<!!!!VOCABVIEWER ASCII 0x00 NUL!!!!>"
-        elif b == 0x01: 
-            ret = "<!!!!VOCABVIEWER ASCII 0x01 SOH!!!!>"
-        elif b == 0x02: 
-            ret = "<!!!!VOCABVIEWER ASCII 0x02 STX!!!!>"
-        elif b == 0x03: 
-            ret = "<!!!!VOCABVIEWER ASCII 0x03 ETX!!!!>"
-        elif b == 0x04: 
-            ret = "<!!!!VOCABVIEWER ASCII 0x04 EOT!!!!>"
-        elif b == 0x05:
-            ret = "<!!!!VOCABVIEWER ASCII 0x05 ENQ!!!!>"
-        elif b == 0x06:
-            ret = "<!!!!VOCABVIEWER ASCII 0x06 ACK!!!!>"
-        elif b == 0x07:
-            ret = "<!!!!VOCABVIEWER ASCII 0x07 BEL!!!!>"
-        elif b == 0x08:
-            ret = "<!!!!VOCABVIEWER ASCII 0x08 BS!!!!>"
-        elif b == 0x0a:
-            ret = "<!!!!VOCABVIEWER ASCII 0x0a LF!!!!>"
-        elif b == 0x0b:
-            ret = "<!!!!VOCABVIEWER ASCII 0x0b VT!!!!>"
-        elif b == 0x0c:
-            ret = "<!!!!VOCABVIEWER ASCII 0x0c FF!!!!>"
-        elif b == 0x0d:
-            ret = "<!!!!VOCABVIEWER ASCII 0x0d CR!!!!>"
-        elif b == 0x0e:
-            ret = "<!!!!VOCABVIEWER ASCII 0x0e SO!!!!>"
-        elif b == 0x0f:
-            ret = "<!!!!VOCABVIEWER ASCII 0x0f SI!!!!>"
-        elif b == 0x10:
-            ret = "<!!!!VOCABVIEWER ASCII 0x10 DLE!!!!>"
-        elif b == 0x11:
-            ret = "<!!!!VOCABVIEWER ASCII 0x11 DC1!!!!>"
-        elif b == 0x12:
-            ret = "<!!!!VOCABVIEWER ASCII 0x12 DC2!!!!>"
-        elif b == 0x13:
-            ret = "<!!!!VOCABVIEWER ASCII 0x13 DC3!!!!>"
-        elif b == 0x14:
-            ret = "<!!!!VOCABVIEWER ASCII 0x14 DC4!!!!>"
-        elif b == 0x15:
-            ret = "<!!!!VOCABVIEWER ASCII 0x15 NAK!!!!>"
-        elif b == 0x16:
-            ret = "<!!!!VOCABVIEWER ASCII 0x16 SYN!!!!>"
-        elif b == 0x17:
-            ret = "<!!!!VOCABVIEWER ASCII 0x17 ETB!!!!>"
-        elif b == 0x18:
-            ret = "<!!!!VOCABVIEWER ASCII 0x18 CAN!!!!>"
-        elif b == 0x19:
-            ret = "<!!!!VOCABVIEWER ASCII 0x19 EM!!!!>"
-        elif b == 0x1a:
-            ret = "<!!!!VOCABVIEWER ASCII 0x1a SUB!!!!>"
-        elif b == 0x1b:
-            ret = "<!!!!VOCABVIEWER ASCII 0x1b ESC!!!!>"
-        elif b == 0x1c:
-            ret = "<!!!!VOCABVIEWER ASCII 0x1c FS!!!!>"
-        elif b == 0x1d:
-            ret = "<!!!!VOCABVIEWER ASCII 0x1d GS!!!!>"
-        elif b == 0x1e:
-            ret = "<!!!!VOCABVIEWER ASCII 0x1e RS!!!!>"
-        elif b == 0x1f:
-            ret = "<!!!!VOCABVIEWER ASCII 0x1f US!!!!>"
-        elif b == 0x20:
-            ret = "<!!!!VOCABVIEWER ASCII 0x20 SPC!!!!>"
+        if b >= 0x00 and b <= 0x08:
+            ret = escape_byte(b)
+        elif b >= 0x0a and b <= 0x19:
+            ret = escape_byte(b)
+        elif b >= 0x1a and b <= 0x20:
+            ret = escape_byte(b)
         elif b == 0x7f:
-            ret = "<!!!!VOCABVIEWER ASCII 0x7f DEL!!!!>"
+            ret = escape_byte(b)
     elif tok_enc == b"\xef\xbf\xbd":
         ret = "<!!!!VOCABVIEWER REPLACEMENT CHARACTER U+FFFD!!!!>"
     elif tok_enc == b"\xe2\x80\xa8" or tok_enc == b"\xe2\x80\xab" or tok_enc == b"\xe2\x80\xac" or tok_enc == b"\xe2\x80\xad" or tok_enc == b"\xe2\x80\xaf":
         ret = "<!!!!VOCABVIEWER ZERO WIDTH SPACE !!!!>"
     elif tok_enc == b"\xe2\x80\x8e" or tok_enc == b"\xe2\x80\x8f" or tok_enc == b"\x2c\xe2\x80\x8e" :
         ret = "<!!!!VOCABVIEWER ZERO WIDTH SPACE !!!!>"
+    else:
+        escape_tok_enc = bytes()
+        for b in tok_enc:
+            es = escape_byte(b)
+            if len(es) > 0:
+                escape_tok_enc += es.encode()
+            else:
+                # add byte
+                escape_tok_enc += b.to_bytes(1, "big")
+        
+        ret = escape_tok_enc.decode()
 
     return ret
 # sort