Change tokenization visualizer

app.py

@@ -147,12 +147,21 @@ def make_html_block(student_tokenizer, teacher_tokenizer, text, idx):
 
     highlighted = highlight_groups(student_tokenizer, teacher_tokenizer, s_ids, t_ids, s_groups, t_groups)
 
-    # Build tokenized views
+    # Build tokenized views with alternating colors
     s_tokens = [student_tokenizer.decode([tid], skip_special_tokens=False, clean_up_tokenization_spaces=False) for tid in s_ids]
     t_tokens = [teacher_tokenizer.decode([tid], skip_special_tokens=False, clean_up_tokenization_spaces=False) for tid in t_ids]
 
-    s_tokens_html = " | ".join(html.escape(f'"{t}"') for t in s_tokens)
-    t_tokens_html = " | ".join(html.escape(f'"{t}"') for t in t_tokens)
+    color1 = "#fff9c4"
+    color2 = "#b2ebf2"
+
+    s_tokens_html = "".join(
+        f'<span style="background-color:{color1 if i % 2 == 0 else color2};">{html.escape(t)}</span>'
+        for i, t in enumerate(s_tokens)
+    )
+    t_tokens_html = "".join(
+        f'<span style="background-color:{color1 if i % 2 == 0 else color2};">{html.escape(t)}</span>'
+        for i, t in enumerate(t_tokens)
+    )
 
     tokenized_section = f'''
     <div style="margin-bottom:15px;">