Upload folder using huggingface_hub

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

.gitignore

@@ -0,0 +1 @@
+venv

README.md

@@ -1,12 +1,7 @@
 ---
-title: Bpe Visualizer
-emoji: 😻
-colorFrom: purple
-colorTo: pink
+title: bpe-visualizer
+app_file: app.py
 sdk: gradio
 sdk_version: 4.42.0
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+An extra-small BPE tokenizer inspired by Karpathy's minbpe.

app.py

@@ -0,0 +1,37 @@
+import gradio as gr
+from xsbpe.basic import BasicTokenizer
+
+tk = BasicTokenizer()
+tk.train(open('dune.txt').read(), 256 + 10000, verbose=False)
+
+def tokenize(text):
+    tokens = tk.encode(text)
+    
+    colors = ['rgba(107,64,216,.3)', 'rgba(104,222,122,.4)', 'rgba(244,172,54,.4)', 'rgba(239,65,70,.4)', 'rgba(39,181,234,.4)']
+    colored_tokens = []
+    
+    for i, token in enumerate(tokens):
+        token = tk.vocab[token].decode('utf-8').replace(' ', ' ')
+        span = f'<span style="background-color: {colors[i % len(colors)]}">{token}</span>'
+        colored_tokens.append(span)
+
+    return '<p style="margin-left: 2px; margin-right: 2px; word-wrap: break-word">' + ''.join(colored_tokens) + '</p>', tokens, len(tokens), len(text)
+
+interface = gr.Interface(
+    fn=tokenize, 
+    inputs=[gr.TextArea(label='Input Text', type='text')], 
+    outputs=[
+        gr.HTML(label='Tokenized Text'),
+        gr.Textbox(label='Token IDs', lines=1, max_lines=5),
+        gr.Textbox(label='Tokens', max_lines=1),
+        gr.Textbox(label='Characters', max_lines=1)
+    ],
+    title="BPE Tokenization Visualizer",
+    live=True,
+    examples=[
+        'BPE, or Byte Pair Encoding, is a method used to compress text by breaking it down into smaller units. In natural language processing, it helps tokenize words by merging the most frequent pairs of characters or symbols, creating more efficient and manageable tokens for analysis.'
+    ],
+    show_progress='hidden',
+    api_name='tokenize',
+    allow_flagging='never'
+).launch()

demo.ipynb

@@ -0,0 +1,123 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/jamino/xsbpe/venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import gradio as gr\n",
+    "from xsbpe.basic import BasicTokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tk = BasicTokenizer()\n",
+    "tk.train(open('dune.txt').read(), 256 + 10000, verbose=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7896\n",
+      "Running on public URL: https://cb2bf07164e5cebb6e.gradio.live\n",
+      "\n",
+      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"https://cb2bf07164e5cebb6e.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def tokenize(text):\n",
+    "    tokens = tk.encode(text)\n",
+    "    \n",
+    "    colors = ['rgba(107,64,216,.3)', 'rgba(104,222,122,.4)', 'rgba(244,172,54,.4)', 'rgba(239,65,70,.4)', 'rgba(39,181,234,.4)']\n",
+    "    colored_tokens = []\n",
+    "    \n",
+    "    for i, token in enumerate(tokens):\n",
+    "        token = tk.vocab[token].decode('utf-8').replace(' ', '&nbsp;')\n",
+    "        span = f'<span style=\"background-color: {colors[i % len(colors)]}\">{token}</span>'\n",
+    "        colored_tokens.append(span)\n",
+    "\n",
+    "    return '<p style=\"margin-left: 2px; margin-right: 2px; word-wrap: break-word\">' + ''.join(colored_tokens) + '</p>', tokens, len(tokens), len(text)\n",
+    "\n",
+    "interface = gr.Interface(\n",
+    "    fn=tokenize, \n",
+    "    inputs=[gr.TextArea(label='Input Text', type='text')], \n",
+    "    outputs=[\n",
+    "        gr.HTML(label='Tokenized Text'),\n",
+    "        gr.Textbox(label='Token IDs', lines=1, max_lines=5),\n",
+    "        gr.Textbox(label='Tokens', max_lines=1),\n",
+    "        gr.Textbox(label='Characters', max_lines=1)\n",
+    "    ],\n",
+    "    title=\"BPE Tokenization Visualizer\",\n",
+    "    live=True,\n",
+    "    examples=[\n",
+    "        'BPE, or Byte Pair Encoding, is a method used to compress text by breaking it down into smaller units. In natural language processing, it helps tokenize words by merging the most frequent pairs of characters or symbols, creating more efficient and manageable tokens for analysis.'\n",
+    "    ],\n",
+    "    show_progress='hidden',\n",
+    "    api_name='tokenize',\n",
+    "    allow_flagging='never'\n",
+    ").launch(share=True, inbrowser=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

test.ipynb

@@ -0,0 +1,910 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "\n",
+    "from xsbpe.basic import BasicTokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentence = \"Jessica, looking down the length of the table, saw a faint trembling at the corners of Leto's mouth, noted the dark flush of anger on his cheeks. What has angered him? she asked herself. Surely not my invitation to the smuggler.\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[74, 101, 115, 115, 105, 99, 97, 44, 32, 108, 111, 111, 107, 105, 110, 103, 32, 100, 111, 119, 110, 32, 116, 104, 101, 32, 108, 101, 110, 103, 116, 104, 32, 111, 102, 32, 116, 104, 101, 32, 116, 97, 98, 108, 101, 44, 32, 115, 97, 119, 32, 97, 32, 102, 97, 105, 110, 116, 32, 116, 114, 101, 109, 98, 108, 105, 110, 103, 32, 97, 116, 32, 116, 104, 101, 32, 99, 111, 114, 110, 101, 114, 115, 32, 111, 102, 32, 76, 101, 116, 111, 39, 115, 32, 109, 111, 117, 116, 104, 44, 32, 110, 111, 116, 101, 100, 32, 116, 104, 101, 32, 100, 97, 114, 107, 32, 102, 108, 117, 115, 104, 32, 111, 102, 32, 97, 110, 103, 101, 114, 32, 111, 110, 32, 104, 105, 115, 32, 99, 104, 101, 101, 107, 115, 46, 32, 87, 104, 97, 116, 32, 104, 97, 115, 32, 97, 110, 103, 101, 114, 101, 100, 32, 104, 105, 109, 63, 32, 115, 104, 101, 32, 97, 115, 107, 101, 100, 32, 104, 101, 114, 115, 101, 108, 102, 46, 32, 83, 117, 114, 101, 108, 121, 32, 110, 111, 116, 32, 109, 121, 32, 105, 110, 118, 105, 116, 97, 116, 105, 111, 110, 32, 116, 111, 32, 116, 104, 101, 32, 115, 109, 117, 103, 103, 108, 101, 114, 46]\n",
+      "LEN: 228\n"
+     ]
+    }
+   ],
+   "source": [
+    "tk = BasicTokenizer()\n",
+    "# only using individual chars, no merging\n",
+    "encoded1 = tk.encode(sentence)\n",
+    "print(encoded1)\n",
+    "print('LEN:', len(encoded1))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "merge 1/44: (101, 32) -> 256 (b'e ') had 37476 occurrences\n",
+      "merge 2/44: (116, 104) -> 257 (b'th') had 26021 occurrences\n",
+      "merge 3/44: (32, 32) -> 258 (b'  ') had 23243 occurrences\n",
+      "merge 4/44: (100, 32) -> 259 (b'd ') had 19923 occurrences\n",
+      "merge 5/44: (115, 32) -> 260 (b's ') had 17700 occurrences\n",
+      "merge 6/44: (116, 32) -> 261 (b't ') had 16514 occurrences\n",
+      "merge 7/44: (105, 110) -> 262 (b'in') had 14704 occurrences\n",
+      "merge 8/44: (101, 114) -> 263 (b'er') had 14496 occurrences\n",
+      "merge 9/44: (257, 256) -> 264 (b'the ') had 12705 occurrences\n",
+      "merge 10/44: (46, 32) -> 265 (b'. ') had 12182 occurrences\n",
+      "merge 11/44: (97, 110) -> 266 (b'an') had 11336 occurrences\n",
+      "merge 12/44: (111, 117) -> 267 (b'ou') had 9967 occurrences\n",
+      "merge 13/44: (44, 32) -> 268 (b', ') had 9853 occurrences\n",
+      "merge 14/44: (101, 110) -> 269 (b'en') had 9128 occurrences\n",
+      "merge 15/44: (111, 110) -> 270 (b'on') had 9008 occurrences\n",
+      "merge 16/44: (101, 259) -> 271 (b'ed ') had 8847 occurrences\n",
+      "merge 17/44: (10, 258) -> 272 (b'\\n  ') had 7751 occurrences\n",
+      "merge 18/44: (272, 258) -> 273 (b'\\n    ') had 7743 occurrences\n",
+      "merge 19/44: (97, 114) -> 274 (b'ar') had 7736 occurrences\n",
+      "merge 20/44: (32, 264) -> 275 (b' the ') had 7594 occurrences\n",
+      "merge 21/44: (32, 115) -> 276 (b' s') had 7368 occurrences\n",
+      "merge 22/44: (116, 111) -> 277 (b'to') had 6860 occurrences\n",
+      "merge 23/44: (121, 32) -> 278 (b'y ') had 6406 occurrences\n",
+      "merge 24/44: (262, 103) -> 279 (b'ing') had 6242 occurrences\n",
+      "merge 25/44: (114, 101) -> 280 (b're') had 5900 occurrences\n",
+      "merge 26/44: (111, 102) -> 281 (b'of') had 5743 occurrences\n",
+      "merge 27/44: (111, 114) -> 282 (b'or') had 5571 occurrences\n",
+      "merge 28/44: (32, 104) -> 283 (b' h') had 5278 occurrences\n",
+      "merge 29/44: (101, 115) -> 284 (b'es') had 5073 occurrences\n",
+      "merge 30/44: (105, 260) -> 285 (b'is ') had 4829 occurrences\n",
+      "merge 31/44: (32, 97) -> 286 (b' a') had 4595 occurrences\n",
+      "merge 32/44: (108, 108) -> 287 (b'll') had 4557 occurrences\n",
+      "merge 33/44: (104, 256) -> 288 (b'he ') had 4393 occurrences\n",
+      "merge 34/44: (116, 105) -> 289 (b'ti') had 4062 occurrences\n",
+      "merge 35/44: (97, 261) -> 290 (b'at ') had 3947 occurrences\n",
+      "merge 36/44: (34, 32) -> 291 (b'\" ') had 3834 occurrences\n",
+      "merge 37/44: (32, 119) -> 292 (b' w') had 3725 occurrences\n",
+      "merge 38/44: (266, 259) -> 293 (b'and ') had 3660 occurrences\n",
+      "merge 39/44: (273, 34) -> 294 (b'\\n    \"') had 3644 occurrences\n",
+      "merge 40/44: (111, 119) -> 295 (b'ow') had 3555 occurrences\n",
+      "merge 41/44: (32, 257) -> 296 (b' th') had 3477 occurrences\n",
+      "merge 42/44: (105, 100) -> 297 (b'id') had 3458 occurrences\n",
+      "merge 43/44: (111, 109) -> 298 (b'om') had 3262 occurrences\n",
+      "merge 44/44: (97, 116) -> 299 (b'at') had 3101 occurrences\n",
+      "Text Length: 203588 | Time to train: 4.710334777832031\n",
+      "[74, 284, 115, 105, 99, 97, 268, 108, 111, 111, 107, 279, 32, 100, 295, 110, 275, 108, 269, 103, 257, 32, 281, 275, 116, 97, 98, 108, 101, 268, 115, 97, 119, 286, 32, 102, 97, 262, 261, 116, 280, 109, 98, 108, 279, 286, 261, 264, 99, 282, 110, 263, 260, 281, 32, 76, 101, 277, 39, 260, 109, 267, 257, 268, 110, 111, 116, 271, 264, 100, 274, 107, 32, 102, 108, 117, 115, 104, 32, 281, 32, 266, 103, 263, 32, 270, 283, 285, 99, 104, 101, 101, 107, 115, 265, 87, 104, 290, 104, 97, 260, 266, 103, 263, 271, 104, 105, 109, 63, 276, 288, 97, 115, 107, 271, 104, 263, 115, 101, 108, 102, 265, 83, 117, 280, 108, 278, 110, 111, 261, 109, 278, 262, 118, 105, 116, 97, 289, 270, 32, 277, 275, 115, 109, 117, 103, 103, 108, 263, 46]\n",
+      "LEN: 150\n"
+     ]
+    }
+   ],
+   "source": [
+    "text = open('dune.txt').read()\n",
+    "\n",
+    "st = time.time()\n",
+    "tk.train(text, 300, verbose=True)\n",
+    "et = time.time()\n",
+    "\n",
+    "print(f'Text Length: {len(text.split())} | Time to train: {et-st}')\n",
+    "encoded2 = tk.encode(sentence)\n",
+    "print(encoded2)\n",
+    "print('LEN:', len(encoded2))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "merge 1/744: (101, 32) -> 256 (b'e ') had 37476 occurrences\n",
+      "merge 2/744: (116, 104) -> 257 (b'th') had 26021 occurrences\n",
+      "merge 3/744: (32, 32) -> 258 (b'  ') had 23243 occurrences\n",
+      "merge 4/744: (100, 32) -> 259 (b'd ') had 19923 occurrences\n",
+      "merge 5/744: (115, 32) -> 260 (b's ') had 17700 occurrences\n",
+      "merge 6/744: (116, 32) -> 261 (b't ') had 16514 occurrences\n",
+      "merge 7/744: (105, 110) -> 262 (b'in') had 14704 occurrences\n",
+      "merge 8/744: (101, 114) -> 263 (b'er') had 14496 occurrences\n",
+      "merge 9/744: (257, 256) -> 264 (b'the ') had 12705 occurrences\n",
+      "merge 10/744: (46, 32) -> 265 (b'. ') had 12182 occurrences\n",
+      "merge 11/744: (97, 110) -> 266 (b'an') had 11336 occurrences\n",
+      "merge 12/744: (111, 117) -> 267 (b'ou') had 9967 occurrences\n",
+      "merge 13/744: (44, 32) -> 268 (b', ') had 9853 occurrences\n",
+      "merge 14/744: (101, 110) -> 269 (b'en') had 9128 occurrences\n",
+      "merge 15/744: (111, 110) -> 270 (b'on') had 9008 occurrences\n",
+      "merge 16/744: (101, 259) -> 271 (b'ed ') had 8847 occurrences\n",
+      "merge 17/744: (10, 258) -> 272 (b'\\n  ') had 7751 occurrences\n",
+      "merge 18/744: (272, 258) -> 273 (b'\\n    ') had 7743 occurrences\n",
+      "merge 19/744: (97, 114) -> 274 (b'ar') had 7736 occurrences\n",
+      "merge 20/744: (32, 264) -> 275 (b' the ') had 7594 occurrences\n",
+      "merge 21/744: (32, 115) -> 276 (b' s') had 7368 occurrences\n",
+      "merge 22/744: (116, 111) -> 277 (b'to') had 6860 occurrences\n",
+      "merge 23/744: (121, 32) -> 278 (b'y ') had 6406 occurrences\n",
+      "merge 24/744: (262, 103) -> 279 (b'ing') had 6242 occurrences\n",
+      "merge 25/744: (114, 101) -> 280 (b're') had 5900 occurrences\n",
+      "merge 26/744: (111, 102) -> 281 (b'of') had 5743 occurrences\n",
+      "merge 27/744: (111, 114) -> 282 (b'or') had 5571 occurrences\n",
+      "merge 28/744: (32, 104) -> 283 (b' h') had 5278 occurrences\n",
+      "merge 29/744: (101, 115) -> 284 (b'es') had 5073 occurrences\n",
+      "merge 30/744: (105, 260) -> 285 (b'is ') had 4829 occurrences\n",
+      "merge 31/744: (32, 97) -> 286 (b' a') had 4595 occurrences\n",
+      "merge 32/744: (108, 108) -> 287 (b'll') had 4557 occurrences\n",
+      "merge 33/744: (104, 256) -> 288 (b'he ') had 4393 occurrences\n",
+      "merge 34/744: (116, 105) -> 289 (b'ti') had 4062 occurrences\n",
+      "merge 35/744: (97, 261) -> 290 (b'at ') had 3947 occurrences\n",
+      "merge 36/744: (34, 32) -> 291 (b'\" ') had 3834 occurrences\n",
+      "merge 37/744: (32, 119) -> 292 (b' w') had 3725 occurrences\n",
+      "merge 38/744: (266, 259) -> 293 (b'and ') had 3660 occurrences\n",
+      "merge 39/744: (273, 34) -> 294 (b'\\n    \"') had 3644 occurrences\n",
+      "merge 40/744: (111, 119) -> 295 (b'ow') had 3555 occurrences\n",
+      "merge 41/744: (32, 257) -> 296 (b' th') had 3477 occurrences\n",
+      "merge 42/744: (105, 100) -> 297 (b'id') had 3458 occurrences\n",
+      "merge 43/744: (111, 109) -> 298 (b'om') had 3262 occurrences\n",
+      "merge 44/744: (97, 116) -> 299 (b'at') had 3101 occurrences\n",
+      "merge 45/744: (32, 109) -> 300 (b' m') had 2939 occurrences\n",
+      "merge 46/744: (105, 116) -> 301 (b'it') had 2934 occurrences\n",
+      "merge 47/744: (97, 99) -> 302 (b'ac') had 2889 occurrences\n",
+      "merge 48/744: (103, 104) -> 303 (b'gh') had 2824 occurrences\n",
+      "merge 49/744: (101, 100) -> 304 (b'ed') had 2824 occurrences\n",
+      "merge 50/744: (32, 99) -> 305 (b' c') had 2740 occurrences\n",
+      "merge 51/744: (105, 99) -> 306 (b'ic') had 2721 occurrences\n",
+      "merge 52/744: (105, 115) -> 307 (b'is') had 2678 occurrences\n",
+      "merge 53/744: (97, 117) -> 308 (b'au') had 2641 occurrences\n",
+      "merge 54/744: (101, 108) -> 309 (b'el') had 2605 occurrences\n",
+      "merge 55/744: (97, 108) -> 310 (b'al') had 2586 occurrences\n",
+      "merge 56/744: (32, 102) -> 311 (b' f') had 2539 occurrences\n",
+      "merge 57/744: (32, 98) -> 312 (b' b') had 2492 occurrences\n",
+      "merge 58/744: (121, 267) -> 313 (b'you') had 2475 occurrences\n",
+      "merge 59/744: (97, 260) -> 314 (b'as ') had 2357 occurrences\n",
+      "merge 60/744: (117, 114) -> 315 (b'ur') had 2200 occurrences\n",
+      "merge 61/744: (97, 115) -> 316 (b'as') had 2194 occurrences\n",
+      "merge 62/744: (114, 111) -> 317 (b'ro') had 2176 occurrences\n",
+      "merge 63/744: (46, 273) -> 318 (b'.\\n    ') had 2169 occurrences\n",
+      "merge 64/744: (97, 297) -> 319 (b'aid') had 2135 occurrences\n",
+      "merge 65/744: (101, 260) -> 320 (b'es ') had 2112 occurrences\n",
+      "merge 66/744: (39, 260) -> 321 (b\"'s \") had 2111 occurrences\n",
+      "merge 67/744: (32, 100) -> 322 (b' d') had 2089 occurrences\n",
+      "merge 68/744: (104, 97) -> 323 (b'ha') had 2087 occurrences\n",
+      "merge 69/744: (265, 34) -> 324 (b'. \"') had 2076 occurrences\n",
+      "merge 70/744: (118, 256) -> 325 (b've ') had 2056 occurrences\n",
+      "merge 71/744: (105, 109) -> 326 (b'im') had 2038 occurrences\n",
+      "merge 72/744: (279, 32) -> 327 (b'ing ') had 1976 occurrences\n",
+      "merge 73/744: (263, 32) -> 328 (b'er ') had 1961 occurrences\n",
+      "merge 74/744: (117, 115) -> 329 (b'us') had 1859 occurrences\n",
+      "merge 75/744: (108, 101) -> 330 (b'le') had 1856 occurrences\n",
+      "merge 76/744: (110, 111) -> 331 (b'no') had 1837 occurrences\n",
+      "merge 77/744: (84, 288) -> 332 (b'The ') had 1830 occurrences\n",
+      "merge 78/744: (105, 257) -> 333 (b'ith') had 1792 occurrences\n",
+      "merge 79/744: (101, 109) -> 334 (b'em') had 1787 occurrences\n",
+      "merge 80/744: (97, 100) -> 335 (b'ad') had 1733 occurrences\n",
+      "merge 81/744: (308, 108) -> 336 (b'aul') had 1733 occurrences\n",
+      "merge 82/744: (116, 114) -> 337 (b'tr') had 1730 occurrences\n",
+      "merge 83/744: (108, 105) -> 338 (b'li') had 1725 occurrences\n",
+      "merge 84/744: (80, 336) -> 339 (b'Paul') had 1722 occurrences\n",
+      "merge 85/744: (263, 256) -> 340 (b'ere ') had 1718 occurrences\n",
+      "merge 86/744: (84, 104) -> 341 (b'Th') had 1704 occurrences\n",
+      "merge 87/744: (99, 104) -> 342 (b'ch') had 1690 occurrences\n",
+      "merge 88/744: (105, 114) -> 343 (b'ir') had 1688 occurrences\n",
+      "merge 89/744: (32, 112) -> 344 (b' p') had 1670 occurrences\n",
+      "merge 90/744: (108, 259) -> 345 (b'ld ') had 1635 occurrences\n",
+      "merge 91/744: (265, 265) -> 346 (b'. . ') had 1604 occurrences\n",
+      "merge 92/744: (117, 110) -> 347 (b'un') had 1590 occurrences\n",
+      "merge 93/744: (284, 115) -> 348 (b'ess') had 1585 occurrences\n",
+      "merge 94/744: (256, 115) -> 349 (b'e s') had 1529 occurrences\n",
+      "merge 95/744: (46, 291) -> 350 (b'.\" ') had 1514 occurrences\n",
+      "merge 96/744: (281, 32) -> 351 (b'of ') had 1509 occurrences\n",
+      "merge 97/744: (105, 261) -> 352 (b'it ') had 1472 occurrences\n",
+      "merge 98/744: (108, 111) -> 353 (b'lo') had 1467 occurrences\n",
+      "merge 99/744: (289, 270) -> 354 (b'tion') had 1463 occurrences\n",
+      "merge 100/744: (101, 99) -> 355 (b'ec') had 1452 occurrences\n",
+      "merge 101/744: (97, 287) -> 356 (b'all') had 1428 occurrences\n",
+      "merge 102/744: (277, 32) -> 357 (b'to ') had 1379 occurrences\n",
+      "merge 103/744: (116, 263) -> 358 (b'ter') had 1373 occurrences\n",
+      "merge 104/744: (267, 303) -> 359 (b'ough') had 1371 occurrences\n",
+      "merge 105/744: (102, 282) -> 360 (b'for') had 1358 occurrences\n",
+      "merge 106/744: (118, 263) -> 361 (b'ver') had 1353 occurrences\n",
+      "merge 107/744: (281, 275) -> 362 (b'of the ') had 1340 occurrences\n",
+      "merge 108/744: (101, 121) -> 363 (b'ey') had 1323 occurrences\n",
+      "merge 109/744: (46, 294) -> 364 (b'.\\n    \"') had 1291 occurrences\n",
+      "merge 110/744: (97, 119) -> 365 (b'aw') had 1274 occurrences\n",
+      "merge 111/744: (97, 262) -> 366 (b'ain') had 1247 occurrences\n",
+      "merge 112/744: (101, 268) -> 367 (b'e, ') had 1240 occurrences\n",
+      "merge 113/744: (97, 107) -> 368 (b'ak') had 1216 occurrences\n",
+      "merge 114/744: (283, 285) -> 369 (b' his ') had 1201 occurrences\n",
+      "merge 115/744: (110, 259) -> 370 (b'nd ') had 1187 occurrences\n",
+      "merge 116/744: (97, 109) -> 371 (b'am') had 1181 occurrences\n",
+      "merge 117/744: (97, 32) -> 372 (b'a ') had 1177 occurrences\n",
+      "merge 118/744: (108, 256) -> 373 (b'le ') had 1177 occurrences\n",
+      "merge 119/744: (119, 104) -> 374 (b'wh') had 1176 occurrences\n",
+      "merge 120/744: (257, 263) -> 375 (b'ther') had 1154 occurrences\n",
+      "merge 121/744: (45, 45) -> 376 (b'--') had 1153 occurrences\n",
+      "merge 122/744: (269, 32) -> 377 (b'en ') had 1142 occurrences\n",
+      "merge 123/744: (98, 101) -> 378 (b'be') had 1117 occurrences\n",
+      "merge 124/744: (44, 291) -> 379 (b',\" ') had 1109 occurrences\n",
+      "merge 125/744: (266, 100) -> 380 (b'and') had 1097 occurrences\n",
+      "merge 126/744: (276, 319) -> 381 (b' said') had 1097 occurrences\n",
+      "merge 127/744: (257, 290) -> 382 (b'that ') had 1087 occurrences\n",
+      "merge 128/744: (104, 285) -> 383 (b'his ') had 1087 occurrences\n",
+      "merge 129/744: (115, 116) -> 384 (b'st') had 1079 occurrences\n",
+      "merge 130/744: (119, 314) -> 385 (b'was ') had 1068 occurrences\n",
+      "merge 131/744: (111, 112) -> 386 (b'op') had 1063 occurrences\n",
+      "merge 132/744: (105, 108) -> 387 (b'il') had 1062 occurrences\n",
+      "merge 133/744: (117, 261) -> 388 (b'ut ') had 1059 occurrences\n",
+      "merge 134/744: (119, 333) -> 389 (b'with') had 1047 occurrences\n",
+      "merge 135/744: (265, 72) -> 390 (b'. H') had 1032 occurrences\n",
+      "merge 136/744: (58, 32) -> 391 (b': ') had 1024 occurrences\n",
+      "merge 137/744: (114, 105) -> 392 (b'ri') had 1023 occurrences\n",
+      "merge 138/744: (114, 298) -> 393 (b'rom') had 1022 occurrences\n",
+      "merge 139/744: (110, 101) -> 394 (b'ne') had 1022 occurrences\n",
+      "merge 140/744: (266, 32) -> 395 (b'an ') had 1016 occurrences\n",
+      "merge 141/744: (267, 261) -> 396 (b'out ') had 1009 occurrences\n",
+      "merge 142/744: (112, 263) -> 397 (b'per') had 998 occurrences\n",
+      "merge 143/744: (44, 34) -> 398 (b',\"') had 993 occurrences\n",
+      "merge 144/744: (46, 34) -> 399 (b'.\"') had 992 occurrences\n",
+      "merge 145/744: (111, 107) -> 400 (b'ok') had 985 occurrences\n",
+      "merge 146/744: (97, 121) -> 401 (b'ay') had 952 occurrences\n",
+      "merge 147/744: (280, 109) -> 402 (b'rem') had 951 occurrences\n",
+      "merge 148/744: (97, 98) -> 403 (b'ab') had 941 occurrences\n",
+      "merge 149/744: (110, 295) -> 404 (b'now') had 931 occurrences\n",
+      "merge 150/744: (111, 115) -> 405 (b'os') had 927 occurrences\n",
+      "merge 151/744: (32, 262) -> 406 (b' in') had 918 occurrences\n",
+      "merge 152/744: (267, 345) -> 407 (b'ould ') had 917 occurrences\n",
+      "merge 153/744: (265, 73) -> 408 (b'. I') had 917 occurrences\n",
+      "merge 154/744: (269, 116) -> 409 (b'ent') had 912 occurrences\n",
+      "merge 155/744: (105, 287) -> 410 (b'ill') had 901 occurrences\n",
+      "merge 156/744: (74, 348) -> 411 (b'Jess') had 899 occurrences\n",
+      "merge 157/744: (411, 306) -> 412 (b'Jessic') had 899 occurrences\n",
+      "merge 158/744: (97, 103) -> 413 (b'ag') had 892 occurrences\n",
+      "merge 159/744: (97, 112) -> 414 (b'ap') had 890 occurrences\n",
+      "merge 160/744: (268, 115) -> 415 (b', s') had 888 occurrences\n",
+      "merge 161/744: (116, 271) -> 416 (b'ted ') had 887 occurrences\n",
+      "merge 162/744: (108, 278) -> 417 (b'ly ') had 872 occurrences\n",
+      "merge 163/744: (277, 275) -> 418 (b'to the ') had 869 occurrences\n",
+      "merge 164/744: (101, 274) -> 419 (b'ear') had 866 occurrences\n",
+      "merge 165/744: (331, 261) -> 420 (b'not ') had 857 occurrences\n",
+      "merge 166/744: (97, 278) -> 421 (b'ay ') had 851 occurrences\n",
+      "merge 167/744: (101, 116) -> 422 (b'et') had 832 occurrences\n",
+      "merge 168/744: (101, 112) -> 423 (b'ep') had 815 occurrences\n",
+      "merge 169/744: (104, 326) -> 424 (b'him') had 814 occurrences\n",
+      "merge 170/744: (89, 267) -> 425 (b'You') had 812 occurrences\n",
+      "merge 171/744: (269, 261) -> 426 (b'ent ') had 786 occurrences\n",
+      "merge 172/744: (97, 259) -> 427 (b'ad ') had 777 occurrences\n",
+      "merge 173/744: (111, 32) -> 428 (b'o ') had 771 occurrences\n",
+      "merge 174/744: (101, 278) -> 429 (b'ey ') had 770 occurrences\n",
+      "merge 175/744: (117, 112) -> 430 (b'up') had 765 occurrences\n",
+      "merge 176/744: (412, 97) -> 431 (b'Jessica') had 757 occurrences\n",
+      "merge 177/744: (302, 107) -> 432 (b'ack') had 757 occurrences\n",
+      "merge 178/744: (295, 110) -> 433 (b'own') had 753 occurrences\n",
+      "merge 179/744: (323, 259) -> 434 (b'had ') had 752 occurrences\n",
+      "merge 180/744: (270, 32) -> 435 (b'on ') had 749 occurrences\n",
+      "merge 181/744: (346, 265) -> 436 (b'. . . ') had 738 occurrences\n",
+      "merge 182/744: (270, 256) -> 437 (b'one ') had 726 occurrences\n",
+      "merge 183/744: (280, 115) -> 438 (b'res') had 716 occurrences\n",
+      "merge 184/744: (265, 332) -> 439 (b'. The ') had 711 occurrences\n",
+      "merge 185/744: (350, 294) -> 440 (b'.\" \\n    \"') had 710 occurrences\n",
+      "merge 186/744: (101, 261) -> 441 (b'et ') had 701 occurrences\n",
+      "merge 187/744: (105, 102) -> 442 (b'if') had 699 occurrences\n",
+      "merge 188/744: (108, 266) -> 443 (b'lan') had 694 occurrences\n",
+      "merge 189/744: (296, 290) -> 444 (b' that ') had 693 occurrences\n",
+      "merge 190/744: (265, 341) -> 445 (b'. Th') had 690 occurrences\n",
+      "merge 191/744: (104, 263) -> 446 (b'her') had 688 occurrences\n",
+      "merge 192/744: (315, 110) -> 447 (b'urn') had 687 occurrences\n",
+      "merge 193/744: (105, 303) -> 448 (b'igh') had 685 occurrences\n",
+      "merge 194/744: (262, 275) -> 449 (b'in the ') had 685 occurrences\n",
+      "merge 195/744: (292, 104) -> 450 (b' wh') had 684 occurrences\n",
+      "merge 196/744: (265, 65) -> 451 (b'. A') had 678 occurrences\n",
+      "merge 197/744: (117, 109) -> 452 (b'um') had 673 occurrences\n",
+      "merge 198/744: (271, 264) -> 453 (b'ed the ') had 672 occurrences\n",
+      "merge 199/744: (111, 375) -> 454 (b'other') had 649 occurrences\n",
+      "merge 200/744: (87, 104) -> 455 (b'Wh') had 648 occurrences\n",
+      "merge 201/744: (107, 256) -> 456 (b'ke ') had 646 occurrences\n",
+      "merge 202/744: (63, 291) -> 457 (b'?\" ') had 642 occurrences\n",
+      "merge 203/744: (102, 393) -> 458 (b'from') had 640 occurrences\n",
+      "merge 204/744: (83, 288) -> 459 (b'She ') had 640 occurrences\n",
+      "merge 205/744: (115, 268) -> 460 (b's, ') had 639 occurrences\n",
+      "merge 206/744: (274, 107) -> 461 (b'ark') had 636 occurrences\n",
+      "merge 207/744: (100, 263) -> 462 (b'der') had 635 occurrences\n",
+      "merge 208/744: (111, 108) -> 463 (b'ol') had 632 occurrences\n",
+      "merge 209/744: (104, 101) -> 464 (b'he') had 622 occurrences\n",
+      "merge 210/744: (115, 309) -> 465 (b'sel') had 615 occurrences\n",
+      "merge 211/744: (267, 114) -> 466 (b'our') had 615 occurrences\n",
+      "merge 212/744: (32, 281) -> 467 (b' of') had 614 occurrences\n",
+      "merge 213/744: (270, 103) -> 468 (b'ong') had 606 occurrences\n",
+      "merge 214/744: (97, 354) -> 469 (b'ation') had 605 occurrences\n",
+      "merge 215/744: (292, 333) -> 470 (b' with') had 605 occurrences\n",
+      "merge 216/744: (313, 114) -> 471 (b'your') had 604 occurrences\n",
+      "merge 217/744: (99, 256) -> 472 (b'ce ') had 604 occurrences\n",
+      "merge 218/744: (329, 261) -> 473 (b'ust ') had 598 occurrences\n",
+      "merge 219/744: (66, 274) -> 474 (b'Bar') had 595 occurrences\n",
+      "merge 220/744: (117, 108) -> 475 (b'ul') had 593 occurrences\n",
+      "merge 221/744: (32, 293) -> 476 (b' and ') had 592 occurrences\n",
+      "merge 222/744: (32, 277) -> 477 (b' to') had 591 occurrences\n",
+      "merge 223/744: (391, 34) -> 478 (b': \"') had 589 occurrences\n",
+      "merge 224/744: (113, 117) -> 479 (b'qu') had 584 occurrences\n",
+      "merge 225/744: (274, 256) -> 480 (b'are ') had 583 occurrences\n",
+      "merge 226/744: (390, 256) -> 481 (b'. He ') had 582 occurrences\n",
+      "merge 227/744: (101, 287) -> 482 (b'ell') had 572 occurrences\n",
+      "merge 228/744: (101, 120) -> 483 (b'ex') had 571 occurrences\n",
+      "merge 229/744: (68, 117) -> 484 (b'Du') had 571 occurrences\n",
+      "merge 230/744: (97, 325) -> 485 (b'ave ') had 566 occurrences\n",
+      "merge 231/744: (107, 404) -> 486 (b'know') had 563 occurrences\n",
+      "merge 232/744: (115, 101) -> 487 (b'se') had 559 occurrences\n",
+      "merge 233/744: (115, 256) -> 488 (b'se ') had 557 occurrences\n",
+      "merge 234/744: (101, 343) -> 489 (b'eir') had 556 occurrences\n",
+      "merge 235/744: (116, 269) -> 490 (b'ten') had 553 occurrences\n",
+      "merge 236/744: (465, 102) -> 491 (b'self') had 553 occurrences\n",
+      "merge 237/744: (39, 261) -> 492 (b\"'t \") had 553 occurrences\n",
+      "merge 238/744: (266, 103) -> 493 (b'ang') had 552 occurrences\n",
+      "merge 239/744: (268, 264) -> 494 (b', the ') had 551 occurrences\n",
+      "merge 240/744: (399, 273) -> 495 (b'.\"\\n    ') had 551 occurrences\n",
+      "merge 241/744: (286, 32) -> 496 (b' a ') had 549 occurrences\n",
+      "merge 242/744: (284, 260) -> 497 (b'ess ') had 536 occurrences\n",
+      "merge 243/744: (115, 104) -> 498 (b'sh') had 534 occurrences\n",
+      "merge 244/744: (115, 319) -> 499 (b'said') had 530 occurrences\n",
+      "merge 245/744: (267, 370) -> 500 (b'ound ') had 526 occurrences\n",
+      "merge 246/744: (267, 110) -> 501 (b'oun') had 524 occurrences\n",
+      "merge 247/744: (105, 98) -> 502 (b'ib') had 520 occurrences\n",
+      "merge 248/744: (70, 402) -> 503 (b'Frem') had 520 occurrences\n",
+      "merge 249/744: (283, 263) -> 504 (b' her') had 519 occurrences\n",
+      "merge 250/744: (359, 116) -> 505 (b'ought') had 515 occurrences\n",
+      "merge 251/744: (63, 34) -> 506 (b'?\"') had 512 occurrences\n",
+      "merge 252/744: (289, 108) -> 507 (b'til') had 512 occurrences\n",
+      "merge 253/744: (119, 282) -> 508 (b'wor') had 511 occurrences\n",
+      "merge 254/744: (98, 388) -> 509 (b'but ') had 508 occurrences\n",
+      "merge 255/744: (311, 282) -> 510 (b' for') had 506 occurrences\n",
+      "merge 256/744: (97, 257) -> 511 (b'ath') had 504 occurrences\n",
+      "merge 257/744: (381, 324) -> 512 (b' said. \"') had 502 occurrences\n",
+      "merge 258/744: (115, 112) -> 513 (b'sp') had 501 occurrences\n",
+      "merge 259/744: (304, 268) -> 514 (b'ed, ') had 499 occurrences\n",
+      "merge 260/744: (474, 270) -> 515 (b'Baron') had 497 occurrences\n",
+      "merge 261/744: (98, 256) -> 516 (b'be ') had 494 occurrences\n",
+      "merge 262/744: (105, 103) -> 517 (b'ig') had 493 occurrences\n",
+      "merge 263/744: (283, 326) -> 518 (b' him') had 487 occurrences\n",
+      "merge 264/744: (111, 100) -> 519 (b'od') had 482 occurrences\n",
+      "merge 265/744: (276, 116) -> 520 (b' st') had 481 occurrences\n",
+      "merge 266/744: (279, 275) -> 521 (b'ing the ') had 480 occurrences\n",
+      "merge 267/744: (289, 109) -> 522 (b'tim') had 476 occurrences\n",
+      "merge 268/744: (102, 302) -> 523 (b'fac') had 476 occurrences\n",
+      "merge 269/744: (99, 107) -> 524 (b'ck') had 470 occurrences\n",
+      "merge 270/744: (277, 114) -> 525 (b'tor') had 469 occurrences\n",
+      "merge 271/744: (109, 266) -> 526 (b'man') had 469 occurrences\n",
+      "merge 272/744: (32, 103) -> 527 (b' g') had 467 occurrences\n",
+      "merge 273/744: (261, 264) -> 528 (b't the ') had 466 occurrences\n",
+      "merge 274/744: (112, 108) -> 529 (b'pl') had 464 occurrences\n",
+      "merge 275/744: (276, 104) -> 530 (b' sh') had 461 occurrences\n",
+      "merge 276/744: (99, 298) -> 531 (b'com') had 459 occurrences\n",
+      "merge 277/744: (105, 259) -> 532 (b'id ') had 457 occurrences\n",
+      "merge 278/744: (98, 108) -> 533 (b'bl') had 455 occurrences\n",
+      "merge 279/744: (111, 361) -> 534 (b'over') had 454 occurrences\n",
+      "merge 280/744: (116, 447) -> 535 (b'turn') had 454 occurrences\n",
+      "merge 281/744: (262, 107) -> 536 (b'ink') had 453 occurrences\n",
+      "merge 282/744: (353, 400) -> 537 (b'look') had 452 occurrences\n",
+      "merge 283/744: (108, 121) -> 538 (b'ly') had 451 occurrences\n",
+      "merge 284/744: (257, 285) -> 539 (b'this ') had 451 occurrences\n",
+      "merge 285/744: (109, 111) -> 540 (b'mo') had 449 occurrences\n",
+      "merge 286/744: (72, 256) -> 541 (b'He ') had 445 occurrences\n",
+      "merge 287/744: (119, 274) -> 542 (b'war') had 445 occurrences\n",
+      "merge 288/744: (276, 288) -> 543 (b' she ') had 444 occurrences\n",
+      "merge 289/744: (276, 101) -> 544 (b' se') had 443 occurrences\n",
+      "merge 290/744: (398, 275) -> 545 (b',\" the ') had 441 occurrences\n",
+      "merge 291/744: (105, 270) -> 546 (b'ion') had 440 occurrences\n",
+      "merge 292/744: (262, 32) -> 547 (b'in ') had 439 occurrences\n",
+      "merge 293/744: (301, 260) -> 548 (b'its ') had 439 occurrences\n",
+      "merge 294/744: (100, 101) -> 549 (b'de') had 433 occurrences\n",
+      "merge 295/744: (103, 274) -> 550 (b'gar') had 433 occurrences\n",
+      "merge 296/744: (271, 98) -> 551 (b'ed b') had 430 occurrences\n",
+      "merge 297/744: (296, 285) -> 552 (b' this ') had 429 occurrences\n",
+      "merge 298/744: (97, 118) -> 553 (b'av') had 427 occurrences\n",
+      "merge 299/744: (280, 99) -> 554 (b'rec') had 424 occurrences\n",
+      "merge 300/744: (65, 114) -> 555 (b'Ar') had 424 occurrences\n",
+      "merge 301/744: (39, 287) -> 556 (b\"'ll\") had 422 occurrences\n",
+      "merge 302/744: (99, 270) -> 557 (b'con') had 420 occurrences\n",
+      "merge 303/744: (112, 112) -> 558 (b'pp') had 420 occurrences\n",
+      "merge 304/744: (117, 116) -> 559 (b'ut') had 419 occurrences\n",
+      "merge 305/744: (39, 325) -> 560 (b\"'ve \") had 418 occurrences\n",
+      "merge 306/744: (271, 277) -> 561 (b'ed to') had 417 occurrences\n",
+      "merge 307/744: (83, 507) -> 562 (b'Stil') had 417 occurrences\n",
+      "merge 308/744: (101, 265) -> 563 (b'e. ') had 415 occurrences\n",
+      "merge 309/744: (306, 256) -> 564 (b'ice ') had 414 occurrences\n",
+      "merge 310/744: (262, 259) -> 565 (b'ind ') had 414 occurrences\n",
+      "merge 311/744: (117, 100) -> 566 (b'ud') had 413 occurrences\n",
+      "merge 312/744: (350, 273) -> 567 (b'.\" \\n    ') had 411 occurrences\n",
+      "merge 313/744: (72, 365) -> 568 (b'Haw') had 405 occurrences\n",
+      "merge 314/744: (262, 100) -> 569 (b'ind') had 405 occurrences\n",
+      "merge 315/744: (119, 340) -> 570 (b'were ') had 404 occurrences\n",
+      "merge 316/744: (270, 275) -> 571 (b'on the ') had 404 occurrences\n",
+      "merge 317/744: (399, 294) -> 572 (b'.\"\\n    \"') had 402 occurrences\n",
+      "merge 318/744: (271, 290) -> 573 (b'ed at ') had 402 occurrences\n",
+      "merge 319/744: (562, 550) -> 574 (b'Stilgar') had 401 occurrences\n",
+      "merge 320/744: (63, 32) -> 575 (b'? ') had 399 occurrences\n",
+      "merge 321/744: (101, 361) -> 576 (b'ever') had 396 occurrences\n",
+      "merge 322/744: (256, 264) -> 577 (b'e the ') had 396 occurrences\n",
+      "merge 323/744: (302, 104) -> 578 (b'ach') had 395 occurrences\n",
+      "merge 324/744: (114, 368) -> 579 (b'rak') had 389 occurrences\n",
+      "merge 325/744: (121, 110) -> 580 (b'yn') had 387 occurrences\n",
+      "merge 326/744: (109, 397) -> 581 (b'mper') had 383 occurrences\n",
+      "merge 327/744: (268, 293) -> 582 (b', and ') had 383 occurrences\n",
+      "merge 328/744: (114, 359) -> 583 (b'rough') had 383 occurrences\n",
+      "merge 329/744: (109, 278) -> 584 (b'my ') had 382 occurrences\n",
+      "merge 330/744: (296, 334) -> 585 (b' them') had 377 occurrences\n",
+      "merge 331/744: (116, 119) -> 586 (b'tw') had 376 occurrences\n",
+      "merge 332/744: (111, 259) -> 587 (b'od ') had 375 occurrences\n",
+      "merge 333/744: (276, 112) -> 588 (b' sp') had 374 occurrences\n",
+      "merge 334/744: (100, 111) -> 589 (b'do') had 373 occurrences\n",
+      "merge 335/744: (555, 579) -> 590 (b'Arrak') had 372 occurrences\n",
+      "merge 336/744: (108, 295) -> 591 (b'low') had 372 occurrences\n",
+      "merge 337/744: (503, 269) -> 592 (b'Fremen') had 369 occurrences\n",
+      "merge 338/744: (279, 286) -> 593 (b'ing a') had 367 occurrences\n",
+      "merge 339/744: (115, 107) -> 594 (b'sk') had 367 occurrences\n",
+      "merge 340/744: (32, 313) -> 595 (b' you') had 366 occurrences\n",
+      "merge 341/744: (75, 580) -> 596 (b'Kyn') had 365 occurrences\n",
+      "merge 342/744: (269, 115) -> 597 (b'ens') had 364 occurrences\n",
+      "merge 343/744: (66, 388) -> 598 (b'But ') had 364 occurrences\n",
+      "merge 344/744: (271, 383) -> 599 (b'ed his ') had 362 occurrences\n",
+      "merge 345/744: (118, 111) -> 600 (b'vo') had 362 occurrences\n",
+      "merge 346/744: (312, 256) -> 601 (b' be ') had 362 occurrences\n",
+      "merge 347/744: (103, 114) -> 602 (b'gr') had 361 occurrences\n",
+      "merge 348/744: (274, 100) -> 603 (b'ard') had 359 occurrences\n",
+      "merge 349/744: (115, 261) -> 604 (b'st ') had 358 occurrences\n",
+      "merge 350/744: (292, 314) -> 605 (b' was ') had 358 occurrences\n",
+      "merge 351/744: (349, 319) -> 606 (b'e said') had 358 occurrences\n",
+      "merge 352/744: (101, 118) -> 607 (b'ev') had 355 occurrences\n",
+      "merge 353/744: (111, 345) -> 608 (b'old ') had 350 occurrences\n",
+      "merge 354/744: (270, 110) -> 609 (b'onn') had 350 occurrences\n",
+      "merge 355/744: (101, 321) -> 610 (b\"e's \") had 349 occurrences\n",
+      "merge 356/744: (111, 257) -> 611 (b'oth') had 349 occurrences\n",
+      "merge 357/744: (101, 102) -> 612 (b'ef') had 348 occurrences\n",
+      "merge 358/744: (72, 461) -> 613 (b'Hark') had 346 occurrences\n",
+      "merge 359/744: (33, 32) -> 614 (b'! ') had 346 occurrences\n",
+      "merge 360/744: (303, 261) -> 615 (b'ght ') had 345 occurrences\n",
+      "merge 361/744: (338, 102) -> 616 (b'lif') had 345 occurrences\n",
+      "merge 362/744: (310, 32) -> 617 (b'al ') had 345 occurrences\n",
+      "merge 363/744: (283, 256) -> 618 (b' he ') had 345 occurrences\n",
+      "merge 364/744: (283, 427) -> 619 (b' had ') had 345 occurrences\n",
+      "merge 365/744: (65, 370) -> 620 (b'And ') had 344 occurrences\n",
+      "merge 366/744: (299, 263) -> 621 (b'ater') had 344 occurrences\n",
+      "merge 367/744: (304, 324) -> 622 (b'ed. \"') had 343 occurrences\n",
+      "merge 368/744: (613, 609) -> 623 (b'Harkonn') had 342 occurrences\n",
+      "merge 369/744: (256, 281) -> 624 (b'e of') had 339 occurrences\n",
+      "merge 370/744: (265, 294) -> 625 (b'. \\n    \"') had 338 occurrences\n",
+      "merge 371/744: (115, 288) -> 626 (b'she ') had 337 occurrences\n",
+      "merge 372/744: (115, 117) -> 627 (b'su') had 337 occurrences\n",
+      "merge 373/744: (257, 334) -> 628 (b'them') had 329 occurrences\n",
+      "merge 374/744: (316, 115) -> 629 (b'ass') had 328 occurrences\n",
+      "merge 375/744: (99, 407) -> 630 (b'could ') had 327 occurrences\n",
+      "merge 376/744: (269, 99) -> 631 (b'enc') had 326 occurrences\n",
+      "merge 377/744: (284, 116) -> 632 (b'est') had 325 occurrences\n",
+      "merge 378/744: (323, 325) -> 633 (b'have ') had 323 occurrences\n",
+      "merge 379/744: (371, 256) -> 634 (b'ame ') had 322 occurrences\n",
+      "merge 380/744: (307, 116) -> 635 (b'ist') had 319 occurrences\n",
+      "merge 381/744: (317, 115) -> 636 (b'ros') had 318 occurrences\n",
+      "merge 382/744: (451, 370) -> 637 (b'. And ') had 317 occurrences\n",
+      "merge 383/744: (257, 279) -> 638 (b'thing') had 317 occurrences\n",
+      "merge 384/744: (265, 273) -> 639 (b'. \\n    ') had 317 occurrences\n",
+      "merge 385/744: (318, 332) -> 640 (b'.\\n    The ') had 316 occurrences\n",
+      "merge 386/744: (298, 101) -> 641 (b'ome') had 316 occurrences\n",
+      "merge 387/744: (114, 117) -> 642 (b'ru') had 314 occurrences\n",
+      "merge 388/744: (338, 456) -> 643 (b'like ') had 314 occurrences\n",
+      "merge 389/744: (289, 287) -> 644 (b'till') had 313 occurrences\n",
+      "merge 390/744: (268, 509) -> 645 (b', but ') had 312 occurrences\n",
+      "merge 391/744: (379, 339) -> 646 (b',\" Paul') had 310 occurrences\n",
+      "merge 392/744: (112, 405) -> 647 (b'pos') had 308 occurrences\n",
+      "merge 393/744: (104, 111) -> 648 (b'ho') had 307 occurrences\n",
+      "merge 394/744: (296, 489) -> 649 (b' their') had 304 occurrences\n",
+      "merge 395/744: (311, 393) -> 650 (b' from') had 304 occurrences\n",
+      "merge 396/744: (269, 259) -> 651 (b'end ') had 303 occurrences\n",
+      "merge 397/744: (87, 256) -> 652 (b'We ') had 303 occurrences\n",
+      "merge 398/744: (67, 104) -> 653 (b'Ch') had 303 occurrences\n",
+      "merge 399/744: (297, 256) -> 654 (b'ide ') had 302 occurrences\n",
+      "merge 400/744: (112, 111) -> 655 (b'po') had 298 occurrences\n",
+      "merge 401/744: (298, 256) -> 656 (b'ome ') had 297 occurrences\n",
+      "merge 402/744: (112, 317) -> 657 (b'pro') had 295 occurrences\n",
+      "merge 403/744: (266, 105) -> 658 (b'ani') had 294 occurrences\n",
+      "merge 404/744: (283, 485) -> 659 (b' have ') had 293 occurrences\n",
+      "merge 405/744: (97, 276) -> 660 (b'a s') had 292 occurrences\n",
+      "merge 406/744: (484, 107) -> 661 (b'Duk') had 291 occurrences\n",
+      "merge 407/744: (312, 101) -> 662 (b' be') had 290 occurrences\n",
+      "merge 408/744: (408, 261) -> 663 (b'. It ') had 290 occurrences\n",
+      "merge 409/744: (257, 429) -> 664 (b'they ') had 290 occurrences\n",
+      "merge 410/744: (596, 320) -> 665 (b'Kynes ') had 290 occurrences\n",
+      "merge 411/744: (286, 260) -> 666 (b' as ') had 289 occurrences\n",
+      "merge 412/744: (315, 256) -> 667 (b'ure ') had 287 occurrences\n",
+      "merge 413/744: (355, 107) -> 668 (b'eck') had 286 occurrences\n",
+      "merge 414/744: (119, 101) -> 669 (b'we') had 285 occurrences\n",
+      "merge 415/744: (313, 32) -> 670 (b'you ') had 284 occurrences\n",
+      "merge 416/744: (117, 342) -> 671 (b'uch') had 284 occurrences\n",
+      "merge 417/744: (105, 309) -> 672 (b'iel') had 284 occurrences\n",
+      "merge 418/744: (110, 492) -> 673 (b\"n't \") had 282 occurrences\n",
+      "merge 419/744: (265, 459) -> 674 (b'. She ') had 281 occurrences\n",
+      "merge 420/744: (105, 112) -> 675 (b'ip') had 281 occurrences\n",
+      "merge 421/744: (117, 260) -> 676 (b'us ') had 279 occurrences\n",
+      "merge 422/744: (61, 32) -> 677 (b'= ') had 278 occurrences\n",
+      "merge 423/744: (275, 115) -> 678 (b' the s') had 278 occurrences\n",
+      "merge 424/744: (116, 260) -> 679 (b'ts ') had 278 occurrences\n",
+      "merge 425/744: (271, 262) -> 680 (b'ed in') had 277 occurrences\n",
+      "merge 426/744: (32, 351) -> 681 (b' of ') had 277 occurrences\n",
+      "merge 427/744: (289, 99) -> 682 (b'tic') had 276 occurrences\n",
+      "merge 428/744: (394, 119) -> 683 (b'new') had 274 occurrences\n",
+      "merge 429/744: (484, 456) -> 684 (b'Duke ') had 274 occurrences\n",
+      "merge 430/744: (257, 505) -> 685 (b'thought') had 274 occurrences\n",
+      "merge 431/744: (69, 581) -> 686 (b'Emper') had 273 occurrences\n",
+      "merge 432/744: (686, 282) -> 687 (b'Emperor') had 273 occurrences\n",
+      "merge 433/744: (103, 111) -> 688 (b'go') had 273 occurrences\n",
+      "merge 434/744: (112, 101) -> 689 (b'pe') had 273 occurrences\n",
+      "merge 435/744: (97, 375) -> 690 (b'ather') had 272 occurrences\n",
+      "merge 436/744: (99, 114) -> 691 (b'cr') had 270 occurrences\n",
+      "merge 437/744: (300, 278) -> 692 (b' my ') had 270 occurrences\n",
+      "merge 438/744: (267, 260) -> 693 (b'ous ') had 269 occurrences\n",
+      "merge 439/744: (102, 111) -> 694 (b'fo') had 268 occurrences\n",
+      "merge 440/744: (119, 401) -> 695 (b'way') had 268 occurrences\n",
+      "merge 441/744: (568, 290) -> 696 (b'Hawat ') had 268 occurrences\n",
+      "merge 442/744: (305, 270) -> 697 (b' con') had 268 occurrences\n",
+      "merge 443/744: (114, 256) -> 698 (b're ') had 268 occurrences\n",
+      "merge 444/744: (117, 280) -> 699 (b'ure') had 267 occurrences\n",
+      "merge 445/744: (302, 256) -> 700 (b'ace ') had 266 occurrences\n",
+      "merge 446/744: (542, 259) -> 701 (b'ward ') had 265 occurrences\n",
+      "merge 447/744: (457, 294) -> 702 (b'?\" \\n    \"') had 264 occurrences\n",
+      "merge 448/744: (121, 275) -> 703 (b'y the ') had 263 occurrences\n",
+      "merge 449/744: (266, 261) -> 704 (b'ant ') had 263 occurrences\n",
+      "merge 450/744: (100, 433) -> 705 (b'down') had 262 occurrences\n",
+      "merge 451/744: (257, 340) -> 706 (b'there ') had 262 occurrences\n",
+      "merge 452/744: (100, 260) -> 707 (b'ds ') had 261 occurrences\n",
+      "merge 453/744: (299, 342) -> 708 (b'atch') had 260 occurrences\n",
+      "merge 454/744: (117, 101) -> 709 (b'ue') had 259 occurrences\n",
+      "merge 455/744: (653, 658) -> 710 (b'Chani') had 259 occurrences\n",
+      "merge 456/744: (378, 269) -> 711 (b'been') had 258 occurrences\n",
+      "merge 457/744: (73, 32) -> 712 (b'I ') had 258 occurrences\n",
+      "merge 458/744: (264, 115) -> 713 (b'the s') had 257 occurrences\n",
+      "merge 459/744: (304, 364) -> 714 (b'ed.\\n    \"') had 257 occurrences\n",
+      "merge 460/744: (111, 118) -> 715 (b'ov') had 254 occurrences\n",
+      "merge 461/744: (347, 462) -> 716 (b'under') had 254 occurrences\n",
+      "merge 462/744: (266, 278) -> 717 (b'any ') had 253 occurrences\n",
+      "merge 463/744: (112, 438) -> 718 (b'pres') had 253 occurrences\n",
+      "merge 464/744: (102, 101) -> 719 (b'fe') had 252 occurrences\n",
+      "merge 465/744: (103, 105) -> 720 (b'gi') had 252 occurrences\n",
+      "merge 466/744: (263, 116) -> 721 (b'ert') had 251 occurrences\n",
+      "merge 467/744: (282, 256) -> 722 (b'ore ') had 251 occurrences\n",
+      "merge 468/744: (284, 268) -> 723 (b'es, ') had 251 occurrences\n",
+      "merge 469/744: (76, 101) -> 724 (b'Le') had 250 occurrences\n",
+      "merge 470/744: (267, 116) -> 725 (b'out') had 249 occurrences\n",
+      "merge 471/744: (99, 108) -> 726 (b'cl') had 249 occurrences\n",
+      "merge 472/744: (109, 101) -> 727 (b'me') had 249 occurrences\n",
+      "merge 473/744: (517, 110) -> 728 (b'ign') had 249 occurrences\n",
+      "merge 474/744: (304, 318) -> 729 (b'ed.\\n    ') had 248 occurrences\n",
+      "merge 475/744: (100, 45) -> 730 (b'd-') had 248 occurrences\n",
+      "merge 476/744: (448, 261) -> 731 (b'ight ') had 247 occurrences\n",
+      "merge 477/744: (117, 287) -> 732 (b'ull') had 247 occurrences\n",
+      "merge 478/744: (286, 594) -> 733 (b' ask') had 246 occurrences\n",
+      "merge 479/744: (274, 500) -> 734 (b'around ') had 246 occurrences\n",
+      "merge 480/744: (99, 266) -> 735 (b'can') had 246 occurrences\n",
+      "merge 481/744: (39, 259) -> 736 (b\"'d \") had 245 occurrences\n",
+      "merge 482/744: (270, 417) -> 737 (b'only ') had 245 occurrences\n",
+      "merge 483/744: (108, 468) -> 738 (b'long') had 245 occurrences\n",
+      "merge 484/744: (299, 256) -> 739 (b'ate ') had 244 occurrences\n",
+      "merge 485/744: (368, 256) -> 740 (b'ake ') had 244 occurrences\n",
+      "merge 486/744: (100, 114) -> 741 (b'dr') had 244 occurrences\n",
+      "merge 487/744: (116, 317) -> 742 (b'tro') had 244 occurrences\n",
+      "merge 488/744: (286, 276) -> 743 (b' a s') had 243 occurrences\n",
+      "merge 489/744: (292, 410) -> 744 (b' will') had 243 occurrences\n",
+      "merge 490/744: (458, 275) -> 745 (b'from the ') had 243 occurrences\n",
+      "merge 491/744: (339, 32) -> 746 (b'Paul ') had 242 occurrences\n",
+      "merge 492/744: (307, 104) -> 747 (b'ish') had 242 occurrences\n",
+      "merge 493/744: (119, 256) -> 748 (b'we ') had 241 occurrences\n",
+      "merge 494/744: (100, 284) -> 749 (b'des') had 241 occurrences\n",
+      "merge 495/744: (119, 410) -> 750 (b'will') had 241 occurrences\n",
+      "merge 496/744: (322, 111) -> 751 (b' do') had 240 occurrences\n",
+      "merge 497/744: (116, 280) -> 752 (b'tre') had 238 occurrences\n",
+      "merge 498/744: (32, 357) -> 753 (b' to ') had 238 occurrences\n",
+      "merge 499/744: (32, 362) -> 754 (b' of the ') had 237 occurrences\n",
+      "merge 500/744: (103, 117) -> 755 (b'gu') had 237 occurrences\n",
+      "merge 501/744: (267, 108) -> 756 (b'oul') had 237 occurrences\n",
+      "merge 502/744: (636, 260) -> 757 (b'ross ') had 237 occurrences\n",
+      "merge 503/744: (283, 606) -> 758 (b' he said') had 237 occurrences\n",
+      "merge 504/744: (103, 280) -> 759 (b'gre') had 236 occurrences\n",
+      "merge 505/744: (266, 99) -> 760 (b'anc') had 235 occurrences\n",
+      "merge 506/744: (100, 105) -> 761 (b'di') had 235 occurrences\n",
+      "merge 507/744: (102, 343) -> 762 (b'fir') had 234 occurrences\n",
+      "merge 508/744: (46, 10) -> 763 (b'.\\n') had 232 occurrences\n",
+      "merge 509/744: (267, 115) -> 764 (b'ous') had 232 occurrences\n",
+      "merge 510/744: (97, 102) -> 765 (b'af') had 232 occurrences\n",
+      "merge 511/744: (257, 359) -> 766 (b'though') had 232 occurrences\n",
+      "merge 512/744: (70, 363) -> 767 (b'Fey') had 232 occurrences\n",
+      "merge 513/744: (405, 261) -> 768 (b'ost ') had 231 occurrences\n",
+      "merge 514/744: (398, 543) -> 769 (b',\" she ') had 231 occurrences\n",
+      "merge 515/744: (386, 269) -> 770 (b'open') had 230 occurrences\n",
+      "merge 516/744: (296, 505) -> 771 (b' thought') had 230 occurrences\n",
+      "merge 517/744: (99, 274) -> 772 (b'car') had 229 occurrences\n",
+      "merge 518/744: (33, 34) -> 773 (b'!\"') had 229 occurrences\n",
+      "merge 519/744: (398, 758) -> 774 (b',\" he said') had 229 occurrences\n",
+      "merge 520/744: (300, 101) -> 775 (b' me') had 228 occurrences\n",
+      "merge 521/744: (299, 321) -> 776 (b\"at's \") had 228 occurrences\n",
+      "merge 522/744: (455, 290) -> 777 (b'What ') had 228 occurrences\n",
+      "merge 523/744: (303, 116) -> 778 (b'ght') had 227 occurrences\n",
+      "merge 524/744: (105, 122) -> 779 (b'iz') had 227 occurrences\n",
+      "merge 525/744: (316, 107) -> 780 (b'ask') had 226 occurrences\n",
+      "merge 526/744: (360, 32) -> 781 (b'for ') had 225 occurrences\n",
+      "merge 527/744: (121, 276) -> 782 (b'y s') had 225 occurrences\n",
+      "merge 528/744: (282, 100) -> 783 (b'ord') had 225 occurrences\n",
+      "merge 529/744: (115, 265) -> 784 (b's. ') had 224 occurrences\n",
+      "merge 530/744: (103, 108) -> 785 (b'gl') had 224 occurrences\n",
+      "merge 531/744: (104, 328) -> 786 (b'her ') had 224 occurrences\n",
+      "merge 532/744: (677, 677) -> 787 (b'= = ') had 223 occurrences\n",
+      "merge 533/744: (405, 256) -> 788 (b'ose ') had 221 occurrences\n",
+      "merge 534/744: (317, 298) -> 789 (b'room') had 220 occurrences\n",
+      "merge 535/744: (316, 261) -> 790 (b'ast ') had 220 occurrences\n",
+      "merge 536/744: (116, 321) -> 791 (b\"t's \") had 220 occurrences\n",
+      "merge 537/744: (623, 269) -> 792 (b'Harkonnen') had 218 occurrences\n",
+      "merge 538/744: (104, 380) -> 793 (b'hand') had 218 occurrences\n",
+      "merge 539/744: (72, 356) -> 794 (b'Hall') had 218 occurrences\n",
+      "merge 540/744: (389, 32) -> 795 (b'with ') had 217 occurrences\n",
+      "merge 541/744: (112, 443) -> 796 (b'plan') had 216 occurrences\n",
+      "merge 542/744: (414, 112) -> 797 (b'app') had 216 occurrences\n",
+      "merge 543/744: (111, 98) -> 798 (b'ob') had 216 occurrences\n",
+      "merge 544/744: (260, 281) -> 799 (b's of') had 215 occurrences\n",
+      "merge 545/744: (39, 698) -> 800 (b\"'re \") had 215 occurrences\n",
+      "merge 546/744: (317, 524) -> 801 (b'rock') had 214 occurrences\n",
+      "merge 547/744: (116, 279) -> 802 (b'ting') had 213 occurrences\n",
+      "merge 548/744: (119, 421) -> 803 (b'way ') had 213 occurrences\n",
+      "merge 549/744: (274, 109) -> 804 (b'arm') had 212 occurrences\n",
+      "merge 550/744: (283, 328) -> 805 (b' her ') had 212 occurrences\n",
+      "merge 551/744: (322, 101) -> 806 (b' de') had 212 occurrences\n",
+      "merge 552/744: (116, 373) -> 807 (b'tle ') had 210 occurrences\n",
+      "merge 553/744: (374, 340) -> 808 (b'where ') had 210 occurrences\n",
+      "merge 554/744: (33, 291) -> 809 (b'!\" ') had 210 occurrences\n",
+      "merge 555/744: (97, 300) -> 810 (b'a m') had 209 occurrences\n",
+      "merge 556/744: (282, 32) -> 811 (b'or ') had 209 occurrences\n",
+      "merge 557/744: (257, 583) -> 812 (b'through') had 209 occurrences\n",
+      "merge 558/744: (97, 311) -> 813 (b'a f') had 209 occurrences\n",
+      "merge 559/744: (284, 261) -> 814 (b'est ') had 208 occurrences\n",
+      "merge 560/744: (506, 294) -> 815 (b'?\"\\n    \"') had 208 occurrences\n",
+      "merge 561/744: (374, 290) -> 816 (b'what ') had 208 occurrences\n",
+      "merge 562/744: (73, 261) -> 817 (b'It ') had 206 occurrences\n",
+      "merge 563/744: (260, 293) -> 818 (b's and ') had 206 occurrences\n",
+      "merge 564/744: (121, 268) -> 819 (b'y, ') had 205 occurrences\n",
+      "merge 565/744: (301, 278) -> 820 (b'ity ') had 205 occurrences\n",
+      "merge 566/744: (330, 102) -> 821 (b'lef') had 205 occurrences\n",
+      "merge 567/744: (308, 257) -> 822 (b'auth') had 205 occurrences\n",
+      "merge 568/744: (77, 117) -> 823 (b'Mu') had 204 occurrences\n",
+      "merge 569/744: (97, 532) -> 824 (b'aid ') had 204 occurrences\n",
+      "merge 570/744: (271, 115) -> 825 (b'ed s') had 204 occurrences\n",
+      "merge 571/744: (794, 668) -> 826 (b'Halleck') had 204 occurrences\n",
+      "merge 572/744: (269, 472) -> 827 (b'ence ') had 203 occurrences\n",
+      "merge 573/744: (103, 366) -> 828 (b'gain') had 203 occurrences\n",
+      "merge 574/744: (118, 105) -> 829 (b'vi') had 203 occurrences\n",
+      "merge 575/744: (45, 104) -> 830 (b'-h') had 203 occurrences\n",
+      "merge 576/744: (767, 730) -> 831 (b'Feyd-') had 203 occurrences\n",
+      "merge 577/744: (831, 82) -> 832 (b'Feyd-R') had 203 occurrences\n",
+      "merge 578/744: (98, 111) -> 833 (b'bo') had 202 occurrences\n",
+      "merge 579/744: (337, 117) -> 834 (b'tru') had 202 occurrences\n",
+      "merge 580/744: (269, 100) -> 835 (b'end') had 202 occurrences\n",
+      "merge 581/744: (832, 822) -> 836 (b'Feyd-Rauth') had 202 occurrences\n",
+      "merge 582/744: (39, 68) -> 837 (b\"'D\") had 201 occurrences\n",
+      "merge 583/744: (306, 107) -> 838 (b'ick') had 201 occurrences\n",
+      "merge 584/744: (413, 366) -> 839 (b'again') had 201 occurrences\n",
+      "merge 585/744: (110, 497) -> 840 (b'ness ') had 200 occurrences\n",
+      "merge 586/744: (97, 344) -> 841 (b'a p') had 200 occurrences\n",
+      "merge 587/744: (823, 335) -> 842 (b'Muad') had 199 occurrences\n",
+      "merge 588/744: (305, 104) -> 843 (b' ch') had 199 occurrences\n",
+      "merge 589/744: (306, 104) -> 844 (b'ich') had 199 occurrences\n",
+      "merge 590/744: (305, 266) -> 845 (b' can') had 199 occurrences\n",
+      "merge 591/744: (109, 473) -> 846 (b'must ') had 199 occurrences\n",
+      "merge 592/744: (842, 837) -> 847 (b\"Muad'D\") had 198 occurrences\n",
+      "merge 593/744: (847, 502) -> 848 (b\"Muad'Dib\") had 198 occurrences\n",
+      "merge 594/744: (323, 100) -> 849 (b'had') had 198 occurrences\n",
+      "merge 595/744: (769, 499) -> 850 (b',\" she said') had 198 occurrences\n",
+      "merge 596/744: (323, 287) -> 851 (b'hall') had 195 occurrences\n",
+      "merge 597/744: (271, 424) -> 852 (b'ed him') had 195 occurrences\n",
+      "merge 598/744: (270, 492) -> 853 (b\"on't \") had 195 occurrences\n",
+      "merge 599/744: (107, 274) -> 854 (b'kar') had 195 occurrences\n",
+      "merge 600/744: (269, 256) -> 855 (b'ene ') had 194 occurrences\n",
+      "merge 601/744: (32, 436) -> 856 (b' . . . ') had 194 occurrences\n",
+      "merge 602/744: (271, 446) -> 857 (b'ed her') had 194 occurrences\n",
+      "merge 603/744: (309, 261) -> 858 (b'elt ') had 194 occurrences\n",
+      "merge 604/744: (112, 274) -> 859 (b'par') had 194 occurrences\n",
+      "merge 605/744: (277, 276) -> 860 (b'to s') had 193 occurrences\n",
+      "merge 606/744: (611, 328) -> 861 (b'other ') had 193 occurrences\n",
+      "merge 607/744: (257, 489) -> 862 (b'their') had 192 occurrences\n",
+      "merge 608/744: (305, 298) -> 863 (b' com') had 192 occurrences\n",
+      "merge 609/744: (71, 447) -> 864 (b'Gurn') had 192 occurrences\n",
+      "merge 610/744: (864, 363) -> 865 (b'Gurney') had 192 occurrences\n",
+      "merge 611/744: (286, 261) -> 866 (b' at ') had 191 occurrences\n",
+      "merge 612/744: (98, 263) -> 867 (b'ber') had 191 occurrences\n",
+      "merge 613/744: (83, 603) -> 868 (b'Sard') had 191 occurrences\n",
+      "merge 614/744: (868, 308) -> 869 (b'Sardau') had 191 occurrences\n",
+      "merge 615/744: (869, 854) -> 870 (b'Sardaukar') had 191 occurrences\n",
+      "merge 616/744: (348, 263) -> 871 (b'esser') had 189 occurrences\n",
+      "merge 617/744: (403, 373) -> 872 (b'able ') had 189 occurrences\n",
+      "merge 618/744: (316, 116) -> 873 (b'ast') had 189 occurrences\n",
+      "merge 619/744: (330, 274) -> 874 (b'lear') had 188 occurrences\n",
+      "merge 620/744: (71, 871) -> 875 (b'Gesser') had 186 occurrences\n",
+      "merge 621/744: (66, 855) -> 876 (b'Bene ') had 185 occurrences\n",
+      "merge 622/744: (876, 875) -> 877 (b'Bene Gesser') had 185 occurrences\n",
+      "merge 623/744: (413, 256) -> 878 (b'age ') had 185 occurrences\n",
+      "merge 624/744: (108, 301) -> 879 (b'lit') had 185 occurrences\n",
+      "merge 625/744: (268, 34) -> 880 (b', \"') had 185 occurrences\n",
+      "merge 626/744: (324, 73) -> 881 (b'. \"I') had 185 occurrences\n",
+      "merge 627/744: (112, 114) -> 882 (b'pr') had 185 occurrences\n",
+      "merge 628/744: (32, 73) -> 883 (b' I') had 184 occurrences\n",
+      "merge 629/744: (363, 320) -> 884 (b'eyes ') had 184 occurrences\n",
+      "merge 630/744: (337, 366) -> 885 (b'train') had 184 occurrences\n",
+      "merge 631/744: (105, 345) -> 886 (b'ild ') had 184 occurrences\n",
+      "merge 632/744: (112, 282) -> 887 (b'por') had 184 occurrences\n",
+      "merge 633/744: (263, 118) -> 888 (b'erv') had 183 occurrences\n",
+      "merge 634/744: (110, 442) -> 889 (b'nif') had 183 occurrences\n",
+      "merge 635/744: (501, 100) -> 890 (b'ound') had 183 occurrences\n",
+      "merge 636/744: (119, 298) -> 891 (b'wom') had 182 occurrences\n",
+      "merge 637/744: (443, 99) -> 892 (b'lanc') had 181 occurrences\n",
+      "merge 638/744: (269, 101) -> 893 (b'ene') had 180 occurrences\n",
+      "merge 639/744: (281, 102) -> 894 (b'off') had 180 occurrences\n",
+      "merge 640/744: (257, 327) -> 895 (b'thing ') had 179 occurrences\n",
+      "merge 641/744: (99, 414) -> 896 (b'cap') had 179 occurrences\n",
+      "merge 642/744: (109, 768) -> 897 (b'most ') had 178 occurrences\n",
+      "merge 643/744: (104, 314) -> 898 (b'has ') had 178 occurrences\n",
+      "merge 644/744: (544, 256) -> 899 (b' see ') had 177 occurrences\n",
+      "merge 645/744: (263, 260) -> 900 (b'ers ') had 177 occurrences\n",
+      "merge 646/744: (292, 282) -> 901 (b' wor') had 176 occurrences\n",
+      "merge 647/744: (286, 109) -> 902 (b' am') had 174 occurrences\n",
+      "merge 648/744: (286, 99) -> 903 (b' ac') had 174 occurrences\n",
+      "merge 649/744: (102, 108) -> 904 (b'fl') had 174 occurrences\n",
+      "merge 650/744: (305, 407) -> 905 (b' could ') had 173 occurrences\n",
+      "merge 651/744: (522, 256) -> 906 (b'time ') had 172 occurrences\n",
+      "merge 652/744: (300, 473) -> 907 (b' must ') had 172 occurrences\n",
+      "merge 653/744: (724, 277) -> 908 (b'Leto') had 172 occurrences\n",
+      "merge 654/744: (406, 275) -> 909 (b' in the ') had 171 occurrences\n",
+      "merge 655/744: (97, 343) -> 910 (b'air') had 171 occurrences\n",
+      "merge 656/744: (279, 260) -> 911 (b'ings ') had 171 occurrences\n",
+      "merge 657/744: (101, 318) -> 912 (b'e.\\n    ') had 170 occurrences\n",
+      "merge 658/744: (103, 256) -> 913 (b'ge ') had 170 occurrences\n",
+      "merge 659/744: (282, 110) -> 914 (b'orn') had 169 occurrences\n",
+      "merge 660/744: (419, 259) -> 915 (b'eard ') had 169 occurrences\n",
+      "merge 661/744: (119, 407) -> 916 (b'would ') had 168 occurrences\n",
+      "merge 662/744: (268, 288) -> 917 (b', he ') had 168 occurrences\n",
+      "merge 663/744: (384, 274) -> 918 (b'star') had 168 occurrences\n",
+      "merge 664/744: (341, 429) -> 919 (b'They ') had 167 occurrences\n",
+      "merge 665/744: (458, 32) -> 920 (b'from ') had 166 occurrences\n",
+      "merge 666/744: (600, 564) -> 921 (b'voice ') had 166 occurrences\n",
+      "merge 667/744: (271, 357) -> 922 (b'ed to ') had 166 occurrences\n",
+      "merge 668/744: (445, 429) -> 923 (b'. They ') had 166 occurrences\n",
+      "merge 669/744: (590, 307) -> 924 (b'Arrakis') had 165 occurrences\n",
+      "merge 670/744: (101, 101) -> 925 (b'ee') had 165 occurrences\n",
+      "merge 671/744: (32, 270) -> 926 (b' on') had 165 occurrences\n",
+      "merge 672/744: (79, 110) -> 927 (b'On') had 164 occurrences\n",
+      "merge 673/744: (471, 32) -> 928 (b'your ') had 163 occurrences\n",
+      "merge 674/744: (97, 104) -> 929 (b'ah') had 163 occurrences\n",
+      "merge 675/744: (109, 269) -> 930 (b'men') had 163 occurrences\n",
+      "merge 676/744: (689, 386) -> 931 (b'peop') had 162 occurrences\n",
+      "merge 677/744: (98, 114) -> 932 (b'br') had 162 occurrences\n",
+      "merge 678/744: (403, 396) -> 933 (b'about ') had 161 occurrences\n",
+      "merge 679/744: (299, 271) -> 934 (b'ated ') had 161 occurrences\n",
+      "merge 680/744: (600, 306) -> 935 (b'voic') had 161 occurrences\n",
+      "merge 681/744: (284, 256) -> 936 (b'ese ') had 161 occurrences\n",
+      "merge 682/744: (394, 278) -> 937 (b'ney ') had 161 occurrences\n",
+      "merge 683/744: (99, 105) -> 938 (b'ci') had 160 occurrences\n",
+      "merge 684/744: (108, 262) -> 939 (b'lin') had 160 occurrences\n",
+      "merge 685/744: (32, 352) -> 940 (b' it ') had 160 occurrences\n",
+      "merge 686/744: (100, 269) -> 941 (b'den') had 160 occurrences\n",
+      "merge 687/744: (353, 115) -> 942 (b'los') had 159 occurrences\n",
+      "merge 688/744: (98, 432) -> 943 (b'back') had 159 occurrences\n",
+      "merge 689/744: (266, 107) -> 944 (b'ank') had 158 occurrences\n",
+      "merge 690/744: (89, 709) -> 945 (b'Yue') had 158 occurrences\n",
+      "merge 691/744: (945, 104) -> 946 (b'Yueh') had 158 occurrences\n",
+      "merge 692/744: (265, 83) -> 947 (b'. S') had 158 occurrences\n",
+      "merge 693/744: (309, 278) -> 948 (b'ely ') had 158 occurrences\n",
+      "merge 694/744: (119, 621) -> 949 (b'water') had 158 occurrences\n",
+      "merge 695/744: (115, 337) -> 950 (b'str') had 157 occurrences\n",
+      "merge 696/744: (265, 425) -> 951 (b'. You') had 157 occurrences\n",
+      "merge 697/744: (76, 783) -> 952 (b'Lord') had 157 occurrences\n",
+      "merge 698/744: (267, 257) -> 953 (b'outh') had 156 occurrences\n",
+      "merge 699/744: (448, 116) -> 954 (b'ight') had 156 occurrences\n",
+      "merge 700/744: (71, 315) -> 955 (b'Gur') had 156 occurrences\n",
+      "merge 701/744: (955, 937) -> 956 (b'Gurney ') had 156 occurrences\n",
+      "merge 702/744: (82, 576) -> 957 (b'Rever') had 155 occurrences\n",
+      "merge 703/744: (102, 690) -> 958 (b'father') had 155 occurrences\n",
+      "merge 704/744: (118, 101) -> 959 (b've') had 155 occurrences\n",
+      "merge 705/744: (268, 257) -> 960 (b', th') had 154 occurrences\n",
+      "merge 706/744: (98, 278) -> 961 (b'by ') had 154 occurrences\n",
+      "merge 707/744: (280, 335) -> 962 (b'read') had 154 occurrences\n",
+      "merge 708/744: (32, 116) -> 963 (b' t') had 154 occurrences\n",
+      "merge 709/744: (360, 256) -> 964 (b'fore ') had 153 occurrences\n",
+      "merge 710/744: (590, 285) -> 965 (b'Arrakis ') had 153 occurrences\n",
+      "merge 711/744: (537, 271) -> 966 (b'looked ') had 153 occurrences\n",
+      "merge 712/744: (877, 352) -> 967 (b'Bene Gesserit ') had 152 occurrences\n",
+      "merge 713/744: (117, 98) -> 968 (b'ub') had 152 occurrences\n",
+      "merge 714/744: (107, 683) -> 969 (b'knew') had 152 occurrences\n",
+      "merge 715/744: (302, 757) -> 970 (b'across ') had 152 occurrences\n",
+      "merge 716/744: (286, 287) -> 971 (b' all') had 151 occurrences\n",
+      "merge 717/744: (323, 112) -> 972 (b'hap') had 151 occurrences\n",
+      "merge 718/744: (101, 119) -> 973 (b'ew') had 151 occurrences\n",
+      "merge 719/744: (318, 339) -> 974 (b'.\\n    Paul') had 151 occurrences\n",
+      "merge 720/744: (403, 98) -> 975 (b'abb') had 151 occurrences\n",
+      "merge 721/744: (296, 536) -> 976 (b' think') had 151 occurrences\n",
+      "merge 722/744: (318, 65) -> 977 (b'.\\n    A') had 151 occurrences\n",
+      "merge 723/744: (274, 114) -> 978 (b'arr') had 151 occurrences\n",
+      "merge 724/744: (32, 418) -> 979 (b' to the ') had 151 occurrences\n",
+      "merge 725/744: (312, 432) -> 980 (b' back') had 150 occurrences\n",
+      "merge 726/744: (293, 264) -> 981 (b'and the ') had 150 occurrences\n",
+      "merge 727/744: (107, 410) -> 982 (b'kill') had 150 occurrences\n",
+      "merge 728/744: (109, 256) -> 983 (b'me ') had 150 occurrences\n",
+      "merge 729/744: (312, 108) -> 984 (b' bl') had 149 occurrences\n",
+      "merge 730/744: (756, 100) -> 985 (b'ould') had 149 occurrences\n",
+      "merge 731/744: (300, 256) -> 986 (b' me ') had 149 occurrences\n",
+      "merge 732/744: (957, 651) -> 987 (b'Reverend ') had 148 occurrences\n",
+      "merge 733/744: (987, 77) -> 988 (b'Reverend M') had 148 occurrences\n",
+      "merge 734/744: (100, 307) -> 989 (b'dis') had 148 occurrences\n",
+      "merge 735/744: (99, 280) -> 990 (b'cre') had 148 occurrences\n",
+      "merge 736/744: (32, 280) -> 991 (b' re') had 148 occurrences\n",
+      "merge 737/744: (821, 261) -> 992 (b'left ') had 147 occurrences\n",
+      "merge 738/744: (97, 283) -> 993 (b'a h') had 147 occurrences\n",
+      "merge 739/744: (374, 111) -> 994 (b'who') had 146 occurrences\n",
+      "merge 740/744: (752, 297) -> 995 (b'treid') had 145 occurrences\n",
+      "merge 741/744: (589, 282) -> 996 (b'door') had 145 occurrences\n",
+      "merge 742/744: (279, 276) -> 997 (b'ing s') had 145 occurrences\n",
+      "merge 743/744: (100, 100) -> 998 (b'dd') had 145 occurrences\n",
+      "merge 744/744: (32, 285) -> 999 (b' is ') had 145 occurrences\n",
+      "Text Length: 203588 | Time to train: 52.854437828063965\n",
+      "[431, 268, 537, 279, 322, 433, 275, 108, 269, 103, 257, 754, 116, 403, 330, 415, 365, 286, 311, 366, 261, 116, 402, 533, 593, 528, 99, 914, 900, 351, 908, 321, 109, 953, 268, 331, 416, 264, 100, 461, 311, 108, 329, 104, 681, 493, 328, 270, 369, 342, 925, 107, 784, 777, 898, 493, 263, 852, 63, 543, 780, 857, 491, 947, 699, 417, 420, 584, 262, 118, 301, 469, 979, 115, 109, 117, 103, 785, 263, 46]\n",
+      "LEN: 80\n"
+     ]
+    }
+   ],
+   "source": [
+    "tk2 = BasicTokenizer()\n",
+    "st = time.time()\n",
+    "tk2.train(text, 1000, verbose=True)\n",
+    "et = time.time()\n",
+    "\n",
+    "print(f'Text Length: {len(text.split())} | Time to train: {et-st}')\n",
+    "encoded2 = tk2.encode(sentence)\n",
+    "print(encoded2)\n",
+    "print('LEN:', len(encoded2))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

xsbpe/base.py

@@ -0,0 +1,63 @@
+import unicodedata
+from collections import defaultdict
+from itertools import pairwise
+
+def get_adjacent_pair_counts(ids) -> defaultdict:
+    counts = defaultdict(int)
+    for pair in pairwise(ids):
+        counts[pair] += 1
+    return counts
+
+def merge_pairs(ids, pair, idx):
+    newids = []
+    i = 0
+    n = len(ids)
+    while i < n:
+        if i < n - 1 and ids[i] == pair[0] and ids[i+1] == pair[1]:
+            newids.append(idx)
+            i += 2
+        else:
+            newids.append(ids[i])
+            i += 1
+    return newids
+
+def replace_control_characters(s: str) -> str:
+    chars = []
+    for ch in s:
+        if unicodedata.category(ch)[0] != 'C':
+            chars.append(ch)
+        else:
+            chars.append(f'\\u{ord(ch):04x}') # escape
+    return ''.join(chars)
+
+def render_token(t: bytes) -> str:
+    s = t.decode('utf-8', errors='replace')
+    s = replace_control_characters(s)
+    return s
+
+
+class Tokenizer:
+    """Base class for Tokenizers"""
+
+    def __init__(self):
+        self.merges = {}
+        self.pattern = ''
+        self.special_tokens = {}
+        self.vocab = self._build_vocab()
+
+    def train(self, text, vocab_size, verbose=False):
+        raise NotImplementedError
+    
+    def encode(self, text):
+        raise NotImplementedError
+    
+    def decode(self, ids):
+        raise NotImplementedError
+    
+    def _build_vocab(self):
+        vocab = {idx: bytes([idx]) for idx in range(256)}
+        for (p0, p1), idx in self.merges.items():
+            vocab[idx] = vocab[p0] + vocab[p1]
+        for special, idx in self.special_tokens.items():
+            vocab[idx] = special.encode('utf-8')
+        return vocab

xsbpe/basic.py

@@ -0,0 +1,47 @@
+from xsbpe.base import Tokenizer, get_adjacent_pair_counts, merge_pairs
+
+
+class BasicTokenizer(Tokenizer):
+    def __init__(self):
+        super().__init__()
+
+    def train(self, text, vocab_size, verbose=False):
+        assert vocab_size >= 256
+        num_merges = vocab_size - 256
+
+        text_bytes = text.encode('utf-8')
+        ids = list(text_bytes)
+
+        merges = {}
+        vocab = {idx: bytes([idx]) for idx in range(256)}
+        for i in range(num_merges):
+            stats = get_adjacent_pair_counts(ids)
+            pair = max(stats, key=stats.get)
+            idx = 256 + i
+            ids = merge_pairs(ids, pair, idx)
+
+            merges[pair] = idx
+            vocab[idx] = vocab[pair[0]] + vocab[pair[1]]
+
+            if verbose:
+                print(f"merge {i+1}/{num_merges}: {pair} -> {idx} ({vocab[idx]}) had {stats[pair]} occurrences")
+        
+        self.merges = merges
+        self.vocab = vocab
+
+    def decode(self, ids):
+        text_bytes = b''.join(self.vocab[idx] for idx in ids)
+        text = text_bytes.decode('utf-8', errors='replace')
+        return text
+    
+    def encode(self, text):
+        text_bytes = text.encode('utf-8')
+        ids = list(text_bytes)
+        while len(ids) >= 2:
+            stats = get_adjacent_pair_counts(ids)
+            pair = min(stats, key=lambda p: self.merges.get(p, float('inf')))
+            if pair not in self.merges:
+                break
+            idx = self.merges[pair]
+            ids = merge_pairs(ids, pair, idx)
+        return ids