bpetokenizer upload

tokenizer/.github/workflows/pypi.yml

@@ -0,0 +1,35 @@
+name: python-package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+
+      - name: Build package
+        run: |
+          python setup.py sdist bdist_wheel
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

tokenizer/.github/workflows/tests.yml

@@ -0,0 +1,41 @@
+name: Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+        branches:
+            - main
+    
+jobs:
+    TestBPETokenizer:
+        runs-on: ${{ matrix.os }}
+        strategy:
+            matrix:
+                os: [ubuntu-latest, windows-latest, macos-latest]
+                python-version: ["3.9", "3.10", "3.11"]
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Setup Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v5
+              with:
+                    python-version: ${{ matrix.python-version }}
+            
+            - name: Install dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install pytest
+                pip install regex
+
+            - name: Run Tests
+              run: |
+                python -m pytest tests/test_tokenizer.py
+
+            - name: Upload Test Results
+              uses: actions/upload-artifact@v4
+              with:
+                    name: test-results
+                    path: test-results.xml
+                

tokenizer/.gitignore

@@ -0,0 +1,156 @@
+# Project
+/.vs
+.vscode
+
+# Log files
+*.log
+
+# Python virtualenv
+.venv*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+logs
+
+.idea/*
+.DS_Store
+
+output/
+*.pkl

tokenizer/README.md

@@ -0,0 +1,151 @@
+# bpetokenizer
+
+A Byte Pair Encoding (BPE) tokenizer, which algorithmically follows along the GPT tokenizer. The tokenizer is capable of handling special tokens and uses a customizable regex pattern for tokenization(includes the gpt4 regex pattern). supports `save` and `load` tokenizers in the `json` and `file` format.
+
+
+### Overview
+
+The Byte Pair Encoding (BPE) algorithm is a simple yet powerful method for building a vocabulary of subword units for a given text corpus. This tokenizer can be used for training your tokenizer of the LLM on various languages of text corpus.
+
+this algorithm is first introduced in the paper [Neural Machine Translation of Rare Words with Subword Units](https://arxiv.org/pdf/1508.07909) and then used this in the gpt2 tokenizer([Language Models are Unsupervised Multitask Learners](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf))
+
+The [notebook](notebooks/tokenization.ipynb) which shows the BPE algorithm in detail and how the tokenizers work internally.
+
+Every LLM(LLama, Gemini, Mistral..) use their own Tokenizers trained on their own text dataset.
+
+
+### Features
+
+- Implements Byte Pair Encoding (BPE) algorithm.
+- Handles special tokens.
+- Uses a customizable regex pattern for tokenization.
+- Compatible with Python 3.9 and above
+
+
+#### This repository has 2 different Tokenizers:
+- `BPETokenizer`
+- `Tokenizer`
+
+1. [Tokenizer](bpetokenizer/base.py): This class contains `train`, `encode`, `decode` and functionalities to `save` and `load`. Also contains few helper functions `get_stats`, `merge`, `replace_control_characters`..  to perform the BPE algorithm for the tokenizer.
+
+2. [BPETokenizer](bpetokenizer/tokenizer.py): This class emphasizes the real power of the tokenizer(used in gpt4 tokenizer..[tiktoken](https://github.com/openai/tiktoken)), uses the `GPT4_SPLIT_PATTERN` to split the text as mentioned in the gpt4 tokenizer. also handles the `special_tokens` (refer [sample_bpetokenizer](sample/bpetokenizer/sample_bpetokenizer.py)). which inherits the `save` and `load` functionlities to save and load the tokenizer respectively.
+
+
+### Usage
+
+this tutorial leverages the `special_tokens` usage in the Tokenizer.
+
+Install the package
+
+```shell
+pip install bpetokenizer
+```
+
+
+```py
+from bpetokenizer import BPETokenizer
+
+special_tokens = {
+    "<|endoftext|>": 1001,
+    "<|startoftext|>": 1002,
+    "[SPECIAL1]": 1003,
+    "[SPECIAL2]": 1004,
+}
+
+tokenizer = BPETokenizer(special_tokens=special_tokens) # you can also use the method _special_tokens to register the special tokens (if not passed when intializing)
+texts = "<|startoftext|> Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.<|endoftext|>"
+
+tokenizer.train(texts, vocab_size=310, verbose=True)
+# tokenizer._special_tokens(special_tokens) # if not passed when intialization of the BPETokenizer
+
+encode_text = """
+<|startoftext|>Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.
+Hello, Universe! Another example sentence containing [SPECIAL1] and [SPECIAL2], used to ensure tokenizer's robustness.
+Greetings, Earth! Here we have [SPECIAL1] appearing once again, followed by [SPECIAL2] in the same sentence.
+Hello, World! This is yet another sample text, with [SPECIAL1] and [SPECIAL2] making an appearance.
+Hey there, World! Testing the tokenizer with [SPECIAL1] and [SPECIAL2] to see if it handles special tokens properly.
+Salutations, Planet! The tokenizer should recognize [SPECIAL1] and [SPECIAL2] in this long string of text.
+Hello again, World! [SPECIAL1] and [SPECIAL2] are special tokens that need to be handled correctly by the tokenizer.
+Welcome, World! Including [SPECIAL1] and [SPECIAL2] multiple times in this large text to ensure proper encoding.
+Hi, World! Let's add [SPECIAL1] and [SPECIAL2] in various parts of this long sentence to test the tokenizer thoroughly.
+<|endoftext|>
+"""
+ids = tokenizer.encode(encode_text, special_tokens="all")
+print(ids)
+
+decode_text = tokenizer.decode(ids)
+print(decode_text)
+
+tokenizer.save("sample_bpetokenizer", mode="json") # mode: default is file
+```
+
+refer [sample_bpetokenizer](sample/bpetokenizer) to have an understanding of the `vocab` and the `model` file of the tokenizer trained on the above texts.
+
+
+#### To Load the Tokenizer
+
+```py
+from bpetokenizer import BPETokenizer
+
+tokenizer = BPETokenizer()
+
+tokenizer.load("sample_bpetokenizer.json", mode="json")
+
+encode_text = """
+<|startoftext|>Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.
+Hello, Universe! Another example sentence containing [SPECIAL1] and [SPECIAL2], used to ensure tokenizer's robustness.
+Greetings, Earth! Here we have [SPECIAL1] appearing once again, followed by [SPECIAL2] in the same sentence.<|endoftext|>"""
+
+print("vocab: ", tokenizer.vocab)
+print('---')
+print("merges: ", tokenizer.merges)
+print('---')
+print("special tokens: ", tokenizer.special_tokens)
+
+ids = tokenizer.encode(encode_text, special_tokens="all")
+print('---')
+print(ids)
+
+decode_text = tokenizer.decode(ids)
+print('---')
+print(decode_text)
+
+# you can also print the tokens and the text chunks split with the pattern.
+tokens = tokenizer.tokens(encode_text, verbose=True) # if verbose, prints the text chunks and also the pattern used to split.
+print('---')
+print("tokens: ", tokens)
+
+```
+refer to the [load_json_vocab](sample/load_json_vocab/) and run the `bpetokenizer_json` to get an overview of `vocab`, `merges`, `special_tokens` and to view the tokens that are split by the tokenizer using pattern, look at [tokens](sample/load_json_vocab/tokens.py)
+
+### Run Tests
+
+the tests folder `tests/` include the tests of the tokenizer, uses pytest.
+
+```
+python3 -m pytest
+```
+
+additionally, the workflows are setup to run the tests when made a PR.
+
+
+### Contributing
+
+Contributions to the BPE Tokenizer are most welcomed! If you would like to contribute, please follow these steps:
+
+- Star and Fork the repository.
+- Create a new branch (git checkout -b feature/your-feature).
+- Commit your changes (git commit -am 'Add some feature').
+- Push to the branch (git push origin feature/your-feature).
+- Create a new Pull Request.
+
+Please ensure your code follows the project's coding standards and includes appropriate tests. Also, update the documentation as necessary.
+
+
+### License
+
+This project is licensed under the MIT License.
+
+----
+
+*this tokenizer is inspired from the [minbpe](https://github.com/karpathy/minbpe), but more optimized.

tokenizer/bpetokenizer/__init__.py

@@ -0,0 +1,3 @@
+from .base import Tokenizer
+from .tokenizer import BPETokenizer
+from .version import __version__

tokenizer/bpetokenizer/base.py

@@ -0,0 +1,223 @@
+"""
+This file will contains all the helper functions
+and Base class which has the methods to save/load model,
+also required to build the BPETokenizer.
+"""
+
+import regex as re
+from .version import __version__
+
+def get_stats(tokens, counts=None) -> dict:
+    """Get statistics of the tokens. Includes the frequency of each consecutive pair of tokens"""
+    counts = {} if counts is None else counts
+    for pair in zip(tokens, tokens[1:]):
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+
+def merge(ids, pair, idx) -> list:
+    """Merge the pair of tokens in the ids(list of tokens) with representing it with idx(new token in the vocab)."""
+    newids = []
+    i = 0
+    while i < len(ids):
+        if i < len(ids) -1 and ids[i] == pair[0] and ids[i+1] == pair[1]:
+            newids.append(idx)
+            i += 2
+        else:
+            newids.append(ids[i])
+            i += 1
+    return newids
+
+
+import unicodedata
+
+def replace_control_characters(s: str) -> str:
+    """
+    Replace control characters in a string with their unicode escape sequences. Prevents distortion
+    Example:
+        token = b"hello\nworld\x00"
+        print(token) -> hello
+                        world (and \x00 might not be visible)
+        print(replace_control_characters(token))
+        -> hello\u000aworld\u0000
+
+    """
+    chars = []
+    for ch in s:
+        if unicodedata.category(ch)[0] != "C": # the category of the `\*` chars start with C
+            chars.append(ch)
+        else:
+            chars.append(f"\\u{ord(ch):04x}")
+    return "".join(chars)
+
+
+def render_token(t: bytes) -> str:
+    s = t.decode('utf-8', errors='replace') # this will replace the unkown with a �
+    s = replace_control_characters(s)
+    return s
+
+
+
+class Tokenizer:
+    """A Base class for the tokenizer, used for training and encoding/decoding the text without special tokens."""
+
+    def __init__(self):
+        self.merges = {}
+        self.pattern = "" # the regex pattern
+        self.compiled_pattern = re.compile(self.pattern) if self.pattern else ""
+        self.special_tokens = {}
+        self.vocab = self._build_vocab() if self.merges else {}
+
+    def _build_vocab(self) -> dict:
+        """Build the vocab from the merges and special tokens. This will be used to encode/decode the tokens."""
+        vocab = {idx: bytes([idx]) for idx in range(256)}
+        for (p0, p1), idx in self.merges.items():
+            vocab[idx] = vocab[p0] + vocab[p1]
+        if self.special_tokens:
+            for special, idx in self.special_tokens.items():
+                vocab[idx] = special.encode("utf-8")
+        return vocab
+
+    def save(self, file_name, mode="file"):
+        """
+        Writes metadata and vocabulary information to the model and vocab files.
+        mode: str, default="file" | "json" to save the model and vocab in json format.
+        """
+        if mode == "file":
+            model_file = file_name + ".model"
+            with open(model_file, 'w') as f:
+                f.write(f"{__version__}\n")
+                f.write(f"{self.pattern}\n")
+                f.write(f"{len(self.special_tokens)}\n")
+                if self.special_tokens:
+                    for special, idx in self.special_tokens.items():
+                        f.write(f"{special} {idx}\n")
+
+                for idx1, idx2 in self.merges: # this will give the tokens of pair which are merged
+                    f.write(f"{idx1} {idx2}\n")
+
+                vocab_file = file_name + ".vocab"
+                inverted_merges = {idx: pair for pair, idx in self.merges.items()}
+                with open(vocab_file, "w", encoding="utf-8") as f:
+                    for idx, token in self.vocab.items():
+                        s = render_token(token)
+                        # find the children of this token, if any
+                        if idx in inverted_merges:
+                            # if this token has children, render it nicely as a merge
+                            idx0, idx1 = inverted_merges[idx]
+                            s0 = render_token(self.vocab[idx0])
+                            s1 = render_token(self.vocab[idx1])
+                            f.write(f"[{s0}][{s1}] -> [{s}] {idx}\n")
+                        else:
+                            # otherwise this is leaf token, just print it
+                            # (this should just be the first 256 tokens, the bytes)
+                            f.write(f"[{s}] {idx}\n")
+        elif mode == "json":
+            import json
+            data = {
+                "version": __version__,
+                "pattern": str(self.pattern),
+                "special_tokens": self.special_tokens,
+                "merges": {str(k): v for k, v in self.merges.items()},
+                "vocab": {idx: render_token(token) for idx, token in self.vocab.items()}
+            }
+            with open(file_name + ".json", "w", encoding="utf-8") as f:
+                json.dump(data, f, ensure_ascii=False, indent=4)
+        else:
+            raise ValueError("mode should be either 'file' or 'json'")
+        
+            
+    def load(self, file_name, mode="file"):
+        """
+        Load the model and vocab files to the tokenizer.
+        mode: str, default="file" | "json" to load the model and vocab in json format.
+        """
+        if mode == "file":
+            assert file_name.endswith(".model")
+            merges = {}
+            special_tokens = {}
+            idx = 256
+            with open(file_name, 'r', encoding="utf-8") as f:
+                assert f.readline().strip() == __version__
+                self.pattern = f.readline().strip().split()
+                num_special = int(f.readline().strip()) # no of lines of special_tokens
+                for _ in range(num_special):
+                    special, idx = f.readline().strip().split()
+                    special_tokens[special] = int(idx)
+                for line in f:
+                    idx1, idx2 = map(int, line.strip().split())
+                    merges[(idx1, idx2)] = idx
+                    idx += 1
+                
+            self.merges = merges
+            self.special_tokens = special_tokens
+            self.vocab = self._build_vocab()
+
+        elif mode == "json":
+            assert file_name.endswith(".json")
+
+            import json
+            with open(file_name, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                assert data["version"] == __version__
+                pattern = data["pattern"]
+                pattern_regex = re.compile(r'regex.Regex\("(.+)", flags=(regex\.\w+)\)')
+                match = pattern_regex.match(pattern)
+                if match:
+                    self.pattern = match.group(1)
+                self.special_tokens = data["special_tokens"]
+                self.inverse_special_tokens = {v: k for k, v in self.special_tokens.items()}
+                merges = data["merges"]
+                self.merges = {tuple(map(int, k.strip('()').split(','))): v for k, v in merges.items()}
+                vocab = data["vocab"]
+                self.vocab = {int(k): v.encode("utf-8") for k, v in vocab.items()}
+
+        
+
+    def encode(self, texts):
+        """Method to encode the text to ids."""
+        text_bytes = texts.encode("utf-8") # raw bytes string
+        ids = list(map(int, text_bytes))
+        while len(ids) >= 2:
+            # find the pair with the lowest merge index
+            stats = get_stats(ids)
+            pair = min(stats, key=lambda p: self.merges.get(p, float("inf")))
+
+            if pair not in self.merges:
+                break # nothing else can be merged anymore
+            # otherwise let's merge the best pair (lowest merge index)
+            idx = self.merges[pair]
+            ids = merge(ids, pair, idx)
+        return ids
+
+    def decode(self, ids):
+        """Method to decode the ids to text."""
+        bytes_str = b"".join([self.vocab[idx] for idx in ids])
+        text = bytes_str.decode("utf-8", errors="replace")
+        return text
+
+    def train(self, texts, vocab_size, verbose=False):
+        """Method for training the tokenizer."""
+        assert vocab_size >= 256
+        num_merges = vocab_size - 256
+
+        tokens = texts.encode("utf-8")
+        ids = list(tokens)
+        merges = {}
+        vocab = {idx: bytes([idx]) for idx in range(256)} # vocab for first 255 bytes
+
+        # bpe algorithm
+        for i in range(num_merges):
+            stats = get_stats(ids)
+            pair = max(stats, key=stats.get) # returns the highest frequency pair
+            idx = 256 + i
+
+            ids = merge(ids, pair, idx)
+            merges[pair] = idx
+            vocab[idx] = vocab[pair[0]] + vocab[pair[1]] # concat of bytes
+
+            if verbose:
+                print(f"merging {i+1}/{num_merges}: {pair} -> {idx} ({vocab[idx]}) had {stats[pair]} frequency")
+
+        self.merges = merges
+        self.vocab = vocab

tokenizer/bpetokenizer/tokenizer.py

@@ -0,0 +1,160 @@
+"""
+Byte Pair Encoding tokenizer.
+
+Algorithmically follows along the GPT tokenizer:
+https://github.com/openai/gpt-2/blob/master/src/encoder.py
+
+The Byte Pair Encoding (BPE) algorithm is a simple algorithm that builds a vocabulary
+of subword units for a given text corpus. 
+
+More detailed information could be found in:
+https://github.com/Hk669/bpetokenizer/blob/main/notebooks/tokenization.ipynb
+https://en.wikipedia.org/wiki/Byte_pair_encoding
+https://youtu.be/zduSFxRajkE?si=Qv-yX2NUY69aIjCQ (Andrej Karpathy's tutorial on Tokenizer)
+
+"""
+
+from .base import Tokenizer, get_stats, merge
+import regex as re
+
+# from the openai/tiktoken (used in gpt4 tokenizer)
+GPT4_SPLIT_PATTERN = r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+"""     # raw string
+
+
+class BPETokenizer(Tokenizer):
+    """Byte Pair Encoding tokenizer. Which handles the special tokens and the pattern for tokenization."""
+
+    def __init__(self, pattern=None, special_tokens=None):
+        super().__init__()
+        self.pattern = GPT4_SPLIT_PATTERN if pattern is None else pattern
+        self.compiled_pattern = re.compile(self.pattern)
+        self.special_tokens = {} if special_tokens is None else special_tokens
+        self.inverse_special_tokens = {} if special_tokens is None else {v: k for k, v in special_tokens.items()}
+
+
+    def train(self, texts, vocab_size, verbose=False) -> None:
+        """Train the tokenizer on the given texts and vocab size. The vocab size should be greater than 256."""
+        assert vocab_size >= 256
+        num_merges = vocab_size - 256
+
+        text_chunks = re.findall(self.compiled_pattern, texts) # handles the desired pattern of tokens with regex pattern
+
+        ids = [list(tokens.encode("utf-8")) for tokens in text_chunks]      # List[List[int]]
+        merges = {}
+        vocab = {idx: bytes([idx]) for idx in range(256)} # vocab for first 255 bytes
+
+        # bpe algorithm
+        for i in range(num_merges):
+            stats = {}
+            for chunk in ids:
+                get_stats(chunk, stats)
+
+            pair = max(stats, key=stats.get) # returns the highest frequency pair
+            idx = 256 + i
+
+            ids = [merge(chunk_ids, pair, idx) for chunk_ids in ids] # merge all the max occuring pair in the each chunk in ids
+            merges[pair] = idx
+            vocab[idx] = vocab[pair[0]] + vocab[pair[1]] # concat of bytes
+
+            if verbose:
+                print(f"merging {i+1}/{num_merges}: {pair} -> {idx} ({vocab[idx]}) had {stats[pair]} frequency")
+
+        self.merges = merges
+        self.vocab = vocab
+
+
+    def _encode(self, _bytes) -> list:
+        """Encode the bytes into token ids(BPE algorithm)."""
+        ids = list(_bytes)
+        while len(ids) >= 2:
+            # find the pair with the lowest merge index
+            stats = get_stats(ids)
+            pair = min(stats, key=lambda p: self.merges.get(p, float("inf")))
+
+            if pair not in self.merges:
+                break # nothing else can be merged anymore
+            # otherwise let's merge the best pair (lowest merge index)
+            idx = self.merges[pair]
+            ids = merge(ids, pair, idx)
+        return ids
+    
+
+    def encode_ord(self, text) -> list:
+        text_chunks = re.findall(self.compiled_pattern, text)
+        ids = []
+        for chunk in text_chunks:
+            _bytes = chunk.encode("utf-8")
+            chunk_ids = self._encode(_bytes)
+            ids.extend(chunk_ids)
+        return ids
+
+
+    def encode(self, text, special_tokens="none") -> list:
+        """
+        Encode the text into token ids. 
+        If special_tokens is set to "all", it will include the special tokens in the ids. 
+        If set to "none", it will exclude the special tokens. 
+        If set to "none_raise", it will raise an error if the text contains any special tokens.
+        """
+        special = None
+        if special_tokens == "all":
+            special = self.special_tokens
+        elif special_tokens == "none":
+            special = {}
+        elif special_tokens == "none_raise":
+            special = {}
+            assert all(token not in text for token in self.special_tokens)
+        else:
+            raise ValueError(f"invalid special tokens argument: {special_tokens}")
+        
+        if not special:
+            return self.encode_ord(text)
+        
+        special_pattern = "(" + "|".join(re.escape(k) for k in special) + ")"
+        text_chunks = re.split(special_pattern, text)
+        ids = []
+        for chunk in text_chunks:
+            if chunk in special:
+                ids.append(special[chunk])
+            else:
+                chunkids = self._encode(chunk.encode("utf-8"))
+                ids.extend(chunkids)
+        return ids
+
+
+    def decode(self, ids) -> str:
+        part_bytes = []
+        for idx in ids:
+            if idx in self.vocab: #str conversion because vocab keys are strings when loaded from json
+                part_bytes.append(self.vocab[idx])
+            elif idx in self.inverse_special_tokens:
+                part_bytes.append(self.inverse_special_tokens[idx].encode("utf-8")) # special tokens are not encoded in vocab
+            elif idx in self.merges:
+                pair = self.merges[idx]
+                part_bytes.append(self.vocab[pair[0]] + self.vocab[pair[1]])
+            else:
+                raise ValueError(f"invalid token id: {idx}")
+        text_bytes = b"".join(part_bytes)
+        text = text_bytes.decode("utf-8", errors="replace")
+        return text
+
+
+    def _special_tokens(self, special_tokens) -> None:
+        """Set the special tokens for the tokenizer. If not passed when initializing, it will be empty."""
+        self.special_tokens = special_tokens
+        self.inverse_special_tokens = {v: k for k, v in special_tokens.items()}
+    
+
+    def tokens(self, text, verbose=False) -> list:
+        text_chunks = re.findall(self.compiled_pattern, text)
+        
+        _tokens = []
+        for chunk in text_chunks:
+            _bytes = chunk.encode("utf-8")
+            chunk_ids = self._encode(_bytes)
+            chunk_tokens = [self.vocab[idx].decode("utf-8", errors="replace") if idx in self.vocab else f"[UNK{idx}]" for idx in chunk_ids]
+            _tokens.extend(chunk_tokens)
+        if verbose:
+            print(f"---\ntext chunks: {text_chunks}\n")
+            print(f"---\npattern: {self.pattern}\n")
+        return _tokens

tokenizer/bpetokenizer/version.py

@@ -0,0 +1 @@
+__version__ = "1.0.31"

tokenizer/notebooks/tokenization.ipynb

@@ -0,0 +1,764 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Tokenizer\n",
+    "\n",
+    "A completely seperate, independent module from the LLM. which has its own training dataset of text, on which you train the vocabulary using the BPE(Byte pair encoding) algorithm. It then translates back and forth between the raw text and the sequence of integers/tokens. LLM only deals with the tokens and never directly deals with the text.\n",
+    "\n",
+    "![image.png](../public/tokenizer.png)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "97"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# the unicode code point of the character\n",
+    "ord('a')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[3118,\n",
+       " 3136,\n",
+       " 3120,\n",
+       " 3137,\n",
+       " 32,\n",
+       " 3086,\n",
+       " 3122,\n",
+       " 3134,\n",
+       " 32,\n",
+       " 3081,\n",
+       " 3112,\n",
+       " 3149,\n",
+       " 3112,\n",
+       " 3134,\n",
+       " 3120,\n",
+       " 3137,\n",
+       " 63,\n",
+       " 32,\n",
+       " 40,\n",
+       " 72,\n",
+       " 111,\n",
+       " 119,\n",
+       " 32,\n",
+       " 97,\n",
+       " 114,\n",
+       " 101,\n",
+       " 32,\n",
+       " 121,\n",
+       " 111,\n",
+       " 117,\n",
+       " 63,\n",
+       " 41]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokens = [ord(c) for c in \"మీరు ఎలా ఉన్నారు? (How are you?)\"]\n",
+    "tokens"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "but having the token for each letter will increase the computation cost to generate and also train the model. so the BPE algorithm to introduced in the [GPT2 paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Byte pair encoding algorithm\n",
+    "\n",
+    "consider the string:\n",
+    "\n",
+    "`aaabdaaabac`\n",
+    "\n",
+    "the byte pair \"aa\" is the most occuring in the string, so we replace that with a new byte which is not used in the `vocab`, let's say \"Z\".\n",
+    "Now the following string will be\n",
+    "\n",
+    "```\n",
+    "ZabdZabac\n",
+    "Z = aa\n",
+    "```\n",
+    "\n",
+    "this process will be continued with recursive byte pair encoding replacing all the byte pairs till the string/data cannot be compressed further.\n",
+    "\n",
+    "\n",
+    "Then the process is repeated with byte pair \"ab\", replacing it with \"Y\"\n",
+    "```\n",
+    "ZYdZYac\n",
+    "Y=ab\n",
+    "Z=aa\n",
+    "```\n",
+    "replacing \"ZY\" with \"X\"\n",
+    "```\n",
+    "XdXac\n",
+    "X=ZY\n",
+    "Y=ab\n",
+    "Z=aa\n",
+    "```\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "----\n",
+      "Autogen enables the next-gen LLM applications with a generic [multi-agent conversation](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat) framework. It offers customizable and conversable agents that integrate LLMs, tools, and humans.\n",
+      "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code.\n",
+      "\n",
+      "Features of this use case include:\n",
+      "\n",
+      "- **Multi-agent conversations**: AutoGen agents can communicate with each other to solve tasks. This allows for more complex and sophisticated applications than would be possible with a single LLM.\n",
+      "- **Customization**: AutoGen agents can be customized to meet the specific needs of an application. This includes the ability to choose the LLMs to use, the types of human input to allow, and the tools to employ.\n",
+      "- **Human participation**: AutoGen seamlessly allows human participation. This means that humans can provide input and feedback to the agents as needed.\n",
+      "\n",
+      "For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py),\n",
+      "\n",
+      "```python\n",
+      "from autogen import AssistantAgent, UserProxyAgent, config_list_from_json\n",
+      "# Load LLM inference endpoints from an env variable or a file\n",
+      "# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints\n",
+      "# and OAI_CONFIG_LIST_sample\n",
+      "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n",
+      "# You can also set config_list directly as a list, for example, config_list = [{'model': 'gpt-4', 'api_key': '<your OpenAI API key here>'},]\n",
+      "assistant = AssistantAgent(\"assistant\", llm_config={\"config_list\": config_list})\n",
+      "user_proxy = UserProxyAgent(\"user_proxy\", code_execution_config={\"work_dir\": \"coding\", \"use_docker\": False}) # IMPORTANT: set to True to run code in docker, recommended\n",
+      "user_proxy.initiate_chat(assistant, message=\"Plot a chart of NVDA and TESLA stock price change YTD.\")\n",
+      "# This initiates an automated chat between the two agents to solve the task\n",
+      "```\n",
+      "\n",
+      "more python code:\n",
+      "\n",
+      "```python\n",
+      "    def create(\n",
+      "        self,\n",
+      "        *,\n",
+      "        messages: Iterable[ChatCompletionMessageParam],\n",
+      "        model: Union[str, ChatModel],\n",
+      "        frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN,\n",
+      "        functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN,\n",
+      "        logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN,\n",
+      "        logprobs: Optional[bool] | NotGiven = NOT_GIVEN,\n",
+      "        max_tokens: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        n: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,\n",
+      "        seed: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,\n",
+      "        stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN,\n",
+      "        stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,\n",
+      "        temperature: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,\n",
+      "        tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,\n",
+      "        top_logprobs: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        top_p: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        user: str | NotGiven = NOT_GIVEN,\n",
+      "        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.\n",
+      "        # The extra values given here take precedence over values defined on the client or passed to this method.\n",
+      "        extra_headers: Headers | None = None,\n",
+      "        extra_query: Query | None = None,\n",
+      "        extra_body: Body | None = None,\n",
+      "        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n",
+      "    ) -> ChatCompletion | Stream[ChatCompletionChunk]:\n",
+      "        return self._post(\n",
+      "            \"/chat/completions\",\n",
+      "            body=maybe_transform(\n",
+      "                {\n",
+      "                    \"messages\": messages,\n",
+      "                    \"model\": model,\n",
+      "                    \"frequency_penalty\": frequency_penalty,\n",
+      "                    \"function_call\": function_call,\n",
+      "                    \"functions\": functions,\n",
+      "                    \"logit_bias\": logit_bias,\n",
+      "                    \"logprobs\": logprobs,\n",
+      "                    \"max_tokens\": max_tokens,\n",
+      "                    \"n\": n,\n",
+      "                    \"presence_penalty\": presence_penalty,\n",
+      "                    \"response_format\": response_format,\n",
+      "                    \"seed\": seed,\n",
+      "                    \"stop\": stop,\n",
+      "                    \"stream\": stream,\n",
+      "                    \"stream_options\": stream_options,\n",
+      "                    \"temperature\": temperature,\n",
+      "                    \"tool_choice\": tool_choice,\n",
+      "                    \"tools\": tools,\n",
+      "                    \"top_logprobs\": top_logprobs,\n",
+      "                    \"top_p\": top_p,\n",
+      "                    \"user\": user,\n",
+      "                },\n",
+      "                completion_create_params.CompletionCreateParams,\n",
+      "            ),\n",
+      "            options=make_request_options(\n",
+      "                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout\n",
+      "            ),\n",
+      "            cast_to=ChatCompletion,\n",
+      "            stream=stream or False,\n",
+      "            stream_cls=Stream[ChatCompletionChunk],\n",
+      "        )\n",
+      "```\n",
+      "\n",
+      "length: 5397\n",
+      "----\n",
+      "[65, 117, 116, 111, 103, 101, 110, 32, 101, 110, 97, 98, 108, 101, 115, 32, 116, 104, 101, 32, 110, 101, 120, 116, 45, 103, 101, 110, 32, 76, 76, 77, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 115, 32, 119, 105, 116, 104, 32, 97, 32, 103, 101, 110, 101, 114, 105, 99, 32, 91, 109, 117, 108, 116, 105, 45, 97, 103, 101, 110, 116, 32, 99, 111, 110, 118, 101, 114, 115, 97, 116, 105, 111, 110, 93, 40, 104, 116, 116, 112, 115, 58, 47, 47, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 103, 105, 116, 104, 117, 98, 46, 105, 111, 47, 97, 117, 116, 111, 103, 101, 110, 47, 100, 111, 99, 115, 47, 85, 115, 101, 45, 67, 97, 115, 101, 115, 47, 97, 103, 101, 110, 116, 95, 99, 104, 97, 116, 41, 32, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 32, 73, 116, 32, 111, 102, 102, 101, 114, 115, 32, 99, 117, 115, 116, 111, 109, 105, 122, 97, 98, 108, 101, 32, 97, 110, 100, 32, 99, 111, 110, 118, 101, 114, 115, 97, 98, 108, 101, 32, 97, 103, 101, 110, 116, 115, 32, 116, 104, 97, 116, 32, 105, 110, 116, 101, 103, 114, 97, 116, 101, 32, 76, 76, 77, 115, 44, 32, 116, 111, 111, 108, 115, 44, 32, 97, 110, 100, 32, 104, 117, 109, 97, 110, 115, 46, 10, 66, 121, 32, 97, 117, 116, 111, 109, 97, 116, 105, 110, 103, 32, 99, 104, 97, 116, 32, 97, 109, 111, 110, 103, 32, 109, 117, 108, 116, 105, 112, 108, 101, 32, 99, 97, 112, 97, 98, 108, 101, 32, 97, 103, 101, 110, 116, 115, 44, 32, 111, 110, 101, 32, 99, 97, 110, 32, 101, 97, 115, 105, 108, 121, 32, 109, 97, 107, 101, 32, 116, 104, 101, 109, 32, 99, 111, 108, 108, 101, 99, 116, 105, 118, 101, 108, 121, 32, 112, 101, 114, 102, 111, 114, 109, 32, 116, 97, 115, 107, 115, 32, 97, 117, 116, 111, 110, 111, 109, 111, 117, 115, 108, 121, 32, 111, 114, 32, 119, 105, 116, 104, 32, 104, 117, 109, 97, 110, 32, 102, 101, 101, 100, 98, 97, 99, 107, 44, 32, 105, 110, 99, 108, 117, 100, 105, 110, 103, 32, 116, 97, 115, 107, 115, 32, 116, 104, 97, 116, 32, 114, 101, 113, 117, 105, 114, 101, 32, 117, 115, 105, 110, 103, 32, 116, 111, 111, 108, 115, 32, 118, 105, 97, 32, 99, 111, 100, 101, 46, 10, 10, 70, 101, 97, 116, 117, 114, 101, 115, 32, 111, 102, 32, 116, 104, 105, 115, 32, 117, 115, 101, 32, 99, 97, 115, 101, 32, 105, 110, 99, 108, 117, 100, 101, 58, 10, 10, 45, 32, 42, 42, 77, 117, 108, 116, 105, 45, 97, 103, 101, 110, 116, 32, 99, 111, 110, 118, 101, 114, 115, 97, 116, 105, 111, 110, 115, 42, 42, 58, 32, 65, 117, 116, 111, 71, 101, 110, 32, 97, 103, 101, 110, 116, 115, 32, 99, 97, 110, 32, 99, 111, 109, 109, 117, 110, 105, 99, 97, 116, 101, 32, 119, 105, 116, 104, 32, 101, 97, 99, 104, 32, 111, 116, 104, 101, 114, 32, 116, 111, 32, 115, 111, 108, 118, 101, 32, 116, 97, 115, 107, 115, 46, 32, 84, 104, 105, 115, 32, 97, 108, 108, 111, 119, 115, 32, 102, 111, 114, 32, 109, 111, 114, 101, 32, 99, 111, 109, 112, 108, 101, 120, 32, 97, 110, 100, 32, 115, 111, 112, 104, 105, 115, 116, 105, 99, 97, 116, 101, 100, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 115, 32, 116, 104, 97, 110, 32, 119, 111, 117, 108, 100, 32, 98, 101, 32, 112, 111, 115, 115, 105, 98, 108, 101, 32, 119, 105, 116, 104, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 76, 76, 77, 46, 10, 45, 32, 42, 42, 67, 117, 115, 116, 111, 109, 105, 122, 97, 116, 105, 111, 110, 42, 42, 58, 32, 65, 117, 116, 111, 71, 101, 110, 32, 97, 103, 101, 110, 116, 115, 32, 99, 97, 110, 32, 98, 101, 32, 99, 117, 115, 116, 111, 109, 105, 122, 101, 100, 32, 116, 111, 32, 109, 101, 101, 116, 32, 116, 104, 101, 32, 115, 112, 101, 99, 105, 102, 105, 99, 32, 110, 101, 101, 100, 115, 32, 111, 102, 32, 97, 110, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 46, 32, 84, 104, 105, 115, 32, 105, 110, 99, 108, 117, 100, 101, 115, 32, 116, 104, 101, 32, 97, 98, 105, 108, 105, 116, 121, 32, 116, 111, 32, 99, 104, 111, 111, 115, 101, 32, 116, 104, 101, 32, 76, 76, 77, 115, 32, 116, 111, 32, 117, 115, 101, 44, 32, 116, 104, 101, 32, 116, 121, 112, 101, 115, 32, 111, 102, 32, 104, 117, 109, 97, 110, 32, 105, 110, 112, 117, 116, 32, 116, 111, 32, 97, 108, 108, 111, 119, 44, 32, 97, 110, 100, 32, 116, 104, 101, 32, 116, 111, 111, 108, 115, 32, 116, 111, 32, 101, 109, 112, 108, 111, 121, 46, 10, 45, 32, 42, 42, 72, 117, 109, 97, 110, 32, 112, 97, 114, 116, 105, 99, 105, 112, 97, 116, 105, 111, 110, 42, 42, 58, 32, 65, 117, 116, 111, 71, 101, 110, 32, 115, 101, 97, 109, 108, 101, 115, 115, 108, 121, 32, 97, 108, 108, 111, 119, 115, 32, 104, 117, 109, 97, 110, 32, 112, 97, 114, 116, 105, 99, 105, 112, 97, 116, 105, 111, 110, 46, 32, 84, 104, 105, 115, 32, 109, 101, 97, 110, 115, 32, 116, 104, 97, 116, 32, 104, 117, 109, 97, 110, 115, 32, 99, 97, 110, 32, 112, 114, 111, 118, 105, 100, 101, 32, 105, 110, 112, 117, 116, 32, 97, 110, 100, 32, 102, 101, 101, 100, 98, 97, 99, 107, 32, 116, 111, 32, 116, 104, 101, 32, 97, 103, 101, 110, 116, 115, 32, 97, 115, 32, 110, 101, 101, 100, 101, 100, 46, 10, 10, 70, 111, 114, 32, 91, 101, 120, 97, 109, 112, 108, 101, 93, 40, 104, 116, 116, 112, 115, 58, 47, 47, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109, 47, 109, 105, 99, 114, 111, 115, 111, 102, 116, 47, 97, 117, 116, 111, 103, 101, 110, 47, 98, 108, 111, 98, 47, 109, 97, 105, 110, 47, 116, 101, 115, 116, 47, 116, 119, 111, 97, 103, 101, 110, 116, 46, 112, 121, 41, 44, 10, 10, 96, 96, 96, 112, 121, 116, 104, 111, 110, 10, 102, 114, 111, 109, 32, 97, 117, 116, 111, 103, 101, 110, 32, 105, 109, 112, 111, 114, 116, 32, 65, 115, 115, 105, 115, 116, 97, 110, 116, 65, 103, 101, 110, 116, 44, 32, 85, 115, 101, 114, 80, 114, 111, 120, 121, 65, 103, 101, 110, 116, 44, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 95, 102, 114, 111, 109, 95, 106, 115, 111, 110, 10, 35, 32, 76, 111, 97, 100, 32, 76, 76, 77, 32, 105, 110, 102, 101, 114, 101, 110, 99, 101, 32, 101, 110, 100, 112, 111, 105, 110, 116, 115, 32, 102, 114, 111, 109, 32, 97, 110, 32, 101, 110, 118, 32, 118, 97, 114, 105, 97, 98, 108, 101, 32, 111, 114, 32, 97, 32, 102, 105, 108, 101, 10, 35, 32, 83, 101, 101, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 103, 105, 116, 104, 117, 98, 46, 105, 111, 47, 97, 117, 116, 111, 103, 101, 110, 47, 100, 111, 99, 115, 47, 70, 65, 81, 35, 115, 101, 116, 45, 121, 111, 117, 114, 45, 97, 112, 105, 45, 101, 110, 100, 112, 111, 105, 110, 116, 115, 10, 35, 32, 97, 110, 100, 32, 79, 65, 73, 95, 67, 79, 78, 70, 73, 71, 95, 76, 73, 83, 84, 95, 115, 97, 109, 112, 108, 101, 10, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 32, 61, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 95, 102, 114, 111, 109, 95, 106, 115, 111, 110, 40, 101, 110, 118, 95, 111, 114, 95, 102, 105, 108, 101, 61, 34, 79, 65, 73, 95, 67, 79, 78, 70, 73, 71, 95, 76, 73, 83, 84, 34, 41, 10, 35, 32, 89, 111, 117, 32, 99, 97, 110, 32, 97, 108, 115, 111, 32, 115, 101, 116, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 32, 100, 105, 114, 101, 99, 116, 108, 121, 32, 97, 115, 32, 97, 32, 108, 105, 115, 116, 44, 32, 102, 111, 114, 32, 101, 120, 97, 109, 112, 108, 101, 44, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 32, 61, 32, 91, 123, 39, 109, 111, 100, 101, 108, 39, 58, 32, 39, 103, 112, 116, 45, 52, 39, 44, 32, 39, 97, 112, 105, 95, 107, 101, 121, 39, 58, 32, 39, 60, 121, 111, 117, 114, 32, 79, 112, 101, 110, 65, 73, 32, 65, 80, 73, 32, 107, 101, 121, 32, 104, 101, 114, 101, 62, 39, 125, 44, 93, 10, 97, 115, 115, 105, 115, 116, 97, 110, 116, 32, 61, 32, 65, 115, 115, 105, 115, 116, 97, 110, 116, 65, 103, 101, 110, 116, 40, 34, 97, 115, 115, 105, 115, 116, 97, 110, 116, 34, 44, 32, 108, 108, 109, 95, 99, 111, 110, 102, 105, 103, 61, 123, 34, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 34, 58, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 125, 41, 10, 117, 115, 101, 114, 95, 112, 114, 111, 120, 121, 32, 61, 32, 85, 115, 101, 114, 80, 114, 111, 120, 121, 65, 103, 101, 110, 116, 40, 34, 117, 115, 101, 114, 95, 112, 114, 111, 120, 121, 34, 44, 32, 99, 111, 100, 101, 95, 101, 120, 101, 99, 117, 116, 105, 111, 110, 95, 99, 111, 110, 102, 105, 103, 61, 123, 34, 119, 111, 114, 107, 95, 100, 105, 114, 34, 58, 32, 34, 99, 111, 100, 105, 110, 103, 34, 44, 32, 34, 117, 115, 101, 95, 100, 111, 99, 107, 101, 114, 34, 58, 32, 70, 97, 108, 115, 101, 125, 41, 32, 35, 32, 73, 77, 80, 79, 82, 84, 65, 78, 84, 58, 32, 115, 101, 116, 32, 116, 111, 32, 84, 114, 117, 101, 32, 116, 111, 32, 114, 117, 110, 32, 99, 111, 100, 101, 32, 105, 110, 32, 100, 111, 99, 107, 101, 114, 44, 32, 114, 101, 99, 111, 109, 109, 101, 110, 100, 101, 100, 10, 117, 115, 101, 114, 95, 112, 114, 111, 120, 121, 46, 105, 110, 105, 116, 105, 97, 116, 101, 95, 99, 104, 97, 116, 40, 97, 115, 115, 105, 115, 116, 97, 110, 116, 44, 32, 109, 101, 115, 115, 97, 103, 101, 61, 34, 80, 108, 111, 116, 32, 97, 32, 99, 104, 97, 114, 116, 32, 111, 102, 32, 78, 86, 68, 65, 32, 97, 110, 100, 32, 84, 69, 83, 76, 65, 32, 115, 116, 111, 99, 107, 32, 112, 114, 105, 99, 101, 32, 99, 104, 97, 110, 103, 101, 32, 89, 84, 68, 46, 34, 41, 10, 35, 32, 84, 104, 105, 115, 32, 105, 110, 105, 116, 105, 97, 116, 101, 115, 32, 97, 110, 32, 97, 117, 116, 111, 109, 97, 116, 101, 100, 32, 99, 104, 97, 116, 32, 98, 101, 116, 119, 101, 101, 110, 32, 116, 104, 101, 32, 116, 119, 111, 32, 97, 103, 101, 110, 116, 115, 32, 116, 111, 32, 115, 111, 108, 118, 101, 32, 116, 104, 101, 32, 116, 97, 115, 107, 10, 96, 96, 96, 10, 10, 109, 111, 114, 101, 32, 112, 121, 116, 104, 111, 110, 32, 99, 111, 100, 101, 58, 10, 10, 96, 96, 96, 112, 121, 116, 104, 111, 110, 10, 32, 32, 32, 32, 100, 101, 102, 32, 99, 114, 101, 97, 116, 101, 40, 10, 32, 32, 32, 32, 32, 32, 32, 32, 115, 101, 108, 102, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 42, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 109, 101, 115, 115, 97, 103, 101, 115, 58, 32, 73, 116, 101, 114, 97, 98, 108, 101, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 77, 101, 115, 115, 97, 103, 101, 80, 97, 114, 97, 109, 93, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 109, 111, 100, 101, 108, 58, 32, 85, 110, 105, 111, 110, 91, 115, 116, 114, 44, 32, 67, 104, 97, 116, 77, 111, 100, 101, 108, 93, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 102, 114, 101, 113, 117, 101, 110, 99, 121, 95, 112, 101, 110, 97, 108, 116, 121, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 102, 117, 110, 99, 116, 105, 111, 110, 95, 99, 97, 108, 108, 58, 32, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 70, 117, 110, 99, 116, 105, 111, 110, 67, 97, 108, 108, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 102, 117, 110, 99, 116, 105, 111, 110, 115, 58, 32, 73, 116, 101, 114, 97, 98, 108, 101, 91, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 70, 117, 110, 99, 116, 105, 111, 110, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 108, 111, 103, 105, 116, 95, 98, 105, 97, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 68, 105, 99, 116, 91, 115, 116, 114, 44, 32, 105, 110, 116, 93, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 108, 111, 103, 112, 114, 111, 98, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 98, 111, 111, 108, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 109, 97, 120, 95, 116, 111, 107, 101, 110, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 110, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 112, 114, 101, 115, 101, 110, 99, 101, 95, 112, 101, 110, 97, 108, 116, 121, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 114, 101, 115, 112, 111, 110, 115, 101, 95, 102, 111, 114, 109, 97, 116, 58, 32, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 82, 101, 115, 112, 111, 110, 115, 101, 70, 111, 114, 109, 97, 116, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 115, 101, 101, 100, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 115, 116, 111, 112, 58, 32, 85, 110, 105, 111, 110, 91, 79, 112, 116, 105, 111, 110, 97, 108, 91, 115, 116, 114, 93, 44, 32, 76, 105, 115, 116, 91, 115, 116, 114, 93, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 115, 116, 114, 101, 97, 109, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 76, 105, 116, 101, 114, 97, 108, 91, 70, 97, 108, 115, 101, 93, 93, 32, 124, 32, 76, 105, 116, 101, 114, 97, 108, 91, 84, 114, 117, 101, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 115, 116, 114, 101, 97, 109, 95, 111, 112, 116, 105, 111, 110, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 83, 116, 114, 101, 97, 109, 79, 112, 116, 105, 111, 110, 115, 80, 97, 114, 97, 109, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 116, 111, 111, 108, 95, 99, 104, 111, 105, 99, 101, 58, 32, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 84, 111, 111, 108, 67, 104, 111, 105, 99, 101, 79, 112, 116, 105, 111, 110, 80, 97, 114, 97, 109, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 116, 111, 111, 108, 115, 58, 32, 73, 116, 101, 114, 97, 98, 108, 101, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 84, 111, 111, 108, 80, 97, 114, 97, 109, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 116, 111, 112, 95, 108, 111, 103, 112, 114, 111, 98, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 116, 111, 112, 95, 112, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 117, 115, 101, 114, 58, 32, 115, 116, 114, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 35, 32, 85, 115, 101, 32, 116, 104, 101, 32, 102, 111, 108, 108, 111, 119, 105, 110, 103, 32, 97, 114, 103, 117, 109, 101, 110, 116, 115, 32, 105, 102, 32, 121, 111, 117, 32, 110, 101, 101, 100, 32, 116, 111, 32, 112, 97, 115, 115, 32, 97, 100, 100, 105, 116, 105, 111, 110, 97, 108, 32, 112, 97, 114, 97, 109, 101, 116, 101, 114, 115, 32, 116, 111, 32, 116, 104, 101, 32, 65, 80, 73, 32, 116, 104, 97, 116, 32, 97, 114, 101, 110, 39, 116, 32, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 118, 105, 97, 32, 107, 119, 97, 114, 103, 115, 46, 10, 32, 32, 32, 32, 32, 32, 32, 32, 35, 32, 84, 104, 101, 32, 101, 120, 116, 114, 97, 32, 118, 97, 108, 117, 101, 115, 32, 103, 105, 118, 101, 110, 32, 104, 101, 114, 101, 32, 116, 97, 107, 101, 32, 112, 114, 101, 99, 101, 100, 101, 110, 99, 101, 32, 111, 118, 101, 114, 32, 118, 97, 108, 117, 101, 115, 32, 100, 101, 102, 105, 110, 101, 100, 32, 111, 110, 32, 116, 104, 101, 32, 99, 108, 105, 101, 110, 116, 32, 111, 114, 32, 112, 97, 115, 115, 101, 100, 32, 116, 111, 32, 116, 104, 105, 115, 32, 109, 101, 116, 104, 111, 100, 46, 10, 32, 32, 32, 32, 32, 32, 32, 32, 101, 120, 116, 114, 97, 95, 104, 101, 97, 100, 101, 114, 115, 58, 32, 72, 101, 97, 100, 101, 114, 115, 32, 124, 32, 78, 111, 110, 101, 32, 61, 32, 78, 111, 110, 101, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 101, 120, 116, 114, 97, 95, 113, 117, 101, 114, 121, 58, 32, 81, 117, 101, 114, 121, 32, 124, 32, 78, 111, 110, 101, 32, 61, 32, 78, 111, 110, 101, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 101, 120, 116, 114, 97, 95, 98, 111, 100, 121, 58, 32, 66, 111, 100, 121, 32, 124, 32, 78, 111, 110, 101, 32, 61, 32, 78, 111, 110, 101, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 116, 105, 109, 101, 111, 117, 116, 58, 32, 102, 108, 111, 97, 116, 32, 124, 32, 104, 116, 116, 112, 120, 46, 84, 105, 109, 101, 111, 117, 116, 32, 124, 32, 78, 111, 110, 101, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 32, 32, 32, 32, 41, 32, 45, 62, 32, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 32, 124, 32, 83, 116, 114, 101, 97, 109, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 67, 104, 117, 110, 107, 93, 58, 10, 32, 32, 32, 32, 32, 32, 32, 32, 114, 101, 116, 117, 114, 110, 32, 115, 101, 108, 102, 46, 95, 112, 111, 115, 116, 40, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 47, 99, 104, 97, 116, 47, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 115, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 98, 111, 100, 121, 61, 109, 97, 121, 98, 101, 95, 116, 114, 97, 110, 115, 102, 111, 114, 109, 40, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 115, 34, 58, 32, 109, 101, 115, 115, 97, 103, 101, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 111, 100, 101, 108, 34, 58, 32, 109, 111, 100, 101, 108, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 102, 114, 101, 113, 117, 101, 110, 99, 121, 95, 112, 101, 110, 97, 108, 116, 121, 34, 58, 32, 102, 114, 101, 113, 117, 101, 110, 99, 121, 95, 112, 101, 110, 97, 108, 116, 121, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 102, 117, 110, 99, 116, 105, 111, 110, 95, 99, 97, 108, 108, 34, 58, 32, 102, 117, 110, 99, 116, 105, 111, 110, 95, 99, 97, 108, 108, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 102, 117, 110, 99, 116, 105, 111, 110, 115, 34, 58, 32, 102, 117, 110, 99, 116, 105, 111, 110, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 108, 111, 103, 105, 116, 95, 98, 105, 97, 115, 34, 58, 32, 108, 111, 103, 105, 116, 95, 98, 105, 97, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 108, 111, 103, 112, 114, 111, 98, 115, 34, 58, 32, 108, 111, 103, 112, 114, 111, 98, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 97, 120, 95, 116, 111, 107, 101, 110, 115, 34, 58, 32, 109, 97, 120, 95, 116, 111, 107, 101, 110, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 110, 34, 58, 32, 110, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 112, 114, 101, 115, 101, 110, 99, 101, 95, 112, 101, 110, 97, 108, 116, 121, 34, 58, 32, 112, 114, 101, 115, 101, 110, 99, 101, 95, 112, 101, 110, 97, 108, 116, 121, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 114, 101, 115, 112, 111, 110, 115, 101, 95, 102, 111, 114, 109, 97, 116, 34, 58, 32, 114, 101, 115, 112, 111, 110, 115, 101, 95, 102, 111, 114, 109, 97, 116, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 115, 101, 101, 100, 34, 58, 32, 115, 101, 101, 100, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 115, 116, 111, 112, 34, 58, 32, 115, 116, 111, 112, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 115, 116, 114, 101, 97, 109, 34, 58, 32, 115, 116, 114, 101, 97, 109, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 115, 116, 114, 101, 97, 109, 95, 111, 112, 116, 105, 111, 110, 115, 34, 58, 32, 115, 116, 114, 101, 97, 109, 95, 111, 112, 116, 105, 111, 110, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 34, 58, 32, 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 111, 111, 108, 95, 99, 104, 111, 105, 99, 101, 34, 58, 32, 116, 111, 111, 108, 95, 99, 104, 111, 105, 99, 101, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 111, 111, 108, 115, 34, 58, 32, 116, 111, 111, 108, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 111, 112, 95, 108, 111, 103, 112, 114, 111, 98, 115, 34, 58, 32, 116, 111, 112, 95, 108, 111, 103, 112, 114, 111, 98, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 111, 112, 95, 112, 34, 58, 32, 116, 111, 112, 95, 112, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 34, 117, 115, 101, 114, 34, 58, 32, 117, 115, 101, 114, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 67, 114, 101, 97, 116, 101, 80, 97, 114, 97, 109, 115, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 41, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 111, 112, 116, 105, 111, 110, 115, 61, 109, 97, 107, 101, 95, 114, 101, 113, 117, 101, 115, 116, 95, 111, 112, 116, 105, 111, 110, 115, 40, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 101, 120, 116, 114, 97, 95, 104, 101, 97, 100, 101, 114, 115, 61, 101, 120, 116, 114, 97, 95, 104, 101, 97, 100, 101, 114, 115, 44, 32, 101, 120, 116, 114, 97, 95, 113, 117, 101, 114, 121, 61, 101, 120, 116, 114, 97, 95, 113, 117, 101, 114, 121, 44, 32, 101, 120, 116, 114, 97, 95, 98, 111, 100, 121, 61, 101, 120, 116, 114, 97, 95, 98, 111, 100, 121, 44, 32, 116, 105, 109, 101, 111, 117, 116, 61, 116, 105, 109, 101, 111, 117, 116, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 41, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 99, 97, 115, 116, 95, 116, 111, 61, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 115, 116, 114, 101, 97, 109, 61, 115, 116, 114, 101, 97, 109, 32, 111, 114, 32, 70, 97, 108, 115, 101, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 115, 116, 114, 101, 97, 109, 95, 99, 108, 115, 61, 83, 116, 114, 101, 97, 109, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 67, 104, 117, 110, 107, 93, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 41, 10, 96, 96, 96, 10]\n",
+      "length: 5397\n"
+     ]
+    }
+   ],
+   "source": [
+    "text = \"\"\"Autogen enables the next-gen LLM applications with a generic [multi-agent conversation](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat) framework. It offers customizable and conversable agents that integrate LLMs, tools, and humans.\n",
+    "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code.\n",
+    "\n",
+    "Features of this use case include:\n",
+    "\n",
+    "- **Multi-agent conversations**: AutoGen agents can communicate with each other to solve tasks. This allows for more complex and sophisticated applications than would be possible with a single LLM.\n",
+    "- **Customization**: AutoGen agents can be customized to meet the specific needs of an application. This includes the ability to choose the LLMs to use, the types of human input to allow, and the tools to employ.\n",
+    "- **Human participation**: AutoGen seamlessly allows human participation. This means that humans can provide input and feedback to the agents as needed.\n",
+    "\n",
+    "For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py),\n",
+    "\n",
+    "```python\n",
+    "from autogen import AssistantAgent, UserProxyAgent, config_list_from_json\n",
+    "# Load LLM inference endpoints from an env variable or a file\n",
+    "# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints\n",
+    "# and OAI_CONFIG_LIST_sample\n",
+    "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n",
+    "# You can also set config_list directly as a list, for example, config_list = [{'model': 'gpt-4', 'api_key': '<your OpenAI API key here>'},]\n",
+    "assistant = AssistantAgent(\"assistant\", llm_config={\"config_list\": config_list})\n",
+    "user_proxy = UserProxyAgent(\"user_proxy\", code_execution_config={\"work_dir\": \"coding\", \"use_docker\": False}) # IMPORTANT: set to True to run code in docker, recommended\n",
+    "user_proxy.initiate_chat(assistant, message=\"Plot a chart of NVDA and TESLA stock price change YTD.\")\n",
+    "# This initiates an automated chat between the two agents to solve the task\n",
+    "```\n",
+    "\n",
+    "more python code:\n",
+    "\n",
+    "```python\n",
+    "    def create(\n",
+    "        self,\n",
+    "        *,\n",
+    "        messages: Iterable[ChatCompletionMessageParam],\n",
+    "        model: Union[str, ChatModel],\n",
+    "        frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN,\n",
+    "        function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN,\n",
+    "        functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN,\n",
+    "        logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN,\n",
+    "        logprobs: Optional[bool] | NotGiven = NOT_GIVEN,\n",
+    "        max_tokens: Optional[int] | NotGiven = NOT_GIVEN,\n",
+    "        n: Optional[int] | NotGiven = NOT_GIVEN,\n",
+    "        presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,\n",
+    "        response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,\n",
+    "        seed: Optional[int] | NotGiven = NOT_GIVEN,\n",
+    "        stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,\n",
+    "        stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN,\n",
+    "        stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,\n",
+    "        temperature: Optional[float] | NotGiven = NOT_GIVEN,\n",
+    "        tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,\n",
+    "        tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,\n",
+    "        top_logprobs: Optional[int] | NotGiven = NOT_GIVEN,\n",
+    "        top_p: Optional[float] | NotGiven = NOT_GIVEN,\n",
+    "        user: str | NotGiven = NOT_GIVEN,\n",
+    "        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.\n",
+    "        # The extra values given here take precedence over values defined on the client or passed to this method.\n",
+    "        extra_headers: Headers | None = None,\n",
+    "        extra_query: Query | None = None,\n",
+    "        extra_body: Body | None = None,\n",
+    "        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n",
+    "    ) -> ChatCompletion | Stream[ChatCompletionChunk]:\n",
+    "        return self._post(\n",
+    "            \"/chat/completions\",\n",
+    "            body=maybe_transform(\n",
+    "                {\n",
+    "                    \"messages\": messages,\n",
+    "                    \"model\": model,\n",
+    "                    \"frequency_penalty\": frequency_penalty,\n",
+    "                    \"function_call\": function_call,\n",
+    "                    \"functions\": functions,\n",
+    "                    \"logit_bias\": logit_bias,\n",
+    "                    \"logprobs\": logprobs,\n",
+    "                    \"max_tokens\": max_tokens,\n",
+    "                    \"n\": n,\n",
+    "                    \"presence_penalty\": presence_penalty,\n",
+    "                    \"response_format\": response_format,\n",
+    "                    \"seed\": seed,\n",
+    "                    \"stop\": stop,\n",
+    "                    \"stream\": stream,\n",
+    "                    \"stream_options\": stream_options,\n",
+    "                    \"temperature\": temperature,\n",
+    "                    \"tool_choice\": tool_choice,\n",
+    "                    \"tools\": tools,\n",
+    "                    \"top_logprobs\": top_logprobs,\n",
+    "                    \"top_p\": top_p,\n",
+    "                    \"user\": user,\n",
+    "                },\n",
+    "                completion_create_params.CompletionCreateParams,\n",
+    "            ),\n",
+    "            options=make_request_options(\n",
+    "                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout\n",
+    "            ),\n",
+    "            cast_to=ChatCompletion,\n",
+    "            stream=stream or False,\n",
+    "            stream_cls=Stream[ChatCompletionChunk],\n",
+    "        )\n",
+    "```\n",
+    "\"\"\"\n",
+    "tokens = text.encode('utf-8') # which will produce raw byte strings\n",
+    "tokens = list(map(int, tokens)) # convert the byte strings to integers\n",
+    "print('----')\n",
+    "print(text)\n",
+    "print('length:', len(text))\n",
+    "print('----')\n",
+    "print(tokens)\n",
+    "print('length:', len(tokens))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[(770, (32, 32)), (86, (111, 110)), (73, (101, 110)), (66, (10, 32)), (65, (116, 105)), (57, (44, 10)), (56, (105, 111)), (55, (58, 32)), (55, (32, 116)), (52, (97, 116)), (50, (116, 111)), (50, (101, 32)), (48, (32, 78)), (47, (110, 32)), (44, (114, 101)), (40, (115, 116)), (40, (32, 97)), (38, (115, 32)), (38, (101, 114)), (36, (115, 101)), (35, (97, 108)), (35, (32, 99)), (34, (108, 101)), (32, (116, 104)), (31, (114, 97)), (30, (97, 110)), (29, (110, 116)), (28, (118, 101)), (28, (116, 114)), (28, (111, 109)), (28, (97, 109)), (27, (124, 32)), (27, (103, 101)), (27, (101, 97)), (27, (99, 111)), (27, (78, 111)), (27, (61, 32)), (27, (32, 124)), (27, (32, 61)), (26, (116, 32)), (26, (101, 115)), (25, (105, 110)), (24, (110, 115)), (24, (34, 58)), (24, (32, 34)), (23, (116, 101)), (23, (112, 108)), (23, (109, 112)), (22, (111, 116)), (22, (105, 118)), (22, (101, 116)), (22, (44, 32)), (22, (32, 115)), (21, (112, 116)), (21, (110, 97)), (21, (108, 111)), (21, (105, 115)), (21, (104, 97)), (21, (100, 101)), (21, (84, 95)), (20, (116, 71)), (20, (104, 101)), (20, (95, 71)), (20, (86, 69)), (20, (79, 84)), (20, (78, 79)), (20, (78, 44)), (20, (73, 86)), (20, (71, 105)), (20, (71, 73)), (20, (69, 78)), (19, (114, 111)), (19, (111, 114)), (19, (110, 99)), (19, (109, 97)), (18, (117, 116)), (18, (105, 99)), (18, (97, 115)), (17, (97, 114)), (17, (95, 112)), (16, (111, 108)), (16, (111, 100)), (16, (105, 116)), (16, (101, 100)), (16, (99, 97)), (16, (93, 32)), (16, (79, 112)), (15, (112, 114)), (15, (111, 112)), (15, (111, 32)), (15, (109, 101)), (15, (108, 91)), (15, (101, 120)), (15, (101, 95)), (15, (100, 32)), (15, (97, 103)), (15, (95, 99)), (15, (32, 102)), (14, (117, 115)), (14, (115, 115)), (14, (111, 103)), (14, (110, 101)), (14, (32, 111)), (14, (32, 101)), (14, (32, 79)), (13, (115, 44)), (13, (112, 101)), (13, (111, 111)), (13, (108, 105)), (13, (102, 105)), (13, (32, 112)), (13, (32, 109)), (12, (117, 110)), (12, (117, 101)), (12, (115, 58)), (12, (112, 97)), (12, (99, 104)), (12, (67, 104)), (12, (32, 105)), (11, (121, 32)), (11, (120, 116)), (11, (114, 32)), (11, (108, 115)), (11, (101, 101)), (11, (99, 116)), (11, (99, 101)), (11, (98, 108)), (10, (116, 97)), (10, (111, 117)), (10, (110, 102)), (10, (110, 100)), (10, (108, 108)), (10, (107, 101)), (10, (104, 117)), (10, (97, 98)), (10, (95, 108)), (9, (116, 115)), (9, (116, 93)), (9, (115, 111)), (9, (115, 105)), (9, (115, 97)), (9, (115, 34)), (9, (114, 115)), (9, (112, 111)), (9, (108, 116)), (9, (105, 103)), (9, (104, 111)), (9, (97, 95)), (9, (67, 111)), (9, (32, 104)), (8, (116, 121)), (8, (116, 95)), (8, (116, 67)), (8, (113, 117)), (8, (111, 102)), (8, (110, 103)), (8, (110, 95)), (8, (109, 111)), (8, (105, 97)), (8, (102, 114)), (8, (102, 111)), (8, (101, 108)), (8, (101, 44)), (8, (99, 114)), (8, (97, 32)), (8, (96, 96)), (8, (35, 32)), (8, (32, 76)), (7, (117, 114)), (7, (117, 109)), (7, (115, 46)), (7, (111, 98)), (7, (111, 97)), (7, (109, 95)), (7, (104, 105)), (7, (103, 112)), (7, (103, 105)), (7, (103, 95)), (7, (97, 117)), (7, (46, 10)), (7, (32, 84)), (7, (32, 65)), (6, (114, 109)), (6, (112, 95)), (6, (111, 115)), (6, (111, 105)), (6, (109, 105)), (6, (109, 32)), (6, (102, 117)), (6, (102, 32)), (6, (101, 99)), (6, (98, 115)), (6, (97, 112)), (6, (97, 100)), (6, (95, 102)), (6, (95, 98)), (6, (42, 42)), (6, (32, 114)), (6, (32, 110)), (6, (32, 108)), (6, (10, 10)), (5, (120, 121)), (5, (119, 111)), (5, (119, 105)), (5, (116, 117)), (5, (116, 44)), (5, (115, 112)), (5, (111, 120)), (5, (111, 99)), (5, (110, 118)), (5, (110, 105)), (5, (109, 115)), (5, (108, 121)), (5, (108, 117)), (5, (105, 109)), (5, (105, 108)), (5, (104, 32)), (5, (103, 32)), (5, (102, 108)), (5, (101, 113)), (5, (101, 109)), (5, (100, 121)), (5, (100, 105)), (5, (99, 108)), (5, (99, 107)), (5, (98, 111)), (5, (95, 116)), (5, (95, 111)), (5, (91, 67)), (5, (84, 104)), (5, (80, 97)), (5, (76, 77)), (5, (76, 76)), (5, (34, 116)), (5, (32, 119)), (5, (32, 118)), (5, (32, 117)), (5, (32, 85)), (5, (32, 73)), (5, (10, 35)), (4, (121, 58)), (4, (121, 44)), (4, (118, 97)), (4, (117, 108)), (4, (116, 116)), (4, (116, 112)), (4, (116, 40)), (4, (115, 107)), (4, (114, 121)), (4, (114, 116)), (4, (114, 95)), (4, (114, 44)), (4, (112, 121)), (4, (111, 119)), (4, (110, 67)), (4, (110, 47)), (4, (104, 116)), (4, (102, 101)), (4, (101, 111)), (4, (101, 58)), (4, (100, 111)), (4, (98, 105)), (4, (98, 101)), (4, (93, 44)), (4, (91, 115)), (4, (91, 105)), (4, (91, 102)), (4, (85, 115)), (4, (73, 116)), (4, (65, 117)), (4, (65, 103)), (4, (47, 109)), (4, (47, 97)), (4, (46, 32)), (4, (41, 10)), (4, (40, 10)), (4, (34, 115)), (4, (34, 44)), (4, (32, 100)), (4, (32, 98)), (4, (32, 42)), (4, (32, 41)), (4, (10, 96)), (3, (121, 116)), (3, (121, 111)), (3, (121, 95)), (3, (121, 61)), (3, (121, 34)), (3, (120, 95)), (3, (118, 105)), (3, (117, 100)), (3, (117, 98)), (3, (116, 119)), (3, (116, 47)), (3, (116, 46)), (3, (116, 45)), (3, (116, 34)), (3, (115, 61)), (3, (115, 47)), (3, (114, 117)), (3, (114, 105)), (3, (114, 34)), (3, (112, 115)), (3, (112, 112)), (3, (111, 107)), (3, (111, 71)), (3, (110, 10)), (3, (109, 117)), (3, (109, 93)), (3, (108, 95)), (3, (107, 115)), (3, (105, 122)), (3, (105, 114)), (3, (105, 112)), (3, (105, 45)), (3, (102, 116)), (3, (101, 93)), (3, (101, 91)), (3, (99, 121)), (3, (99, 117)), (3, (99, 105)), (3, (98, 46)), (3, (97, 120)), (3, (97, 107)), (3, (97, 99)), (3, (95, 113)), (3, (95, 104)), (3, (93, 93)), (3, (83, 116)), (3, (76, 105)), (3, (73, 32)), (3, (71, 101)), (3, (70, 97)), (3, (65, 73)), (3, (61, 101)), (3, (58, 47)), (3, (58, 10)), (3, (47, 47)), (3, (46, 105)), (3, (45, 97)), (3, (45, 32)), (3, (42, 58)), (3, (41, 44)), (3, (41, 32)), (3, (34, 117)), (3, (34, 109)), (3, (34, 102)), (3, (32, 91)), (3, (32, 67)), (3, (32, 39)), (3, (32, 35)), (3, (10, 45)), (2, (125, 44)), (2, (125, 41)), (2, (123, 34)), (2, (122, 97)), (2, (121, 65)), (2, (121, 46)), (2, (120, 97)), (2, (119, 115)), (2, (117, 32)), (2, (116, 91)), (2, (116, 65)), (2, (116, 58)), (2, (115, 108)), (2, (114, 107)), (2, (114, 103)), (2, (114, 93)), (2, (114, 80)), (2, (112, 117)), (2, (112, 105)), (2, (112, 58)), (2, (112, 44)), (2, (112, 34)), (2, (111, 118)), (2, (111, 47)), (2, (110, 112)), (2, (110, 107)), (2, (110, 93)), (2, (110, 91)), (2, (110, 84)), (2, (110, 46)), (2, (110, 44)), (2, (110, 42)), (2, (109, 109)), (2, (109, 91)), (2, (108, 118)), (2, (108, 102)), (2, (108, 93)), (2, (108, 58)), (2, (108, 44)), (2, (108, 34)), (2, (108, 32)), (2, (107, 93)), (2, (107, 32)), (2, (106, 115)), (2, (105, 102)), (2, (103, 61)), (2, (101, 121)), (2, (101, 102)), (2, (101, 80)), (2, (101, 61)), (2, (101, 34)), (2, (101, 10)), (2, (100, 112)), (2, (100, 98)), (2, (100, 46)), (2, (99, 115)), (2, (99, 32)), (2, (98, 97)), (2, (97, 105)), (2, (96, 112)), (2, (96, 10)), (2, (95, 106)), (2, (95, 100)), (2, (95, 76)), (2, (95, 67)), (2, (93, 40)), (2, (85, 110)), (2, (84, 114)), (2, (84, 111)), (2, (83, 84)), (2, (80, 114)), (2, (80, 73)), (2, (79, 78)), (2, (79, 65)), (2, (78, 70)), (2, (77, 115)), (2, (77, 32)), (2, (76, 73)), (2, (73, 95)), (2, (73, 83)), (2, (73, 71)), (2, (71, 95)), (2, (70, 117)), (2, (70, 111)), (2, (70, 73)), (2, (67, 97)), (2, (67, 79)), (2, (65, 115)), (2, (65, 80)), (2, (65, 32)), (2, (61, 123)), (2, (61, 109)), (2, (61, 34)), (2, (47, 116)), (2, (47, 100)), (2, (47, 99)), (2, (46, 103)), (2, (46, 70)), (2, (40, 104)), (2, (40, 34)), (2, (39, 58)), (2, (34, 108)), (2, (34, 99)), (2, (34, 41)), (2, (32, 107)), (2, (32, 103)), (2, (32, 89)), (2, (32, 83)), (2, (32, 70)), (2, (10, 117)), (2, (10, 70)), (1, (123, 39)), (1, (123, 10)), (1, (122, 101)), (1, (121, 112)), (1, (121, 98)), (1, (121, 41)), (1, (121, 39)), (1, (120, 101)), (1, (120, 46)), (1, (120, 32)), (1, (119, 101)), (1, (119, 97)), (1, (119, 44)), (1, (118, 95)), (1, (118, 32)), (1, (117, 105)), (1, (116, 125)), (1, (116, 108)), (1, (116, 77)), (1, (116, 61)), (1, (116, 41)), (1, (116, 10)), (1, (115, 102)), (1, (115, 80)), (1, (115, 42)), (1, (115, 40)), (1, (115, 10)), (1, (114, 110)), (1, (114, 102)), (1, (114, 58)), (1, (114, 45)), (1, (112, 120)), (1, (112, 104)), (1, (111, 121)), (1, (111, 61)), (1, (110, 111)), (1, (110, 83)), (1, (110, 80)), (1, (110, 77)), (1, (110, 65)), (1, (110, 58)), (1, (110, 40)), (1, (110, 39)), (1, (110, 34)), (1, (109, 108)), (1, (109, 79)), (1, (109, 61)), (1, (109, 58)), (1, (109, 47)), (1, (109, 44)), (1, (109, 40)), (1, (109, 34)), (1, (108, 109)), (1, (108, 100)), (1, (108, 97)), (1, (108, 80)), (1, (108, 67)), (1, (108, 39)), (1, (107, 119)), (1, (107, 95)), (1, (107, 46)), (1, (107, 44)), (1, (107, 10)), (1, (105, 101)), (1, (105, 100)), (1, (105, 98)), (1, (105, 95)), (1, (103, 117)), (1, (103, 115)), (1, (103, 114)), (1, (103, 108)), (1, (103, 34)), (1, (102, 102)), (1, (102, 46)), (1, (102, 44)), (1, (101, 125)), (1, (101, 119)), (1, (101, 103)), (1, (101, 79)), (1, (101, 70)), (1, (101, 62)), (1, (101, 46)), (1, (101, 45)), (1, (101, 40)), (1, (100, 115)), (1, (100, 100)), (1, (100, 58)), (1, (100, 44)), (1, (100, 34)), (1, (100, 10)), (1, (98, 47)), (1, (97, 121)), (1, (97, 118)), (1, (95, 115)), (1, (95, 114)), (1, (95, 107)), (1, (95, 101)), (1, (93, 58)), (1, (93, 10)), (1, (91, 123)), (1, (91, 109)), (1, (91, 101)), (1, (91, 99)), (1, (91, 98)), (1, (91, 84)), (1, (91, 79)), (1, (91, 76)), (1, (91, 70)), (1, (91, 68)), (1, (89, 111)), (1, (89, 84)), (1, (86, 68)), (1, (84, 105)), (1, (84, 69)), (1, (84, 68)), (1, (84, 65)), (1, (84, 58)), (1, (84, 34)), (1, (83, 101)), (1, (83, 76)), (1, (82, 101)), (1, (82, 84)), (1, (81, 117)), (1, (81, 35)), (1, (80, 108)), (1, (80, 79)), (1, (79, 82)), (1, (78, 86)), (1, (78, 84)), (1, (77, 117)), (1, (77, 111)), (1, (77, 101)), (1, (77, 80)), (1, (77, 46)), (1, (76, 111)), (1, (76, 65)), (1, (73, 77)), (1, (72, 117)), (1, (72, 101)), (1, (70, 101)), (1, (70, 65)), (1, (69, 83)), (1, (68, 105)), (1, (68, 65)), (1, (68, 46)), (1, (67, 117)), (1, (67, 114)), (1, (66, 121)), (1, (66, 111)), (1, (65, 81)), (1, (65, 78)), (1, (62, 39)), (1, (62, 32)), (1, (61, 116)), (1, (61, 115)), (1, (61, 83)), (1, (61, 67)), (1, (60, 121)), (1, (52, 39)), (1, (47, 103)), (1, (47, 98)), (1, (47, 85)), (1, (47, 70)), (1, (46, 112)), (1, (46, 99)), (1, (46, 95)), (1, (46, 84)), (1, (46, 82)), (1, (46, 67)), (1, (46, 34)), (1, (45, 121)), (1, (45, 103)), (1, (45, 101)), (1, (45, 67)), (1, (45, 62)), (1, (45, 52)), (1, (44, 93)), (1, (42, 77)), (1, (42, 72)), (1, (42, 67)), (1, (42, 44)), (1, (40, 101)), (1, (40, 97)), (1, (39, 125)), (1, (39, 116)), (1, (39, 109)), (1, (39, 103)), (1, (39, 97)), (1, (39, 60)), (1, (39, 44)), (1, (35, 115)), (1, (34, 119)), (1, (34, 114)), (1, (34, 112)), (1, (34, 110)), (1, (34, 97)), (1, (34, 80)), (1, (34, 79)), (1, (34, 47)), (1, (32, 125)), (1, (32, 123)), (1, (32, 121)), (1, (32, 81)), (1, (32, 72)), (1, (32, 66)), (1, (32, 45)), (1, (10, 109)), (1, (10, 102)), (1, (10, 99)), (1, (10, 97)), (1, (10, 66))]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_stats(ids):\n",
+    "    \"\"\"\n",
+    "    Get statistics of the token ids. includes the most common token pairs.\n",
+    "    \"\"\"\n",
+    "    counts = {}\n",
+    "    for pair in zip(ids, ids[1:]):\n",
+    "        counts[pair] = counts.get(pair, 0) + 1\n",
+    "    return counts\n",
+    "\n",
+    "stats = get_stats(tokens)\n",
+    "# print(stats)\n",
+    "print(sorted(((v,k) for k,v in stats.items()), reverse=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(' ', ' ')"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chr(32), chr(32) # the space character is the most common character in the text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[65, 117, 116, 111, 103, 101, 110, 32, 101, 110, 97, 98, 108, 101, 115, 32, 116, 104, 101, 32, 110, 101, 120, 116, 45, 103, 101, 110, 32, 76, 76, 77, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 115, 32, 119, 105, 116, 104, 32, 97, 32, 103, 101, 110, 101, 114, 105, 99, 32, 91, 109, 117, 108, 116, 105, 45, 97, 103, 101, 110, 116, 32, 99, 111, 110, 118, 101, 114, 115, 97, 116, 105, 111, 110, 93, 40, 104, 116, 116, 112, 115, 58, 47, 47, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 103, 105, 116, 104, 117, 98, 46, 105, 111, 47, 97, 117, 116, 111, 103, 101, 110, 47, 100, 111, 99, 115, 47, 85, 115, 101, 45, 67, 97, 115, 101, 115, 47, 97, 103, 101, 110, 116, 95, 99, 104, 97, 116, 41, 32, 102, 114, 97, 109, 101, 119, 111, 114, 107, 46, 32, 73, 116, 32, 111, 102, 102, 101, 114, 115, 32, 99, 117, 115, 116, 111, 109, 105, 122, 97, 98, 108, 101, 32, 97, 110, 100, 32, 99, 111, 110, 118, 101, 114, 115, 97, 98, 108, 101, 32, 97, 103, 101, 110, 116, 115, 32, 116, 104, 97, 116, 32, 105, 110, 116, 101, 103, 114, 97, 116, 101, 32, 76, 76, 77, 115, 44, 32, 116, 111, 111, 108, 115, 44, 32, 97, 110, 100, 32, 104, 117, 109, 97, 110, 115, 46, 10, 66, 121, 32, 97, 117, 116, 111, 109, 97, 116, 105, 110, 103, 32, 99, 104, 97, 116, 32, 97, 109, 111, 110, 103, 32, 109, 117, 108, 116, 105, 112, 108, 101, 32, 99, 97, 112, 97, 98, 108, 101, 32, 97, 103, 101, 110, 116, 115, 44, 32, 111, 110, 101, 32, 99, 97, 110, 32, 101, 97, 115, 105, 108, 121, 32, 109, 97, 107, 101, 32, 116, 104, 101, 109, 32, 99, 111, 108, 108, 101, 99, 116, 105, 118, 101, 108, 121, 32, 112, 101, 114, 102, 111, 114, 109, 32, 116, 97, 115, 107, 115, 32, 97, 117, 116, 111, 110, 111, 109, 111, 117, 115, 108, 121, 32, 111, 114, 32, 119, 105, 116, 104, 32, 104, 117, 109, 97, 110, 32, 102, 101, 101, 100, 98, 97, 99, 107, 44, 32, 105, 110, 99, 108, 117, 100, 105, 110, 103, 32, 116, 97, 115, 107, 115, 32, 116, 104, 97, 116, 32, 114, 101, 113, 117, 105, 114, 101, 32, 117, 115, 105, 110, 103, 32, 116, 111, 111, 108, 115, 32, 118, 105, 97, 32, 99, 111, 100, 101, 46, 10, 10, 70, 101, 97, 116, 117, 114, 101, 115, 32, 111, 102, 32, 116, 104, 105, 115, 32, 117, 115, 101, 32, 99, 97, 115, 101, 32, 105, 110, 99, 108, 117, 100, 101, 58, 10, 10, 45, 32, 42, 42, 77, 117, 108, 116, 105, 45, 97, 103, 101, 110, 116, 32, 99, 111, 110, 118, 101, 114, 115, 97, 116, 105, 111, 110, 115, 42, 42, 58, 32, 65, 117, 116, 111, 71, 101, 110, 32, 97, 103, 101, 110, 116, 115, 32, 99, 97, 110, 32, 99, 111, 109, 109, 117, 110, 105, 99, 97, 116, 101, 32, 119, 105, 116, 104, 32, 101, 97, 99, 104, 32, 111, 116, 104, 101, 114, 32, 116, 111, 32, 115, 111, 108, 118, 101, 32, 116, 97, 115, 107, 115, 46, 32, 84, 104, 105, 115, 32, 97, 108, 108, 111, 119, 115, 32, 102, 111, 114, 32, 109, 111, 114, 101, 32, 99, 111, 109, 112, 108, 101, 120, 32, 97, 110, 100, 32, 115, 111, 112, 104, 105, 115, 116, 105, 99, 97, 116, 101, 100, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 115, 32, 116, 104, 97, 110, 32, 119, 111, 117, 108, 100, 32, 98, 101, 32, 112, 111, 115, 115, 105, 98, 108, 101, 32, 119, 105, 116, 104, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 76, 76, 77, 46, 10, 45, 32, 42, 42, 67, 117, 115, 116, 111, 109, 105, 122, 97, 116, 105, 111, 110, 42, 42, 58, 32, 65, 117, 116, 111, 71, 101, 110, 32, 97, 103, 101, 110, 116, 115, 32, 99, 97, 110, 32, 98, 101, 32, 99, 117, 115, 116, 111, 109, 105, 122, 101, 100, 32, 116, 111, 32, 109, 101, 101, 116, 32, 116, 104, 101, 32, 115, 112, 101, 99, 105, 102, 105, 99, 32, 110, 101, 101, 100, 115, 32, 111, 102, 32, 97, 110, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 46, 32, 84, 104, 105, 115, 32, 105, 110, 99, 108, 117, 100, 101, 115, 32, 116, 104, 101, 32, 97, 98, 105, 108, 105, 116, 121, 32, 116, 111, 32, 99, 104, 111, 111, 115, 101, 32, 116, 104, 101, 32, 76, 76, 77, 115, 32, 116, 111, 32, 117, 115, 101, 44, 32, 116, 104, 101, 32, 116, 121, 112, 101, 115, 32, 111, 102, 32, 104, 117, 109, 97, 110, 32, 105, 110, 112, 117, 116, 32, 116, 111, 32, 97, 108, 108, 111, 119, 44, 32, 97, 110, 100, 32, 116, 104, 101, 32, 116, 111, 111, 108, 115, 32, 116, 111, 32, 101, 109, 112, 108, 111, 121, 46, 10, 45, 32, 42, 42, 72, 117, 109, 97, 110, 32, 112, 97, 114, 116, 105, 99, 105, 112, 97, 116, 105, 111, 110, 42, 42, 58, 32, 65, 117, 116, 111, 71, 101, 110, 32, 115, 101, 97, 109, 108, 101, 115, 115, 108, 121, 32, 97, 108, 108, 111, 119, 115, 32, 104, 117, 109, 97, 110, 32, 112, 97, 114, 116, 105, 99, 105, 112, 97, 116, 105, 111, 110, 46, 32, 84, 104, 105, 115, 32, 109, 101, 97, 110, 115, 32, 116, 104, 97, 116, 32, 104, 117, 109, 97, 110, 115, 32, 99, 97, 110, 32, 112, 114, 111, 118, 105, 100, 101, 32, 105, 110, 112, 117, 116, 32, 97, 110, 100, 32, 102, 101, 101, 100, 98, 97, 99, 107, 32, 116, 111, 32, 116, 104, 101, 32, 97, 103, 101, 110, 116, 115, 32, 97, 115, 32, 110, 101, 101, 100, 101, 100, 46, 10, 10, 70, 111, 114, 32, 91, 101, 120, 97, 109, 112, 108, 101, 93, 40, 104, 116, 116, 112, 115, 58, 47, 47, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109, 47, 109, 105, 99, 114, 111, 115, 111, 102, 116, 47, 97, 117, 116, 111, 103, 101, 110, 47, 98, 108, 111, 98, 47, 109, 97, 105, 110, 47, 116, 101, 115, 116, 47, 116, 119, 111, 97, 103, 101, 110, 116, 46, 112, 121, 41, 44, 10, 10, 96, 96, 96, 112, 121, 116, 104, 111, 110, 10, 102, 114, 111, 109, 32, 97, 117, 116, 111, 103, 101, 110, 32, 105, 109, 112, 111, 114, 116, 32, 65, 115, 115, 105, 115, 116, 97, 110, 116, 65, 103, 101, 110, 116, 44, 32, 85, 115, 101, 114, 80, 114, 111, 120, 121, 65, 103, 101, 110, 116, 44, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 95, 102, 114, 111, 109, 95, 106, 115, 111, 110, 10, 35, 32, 76, 111, 97, 100, 32, 76, 76, 77, 32, 105, 110, 102, 101, 114, 101, 110, 99, 101, 32, 101, 110, 100, 112, 111, 105, 110, 116, 115, 32, 102, 114, 111, 109, 32, 97, 110, 32, 101, 110, 118, 32, 118, 97, 114, 105, 97, 98, 108, 101, 32, 111, 114, 32, 97, 32, 102, 105, 108, 101, 10, 35, 32, 83, 101, 101, 32, 104, 116, 116, 112, 115, 58, 47, 47, 109, 105, 99, 114, 111, 115, 111, 102, 116, 46, 103, 105, 116, 104, 117, 98, 46, 105, 111, 47, 97, 117, 116, 111, 103, 101, 110, 47, 100, 111, 99, 115, 47, 70, 65, 81, 35, 115, 101, 116, 45, 121, 111, 117, 114, 45, 97, 112, 105, 45, 101, 110, 100, 112, 111, 105, 110, 116, 115, 10, 35, 32, 97, 110, 100, 32, 79, 65, 73, 95, 67, 79, 78, 70, 73, 71, 95, 76, 73, 83, 84, 95, 115, 97, 109, 112, 108, 101, 10, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 32, 61, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 95, 102, 114, 111, 109, 95, 106, 115, 111, 110, 40, 101, 110, 118, 95, 111, 114, 95, 102, 105, 108, 101, 61, 34, 79, 65, 73, 95, 67, 79, 78, 70, 73, 71, 95, 76, 73, 83, 84, 34, 41, 10, 35, 32, 89, 111, 117, 32, 99, 97, 110, 32, 97, 108, 115, 111, 32, 115, 101, 116, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 32, 100, 105, 114, 101, 99, 116, 108, 121, 32, 97, 115, 32, 97, 32, 108, 105, 115, 116, 44, 32, 102, 111, 114, 32, 101, 120, 97, 109, 112, 108, 101, 44, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 32, 61, 32, 91, 123, 39, 109, 111, 100, 101, 108, 39, 58, 32, 39, 103, 112, 116, 45, 52, 39, 44, 32, 39, 97, 112, 105, 95, 107, 101, 121, 39, 58, 32, 39, 60, 121, 111, 117, 114, 32, 79, 112, 101, 110, 65, 73, 32, 65, 80, 73, 32, 107, 101, 121, 32, 104, 101, 114, 101, 62, 39, 125, 44, 93, 10, 97, 115, 115, 105, 115, 116, 97, 110, 116, 32, 61, 32, 65, 115, 115, 105, 115, 116, 97, 110, 116, 65, 103, 101, 110, 116, 40, 34, 97, 115, 115, 105, 115, 116, 97, 110, 116, 34, 44, 32, 108, 108, 109, 95, 99, 111, 110, 102, 105, 103, 61, 123, 34, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 34, 58, 32, 99, 111, 110, 102, 105, 103, 95, 108, 105, 115, 116, 125, 41, 10, 117, 115, 101, 114, 95, 112, 114, 111, 120, 121, 32, 61, 32, 85, 115, 101, 114, 80, 114, 111, 120, 121, 65, 103, 101, 110, 116, 40, 34, 117, 115, 101, 114, 95, 112, 114, 111, 120, 121, 34, 44, 32, 99, 111, 100, 101, 95, 101, 120, 101, 99, 117, 116, 105, 111, 110, 95, 99, 111, 110, 102, 105, 103, 61, 123, 34, 119, 111, 114, 107, 95, 100, 105, 114, 34, 58, 32, 34, 99, 111, 100, 105, 110, 103, 34, 44, 32, 34, 117, 115, 101, 95, 100, 111, 99, 107, 101, 114, 34, 58, 32, 70, 97, 108, 115, 101, 125, 41, 32, 35, 32, 73, 77, 80, 79, 82, 84, 65, 78, 84, 58, 32, 115, 101, 116, 32, 116, 111, 32, 84, 114, 117, 101, 32, 116, 111, 32, 114, 117, 110, 32, 99, 111, 100, 101, 32, 105, 110, 32, 100, 111, 99, 107, 101, 114, 44, 32, 114, 101, 99, 111, 109, 109, 101, 110, 100, 101, 100, 10, 117, 115, 101, 114, 95, 112, 114, 111, 120, 121, 46, 105, 110, 105, 116, 105, 97, 116, 101, 95, 99, 104, 97, 116, 40, 97, 115, 115, 105, 115, 116, 97, 110, 116, 44, 32, 109, 101, 115, 115, 97, 103, 101, 61, 34, 80, 108, 111, 116, 32, 97, 32, 99, 104, 97, 114, 116, 32, 111, 102, 32, 78, 86, 68, 65, 32, 97, 110, 100, 32, 84, 69, 83, 76, 65, 32, 115, 116, 111, 99, 107, 32, 112, 114, 105, 99, 101, 32, 99, 104, 97, 110, 103, 101, 32, 89, 84, 68, 46, 34, 41, 10, 35, 32, 84, 104, 105, 115, 32, 105, 110, 105, 116, 105, 97, 116, 101, 115, 32, 97, 110, 32, 97, 117, 116, 111, 109, 97, 116, 101, 100, 32, 99, 104, 97, 116, 32, 98, 101, 116, 119, 101, 101, 110, 32, 116, 104, 101, 32, 116, 119, 111, 32, 97, 103, 101, 110, 116, 115, 32, 116, 111, 32, 115, 111, 108, 118, 101, 32, 116, 104, 101, 32, 116, 97, 115, 107, 10, 96, 96, 96, 10, 10, 109, 111, 114, 101, 32, 112, 121, 116, 104, 111, 110, 32, 99, 111, 100, 101, 58, 10, 10, 96, 96, 96, 112, 121, 116, 104, 111, 110, 10, 1000, 1000, 100, 101, 102, 32, 99, 114, 101, 97, 116, 101, 40, 10, 1000, 1000, 1000, 1000, 115, 101, 108, 102, 44, 10, 1000, 1000, 1000, 1000, 42, 44, 10, 1000, 1000, 1000, 1000, 109, 101, 115, 115, 97, 103, 101, 115, 58, 32, 73, 116, 101, 114, 97, 98, 108, 101, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 77, 101, 115, 115, 97, 103, 101, 80, 97, 114, 97, 109, 93, 44, 10, 1000, 1000, 1000, 1000, 109, 111, 100, 101, 108, 58, 32, 85, 110, 105, 111, 110, 91, 115, 116, 114, 44, 32, 67, 104, 97, 116, 77, 111, 100, 101, 108, 93, 44, 10, 1000, 1000, 1000, 1000, 102, 114, 101, 113, 117, 101, 110, 99, 121, 95, 112, 101, 110, 97, 108, 116, 121, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 102, 117, 110, 99, 116, 105, 111, 110, 95, 99, 97, 108, 108, 58, 32, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 70, 117, 110, 99, 116, 105, 111, 110, 67, 97, 108, 108, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 102, 117, 110, 99, 116, 105, 111, 110, 115, 58, 32, 73, 116, 101, 114, 97, 98, 108, 101, 91, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 70, 117, 110, 99, 116, 105, 111, 110, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 108, 111, 103, 105, 116, 95, 98, 105, 97, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 68, 105, 99, 116, 91, 115, 116, 114, 44, 32, 105, 110, 116, 93, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 108, 111, 103, 112, 114, 111, 98, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 98, 111, 111, 108, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 109, 97, 120, 95, 116, 111, 107, 101, 110, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 110, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 112, 114, 101, 115, 101, 110, 99, 101, 95, 112, 101, 110, 97, 108, 116, 121, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 114, 101, 115, 112, 111, 110, 115, 101, 95, 102, 111, 114, 109, 97, 116, 58, 32, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 82, 101, 115, 112, 111, 110, 115, 101, 70, 111, 114, 109, 97, 116, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 115, 101, 101, 100, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 115, 116, 111, 112, 58, 32, 85, 110, 105, 111, 110, 91, 79, 112, 116, 105, 111, 110, 97, 108, 91, 115, 116, 114, 93, 44, 32, 76, 105, 115, 116, 91, 115, 116, 114, 93, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 115, 116, 114, 101, 97, 109, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 76, 105, 116, 101, 114, 97, 108, 91, 70, 97, 108, 115, 101, 93, 93, 32, 124, 32, 76, 105, 116, 101, 114, 97, 108, 91, 84, 114, 117, 101, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 115, 116, 114, 101, 97, 109, 95, 111, 112, 116, 105, 111, 110, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 83, 116, 114, 101, 97, 109, 79, 112, 116, 105, 111, 110, 115, 80, 97, 114, 97, 109, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 116, 111, 111, 108, 95, 99, 104, 111, 105, 99, 101, 58, 32, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 84, 111, 111, 108, 67, 104, 111, 105, 99, 101, 79, 112, 116, 105, 111, 110, 80, 97, 114, 97, 109, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 116, 111, 111, 108, 115, 58, 32, 73, 116, 101, 114, 97, 98, 108, 101, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 84, 111, 111, 108, 80, 97, 114, 97, 109, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 116, 111, 112, 95, 108, 111, 103, 112, 114, 111, 98, 115, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 105, 110, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 116, 111, 112, 95, 112, 58, 32, 79, 112, 116, 105, 111, 110, 97, 108, 91, 102, 108, 111, 97, 116, 93, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 117, 115, 101, 114, 58, 32, 115, 116, 114, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 1000, 1000, 35, 32, 85, 115, 101, 32, 116, 104, 101, 32, 102, 111, 108, 108, 111, 119, 105, 110, 103, 32, 97, 114, 103, 117, 109, 101, 110, 116, 115, 32, 105, 102, 32, 121, 111, 117, 32, 110, 101, 101, 100, 32, 116, 111, 32, 112, 97, 115, 115, 32, 97, 100, 100, 105, 116, 105, 111, 110, 97, 108, 32, 112, 97, 114, 97, 109, 101, 116, 101, 114, 115, 32, 116, 111, 32, 116, 104, 101, 32, 65, 80, 73, 32, 116, 104, 97, 116, 32, 97, 114, 101, 110, 39, 116, 32, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 118, 105, 97, 32, 107, 119, 97, 114, 103, 115, 46, 10, 1000, 1000, 1000, 1000, 35, 32, 84, 104, 101, 32, 101, 120, 116, 114, 97, 32, 118, 97, 108, 117, 101, 115, 32, 103, 105, 118, 101, 110, 32, 104, 101, 114, 101, 32, 116, 97, 107, 101, 32, 112, 114, 101, 99, 101, 100, 101, 110, 99, 101, 32, 111, 118, 101, 114, 32, 118, 97, 108, 117, 101, 115, 32, 100, 101, 102, 105, 110, 101, 100, 32, 111, 110, 32, 116, 104, 101, 32, 99, 108, 105, 101, 110, 116, 32, 111, 114, 32, 112, 97, 115, 115, 101, 100, 32, 116, 111, 32, 116, 104, 105, 115, 32, 109, 101, 116, 104, 111, 100, 46, 10, 1000, 1000, 1000, 1000, 101, 120, 116, 114, 97, 95, 104, 101, 97, 100, 101, 114, 115, 58, 32, 72, 101, 97, 100, 101, 114, 115, 32, 124, 32, 78, 111, 110, 101, 32, 61, 32, 78, 111, 110, 101, 44, 10, 1000, 1000, 1000, 1000, 101, 120, 116, 114, 97, 95, 113, 117, 101, 114, 121, 58, 32, 81, 117, 101, 114, 121, 32, 124, 32, 78, 111, 110, 101, 32, 61, 32, 78, 111, 110, 101, 44, 10, 1000, 1000, 1000, 1000, 101, 120, 116, 114, 97, 95, 98, 111, 100, 121, 58, 32, 66, 111, 100, 121, 32, 124, 32, 78, 111, 110, 101, 32, 61, 32, 78, 111, 110, 101, 44, 10, 1000, 1000, 1000, 1000, 116, 105, 109, 101, 111, 117, 116, 58, 32, 102, 108, 111, 97, 116, 32, 124, 32, 104, 116, 116, 112, 120, 46, 84, 105, 109, 101, 111, 117, 116, 32, 124, 32, 78, 111, 110, 101, 32, 124, 32, 78, 111, 116, 71, 105, 118, 101, 110, 32, 61, 32, 78, 79, 84, 95, 71, 73, 86, 69, 78, 44, 10, 1000, 1000, 41, 32, 45, 62, 32, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 32, 124, 32, 83, 116, 114, 101, 97, 109, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 67, 104, 117, 110, 107, 93, 58, 10, 1000, 1000, 1000, 1000, 114, 101, 116, 117, 114, 110, 32, 115, 101, 108, 102, 46, 95, 112, 111, 115, 116, 40, 10, 1000, 1000, 1000, 1000, 1000, 1000, 34, 47, 99, 104, 97, 116, 47, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 115, 34, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 98, 111, 100, 121, 61, 109, 97, 121, 98, 101, 95, 116, 114, 97, 110, 115, 102, 111, 114, 109, 40, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 123, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 109, 101, 115, 115, 97, 103, 101, 115, 34, 58, 32, 109, 101, 115, 115, 97, 103, 101, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 109, 111, 100, 101, 108, 34, 58, 32, 109, 111, 100, 101, 108, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 102, 114, 101, 113, 117, 101, 110, 99, 121, 95, 112, 101, 110, 97, 108, 116, 121, 34, 58, 32, 102, 114, 101, 113, 117, 101, 110, 99, 121, 95, 112, 101, 110, 97, 108, 116, 121, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 102, 117, 110, 99, 116, 105, 111, 110, 95, 99, 97, 108, 108, 34, 58, 32, 102, 117, 110, 99, 116, 105, 111, 110, 95, 99, 97, 108, 108, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 102, 117, 110, 99, 116, 105, 111, 110, 115, 34, 58, 32, 102, 117, 110, 99, 116, 105, 111, 110, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 108, 111, 103, 105, 116, 95, 98, 105, 97, 115, 34, 58, 32, 108, 111, 103, 105, 116, 95, 98, 105, 97, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 108, 111, 103, 112, 114, 111, 98, 115, 34, 58, 32, 108, 111, 103, 112, 114, 111, 98, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 109, 97, 120, 95, 116, 111, 107, 101, 110, 115, 34, 58, 32, 109, 97, 120, 95, 116, 111, 107, 101, 110, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 110, 34, 58, 32, 110, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 112, 114, 101, 115, 101, 110, 99, 101, 95, 112, 101, 110, 97, 108, 116, 121, 34, 58, 32, 112, 114, 101, 115, 101, 110, 99, 101, 95, 112, 101, 110, 97, 108, 116, 121, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 114, 101, 115, 112, 111, 110, 115, 101, 95, 102, 111, 114, 109, 97, 116, 34, 58, 32, 114, 101, 115, 112, 111, 110, 115, 101, 95, 102, 111, 114, 109, 97, 116, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 115, 101, 101, 100, 34, 58, 32, 115, 101, 101, 100, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 115, 116, 111, 112, 34, 58, 32, 115, 116, 111, 112, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 115, 116, 114, 101, 97, 109, 34, 58, 32, 115, 116, 114, 101, 97, 109, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 115, 116, 114, 101, 97, 109, 95, 111, 112, 116, 105, 111, 110, 115, 34, 58, 32, 115, 116, 114, 101, 97, 109, 95, 111, 112, 116, 105, 111, 110, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 34, 58, 32, 116, 101, 109, 112, 101, 114, 97, 116, 117, 114, 101, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 116, 111, 111, 108, 95, 99, 104, 111, 105, 99, 101, 34, 58, 32, 116, 111, 111, 108, 95, 99, 104, 111, 105, 99, 101, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 116, 111, 111, 108, 115, 34, 58, 32, 116, 111, 111, 108, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 116, 111, 112, 95, 108, 111, 103, 112, 114, 111, 98, 115, 34, 58, 32, 116, 111, 112, 95, 108, 111, 103, 112, 114, 111, 98, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 116, 111, 112, 95, 112, 34, 58, 32, 116, 111, 112, 95, 112, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 34, 117, 115, 101, 114, 34, 58, 32, 117, 115, 101, 114, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 125, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 99, 111, 109, 112, 108, 101, 116, 105, 111, 110, 95, 99, 114, 101, 97, 116, 101, 95, 112, 97, 114, 97, 109, 115, 46, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 67, 114, 101, 97, 116, 101, 80, 97, 114, 97, 109, 115, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 41, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 111, 112, 116, 105, 111, 110, 115, 61, 109, 97, 107, 101, 95, 114, 101, 113, 117, 101, 115, 116, 95, 111, 112, 116, 105, 111, 110, 115, 40, 10, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 101, 120, 116, 114, 97, 95, 104, 101, 97, 100, 101, 114, 115, 61, 101, 120, 116, 114, 97, 95, 104, 101, 97, 100, 101, 114, 115, 44, 32, 101, 120, 116, 114, 97, 95, 113, 117, 101, 114, 121, 61, 101, 120, 116, 114, 97, 95, 113, 117, 101, 114, 121, 44, 32, 101, 120, 116, 114, 97, 95, 98, 111, 100, 121, 61, 101, 120, 116, 114, 97, 95, 98, 111, 100, 121, 44, 32, 116, 105, 109, 101, 111, 117, 116, 61, 116, 105, 109, 101, 111, 117, 116, 10, 1000, 1000, 1000, 1000, 1000, 1000, 41, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 99, 97, 115, 116, 95, 116, 111, 61, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 115, 116, 114, 101, 97, 109, 61, 115, 116, 114, 101, 97, 109, 32, 111, 114, 32, 70, 97, 108, 115, 101, 44, 10, 1000, 1000, 1000, 1000, 1000, 1000, 115, 116, 114, 101, 97, 109, 95, 99, 108, 115, 61, 83, 116, 114, 101, 97, 109, 91, 67, 104, 97, 116, 67, 111, 109, 112, 108, 101, 116, 105, 111, 110, 67, 104, 117, 110, 107, 93, 44, 10, 1000, 1000, 1000, 1000, 41, 10, 96, 96, 96, 10]\n",
+      "length:  4979\n"
+     ]
+    }
+   ],
+   "source": [
+    "def merge(ids, pair, idx):\n",
+    "    \"\"\"\n",
+    "    BPE algorithm\n",
+    "    ids: list of integers(tokens)\n",
+    "    pair: tuple of consecutive integers\n",
+    "    idx: new vocab token to replace the pair\n",
+    "    \"\"\"\n",
+    "    new_ids = []\n",
+    "    i = 0\n",
+    "    while i < len(ids):\n",
+    "        if i < len(ids) - 1 and ids[i] == pair[0] and ids[i+1] == pair[1]:\n",
+    "            new_ids.append(idx)\n",
+    "            i += 2\n",
+    "        else:\n",
+    "            new_ids.append(ids[i])\n",
+    "            i += 1\n",
+    "    return new_ids\n",
+    "\n",
+    "# merge the most common pair\n",
+    "tokens2 = merge(tokens, (32, 32), 1000)\n",
+    "print(tokens2)\n",
+    "print('length: ',len(tokens2))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "merge (32, 32) to 256\n",
+      "merge (256, 256) to 257\n",
+      "merge (257, 257) to 258\n",
+      "merge (111, 110) to 259\n",
+      "merge (101, 110) to 260\n",
+      "merge (116, 105) to 261\n",
+      "merge (10, 258) to 262\n",
+      "merge (58, 32) to 263\n",
+      "merge (44, 262) to 264\n",
+      "merge (261, 259) to 265\n",
+      "merge (101, 32) to 266\n",
+      "merge (116, 111) to 267\n",
+      "merge (32, 78) to 268\n",
+      "merge (97, 116) to 269\n",
+      "merge (115, 32) to 270\n",
+      "merge (101, 114) to 271\n",
+      "merge (114, 101) to 272\n",
+      "merge (97, 108) to 273\n",
+      "merge (116, 104) to 274\n",
+      "merge (115, 116) to 275\n",
+      "merge (97, 110) to 276\n",
+      "merge (260, 32) to 277\n",
+      "merge (97, 109) to 278\n",
+      "merge (108, 101) to 279\n",
+      "merge (32, 124) to 280\n",
+      "merge (105, 110) to 281\n",
+      "merge (34, 263) to 282\n",
+      "merge (111, 109) to 283\n",
+      "merge (61, 268) to 284\n",
+      "merge (44, 32) to 285\n",
+      "merge (280, 268) to 286\n",
+      "merge (257, 34) to 287\n",
+      "merge (264, 258) to 288\n",
+      "merge (115, 101) to 289\n",
+      "merge (108, 111) to 290\n",
+      "merge (84, 95) to 291\n",
+      "merge (105, 118) to 292\n",
+      "merge (292, 277) to 293\n",
+      "merge (112, 265) to 294\n",
+      "merge (111, 116) to 295\n"
+     ]
+    }
+   ],
+   "source": [
+    "# complete cycle\n",
+    "def get_stats(ids):\n",
+    "    counts = {}\n",
+    "    for pair in zip(ids, ids[1:]):\n",
+    "        counts[pair] = counts.get(pair, 0) +1 \n",
+    "    return counts\n",
+    "\n",
+    "def merge(ids, pair, idx):\n",
+    "    newids = []\n",
+    "    i = 0\n",
+    "    while i < len(ids):\n",
+    "        if i < len(ids) - 1 and ids[i] == pair[0] and ids [i+1] == pair[1]:\n",
+    "            newids.append(idx)\n",
+    "            i += 2\n",
+    "        else:\n",
+    "            newids.append(ids[i])\n",
+    "            i += 1\n",
+    "    return newids\n",
+    "\n",
+    "# merge all the common pairs and create a new vocab\n",
+    "vocab_size = 296\n",
+    "num_merges = vocab_size - 256 # the utf-8 vocab size is 256\n",
+    "ids = list(tokens)\n",
+    "\n",
+    "\n",
+    "merges = {}\n",
+    "for i in range(num_merges):\n",
+    "    stats = get_stats(ids)\n",
+    "    pair = max(stats, key = stats.get) # get the most common pair\n",
+    "    idx = 256 + i # new vocab token\n",
+    "    print(f'merge {pair} to {idx}')\n",
+    "    ids = merge(ids, pair, idx)\n",
+    "    merges[pair] = idx\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tokens length:  5397\n",
+      "new tokens length:  3365\n",
+      "compression rate: 1.60X\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"tokens length: \", len(tokens))\n",
+    "print(\"new tokens length: \", len(ids))\n",
+    "print(f\"compression rate: {len(tokens) / len(ids):.2f}X\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### decoding\n",
+    "\n",
+    "Given the sequence of integers [0, vocab_size], converting it into a string."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---\n",
+      "Autogen enables the next-gen LLM applications with a generic [multi-agent conversation](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat) framework. It offers customizable and conversable agents that integrate LLMs, tools, and humans.\n",
+      "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code.\n",
+      "\n",
+      "Features of this use case include:\n",
+      "\n",
+      "- **Multi-agent conversations**: AutoGen agents can communicate with each other to solve tasks. This allows for more complex and sophisticated applications than would be possible with a single LLM.\n",
+      "- **Customization**: AutoGen agents can be customized to meet the specific needs of an application. This includes the ability to choose the LLMs to use, the types of human input to allow, and the tools to employ.\n",
+      "- **Human participation**: AutoGen seamlessly allows human participation. This means that humans can provide input and feedback to the agents as needed.\n",
+      "\n",
+      "For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py),\n",
+      "\n",
+      "```python\n",
+      "from autogen import AssistantAgent, UserProxyAgent, config_list_from_json\n",
+      "# Load LLM inference endpoints from an env variable or a file\n",
+      "# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints\n",
+      "# and OAI_CONFIG_LIST_sample\n",
+      "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n",
+      "# You can also set config_list directly as a list, for example, config_list = [{'model': 'gpt-4', 'api_key': '<your OpenAI API key here>'},]\n",
+      "assistant = AssistantAgent(\"assistant\", llm_config={\"config_list\": config_list})\n",
+      "user_proxy = UserProxyAgent(\"user_proxy\", code_execution_config={\"work_dir\": \"coding\", \"use_docker\": False}) # IMPORTANT: set to True to run code in docker, recommended\n",
+      "user_proxy.initiate_chat(assistant, message=\"Plot a chart of NVDA and TESLA stock price change YTD.\")\n",
+      "# This initiates an automated chat between the two agents to solve the task\n",
+      "```\n",
+      "\n",
+      "more python code:\n",
+      "\n",
+      "```python\n",
+      "    def create(\n",
+      "        self,\n",
+      "        *,\n",
+      "        messages: Iterable[ChatCompletionMessageParam],\n",
+      "        model: Union[str, ChatModel],\n",
+      "        frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        function_call: completion_create_params.FunctionCall | NotGiven = NOT_GIVEN,\n",
+      "        functions: Iterable[completion_create_params.Function] | NotGiven = NOT_GIVEN,\n",
+      "        logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN,\n",
+      "        logprobs: Optional[bool] | NotGiven = NOT_GIVEN,\n",
+      "        max_tokens: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        n: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,\n",
+      "        seed: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,\n",
+      "        stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN,\n",
+      "        stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,\n",
+      "        temperature: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,\n",
+      "        tools: Iterable[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,\n",
+      "        top_logprobs: Optional[int] | NotGiven = NOT_GIVEN,\n",
+      "        top_p: Optional[float] | NotGiven = NOT_GIVEN,\n",
+      "        user: str | NotGiven = NOT_GIVEN,\n",
+      "        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.\n",
+      "        # The extra values given here take precedence over values defined on the client or passed to this method.\n",
+      "        extra_headers: Headers | None = None,\n",
+      "        extra_query: Query | None = None,\n",
+      "        extra_body: Body | None = None,\n",
+      "        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n",
+      "    ) -> ChatCompletion | Stream[ChatCompletionChunk]:\n",
+      "        return self._post(\n",
+      "            \"/chat/completions\",\n",
+      "            body=maybe_transform(\n",
+      "                {\n",
+      "                    \"messages\": messages,\n",
+      "                    \"model\": model,\n",
+      "                    \"frequency_penalty\": frequency_penalty,\n",
+      "                    \"function_call\": function_call,\n",
+      "                    \"functions\": functions,\n",
+      "                    \"logit_bias\": logit_bias,\n",
+      "                    \"logprobs\": logprobs,\n",
+      "                    \"max_tokens\": max_tokens,\n",
+      "                    \"n\": n,\n",
+      "                    \"presence_penalty\": presence_penalty,\n",
+      "                    \"response_format\": response_format,\n",
+      "                    \"seed\": seed,\n",
+      "                    \"stop\": stop,\n",
+      "                    \"stream\": stream,\n",
+      "                    \"stream_options\": stream_options,\n",
+      "                    \"temperature\": temperature,\n",
+      "                    \"tool_choice\": tool_choice,\n",
+      "                    \"tools\": tools,\n",
+      "                    \"top_logprobs\": top_logprobs,\n",
+      "                    \"top_p\": top_p,\n",
+      "                    \"user\": user,\n",
+      "                },\n",
+      "                completion_create_params.CompletionCreateParams,\n",
+      "            ),\n",
+      "            options=make_request_options(\n",
+      "                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout\n",
+      "            ),\n",
+      "            cast_to=ChatCompletion,\n",
+      "            stream=stream or False,\n",
+      "            stream_cls=Stream[ChatCompletionChunk],\n",
+      "        )\n",
+      "```\n",
+      "\n",
+      "length:  5397\n"
+     ]
+    }
+   ],
+   "source": [
+    "vocab = {idx: bytes([idx]) for idx in range(256)} # utf-8 vocab\n",
+    "for (p0, p1), idx in merges.items():\n",
+    "    vocab[idx] = vocab[p0] + vocab[p1] # adding the extra vocab tokens (256 - 296)\n",
+    "\n",
+    "def decode(ids):\n",
+    "    bytetokens = b\"\".join(vocab[i] for i in ids)\n",
+    "    text = bytetokens.decode(\"utf-8\", errors=\"replace\") # if there are any errors, replace them with a question mark\n",
+    "    return text\n",
+    "\n",
+    "print('---')\n",
+    "print(decode(ids))\n",
+    "print('length: ', len(decode(ids)))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### encoding\n",
+    "convert the string into the tokens"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[104, 107]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def encode(texts):\n",
+    "    tokens = list(texts.encode('utf-8'))\n",
+    "    while len(tokens) >=2:\n",
+    "        stats = get_stats(tokens)\n",
+    "        pair = min(stats, key=lambda p: merges.get(p, float('inf'))) # selects the pair with minimum prioroty\n",
+    "        if pair not in merges:\n",
+    "            break\n",
+    "        idx = merges[pair]\n",
+    "        tokens = merge(tokens, pair, idx)\n",
+    "    return tokens\n",
+    "\n",
+    "print(encode(\"hk\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "*the line ensures the algorithm respects the merge priorities defined\n",
+    "```\n",
+    "pair = min(stats, key=lambda p: merges.get(p, float('inf')))\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   presence_penalty \n"
+     ]
+    }
+   ],
+   "source": [
+    "print(decode(encode(\"   presence_penalty \")))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tokenizer/sample/bpetokenizer/sample_bpetokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7204cc27b30a3ec11d1c6bb741376eabc5345f4660def0d564cff1cda29a6a28
+size 720

tokenizer/sample/bpetokenizer/sample_bpetokenizer.py

@@ -0,0 +1,36 @@
+import sys
+sys.path.append("../")
+
+from bpetokenizer import BPETokenizer
+
+special_tokens = {
+    "<|endoftext|>": 1001,
+    "<|startoftext|>": 1002,
+    "[SPECIAL1]": 1003,
+    "[SPECIAL2]": 1004,
+}
+
+tokenizer = BPETokenizer(special_tokens=special_tokens)
+texts = "<|startoftext|> Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.<|endoftext|>"
+
+tokenizer.train(texts, vocab_size=310, verbose=True)
+
+encode_text = """
+<|startoftext|>Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.
+Hello, Universe! Another example sentence containing [SPECIAL1] and [SPECIAL2], used to ensure tokenizer's robustness.
+Greetings, Earth! Here we have [SPECIAL1] appearing once again, followed by [SPECIAL2] in the same sentence.
+Hello, World! This is yet another sample text, with [SPECIAL1] and [SPECIAL2] making an appearance.
+Hey there, World! Testing the tokenizer with [SPECIAL1] and [SPECIAL2] to see if it handles special tokens properly.
+Salutations, Planet! The tokenizer should recognize [SPECIAL1] and [SPECIAL2] in this long string of text.
+Hello again, World! [SPECIAL1] and [SPECIAL2] are special tokens that need to be handled correctly by the tokenizer.
+Welcome, World! Including [SPECIAL1] and [SPECIAL2] multiple times in this large text to ensure proper encoding.
+Hi, World! Let's add [SPECIAL1] and [SPECIAL2] in various parts of this long sentence to test the tokenizer thoroughly.
+<|endoftext|>
+"""
+ids = tokenizer.encode(encode_text, special_tokens="all")
+print(ids)
+
+decode_text = tokenizer.decode(ids)
+print(decode_text)
+
+tokenizer.save("sample_bpetokenizer")

tokenizer/sample/bpetokenizer/sample_bpetokenizer.vocab

@@ -0,0 +1,310 @@
+[\u0000] 0
+[\u0001] 1
+[\u0002] 2
+[\u0003] 3
+[\u0004] 4
+[\u0005] 5
+[\u0006] 6
+[\u0007] 7
+[\u0008] 8
+[\u0009] 9
+[\u000a] 10
+[\u000b] 11
+[\u000c] 12
+[\u000d] 13
+[\u000e] 14
+[\u000f] 15
+[\u0010] 16
+[\u0011] 17
+[\u0012] 18
+[\u0013] 19
+[\u0014] 20
+[\u0015] 21
+[\u0016] 22
+[\u0017] 23
+[\u0018] 24
+[\u0019] 25
+[\u001a] 26
+[\u001b] 27
+[\u001c] 28
+[\u001d] 29
+[\u001e] 30
+[\u001f] 31
+[ ] 32
+[!] 33
+["] 34
+[#] 35
+[$] 36
+[%] 37
+[&] 38
+['] 39
+[(] 40
+[)] 41
+[*] 42
+[+] 43
+[,] 44
+[-] 45
+[.] 46
+[/] 47
+[0] 48
+[1] 49
+[2] 50
+[3] 51
+[4] 52
+[5] 53
+[6] 54
+[7] 55
+[8] 56
+[9] 57
+[:] 58
+[;] 59
+[<] 60
+[=] 61
+[>] 62
+[?] 63
+[@] 64
+[A] 65
+[B] 66
+[C] 67
+[D] 68
+[E] 69
+[F] 70
+[G] 71
+[H] 72
+[I] 73
+[J] 74
+[K] 75
+[L] 76
+[M] 77
+[N] 78
+[O] 79
+[P] 80
+[Q] 81
+[R] 82
+[S] 83
+[T] 84
+[U] 85
+[V] 86
+[W] 87
+[X] 88
+[Y] 89
+[Z] 90
+[[] 91
+[\] 92
+[]] 93
+[^] 94
+[_] 95
+[`] 96
+[a] 97
+[b] 98
+[c] 99
+[d] 100
+[e] 101
+[f] 102
+[g] 103
+[h] 104
+[i] 105
+[j] 106
+[k] 107
+[l] 108
+[m] 109
+[n] 110
+[o] 111
+[p] 112
+[q] 113
+[r] 114
+[s] 115
+[t] 116
+[u] 117
+[v] 118
+[w] 119
+[x] 120
+[y] 121
+[z] 122
+[{] 123
+[|] 124
+[}] 125
+[~] 126
+[\u007f] 127
+[�] 128
+[�] 129
+[�] 130
+[�] 131
+[�] 132
+[�] 133
+[�] 134
+[�] 135
+[�] 136
+[�] 137
+[�] 138
+[�] 139
+[�] 140
+[�] 141
+[�] 142
+[�] 143
+[�] 144
+[�] 145
+[�] 146
+[�] 147
+[�] 148
+[�] 149
+[�] 150
+[�] 151
+[�] 152
+[�] 153
+[�] 154
+[�] 155
+[�] 156
+[�] 157
+[�] 158
+[�] 159
+[�] 160
+[�] 161
+[�] 162
+[�] 163
+[�] 164
+[�] 165
+[�] 166
+[�] 167
+[�] 168
+[�] 169
+[�] 170
+[�] 171
+[�] 172
+[�] 173
+[�] 174
+[�] 175
+[�] 176
+[�] 177
+[�] 178
+[�] 179
+[�] 180
+[�] 181
+[�] 182
+[�] 183
+[�] 184
+[�] 185
+[�] 186
+[�] 187
+[�] 188
+[�] 189
+[�] 190
+[�] 191
+[�] 192
+[�] 193
+[�] 194
+[�] 195
+[�] 196
+[�] 197
+[�] 198
+[�] 199
+[�] 200
+[�] 201
+[�] 202
+[�] 203
+[�] 204
+[�] 205
+[�] 206
+[�] 207
+[�] 208
+[�] 209
+[�] 210
+[�] 211
+[�] 212
+[�] 213
+[�] 214
+[�] 215
+[�] 216
+[�] 217
+[�] 218
+[�] 219
+[�] 220
+[�] 221
+[�] 222
+[�] 223
+[�] 224
+[�] 225
+[�] 226
+[�] 227
+[�] 228
+[�] 229
+[�] 230
+[�] 231
+[�] 232
+[�] 233
+[�] 234
+[�] 235
+[�] 236
+[�] 237
+[�] 238
+[�] 239
+[�] 240
+[�] 241
+[�] 242
+[�] 243
+[�] 244
+[�] 245
+[�] 246
+[�] 247
+[�] 248
+[�] 249
+[�] 250
+[�] 251
+[�] 252
+[�] 253
+[�] 254
+[�] 255
+[ ][t] -> [ t] 256
+[e][x] -> [ex] 257
+[ex][t] -> [ext] 258
+[ t][o] -> [ to] 259
+[e][n] -> [en] 260
+[<][|] -> [<|] 261
+[s][t] -> [st] 262
+[o][f] -> [of] 263
+[of][t] -> [oft] 264
+[oft][ext] -> [oftext] 265
+[|][>] -> [|>] 266
+[i][s] -> [is] 267
+[ ][a] -> [ a] 268
+[ ][s] -> [ s] 269
+[ t][h] -> [ th] 270
+[ th][e] -> [ the] 271
+[ to][k] -> [ tok] 272
+[ tok][en] -> [ token] 273
+[ ][[] -> [ [] 274
+[S][P] -> [SP] 275
+[SP][E] -> [SPE] 276
+[SPE][C] -> [SPEC] 277
+[SPEC][I] -> [SPECI] 278
+[SPECI][A] -> [SPECIA] 279
+[SPECIA][L] -> [SPECIAL] 280
+[st][a] -> [sta] 281
+[sta][r] -> [star] 282
+[star][t] -> [start] 283
+[start][oftext] -> [startoftext] 284
+[ ][H] -> [ H] 285
+[ H][e] -> [ He] 286
+[ He][l] -> [ Hel] 287
+[ Hel][l] -> [ Hell] 288
+[ Hell][o] -> [ Hello] 289
+[ ][W] -> [ W] 290
+[ W][o] -> [ Wo] 291
+[ Wo][r] -> [ Wor] 292
+[ Wor][l] -> [ Worl] 293
+[ Worl][d] -> [ World] 294
+[ ][T] -> [ T] 295
+[ T][h] -> [ Th] 296
+[ Th][is] -> [ This] 297
+[ ][is] -> [ is] 298
+[ s][a] -> [ sa] 299
+[ sa][m] -> [ sam] 300
+[ sam][p] -> [ samp] 301
+[ samp][l] -> [ sampl] 302
+[ sampl][e] -> [ sample] 303
+[ t][ext] -> [ text] 304
+[ ][w] -> [ w] 305
+[ w][i] -> [ wi] 306
+[ wi][t] -> [ wit] 307
+[ wit][h] -> [ with] 308
+[ s][p] -> [ sp] 309

tokenizer/sample/load_json_vocab/bpetokenizer_json.py

@@ -0,0 +1,24 @@
+from bpetokenizer import BPETokenizer
+
+tokenizer = BPETokenizer()
+
+tokenizer.load("sample_bpetokenizer.json", mode="json")
+
+encode_text = """
+<|startoftext|>Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.
+Hello, Universe! Another example sentence containing [SPECIAL1] and [SPECIAL2], used to ensure tokenizer's robustness.
+Greetings, Earth! Here we have [SPECIAL1] appearing once again, followed by [SPECIAL2] in the same sentence.<|endoftext|>"""
+
+print("vocab: ", tokenizer.vocab)
+print('---')
+print("merges: ", tokenizer.merges)
+print('---')
+print("special tokens: ", tokenizer.special_tokens)
+
+ids = tokenizer.encode(encode_text, special_tokens="all")
+print('---')
+print('Ids: ', ids)
+
+decode_text = tokenizer.decode(ids)
+print('---')
+print(decode_text)

tokenizer/sample/load_json_vocab/sample_bpetokenizer.json

@@ -0,0 +1,378 @@
+{
+    "version": "1.0.31",
+    "pattern": "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+",
+    "special_tokens": {
+        "<|endoftext|>": 311,
+        "<|startoftext|>": 312,
+        "[SPECIAL1]": 313,
+        "[SPECIAL2]": 314
+    },
+    "merges": {
+        "(32, 116)": 256,
+        "(101, 120)": 257,
+        "(257, 116)": 258,
+        "(256, 111)": 259,
+        "(101, 110)": 260,
+        "(60, 124)": 261,
+        "(115, 116)": 262,
+        "(111, 102)": 263,
+        "(263, 116)": 264,
+        "(264, 258)": 265,
+        "(124, 62)": 266,
+        "(105, 115)": 267,
+        "(32, 97)": 268,
+        "(32, 115)": 269,
+        "(256, 104)": 270,
+        "(270, 101)": 271,
+        "(259, 107)": 272,
+        "(272, 260)": 273,
+        "(32, 91)": 274,
+        "(83, 80)": 275,
+        "(275, 69)": 276,
+        "(276, 67)": 277,
+        "(277, 73)": 278,
+        "(278, 65)": 279,
+        "(279, 76)": 280,
+        "(262, 97)": 281,
+        "(281, 114)": 282,
+        "(282, 116)": 283,
+        "(283, 265)": 284,
+        "(32, 72)": 285,
+        "(285, 101)": 286,
+        "(286, 108)": 287,
+        "(287, 108)": 288,
+        "(288, 111)": 289,
+        "(32, 87)": 290,
+        "(290, 111)": 291,
+        "(291, 114)": 292,
+        "(292, 108)": 293,
+        "(293, 100)": 294,
+        "(32, 84)": 295,
+        "(295, 104)": 296,
+        "(296, 267)": 297,
+        "(32, 267)": 298,
+        "(269, 97)": 299,
+        "(299, 109)": 300,
+        "(300, 112)": 301,
+        "(301, 108)": 302,
+        "(302, 101)": 303,
+        "(256, 258)": 304,
+        "(32, 119)": 305,
+        "(305, 105)": 306,
+        "(306, 116)": 307,
+        "(307, 104)": 308,
+        "(269, 112)": 309
+    },
+    "vocab": {
+        "0": "\\u0000",
+        "1": "\\u0001",
+        "2": "\\u0002",
+        "3": "\\u0003",
+        "4": "\\u0004",
+        "5": "\\u0005",
+        "6": "\\u0006",
+        "7": "\\u0007",
+        "8": "\\u0008",
+        "9": "\\u0009",
+        "10": "\\u000a",
+        "11": "\\u000b",
+        "12": "\\u000c",
+        "13": "\\u000d",
+        "14": "\\u000e",
+        "15": "\\u000f",
+        "16": "\\u0010",
+        "17": "\\u0011",
+        "18": "\\u0012",
+        "19": "\\u0013",
+        "20": "\\u0014",
+        "21": "\\u0015",
+        "22": "\\u0016",
+        "23": "\\u0017",
+        "24": "\\u0018",
+        "25": "\\u0019",
+        "26": "\\u001a",
+        "27": "\\u001b",
+        "28": "\\u001c",
+        "29": "\\u001d",
+        "30": "\\u001e",
+        "31": "\\u001f",
+        "32": " ",
+        "33": "!",
+        "34": "\"",
+        "35": "#",
+        "36": "$",
+        "37": "%",
+        "38": "&",
+        "39": "'",
+        "40": "(",
+        "41": ")",
+        "42": "*",
+        "43": "+",
+        "44": ",",
+        "45": "-",
+        "46": ".",
+        "47": "/",
+        "48": "0",
+        "49": "1",
+        "50": "2",
+        "51": "3",
+        "52": "4",
+        "53": "5",
+        "54": "6",
+        "55": "7",
+        "56": "8",
+        "57": "9",
+        "58": ":",
+        "59": ";",
+        "60": "<",
+        "61": "=",
+        "62": ">",
+        "63": "?",
+        "64": "@",
+        "65": "A",
+        "66": "B",
+        "67": "C",
+        "68": "D",
+        "69": "E",
+        "70": "F",
+        "71": "G",
+        "72": "H",
+        "73": "I",
+        "74": "J",
+        "75": "K",
+        "76": "L",
+        "77": "M",
+        "78": "N",
+        "79": "O",
+        "80": "P",
+        "81": "Q",
+        "82": "R",
+        "83": "S",
+        "84": "T",
+        "85": "U",
+        "86": "V",
+        "87": "W",
+        "88": "X",
+        "89": "Y",
+        "90": "Z",
+        "91": "[",
+        "92": "\\",
+        "93": "]",
+        "94": "^",
+        "95": "_",
+        "96": "`",
+        "97": "a",
+        "98": "b",
+        "99": "c",
+        "100": "d",
+        "101": "e",
+        "102": "f",
+        "103": "g",
+        "104": "h",
+        "105": "i",
+        "106": "j",
+        "107": "k",
+        "108": "l",
+        "109": "m",
+        "110": "n",
+        "111": "o",
+        "112": "p",
+        "113": "q",
+        "114": "r",
+        "115": "s",
+        "116": "t",
+        "117": "u",
+        "118": "v",
+        "119": "w",
+        "120": "x",
+        "121": "y",
+        "122": "z",
+        "123": "{",
+        "124": "|",
+        "125": "}",
+        "126": "~",
+        "127": "\\u007f",
+        "128": "�",
+        "129": "�",
+        "130": "�",
+        "131": "�",
+        "132": "�",
+        "133": "�",
+        "134": "�",
+        "135": "�",
+        "136": "�",
+        "137": "�",
+        "138": "�",
+        "139": "�",
+        "140": "�",
+        "141": "�",
+        "142": "�",
+        "143": "�",
+        "144": "�",
+        "145": "�",
+        "146": "�",
+        "147": "�",
+        "148": "�",
+        "149": "�",
+        "150": "�",
+        "151": "�",
+        "152": "�",
+        "153": "�",
+        "154": "�",
+        "155": "�",
+        "156": "�",
+        "157": "�",
+        "158": "�",
+        "159": "�",
+        "160": "�",
+        "161": "�",
+        "162": "�",
+        "163": "�",
+        "164": "�",
+        "165": "�",
+        "166": "�",
+        "167": "�",
+        "168": "�",
+        "169": "�",
+        "170": "�",
+        "171": "�",
+        "172": "�",
+        "173": "�",
+        "174": "�",
+        "175": "�",
+        "176": "�",
+        "177": "�",
+        "178": "�",
+        "179": "�",
+        "180": "�",
+        "181": "�",
+        "182": "�",
+        "183": "�",
+        "184": "�",
+        "185": "�",
+        "186": "�",
+        "187": "�",
+        "188": "�",
+        "189": "�",
+        "190": "�",
+        "191": "�",
+        "192": "�",
+        "193": "�",
+        "194": "�",
+        "195": "�",
+        "196": "�",
+        "197": "�",
+        "198": "�",
+        "199": "�",
+        "200": "�",
+        "201": "�",
+        "202": "�",
+        "203": "�",
+        "204": "�",
+        "205": "�",
+        "206": "�",
+        "207": "�",
+        "208": "�",
+        "209": "�",
+        "210": "�",
+        "211": "�",
+        "212": "�",
+        "213": "�",
+        "214": "�",
+        "215": "�",
+        "216": "�",
+        "217": "�",
+        "218": "�",
+        "219": "�",
+        "220": "�",
+        "221": "�",
+        "222": "�",
+        "223": "�",
+        "224": "�",
+        "225": "�",
+        "226": "�",
+        "227": "�",
+        "228": "�",
+        "229": "�",
+        "230": "�",
+        "231": "�",
+        "232": "�",
+        "233": "�",
+        "234": "�",
+        "235": "�",
+        "236": "�",
+        "237": "�",
+        "238": "�",
+        "239": "�",
+        "240": "�",
+        "241": "�",
+        "242": "�",
+        "243": "�",
+        "244": "�",
+        "245": "�",
+        "246": "�",
+        "247": "�",
+        "248": "�",
+        "249": "�",
+        "250": "�",
+        "251": "�",
+        "252": "�",
+        "253": "�",
+        "254": "�",
+        "255": "�",
+        "256": " t",
+        "257": "ex",
+        "258": "ext",
+        "259": " to",
+        "260": "en",
+        "261": "<|",
+        "262": "st",
+        "263": "of",
+        "264": "oft",
+        "265": "oftext",
+        "266": "|>",
+        "267": "is",
+        "268": " a",
+        "269": " s",
+        "270": " th",
+        "271": " the",
+        "272": " tok",
+        "273": " token",
+        "274": " [",
+        "275": "SP",
+        "276": "SPE",
+        "277": "SPEC",
+        "278": "SPECI",
+        "279": "SPECIA",
+        "280": "SPECIAL",
+        "281": "sta",
+        "282": "star",
+        "283": "start",
+        "284": "startoftext",
+        "285": " H",
+        "286": " He",
+        "287": " Hel",
+        "288": " Hell",
+        "289": " Hello",
+        "290": " W",
+        "291": " Wo",
+        "292": " Wor",
+        "293": " Worl",
+        "294": " World",
+        "295": " T",
+        "296": " Th",
+        "297": " This",
+        "298": " is",
+        "299": " sa",
+        "300": " sam",
+        "301": " samp",
+        "302": " sampl",
+        "303": " sample",
+        "304": " text",
+        "305": " w",
+        "306": " wi",
+        "307": " wit",
+        "308": " with",
+        "309": " sp"
+    }
+}

tokenizer/sample/load_json_vocab/tokens.py

@@ -0,0 +1,26 @@
+import sys
+sys.path.append('../')
+
+from bpetokenizer import BPETokenizer
+
+# intializing the tokenizer
+tokenizer = BPETokenizer()
+
+# load the vocab which is pretrained
+tokenizer.load("sample_bpetokenizer.json", mode="json")
+
+text = "<|startoftext|>This method? generates the tokens! which are split, before the tokenization using the pattern: default we use the gpt4 split pattern mentioned in the tiktoken.<|endoftext|>"
+
+
+# this method returns a list of tokens of the text passed.
+tokens = tokenizer.tokens(text, verbose=True) # if verbose, prints the text chunks and also the pattern used to split.
+print('---')
+print("tokens: ", tokens)
+
+"""
+tokens:  ['<|', 'st', 'ar', 't', 'oftext', '|>', 'T', 'h', 'is', ' ', 'm', 'e', 'th', 'o', 'd', '?', ' ', 'g', 'en', 'er', 'a', 't', 'e', 's', ' the', ' token',
+ 's', '!', ' w', 'h', 'i', 'c', 'h', ' a', 'r', 'e', ' s', 'pl', 'i', 't', ',', ' ', 'b', 'e', 'f', 'o', 'r', 'e', ' the',
+ ' tokeniz', 'a', 't', 'i', 'on', ' ', 'u', 's', 'ing', ' the', ' ', 'p', 'a', 't', 't', 'er', 'n', ':', ' ', 'd', 'e', 'f', 'a', 'u', 'l', 't', ' w', 'e', ' ', 
+ 'u', 'se', ' the', ' ', 'g', 'p', 't', '4', ' s', 'pl', 'i', 't', ' ', 'p', 'a', 't', 't', 'er', 'n', ' ',
+ 'm', 'en', 't', 'i', 'on', 'e', 'd', ' ', 'in', ' the', ' t', 'i', 'k', 't', 'o', 'k', 'en', '.', '<|', 'en', 'd', 'oftext', '|>']
+"""

tokenizer/sample/tokenizer/wiki.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8329f536bb6d37b88b4bd5b75770c5240a06d343f32756dd34eea37c49d69e2
+size 36

tokenizer/sample/tokenizer/wiki.py

@@ -0,0 +1,14 @@
+from bpetokenizer import Tokenizer
+
+text = "aaabdaaabac"
+tokenizer = Tokenizer()
+tokenizer.train(text, 259, verbose=True)
+
+ids = tokenizer.encode(text)
+print(ids)
+print('---')
+
+decoded_text = tokenizer.decode(ids)
+print(decoded_text)
+
+tokenizer.save("wiki")

tokenizer/sample/tokenizer/wiki.vocab

@@ -0,0 +1,259 @@
+[\u0000] 0
+[\u0001] 1
+[\u0002] 2
+[\u0003] 3
+[\u0004] 4
+[\u0005] 5
+[\u0006] 6
+[\u0007] 7
+[\u0008] 8
+[\u0009] 9
+[\u000a] 10
+[\u000b] 11
+[\u000c] 12
+[\u000d] 13
+[\u000e] 14
+[\u000f] 15
+[\u0010] 16
+[\u0011] 17
+[\u0012] 18
+[\u0013] 19
+[\u0014] 20
+[\u0015] 21
+[\u0016] 22
+[\u0017] 23
+[\u0018] 24
+[\u0019] 25
+[\u001a] 26
+[\u001b] 27
+[\u001c] 28
+[\u001d] 29
+[\u001e] 30
+[\u001f] 31
+[ ] 32
+[!] 33
+["] 34
+[#] 35
+[$] 36
+[%] 37
+[&] 38
+['] 39
+[(] 40
+[)] 41
+[*] 42
+[+] 43
+[,] 44
+[-] 45
+[.] 46
+[/] 47
+[0] 48
+[1] 49
+[2] 50
+[3] 51
+[4] 52
+[5] 53
+[6] 54
+[7] 55
+[8] 56
+[9] 57
+[:] 58
+[;] 59
+[<] 60
+[=] 61
+[>] 62
+[?] 63
+[@] 64
+[A] 65
+[B] 66
+[C] 67
+[D] 68
+[E] 69
+[F] 70
+[G] 71
+[H] 72
+[I] 73
+[J] 74
+[K] 75
+[L] 76
+[M] 77
+[N] 78
+[O] 79
+[P] 80
+[Q] 81
+[R] 82
+[S] 83
+[T] 84
+[U] 85
+[V] 86
+[W] 87
+[X] 88
+[Y] 89
+[Z] 90
+[[] 91
+[\] 92
+[]] 93
+[^] 94
+[_] 95
+[`] 96
+[a] 97
+[b] 98
+[c] 99
+[d] 100
+[e] 101
+[f] 102
+[g] 103
+[h] 104
+[i] 105
+[j] 106
+[k] 107
+[l] 108
+[m] 109
+[n] 110
+[o] 111
+[p] 112
+[q] 113
+[r] 114
+[s] 115
+[t] 116
+[u] 117
+[v] 118
+[w] 119
+[x] 120
+[y] 121
+[z] 122
+[{] 123
+[|] 124
+[}] 125
+[~] 126
+[\u007f] 127
+[�] 128
+[�] 129
+[�] 130
+[�] 131
+[�] 132
+[�] 133
+[�] 134
+[�] 135
+[�] 136
+[�] 137
+[�] 138
+[�] 139
+[�] 140
+[�] 141
+[�] 142
+[�] 143
+[�] 144
+[�] 145
+[�] 146
+[�] 147
+[�] 148
+[�] 149
+[�] 150
+[�] 151
+[�] 152
+[�] 153
+[�] 154
+[�] 155
+[�] 156
+[�] 157
+[�] 158
+[�] 159
+[�] 160
+[�] 161
+[�] 162
+[�] 163
+[�] 164
+[�] 165
+[�] 166
+[�] 167
+[�] 168
+[�] 169
+[�] 170
+[�] 171
+[�] 172
+[�] 173
+[�] 174
+[�] 175
+[�] 176
+[�] 177
+[�] 178
+[�] 179
+[�] 180
+[�] 181
+[�] 182
+[�] 183
+[�] 184
+[�] 185
+[�] 186
+[�] 187
+[�] 188
+[�] 189
+[�] 190
+[�] 191
+[�] 192
+[�] 193
+[�] 194
+[�] 195
+[�] 196
+[�] 197
+[�] 198
+[�] 199
+[�] 200
+[�] 201
+[�] 202
+[�] 203
+[�] 204
+[�] 205
+[�] 206
+[�] 207
+[�] 208
+[�] 209
+[�] 210
+[�] 211
+[�] 212
+[�] 213
+[�] 214
+[�] 215
+[�] 216
+[�] 217
+[�] 218
+[�] 219
+[�] 220
+[�] 221
+[�] 222
+[�] 223
+[�] 224
+[�] 225
+[�] 226
+[�] 227
+[�] 228
+[�] 229
+[�] 230
+[�] 231
+[�] 232
+[�] 233
+[�] 234
+[�] 235
+[�] 236
+[�] 237
+[�] 238
+[�] 239
+[�] 240
+[�] 241
+[�] 242
+[�] 243
+[�] 244
+[�] 245
+[�] 246
+[�] 247
+[�] 248
+[�] 249
+[�] 250
+[�] 251
+[�] 252
+[�] 253
+[�] 254
+[�] 255
+[a][a] -> [aa] 256
+[aa][a] -> [aaa] 257
+[aaa][b] -> [aaab] 258

tokenizer/setup.py

@@ -0,0 +1,38 @@
+import os
+from setuptools import find_packages, setup
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+# Get the code version
+version = {}
+with open(os.path.join(here, "bpetokenizer/version.py")) as f:
+    exec(f.read(), version)
+__version__ = version["__version__"]
+
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+
+setup(
+    name="bpetokenizer",
+    version=__version__,
+    description="Byte Pair Encoding Tokenizer with special tokens and regex pattern",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/Hk669/bpetokenizer",
+    author="Hrushikesh Dokala",
+    author_email="hrushi669@gmail.com",
+    license="MIT",
+    packages=find_packages(include=["bpetokenizer"]),
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3",
+        "Operating System :: OS Independent",
+    ],
+    install_requires=["regex"],
+    extras_require={
+        "dev": ["pytest", "twine"],
+    },
+    python_requires=">=3.9,<3.13",
+)

tokenizer/tests/test_tokenizer.py

@@ -0,0 +1,69 @@
+import os
+import pytest
+from bpetokenizer import BPETokenizer, Tokenizer
+
+@pytest.fixture
+def tokenizer():
+    return Tokenizer()
+
+@pytest.fixture
+def bpe_tokenizer():
+    return BPETokenizer()
+
+
+def test_train():
+    """Test the training of the tokenizer."""
+    text = "aaabdaaabac"
+    tokenizer = Tokenizer()
+    tokenizer.train(text, 259, verbose=False)
+    assert len(tokenizer.vocab) == 259
+    assert len(tokenizer.merges) == 3
+    assert tokenizer.decode(tokenizer.encode(text)) == "aaabdaaabac"
+
+
+def test_encode():
+    """Test the encoding of the tokenizer."""
+    text = "aaabdaaabac"
+    tokenizer = Tokenizer()
+    tokenizer.train(text, 259, verbose=False)
+    assert tokenizer.encode("aaabdaaabac") == [258, 100, 258, 97, 99]
+
+
+def test_decode():
+    """Test the decoding of the tokenizer."""
+    text = "aaabdaaabac"
+    tokenizer = Tokenizer()
+    tokenizer.train(text, 259, verbose=False)
+    assert tokenizer.decode([258, 100, 258, 97, 99]) == "aaabdaaabac"
+
+
+def test_train_bpe():
+    """Test the training of the BPE tokenizer."""
+    text = "aaabdaaabac"
+    tokenizer = BPETokenizer()
+    tokenizer.train(text, 256 + 3, verbose=False)
+    assert len(tokenizer.vocab) == 259
+    assert len(tokenizer.merges) == 3
+    assert tokenizer.decode(tokenizer.encode(text)) == "aaabdaaabac"
+
+
+def test_train_bpe_w_special_tokens():
+    """Test the bpetokenizer with special tokens"""
+    special_tokens = {
+        "<|endoftext|>": 1001,
+        "<|startoftext|>": 1002,
+        "[SPECIAL1]": 1003,
+        "[SPECIAL2]": 1004,
+    }
+
+    PATTERN = r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+"""
+    tokenizer = BPETokenizer(special_tokens=special_tokens, pattern=PATTERN)
+    texts = "<|startoftext|> Hello, World! This is a sample text with the special tokens [SPECIAL1] and [SPECIAL2] to test the tokenizer.<|endoftext|>"
+    tokenizer.train(texts, vocab_size=310, verbose=False)
+
+    assert len(tokenizer.vocab) == 310
+    assert len(tokenizer.merges) == 310 - 256
+    assert tokenizer.decode(tokenizer.encode(texts)) == texts
+    assert tokenizer.inverse_special_tokens == {v: k for k,v in special_tokens.items()}
+    assert tokenizer.special_tokens == special_tokens
+    assert tokenizer.pattern == PATTERN