Following this instruction :
1. Strictly adhere to PEP-8 standards, ensuring code is clean, well-structured, and follows Python's style guide for readability.
2. Use appropriate modules like `typing` for type hints, ensuring code clarity, maintainability, and leveraging Python's standard libraries effectively.
3. Focus on time, space, robustness, scalability; write optimized, efficient code that can handle large datasets and scale seamlessly.
4. Implement comprehensive error handling, catching exceptions gracefully, providing meaningful error messages, and ensuring code robustness under unexpected conditions.
5. Ensure the developer agent writes advanced, maintainable Python code, balancing performance and readability while adhering to best practices.

TASK : Generalised robustness tokenization  Pipeline , (we rewrite using transformers module ) avoid using tiketoken

all feature we add to encode_plus

  text: TextInput | PreTokenizedInput | EncodedInput,
    text_pair: TextInput | PreTokenizedInput | EncodedInput | None = None,
    add_special_tokens: bool = True,
    padding: bool | str | PaddingStrategy = False,
    truncation: bool | str | TruncationStrategy = None,
    max_length: int | None = None,
    stride: int = 0,
    is_split_into_words: bool = False,
    pad_to_multiple_of: int | None = None,
    return_tensors: str | TensorType | None = None,
    return_token_type_ids: bool | None = None,
    return_attention_mask: bool | None = None,
    return_overflowing_tokens: bool = False,
    return_special_tokens_mask: bool = False,
    return_offsets_mapping: bool = False,
    return_length: bool = False,
    verbose: bool = True,
    **kwargs: Any


decode
  token_ids: int | List[int] | ndarray | Tensor | Any,
    skip_special_tokens: bool = False,
    clean_up_tokenization_spaces: bool = None,
    **kwargs: Any

3)
apply_chat_template(
    conversation: List[Dict[str, str]] | List[List[Dict[str, str]]],
    tools: List[Dict] | None = None,
    documents: List[Dict[str, str]] | None = None,
    chat_template: str | None = None,
    add_generation_prompt: bool = False,
    tokenize: bool = True,
    padding: bool = False,
    truncation: bool = False,
    max_length: int | None = None,
    return_tensors: str | TensorType | None = None,
    return_dict: bool = False,
    return_assistant_tokens_mask: bool = False,
    tokenizer_kwargs: Dict[str, Any] | None = None,
    **kwargs: Any
) -> (str | List[int] | List[str] | List[List[int]] | BatchEncoding)


import os
import logging
from pathlib import Path
from typing import (
    AbstractSet,
    Collection,
    Dict,
    Iterator,
    List,
    Literal,
    Sequence,
    TypedDict,
    Union,
    Optional,
)

from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, AutoTokenizer
from transformers.tokenization_utils import BatchEncoding

logger = logging.getLogger(__name__)

Role = Literal["system", "user", "assistant"]


class Message(TypedDict):
    role: Role
    content: str


Dialog = Sequence[Message]


class Tokenizer:
    """
    Tokenizing and encoding/decoding text using the Hugging Face Transformers tokenizer.
    """

    def __init__(self, model_name_or_path: str):
        """
        Initializes the Tokenizer with a Transformers model.

        Args:
            model_name_or_path (str): The name or path to the Hugging Face Transformers model.
        """
        # if not isinstance(model_name_or_path, str):
        #     raise TypeError("model_name_or_path must be a string.")

        # if not os.path.isdir(model_name_or_path) and not os.path.isfile(model_name_or_path):
        #     raise ValueError(f"Invalid model path or name: {model_name_or_path}")

        try:
            self.tokenizer: PreTrainedTokenizerFast = AutoTokenizer.from_pretrained(
                pretrained_model_name_or_path=model_name_or_path, use_fast=True
            )
            if not isinstance(self.tokenizer, PreTrainedTokenizerFast):
                self.tokenizer = PreTrainedTokenizerFast.from_pretrained(pretrained_model_name_or_path=model_name_or_path)
            logger.info(f"Loaded tokenizer from {model_name_or_path}")
        except Exception as e:
            logger.error(f"Failed to load tokenizer from {model_name_or_path}: {e}")
            raise

        self.bos_token_id: Optional[int] = self.tokenizer.bos_token_id
        self.eos_token_id: Optional[int] = self.tokenizer.eos_token_id
        self.pad_token_id: Optional[int] = self.tokenizer.pad_token_id

        self.stop_tokens = set(filter(None, [self.eos_token_id]))
        logger.info(
            f"Tokenizer initialized with BOS ID: {self.bos_token_id}, "
            f"EOS ID: {self.eos_token_id}, PAD ID: {self.pad_token_id}"
        )

    def encode(
        self,
        text: str,
        add_bos: bool = False,
        add_eos: bool = False,
        allowed_special: Union[Literal["all"], AbstractSet[str]] = set(),
        disallowed_special: Union[Literal["all"], Collection[str]] = (),
        max_length: Optional[int] = None,
        truncation: bool = True,
    ) -> List[int]:
        """
        Encodes a string into a list of token IDs.

        Args:
            text (str): The input string to be encoded.
            add_bos (bool): Whether to prepend the beginning-of-sequence token.
            add_eos (bool): Whether to append the end-of-sequence token.
            allowed_special ("all" | set[str]): Allowed special tokens in string.
            disallowed_special ("all" | Collection[str]): Special tokens that raise an error when present in string.
            max_length (Optional[int]): Maximum length of the encoded sequence.
            truncation (bool): Whether to truncate the sequence to max_length.

        Returns:
            List[int]: A list of token IDs.
        """
        if not isinstance(text, str):
            raise TypeError("Input text must be a string.")

        try:
            encoding: BatchEncoding = self.tokenizer.encode_plus(
                text,
                add_special_tokens=False,
                return_attention_mask=False,
                return_token_type_ids=False,
                max_length=max_length,
                truncation=truncation,
                # allow_special_tokens=True,
            )
            tokens = encoding['input_ids']
        except Exception as e:
            logger.error(f"Encoding failed for text: {text[:50]}... Error: {e}")
            raise

        if add_bos and self.bos_token_id is not None:
            tokens.insert(0, self.bos_token_id)

        if add_eos and self.eos_token_id is not None:
            tokens.append(self.eos_token_id)

        return tokens

    def decode(self, tokens: Sequence[int], skip_special_tokens: bool = True) -> str:
        """
        Decodes a list of token IDs into a string.

        Args:
            tokens (Sequence[int]): The list of token IDs to be decoded.
            skip_special_tokens (bool): Whether to skip special tokens during decoding.

        Returns:
            str: The decoded string.
        """
        if not all(isinstance(token, int) for token in tokens):
            raise TypeError("All tokens must be integers.")

        try:
            decoded_text: str = self.tokenizer.decode(
                tokens, skip_special_tokens=skip_special_tokens
            )
            return decoded_text
        except Exception as e:
            logger.error(f"Decoding failed for tokens: {tokens[:10]}... Error: {e}")
            raise


class ChatFormat:
    """
    Formats chat dialogs into token sequences suitable for model input.
    """

    def __init__(self, tokenizer: Tokenizer):
        if not isinstance(tokenizer, Tokenizer):
            raise TypeError("tokenizer must be an instance of Tokenizer.")
        self.tokenizer = tokenizer

    def encode_header(self, message: Message) -> List[int]:
        """
        Encodes the header of a message (e.g., role).

        Args:
            message (Message): The message containing role and content.

        Returns:
            List[int]: Token IDs representing the encoded header.
        """
        tokens = []
        start_header = "
"
        end_header = "
"

        start_header_id = self.tokenizer.tokenizer.convert_tokens_to_ids(start_header)
        end_header_id = self.tokenizer.tokenizer.convert_tokens_to_ids(end_header)

        if start_header_id is None or end_header_id is None:
            logger.error("Special tokens for headers are not defined in the tokenizer.")
            raise ValueError("Missing special header tokens in tokenizer.")

        tokens.append(start_header_id)
        role_encoded = self.tokenizer.encode(message["role"], add_bos=False, add_eos=False)
        tokens.extend(role_encoded)
        tokens.append(end_header_id)
        newline_encoded = self.tokenizer.encode("\n\n", add_bos=False, add_eos=False)
        tokens.extend(newline_encoded)

        return tokens

    def encode_message(self, message: Message) -> List[int]:
        """
        Encodes a single message into token IDs.

        Args:
            message (Message): The message to encode.

        Returns:
            List[int]: Token IDs representing the encoded message.
        """
        tokens = self.encode_header(message)
        content_encoded = self.tokenizer.encode(message["content"].strip(), add_bos=False, add_eos=False)
        tokens.extend(content_encoded)

        eot_token = "
"
        eot_id = self.tokenizer.tokenizer.convert_tokens_to_ids(eot_token)

        if eot_id is None:
            logger.error(f"Special token {eot_token} is not defined in the tokenizer.")
            raise ValueError("Missing end-of-turn token in tokenizer.")

        tokens.append(eot_id)
        return tokens

    def encode_dialog_prompt(self, dialog: Dialog) -> List[int]:
        """
        Encodes an entire dialog into a prompt suitable for model input.

        Args:
            dialog (Dialog): A sequence of messages forming the dialog.

        Returns:
            List[int]: Token IDs representing the encoded dialog prompt.
        """
        tokens = []

        begin_of_text = "
"
        begin_of_text_id = self.tokenizer.tokenizer.convert_tokens_to_ids(begin_of_text)

        if begin_of_text_id is None:
            logger.error("Special token 
 is not defined in the tokenizer.")
            raise ValueError("Missing 
 token in tokenizer.")

        tokens.append(begin_of_text_id)

        for message in dialog:
            tokens.extend(self.encode_message(message))

        # Add the start of an assistant message for the model to complete.
        assistant_message: Message = {"role": "assistant", "content": ""}
        tokens.extend(self.encode_header(assistant_message))

        return tokens



import logging

# Configure logging
logging.basicConfig(level=logging.INFO)

# Initialize the tokenizer with a pre-trained model or a local path
tokenizer = Tokenizer("gpt2")  # Replace with your model path or name

# Initialize ChatFormat with the tokenizer
chat_format = ChatFormat(tokenizer=tokenizer)

# Define a sample dialog
dialog = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Hello, how are you?"},
    {"role": "assistant", "content": "I'm doing well, thank you! How can I assist you today?"},
]

# Encode the dialog prompt
encoded_prompt = chat_format.encode_dialog_prompt(dialog)
print(f"Encoded Prompt: {encoded_prompt}")

# Decode the encoded prompt back to text
decoded_prompt = tokenizer.decode(encoded_prompt)
print(f"Decoded Prompt:\n{decoded_prompt}")