below code tokenization not happing properly 
/scratch/hemanth/anaconda/envs/chatbot/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884
  warnings.warn(
INFO:root:Found 144 .txt files in /scratch/hemanth/Hemanth/localdataset
INFO:root:DataLoader created successfully.
Token indices sequence length is longer than the specified maximum sequence length for this model (4090 > 512). Running this sequence through the model will result in indexing errors
Token indices sequence length is longer than the specified maximum sequence length for this model (18209 > 512). Running this sequence through the model will result in indexing errors
Token indices sequence length is longer than the specified maximum sequence length for this model (19751 > 512). Running this sequence through the model will result in indexing errors
Token indices sequence length is longer than the specified maximum sequence length for this model (21981 > 512). Running this sequence through the model will result in indexing errors
WARNING:root:File /scratch/hemanth/Hemanth/localdataset/requirements.txt is empty.
WARNING:root:File /scratch/hemanth/Hemanth/localdataset/__init__.txt is empty.
WARNING:root:File /scratch/hemanth/Hemanth/localdataset/Securing AI systems with adversarial robustness.txt is empty.
INFO:root:Batch input_ids shape: torch.Size([32, 512])


import os
import logging
from typing import Any, Dict, List, Optional

import torch
from torch.utils.data import DataLoader, Dataset
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from transformers import DataCollatorWithPadding

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[logging.StreamHandler()],
)


class TextFileDataset(Dataset):
    """
    A PyTorch Dataset for loading and processing text files from a directory.
    Each file can have a variable number of lines and variable line lengths.
    """

    def __init__(
        self,
        directory: str,
        tokenizer: PreTrainedTokenizerBase,
        max_length: Optional[int] = None,
        bos_token: Optional[str] = None,
        eos_token: Optional[str] = None,
    ) -> None:
        """
        Initializes the dataset by listing all .txt files in the provided directory.

        Args:
            directory (str): Path to the directory containing .txt files.
            tokenizer (PreTrainedTokenizerBase): Tokenizer for processing text.
            max_length (Optional[int], optional): Maximum token length. Defaults to None.
            bos_token (Optional[str], optional): Beginning of sequence token. Defaults to None.
            eos_token (Optional[str], optional): End of sequence token. Defaults to None.
        """
        super().__init__()
        self.directory = directory
        self.tokenizer = tokenizer
        self.max_length = max_length

        # Special tokens
        if bos_token:
            self.tokenizer.add_special_tokens({'bos_token': bos_token})
        if eos_token:
            self.tokenizer.add_special_tokens({'eos_token': eos_token})

        self.bos_token_id = self.tokenizer.bos_token_id
        self.eos_token_id = self.tokenizer.eos_token_id

        # Gather all .txt file paths
        self.file_paths = self._get_txt_file_paths()
        if not self.file_paths:
            logging.warning(f"No .txt files found in directory: {directory}")

    def _get_txt_file_paths(self) -> List[str]:
        """
        Retrieves all .txt file paths from the specified directory.

        Returns:
            List[str]: List of .txt file paths.
        """
        try:
            files = [
                os.path.join(self.directory, fname)
                for fname in os.listdir(self.directory)
                if fname.lower().endswith(".txt")
            ]
            logging.info(f"Found {len(files)} .txt files in {self.directory}")
            return files
        except FileNotFoundError as e:
            logging.error(f"Directory not found: {self.directory}")
            raise e
        except Exception as e:
            logging.error(f"Error accessing directory {self.directory}: {e}")
            raise e

    def __len__(self) -> int:
        return len(self.file_paths)

    def _read_file(self, file_path: str) -> str:
        """
        Reads the content of a text file.

        Args:
            file_path (str): Path to the text file.

        Returns:
            str: Content of the file.
        """
        try:
            with open(file_path, "r", encoding="utf-8") as file:
                content = file.read()
            return content
        except Exception as e:
            logging.error(f"Error reading file {file_path}: {e}")
            return ""

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        """
        Retrieves and tokenizes the text from the specified file index.

        Args:
            idx (int): Index of the file.

        Returns:
            Dict[str, Any]: Tokenized representation of the text.
        """
        file_path = self.file_paths[idx]
        text = self._read_file(file_path)

        if not text.strip():
            logging.warning(f"File {file_path} is empty.")
            text = self.tokenizer.pad_token or ""

        # Add special tokens
        tokens = []
        if self.bos_token_id is not None:
            tokens.append(self.bos_token_id)
        encoded = self.tokenizer.encode(
            text,
            add_special_tokens=False,
            truncation=False,
            max_length=None,
        )
        tokens.extend(encoded)
        if self.eos_token_id is not None:
            tokens.append(self.eos_token_id)

        if self.max_length:
            if len(tokens) > self.max_length:
                logging.debug(
                    f"Truncating tokens from {len(tokens)} to {self.max_length}"
                )
                tokens = tokens[: self.max_length]
            elif len(tokens) < self.max_length:
                padding_length = self.max_length - len(tokens)
                tokens.extend([self.tokenizer.pad_token_id] * padding_length)

        return {"input_ids": torch.tensor(tokens, dtype=torch.long)}


def create_dataloader(
    dataset: Dataset,
    tokenizer: PreTrainedTokenizerBase,
    batch_size: int = 32,
    shuffle: bool = True,
    num_workers: int = 4,
) -> DataLoader:
    """
    Creates a DataLoader with appropriate settings and a data collator.

    Args:
        dataset (Dataset): The dataset to load.
        tokenizer (PreTrainedTokenizerBase): Tokenizer for data collation.
        batch_size (int, optional): Number of samples per batch. Defaults to 32.
        shuffle (bool, optional): Whether to shuffle the data. Defaults to True.
        num_workers (int, optional): Number of subprocesses for data loading. Defaults to 4.

    Returns:
        DataLoader: Configured DataLoader.
    """
    try:
        data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding=True)

        dataloader = DataLoader(
            dataset,
            batch_size=batch_size,
            shuffle=shuffle,
            num_workers=num_workers,
            collate_fn=data_collator,
        )
        logging.info("DataLoader created successfully.")
        return dataloader
    except Exception as e:
        logging.error(f"Failed to create DataLoader: {e}")
        raise e


def main(
    data_directory: str,
    tokenizer_name: str = "bert-base-uncased",
    batch_size: int = 32,
    max_length: Optional[int] = None,
) -> None:
    """
    Main function to set up the dataset and DataLoader.

    Args:
        data_directory (str): Path to the directory containing .txt files.
        tokenizer_name (str, optional): Name of the pretrained tokenizer. Defaults to "bert-base-uncased".
        batch_size (int, optional): Batch size for DataLoader. Defaults to 32.
        max_length (Optional[int], optional): Maximum token length. Defaults to None.
    """
    try:
        tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
        if not tokenizer.pad_token:
            tokenizer.add_special_tokens({'pad_token': '[PAD]'})  # Ensure pad token exists

        dataset = TextFileDataset(
            directory=data_directory,
            tokenizer=tokenizer,
            max_length=max_length,
            bos_token=tokenizer.cls_token,  # Typically [CLS] token
            eos_token=tokenizer.sep_token,  # Typically [SEP] token
        )

        dataloader = create_dataloader(
            dataset=dataset,
            tokenizer=tokenizer,
            batch_size=batch_size,
            shuffle=True,
            num_workers=4,
        )

        # Example iteration
        for batch in dataloader:
            input_ids = batch["input_ids"]
            # Here you would typically pass input_ids to your model
            logging.info(f"Batch input_ids shape: {input_ids.shape}")
            break  # Remove this break to iterate through the entire dataset

    except Exception as e:
        logging.error(f"An error occurred in the main pipeline: {e}")

main(
    data_directory='/scratch/hemanth/Hemanth/localdataset',
    tokenizer_name='bert-base-uncased',
    batch_size=32,
    max_length=512,

)