Add HindiCausalLM: A specialized Hindi language model (~102M parameters)
Author: NandhaKishorM
Pull request: no
Labels: name: New model

### Model description

HindiCausalLM model, a causal language model specifically designed for Hindi text generation. The model was developed by ConvaiInnovations and the original implementation is available on the [Hugging Face Hub](https://huggingface.co/convaiinnovations/hindi-causal-lm).

HindiCausalLM addresses the need for specialized language models for the Hindi language, which is one of the most widely spoken languages in the world with over 600 million speakers. The model provides improved performance on Hindi text generation tasks compared to multilingual models of similar size.

## Model Architecture and Technical Specifications

HindiCausalLM is based on a decoder-only transformer architecture with the following specifications:

- **Base Architecture**: Transformer decoder-only, similar to LLaMA but with architecture modifications
- **Size**: 12 layers, 768 hidden dimensions, ~102M parameters
- **Attention Mechanism**: Implements Grouped Query Attention (GQA) with 16 attention heads and 4 key-value heads for efficient inference
- **Feed-Forward Network**: Uses intermediate size of 3072 with SiLU activation functions
- **Normalization**: Layer normalization applied before attention and feed-forward blocks (pre-normalization)
- **Vocabulary Size**: 16,000 tokens using a SentencePiece tokenizer optimized for Hindi text
- **Context Window**: Supports sequences up to 512 tokens in length


## Usage

You can use this model with the following code after cloning [Hugging Face Hub](https://huggingface.co/convaiinnovations/hindi-causal-lm)

```python
import torch
from hindi_embeddings import SentencePieceTokenizerWrapper
from convaicausallm_model import ConvaiCausalLM, ConvaiCausalLMConfig
from safetensors.torch import load_file
import os
class HindiLLMGenerator:
    def __init__(self, model_path, device=None):
        # Set device
        if device is None:
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            self.device = torch.device(device)
            
        print(f"Using device: {self.device}")
        
        # Load tokenizer
        tokenizer_path = os.path.join(model_path, "tokenizer.model")
        self.tokenizer = SentencePieceTokenizerWrapper(tokenizer_path)
        
        # Load model config
        config_path = os.path.join(model_path, "config.json")
        import json
        with open(config_path, 'r') as f:
            config_dict = json.load(f)
            
        self.config = ConvaiCausalLMConfig(**config_dict)
        
        # Load model - try safetensors first, fall back to PyTorch bin if needed
        safetensors_path = os.path.join(model_path, "model.safetensors")
        pytorch_path = os.path.join(model_path, "pytorch_model.bin")
        
        self.model = ConvaiCausalLM(self.config)
        
        # Check which format is available and load accordingly
        if os.path.exists(safetensors_path):
            print(f"Loading model from SafeTensors")
            state_dict = load_file(safetensors_path, device="cpu")
            self.model.load_state_dict(state_dict)
        elif os.path.exists(pytorch_path):
            print(f"Loading model from PyTorch bin")
            self.model.load_state_dict(torch.load(pytorch_path, map_location="cpu"))
        
        # Move model to device and set to evaluation mode
        self.model.to(self.device)
        self.model.eval()
    
    def generate(self, prompt, max_length=100, temperature=0.8, top_k=50, top_p=0.9, 
                 repetition_penalty=1.1, do_sample=True):
        # Tokenize the prompt
        input_ids = self.tokenizer.sp_model.EncodeAsIds(prompt)
        input_tensor = torch.tensor([input_ids], dtype=torch.long).to(self.device)
        
        # Start with the input tensor
        output_sequence = input_tensor.clone()
        
        # Generate tokens one by one
        for _ in range(max_length - len(input_ids)):
            with torch.no_grad():
                # Get the model's output for the current sequence
                outputs = self.model(output_sequence)
                next_token_logits = outputs[0, -1, :]
                
                # Apply temperature
                if temperature > 0:
                    next_token_logits = next_token_logits / temperature
                
                # Apply repetition penalty
                if repetition_penalty > 1.0:
                    for token_id in output_sequence[0].tolist():
                        next_token_logits[token_id] /= repetition_penalty
                
                # Filter with top-k sampling
                if top_k > 0:
                    top_k_values, top_k_indices = torch.topk(next_token_logits, top_k)
                    next_token_logits = torch.full_like(next_token_logits, float('-inf'))
                    next_token_logits.scatter_(0, top_k_indices, top_k_values)
                
                # Filter with top-p/nucleus sampling
                if top_p < 1.0 and do_sample:
                    sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
                    cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
                    
                    # Remove tokens with cumulative probability above the threshold
                    sorted_indices_to_remove = cumulative_probs > top_p
                    # Shift the indices to the right to keep the first token above the threshold
                    sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
                    sorted_indices_to_remove[..., 0] = 0
                    
                    indices_to_remove = sorted_indices[sorted_indices_to_remove]
                    next_token_logits[indices_to_remove] = float('-inf')
                
                # Sample or choose the next token
                if do_sample:
                    probs = torch.softmax(next_token_logits, dim=-1)
                    next_token = torch.multinomial(probs, num_samples=1)
                else:
                    next_token = torch.argmax(next_token_logits, dim=-1).unsqueeze(0)
                
                # Add the next token to the sequence
                output_sequence = torch.cat([output_sequence, next_token.unsqueeze(0)], dim=1)
                
                # Check if we've generated an end token
                if next_token.item() == self.tokenizer.eos_token_id:
                    break
        
        # Decode the generated sequence
        generated_ids = output_sequence[0].tolist()
        generated_text = self.tokenizer.sp_model.DecodeIds(generated_ids)
        
        return generated_text

# Example usage
if __name__ == "__main__":
    generator = HindiLLMGenerator("path/to/model")
    result = generator.generate("भारत एक विशाल देश है")
    print(result)

Comments:
Inference colab notebook at [Colab_Link](https://colab.research.google.com/drive/1kWjqTXDu0ah_yi2LdsHTwDBjFB0KDh2Y?usp=sharing)