sarvam1_2b.toml

[model]
dim = 2048
n_vocab = 68096
hf_model_id = "sarvamai/sarvam-1"
pad_token = "</s>"

[tokenizer]
return_tensors = "pt"
return_attention_mask = true
max_length = 256
padding = "longest"
truncation = true
add_special_tokens = false

[training]
output_dir = "output/matryoshka_sarvam1"
num_train_epochs = 20
per_device_train_batch_size = 128
warmup_steps = 256
evaluation_strategy = "steps"
eval_steps = 2000
save_steps = 2000
fp16 = true
include_num_input_tokens_seen = false
learning_rate = 3e-4
multi_dataset_batch_sampler = "PROPORTIONAL"
binarizer_ste = "tanh"

[matryoshka]
dims = [1024, 512, 256, 128, 64]