inference.yaml

transforms: [sentencepiece]

#### Subword
src_subword_model: "<model_dir>/tokenizer.model"
tgt_subword_model: "<model_dir>/tokenizer.model"

# Model info
model: "<model_dir>/mistral-instruct-v0.2-onmt-awq-gemv.pt"


# Inference
seed: 42
max_length: 256
gpu: 0
batch_type: sents
batch_size: 4
world_size: 1
gpu_ranks: [0]
#parallel_mode: "tensor_parallel"
precision: fp16
#random_sampling_topk: 1
#random_sampling_topp: 0.6
#random_sampling_temp: 0.9
beam_size: 1
n_best: 1
profile: false
report_time: true
src: None