Initial commit

inference.yaml

@@ -0,0 +1,28 @@
+transforms: [sentencepiece]
+
+#### Subword
+src_subword_model: "<model_dir>/tokenizer.model"
+tgt_subword_model: "<model_dir>/tokenizer.model"
+
+# Model info
+model: "<model_dir>/mistral-instruct-v0.2-onmt-awq-gemv.pt"
+
+
+# Inference
+seed: 42
+max_length: 256
+gpu: 0
+batch_type: sents
+batch_size: 4
+world_size: 1
+gpu_ranks: [0]
+#parallel_mode: "tensor_parallel"
+precision: fp16
+#random_sampling_topk: 1
+#random_sampling_topp: 0.6
+#random_sampling_temp: 0.9
+beam_size: 1
+n_best: 1
+profile: false
+report_time: true
+src: None

mistral-instruct-v0.2-onmt-awq-gemv.00.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7c24e774166aa5bf5293f6a2c603336dfec29d4f3fdcf8733a2de679622d4bf
+size 4150957864

mistral-instruct-v0.2-onmt-awq-gemv.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17a185e64df160df9049ef26adf67c1b80d00d5a64b497e19fa313f3d09fad19
+size 1054160

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443