Use AutoTokenizer for redpajama example
Browse files
examples/redpajama/config-3b.yml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
base_model: togethercomputer/RedPajama-INCITE-Chat-3B-v1
|
| 2 |
base_model_config: togethercomputer/RedPajama-INCITE-Chat-3B-v1
|
| 3 |
model_type: GPTNeoXForCausalLM
|
| 4 |
-
tokenizer_type:
|
| 5 |
trust_remote_code:
|
| 6 |
load_in_8bit: false
|
| 7 |
datasets:
|
|
|
|
| 1 |
base_model: togethercomputer/RedPajama-INCITE-Chat-3B-v1
|
| 2 |
base_model_config: togethercomputer/RedPajama-INCITE-Chat-3B-v1
|
| 3 |
model_type: GPTNeoXForCausalLM
|
| 4 |
+
tokenizer_type: AutoTokenizer
|
| 5 |
trust_remote_code:
|
| 6 |
load_in_8bit: false
|
| 7 |
datasets:
|