Spaces:
Runtime error
Runtime error
Update train.py
Browse files
train.py
CHANGED
|
@@ -7,11 +7,11 @@ from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM, TrainingA
|
|
| 7 |
from datasets import load_dataset, DatasetDict, Dataset
|
| 8 |
from tokenizers import ByteLevelBPETokenizer
|
| 9 |
|
| 10 |
-
MAX_SEQ_LENGTH = 128
|
| 11 |
BATCH_SIZE = 64
|
| 12 |
EPOCHS = 20
|
| 13 |
LEARNING_RATE = 2e-4
|
| 14 |
-
FACTOR =
|
|
|
|
| 15 |
VOCAB_SIZE = 32000
|
| 16 |
INPUT_DATASET = "HuggingFaceTB/smollm-corpus"
|
| 17 |
INSTRUCT_DATASET = "nroggendorff/elephant"
|
|
|
|
| 7 |
from datasets import load_dataset, DatasetDict, Dataset
|
| 8 |
from tokenizers import ByteLevelBPETokenizer
|
| 9 |
|
|
|
|
| 10 |
BATCH_SIZE = 64
|
| 11 |
EPOCHS = 20
|
| 12 |
LEARNING_RATE = 2e-4
|
| 13 |
+
FACTOR = 1440
|
| 14 |
+
MAX_SEQ_LENGTH = FACTOR // 8
|
| 15 |
VOCAB_SIZE = 32000
|
| 16 |
INPUT_DATASET = "HuggingFaceTB/smollm-corpus"
|
| 17 |
INSTRUCT_DATASET = "nroggendorff/elephant"
|