Kevin Fink
commited on
Commit
·
2495767
1
Parent(s):
40f1713
deve
Browse files
app.py
CHANGED
@@ -107,7 +107,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
|
|
107 |
# Assuming 'text' is the input and 'target' is the expected output
|
108 |
model_inputs = tokenizer(
|
109 |
examples['text'],
|
110 |
-
|
111 |
truncation=True,
|
112 |
padding='max_length',
|
113 |
#return_tensors='pt',
|
@@ -117,7 +117,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
|
|
117 |
# Setup the decoder input IDs (shifted right)
|
118 |
labels = tokenizer(
|
119 |
examples['target'],
|
120 |
-
|
121 |
truncation=True,
|
122 |
padding='max_length',
|
123 |
#text_target=examples['target'],
|
|
|
107 |
# Assuming 'text' is the input and 'target' is the expected output
|
108 |
model_inputs = tokenizer(
|
109 |
examples['text'],
|
110 |
+
max_length=max_length, # Set to None for dynamic padding
|
111 |
truncation=True,
|
112 |
padding='max_length',
|
113 |
#return_tensors='pt',
|
|
|
117 |
# Setup the decoder input IDs (shifted right)
|
118 |
labels = tokenizer(
|
119 |
examples['target'],
|
120 |
+
max_length=128, # Set to None for dynamic padding
|
121 |
truncation=True,
|
122 |
padding='max_length',
|
123 |
#text_target=examples['target'],
|