Commit
•
1578b2b
1
Parent(s):
9638d60
Fix minor bug in auto-chunking tokenization
Browse files- tokenization_hat.py +1 -1
tokenization_hat.py
CHANGED
|
@@ -157,7 +157,7 @@ class HATTokenizer:
|
|
| 157 |
if isinstance(key_inputs_list[0], list):
|
| 158 |
key_inputs_list = [token for sentence in key_inputs_list for token in sentence]
|
| 159 |
else:
|
| 160 |
-
key_inputs_list = torch.stack(key_inputs_list)
|
| 161 |
if input_key in batch:
|
| 162 |
batch[input_key].append(key_inputs_list)
|
| 163 |
else:
|
| 157 |
if isinstance(key_inputs_list[0], list):
|
| 158 |
key_inputs_list = [token for sentence in key_inputs_list for token in sentence]
|
| 159 |
else:
|
| 160 |
+
key_inputs_list = torch.stack([token for sentence in key_inputs_list for token in sentence])
|
| 161 |
if input_key in batch:
|
| 162 |
batch[input_key].append(key_inputs_list)
|
| 163 |
else:
|