Commit
•
0a226a4
1
Parent(s):
3efdd53
model improved
Browse files- maker.py +2 -2
- pytorch_model.bin +1 -1
- tokenizer.json +0 -0
- tokenizer_config.json +1 -1
maker.py
CHANGED
@@ -41,8 +41,8 @@ class UDgoeswithDataset(object):
|
|
41 |
return lid
|
42 |
__len__=lambda self:len(self.ids)
|
43 |
__getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
|
44 |
-
from transformers import
|
45 |
-
tkz=
|
46 |
trainDS=UDgoeswithDataset("train.conllu",tkz)
|
47 |
devDS=UDgoeswithDataset("dev.conllu",tkz)
|
48 |
testDS=UDgoeswithDataset("test.conllu",tkz)
|
|
|
41 |
return lid
|
42 |
__len__=lambda self:len(self.ids)
|
43 |
__getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
|
44 |
+
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
|
45 |
+
tkz=AutoTokenizer.from_pretrained(src)
|
46 |
trainDS=UDgoeswithDataset("train.conllu",tkz)
|
47 |
devDS=UDgoeswithDataset("dev.conllu",tkz)
|
48 |
testDS=UDgoeswithDataset("test.conllu",tkz)
|
pytorch_model.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 441306865
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:53a888fb108192c80f8a0e937ca661808334b94961eee95832a70f7181b8e93c
|
3 |
size 441306865
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
{
|
2 |
"cls_token": "[CLS]",
|
3 |
"do_basic_tokenize": true,
|
4 |
-
"do_lower_case":
|
5 |
"do_lowercase": true,
|
6 |
"mask_token": "[MASK]",
|
7 |
"model_max_length": 512,
|
|
|
1 |
{
|
2 |
"cls_token": "[CLS]",
|
3 |
"do_basic_tokenize": true,
|
4 |
+
"do_lower_case": true,
|
5 |
"do_lowercase": true,
|
6 |
"mask_token": "[MASK]",
|
7 |
"model_max_length": 512,
|