Commit
•
a66a53d
1
Parent(s):
6cf32b2
token-classification pipeline
Browse files- config.json +8 -0
- upos.py +4 -0
config.json
CHANGED
@@ -12,6 +12,14 @@
|
|
12 |
"upos": {
|
13 |
"impl": "upos.BellmanFordTokenClassificationPipeline",
|
14 |
"pt": "AutoModelForTokenClassification"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
15 |
}
|
16 |
},
|
17 |
"eos_token_id": 2,
|
|
|
12 |
"upos": {
|
13 |
"impl": "upos.BellmanFordTokenClassificationPipeline",
|
14 |
"pt": "AutoModelForTokenClassification"
|
15 |
+
},
|
16 |
+
"token-classification": {
|
17 |
+
"impl": "upos.RawTokenClassificationPipeline",
|
18 |
+
"pt": "AutoModelForTokenClassification"
|
19 |
+
},
|
20 |
+
"ner": {
|
21 |
+
"impl": "upos.RawTokenClassificationPipeline",
|
22 |
+
"pt": "AutoModelForTokenClassification"
|
23 |
}
|
24 |
},
|
25 |
"eos_token_id": 2,
|
upos.py
CHANGED
@@ -40,6 +40,10 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
|
|
40 |
t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
|
41 |
return w
|
42 |
|
|
|
|
|
|
|
|
|
43 |
class MistralForTokenClassification(MistralPreTrainedModel):
|
44 |
def __init__(self,config):
|
45 |
from torch import nn
|
|
|
40 |
t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
|
41 |
return w
|
42 |
|
43 |
+
class RawTokenClassificationPipeline(TokenClassificationPipeline):
|
44 |
+
def check_model_type(self,supported_models):
|
45 |
+
pass
|
46 |
+
|
47 |
class MistralForTokenClassification(MistralPreTrainedModel):
|
48 |
def __init__(self,config):
|
49 |
from torch import nn
|