KoichiYasuoka commited on
Commit
a66a53d
1 Parent(s): 6cf32b2

token-classification pipeline

Browse files
Files changed (2) hide show
  1. config.json +8 -0
  2. upos.py +4 -0
config.json CHANGED
@@ -12,6 +12,14 @@
12
  "upos": {
13
  "impl": "upos.BellmanFordTokenClassificationPipeline",
14
  "pt": "AutoModelForTokenClassification"
 
 
 
 
 
 
 
 
15
  }
16
  },
17
  "eos_token_id": 2,
 
12
  "upos": {
13
  "impl": "upos.BellmanFordTokenClassificationPipeline",
14
  "pt": "AutoModelForTokenClassification"
15
+ },
16
+ "token-classification": {
17
+ "impl": "upos.RawTokenClassificationPipeline",
18
+ "pt": "AutoModelForTokenClassification"
19
+ },
20
+ "ner": {
21
+ "impl": "upos.RawTokenClassificationPipeline",
22
+ "pt": "AutoModelForTokenClassification"
23
  }
24
  },
25
  "eos_token_id": 2,
upos.py CHANGED
@@ -40,6 +40,10 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
40
  t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
41
  return w
42
 
 
 
 
 
43
  class MistralForTokenClassification(MistralPreTrainedModel):
44
  def __init__(self,config):
45
  from torch import nn
 
40
  t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
41
  return w
42
 
43
+ class RawTokenClassificationPipeline(TokenClassificationPipeline):
44
+ def check_model_type(self,supported_models):
45
+ pass
46
+
47
  class MistralForTokenClassification(MistralPreTrainedModel):
48
  def __init__(self,config):
49
  from torch import nn