token-classification pipeline

config.json

@@ -11,6 +11,14 @@
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
+    },
+    "token-classification":{
+      "impl": "upos.RawTokenClassificationPipeline",
+      "pt": "AutoModelForTokenClassification"
+    },
+    "ner":{
+      "impl": "upos.RawTokenClassificationPipeline",
+      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 151643,

upos.py

@@ -40,6 +40,10 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
+class RawTokenClassificationPipeline(TokenClassificationPipeline):
+  def check_model_type(self,supported_models):
+    pass
+
 class Qwen2ForTokenClassification(Qwen2PreTrainedModel):
   def __init__(self,config):
     from torch import nn