File size: 1,893 Bytes
d8f96d2 5c5407c d8f96d2 82ec9f7 d8f96d2 82ec9f7 d8f96d2 82ec9f7 d8f96d2 82ec9f7 d8f96d2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
import typing as tp
from collections import namedtuple
import torch
from transformers import Pipeline, AutoModelForSequenceClassification
from transformers.pipelines import PIPELINE_REGISTRY
class PapersClassificationPipeline(Pipeline):
def _sanitize_parameters(self, **kwargs):
return {}, {}, {}
def preprocess(self, inputs):
if (
not isinstance(inputs, tp.Iterable)
or isinstance(inputs, tp.Dict)
or isinstance(inputs, str)
):
inputs = [inputs]
title = "title"
authors = "authors"
abstract = "abstract"
texts = [
(
f"AUTHORS: {' '.join(paper[title]) if isinstance(paper[authors], list) else paper[authors]} "
f"TITLE: {paper[title]} ABSTRACT: {paper[abstract]}"
if not isinstance(paper, str)
else paper
)
for paper in inputs
]
inputs = self.tokenizer(
texts, truncation=True, padding=True, max_length=256, return_tensors="pt"
).to(self.device)
return inputs
def _forward(self, model_inputs):
with torch.no_grad():
outputs = self.model(**model_inputs)
return outputs
def postprocess(self, model_outputs):
probs = torch.nn.functional.softmax(model_outputs.logits, dim=-1)
results = []
for prob in probs:
result = [
{"label": self.model.config.id2label[label_idx], "score": score.item()}
for label_idx, score in enumerate(prob)
]
results.append(result)
if 1 == len(results):
return results[0]
return results
PIPELINE_REGISTRY.register_pipeline(
"paper-classification",
pipeline_class=PapersClassificationPipeline,
pt_model=AutoModelForSequenceClassification,
)
|