Update README.md
Browse files
README.md
CHANGED
@@ -1,4 +1,57 @@
|
|
1 |
---
|
2 |
license: mit
|
3 |
pipeline_tag: token-classification
|
4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
---
|
2 |
license: mit
|
3 |
pipeline_tag: token-classification
|
4 |
+
language:
|
5 |
+
- en
|
6 |
+
---\
|
7 |
+
|
8 |
+
|
9 |
+
## Usage
|
10 |
+
|
11 |
+
```python
|
12 |
+
|
13 |
+
from transformers import AutoModelForTokenClassification, AutoTokenizer
|
14 |
+
import torch
|
15 |
+
|
16 |
+
|
17 |
+
model = AutoModelForTokenClassification.from_pretrained('Sinanmz/toxicity_token_classifier')
|
18 |
+
tokenizer = AutoTokenizer.from_pretrained('Sinanmz/toxicity_token_classifier')
|
19 |
+
|
20 |
+
def test_model(text):
|
21 |
+
inputs = tokenizer(text, return_tensors='pt')
|
22 |
+
with torch.no_grad():
|
23 |
+
outputs = model(**inputs)
|
24 |
+
logits = outputs.logits
|
25 |
+
predictions = np.argmax(logits.detach().numpy(), axis=2)
|
26 |
+
tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
|
27 |
+
labels = predictions[0]
|
28 |
+
labels = labels[1:-1]
|
29 |
+
tokens = tokens[1:-1]
|
30 |
+
result = []
|
31 |
+
for i in range(len(labels)):
|
32 |
+
if i > 0 and inputs.word_ids()[i+1] == inputs.word_ids()[i]:
|
33 |
+
result.popitem()
|
34 |
+
result.append((tokens[i-1] + tokens[i][2:], model.config.id2label[labels[i-1]]))
|
35 |
+
else:
|
36 |
+
result.append((tokens[i], model.config.id2label[labels[i]]))
|
37 |
+
return result
|
38 |
+
|
39 |
+
|
40 |
+
text1 = 'Your face is disgusting.'
|
41 |
+
print("Result:", test_model(text1))
|
42 |
+
# output:
|
43 |
+
# Result: {'your': 'none', 'face': 'none', 'is': 'none', 'disgusting': 'other toxicity', '.': 'none'}
|
44 |
+
|
45 |
+
|
46 |
+
text2 = 'What an ugly person you are.'
|
47 |
+
print("Result:", test_model(text2))
|
48 |
+
# output:
|
49 |
+
# Result: {'what': 'none', 'an': 'none', 'ugly': 'insult', 'person': 'none', 'you': 'none', 'are': 'none', '.': 'none'}
|
50 |
+
|
51 |
+
|
52 |
+
text3 = 'Nice to meet you, sir.'
|
53 |
+
print("Result:", test_model(text3))
|
54 |
+
# output:
|
55 |
+
# Result: {'nice': 'none', 'to': 'none', 'meet': 'none', 'you': 'none', ',': 'none', 'sir': 'none', '.': 'none'}
|
56 |
+
|
57 |
+
```
|