Commit
•
cf89623
1
Parent(s):
6a7e149
Update README.md
Browse files
README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1 |
-
# Twitter-roBERTa-base
|
2 |
|
3 |
This is a roBERTa-base model trained on ~58M tweets and finetuned for the Offensive Language Identification task at Semeval 2019.
|
4 |
For full description: [_TweetEval_ benchmark (Findings of EMNLP 2020)](https://arxiv.org/pdf/2010.12421.pdf).
|
@@ -15,6 +15,15 @@ from scipy.special import softmax
|
|
15 |
import csv
|
16 |
import urllib.request
|
17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18 |
# Tasks:
|
19 |
# emoji, emotion, hate, irony, offensive, sentiment
|
20 |
# stance/abortion, stance/atheism, stance/climate, stance/feminist, stance/hillary
|
@@ -37,6 +46,7 @@ model = AutoModelForSequenceClassification.from_pretrained(MODEL)
|
|
37 |
model.save_pretrained(MODEL)
|
38 |
|
39 |
text = "Good night 😊"
|
|
|
40 |
encoded_input = tokenizer(text, return_tensors='pt')
|
41 |
output = model(**encoded_input)
|
42 |
scores = output[0][0].detach().numpy()
|
|
|
1 |
+
# Twitter-roBERTa-base for Offensive Language Identification
|
2 |
|
3 |
This is a roBERTa-base model trained on ~58M tweets and finetuned for the Offensive Language Identification task at Semeval 2019.
|
4 |
For full description: [_TweetEval_ benchmark (Findings of EMNLP 2020)](https://arxiv.org/pdf/2010.12421.pdf).
|
|
|
15 |
import csv
|
16 |
import urllib.request
|
17 |
|
18 |
+
# Preprocess text (username and link placeholders)
|
19 |
+
def preprocess(text):
|
20 |
+
new_text = []
|
21 |
+
for t in text.split(" "):
|
22 |
+
t = '@user' if t.startswith('@') and len(t) > 1 else t
|
23 |
+
t = 'http' if t.startswith('http') else t
|
24 |
+
new_text.append(t)
|
25 |
+
return " ".join(new_text)
|
26 |
+
|
27 |
# Tasks:
|
28 |
# emoji, emotion, hate, irony, offensive, sentiment
|
29 |
# stance/abortion, stance/atheism, stance/climate, stance/feminist, stance/hillary
|
|
|
46 |
model.save_pretrained(MODEL)
|
47 |
|
48 |
text = "Good night 😊"
|
49 |
+
text = preprocess(text)
|
50 |
encoded_input = tokenizer(text, return_tensors='pt')
|
51 |
output = model(**encoded_input)
|
52 |
scores = output[0][0].detach().numpy()
|