Fernando Carneiro
commited on
Commit
•
1332e64
1
Parent(s):
07fab57
README
Browse files
README.md
CHANGED
|
@@ -30,9 +30,15 @@ with torch.no_grad():
|
|
| 30 |
### Normalize raw input Tweets
|
| 31 |
|
| 32 |
```python
|
|
|
|
| 33 |
import torch
|
| 34 |
from transformers import AutoModel, AutoTokenizer
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
|
| 37 |
```python
|
| 38 |
from transformers import pipeline
|
|
|
|
| 30 |
### Normalize raw input Tweets
|
| 31 |
|
| 32 |
```python
|
| 33 |
+
from emoji import demojize
|
| 34 |
import torch
|
| 35 |
from transformers import AutoModel, AutoTokenizer
|
| 36 |
|
| 37 |
+
model = AutoModel.from_pretrained('melll-uff/bertweetbr')
|
| 38 |
+
tokenizer = AutoTokenizer.from_pretrained('melll-uff/bertweetbr', normalization=False)
|
| 39 |
+
|
| 40 |
+
tokenizer.demojizer = lambda x: demojize(x, language='pt')
|
| 41 |
+
```
|
| 42 |
|
| 43 |
```python
|
| 44 |
from transformers import pipeline
|