Fix
Browse files
README.md
CHANGED
@@ -26,8 +26,11 @@ Simple translation using Python:
|
|
26 |
```python
|
27 |
|
28 |
import ctranslate2
|
29 |
-
|
30 |
-
|
|
|
|
|
|
|
31 |
[[{'tokens': ['▁Hola', '▁món', '!']}]]
|
32 |
|
33 |
```
|
@@ -36,13 +39,16 @@ Simple tokenization & translation using Python:
|
|
36 |
|
37 |
|
38 |
```python
|
39 |
-
|
40 |
import pyonmttok
|
41 |
-
|
|
|
|
|
|
|
42 |
tokenized=tokenizer.tokenize("Hello world!")
|
43 |
|
44 |
import ctranslate2
|
45 |
-
translator = ctranslate2.Translator(
|
46 |
translated = translator.translate_batch([tokenized[0]])
|
47 |
print(tokenizer.detokenize(translated[0][0]['tokens']))
|
48 |
Hola món!
|
|
|
26 |
```python
|
27 |
|
28 |
import ctranslate2
|
29 |
+
from huggingface_hub import snapshot_download
|
30 |
+
model_dir = snapshot_download(repo_id="softcatala/opennmt-eng-cat", revision="main")
|
31 |
+
|
32 |
+
translator = ctranslate2.Translator(model_dir)
|
33 |
+
print(translator.translate_batch([["▁Hello", "▁world", "!"]]))
|
34 |
[[{'tokens': ['▁Hola', '▁món', '!']}]]
|
35 |
|
36 |
```
|
|
|
39 |
|
40 |
|
41 |
```python
|
42 |
+
import ctranslate2
|
43 |
import pyonmttok
|
44 |
+
from huggingface_hub import snapshot_download
|
45 |
+
model_dir = snapshot_download(repo_id="softcatala/opennmt-eng-cat", revision="main")
|
46 |
+
|
47 |
+
tokenizer=pyonmttok.Tokenizer(mode="none", sp_model_path = model_dir + "/sp_m.model")
|
48 |
tokenized=tokenizer.tokenize("Hello world!")
|
49 |
|
50 |
import ctranslate2
|
51 |
+
translator = ctranslate2.Translator(model_dir)
|
52 |
translated = translator.translate_batch([tokenized[0]])
|
53 |
print(tokenizer.detokenize(translated[0][0]['tokens']))
|
54 |
Hola món!
|