Update README.md

README.md

@@ -1,208 +1,208 @@
 ---
 license: cc-by-nc-4.0
 language:
-  - ace   # ace_Arab Acehnese (Arabic script)
-  - ace   # ace_Latn Acehnese (Latin script)
-  - acm  # acm_Arab Mesopotamian Arabic
-  - acq  # acq_Arab Ta’izzi-Adeni Arabic
-  - aeb  # aeb_Arab Tunisian Arabic
-  - af   # afr_Latn Afrikaans
-  - ajp  # ajp_Arab South Levantine Arabic
-  - ak   # aka_Latn Akan
-  - am   # amh_Ethi Amharic
-  - apc  # apc_Arab North Levantine Arabic
-  - ar   # arb_Arab Modern Standard Arabic
-  - ars  # ars_Arab Najdi Arabic
-  - ary  # ary_Arab Moroccan Arabic
-  - arz  # arz_Arab Egyptian Arabic
-  - as   # asm_Beng Assamese
-  - ast  # ast_Latn Asturian
-  - awa  # awa_Deva Awadhi
-  - ay   # ayr_Latn Central Aymara
-  - azb  # azb_Arab South Azerbaijani
-  - azj  # azj_Latn North Azerbaijani
-  - ba   # bak_Cyrl Bashkir
-  - bm   # bam_Latn Bambara
-  - ban  # ban_Latn Balinese
-  - be   # bel_Cyrl Belarusian
-  - bem  # bem_Latn Bemba
-  - bn   # ben_Beng Bengali
-  - bho  # bho_Deva Bhojpuri
-  - bjn  # bjn_Arab Banjar (Arabic script)
-  - bjn  # bjn_Latn Banjar (Latin script)
-  - bo   # bod_Tibt Standard Tibetan
-  - bs   # bos_Latn Bosnian
-  - bug  # bug_Latn Buginese
-  - bg   # bul_Cyrl Bulgarian
-  - ca   # cat_Latn Catalan
-  - ceb  # ceb_Latn Cebuano
-  - cs   # ces_Latn Czech
-  - cjk  # cjk_Latn Chokwe
-  - ckb  # ckb_Arab Central Kurdish
-  - crh  # crh_Latn Crimean Tatar
-  - cy   # cym_Latn Welsh
-  - da   # dan_Latn Danish
-  - de   # deu_Latn German
-  - dik  # dik_Latn Southwestern Dinka
-  - dyu  # dyu_Latn Dyula
-  - dz   # dzo_Tibt Dzongkha
-  - el   # ell_Grek Greek
-  - en   # eng_Latn English
-  - eo   # epo_Latn Esperanto
-  - et   # est_Latn Estonian
-  - eu   # eus_Latn Basque
-  - ee   # ewe_Latn Ewe
-  - fo   # fao_Latn Faroese
-  - fa   # pes_Arab Western Persian
-  - fj   # fij_Latn Fijian
-  - fi   # fin_Latn Finnish
-  - fon  # fon_Latn Fon
-  - fr   # fra_Latn French
-  - fur  # fur_Latn Friulian
-  - ff   # fuv_Latn Nigerian Fulfulde
-  - gd   # gla_Latn Scottish Gaelic
-  - ga   # gle_Latn Irish
-  - gl   # glg_Latn Galician
-  - gn   # grn_Latn Guarani
-  - gu   # guj_Gujr Gujarati
-  - ht   # hat_Latn Haitian Creole
-  - ha   # hau_Latn Hausa
-  - he   # heb_Hebr Hebrew
-  - hi   # hin_Deva Hindi
-  - hne  # hne_Deva Chhattisgarhi
-  - hr   # hrv_Latn Croatian
-  - hu   # hun_Latn Hungarian
-  - hy   # hye_Armn Armenian
-  - ig   # ibo_Latn Igbo
-  - ilo  # ilo_Latn Ilocano
-  - id   # ind_Latn Indonesian
-  - is   # isl_Latn Icelandic
-  - it   # ita_Latn Italian
-  - jv   # jav_Latn Javanese
-  - ja   # jpn_Jpan Japanese
-  - kab  # kab_Latn Kabyle
-  - kac  # kac_Latn Jingpho
-  - kam  # kam_Latn Kamba
-  - kn   # kan_Knda Kannada
-  - ks   # kas_Arab Kashmiri (Arabic script)
-  - ks   # kas_Deva Kashmiri (Devanagari script)
-  - ka   # kat_Geor Georgian
-  - kr   # knc_Arab Central Kanuri (Arabic script)
-  - kr   # knc_Latn Central Kanuri (Latin script)
-  - kk   # kaz_Cyrl Kazakh
-  - kbp  # kbp_Latn Kabiyè
-  - kea  # kea_Latn Kabuverdianu
-  - km   # khm_Khmr Khmer
-  - ki   # kik_Latn Kikuyu
-  - rw   # kin_Latn Kinyarwanda
-  - ky   # kir_Cyrl Kyrgyz
-  - kmb  # kmb_Latn Kimbundu
-  - kg   # kon_Latn Kikongo
-  - ko   # kor_Hang Korean
-  - kmr  # kmr_Latn Northern Kurdish
-  - lo   # lao_Laoo Lao
-  - lv   # lvs_Latn Standard Latvian
-  - lij  # lij_Latn Ligurian
-  - li   # lim_Latn Limburgish
-  - ln   # lin_Latn Lingala
-  - lt   # lit_Latn Lithuanian
-  - lmo  # lmo_Latn Lombard
-  - ltg  # ltg_Latn Latgalian
-  - lb   # ltz_Latn Luxembourgish
-  - lua  # lua_Latn Luba-Kasai
-  - lg   # lug_Latn Ganda
-  - luo  # luo_Latn Luo
-  - lus  # lus_Latn Mizo
-  - mag  # mag_Deva Magahi
-  - mai  # mai_Deva Maithili
-  - ml   # mal_Mlym Malayalam
-  - mr   # mar_Deva Marathi
-  - min  # min_Latn Minangkabau (Latin script)
-  - mk   # mkd_Cyrl Macedonian
-  - plt  # plt_Latn Plateau Malagasy
-  - mt   # mlt_Latn Maltese
-  - mni  # mni_Beng Meitei (Bengali script)
-  - mn   # khk_Cyrl Halh Mongolian
-  - mos  # mos_Latn Mossi
-  - mi   # mri_Latn Maori
-  - ms   # zsm_Latn Standard Malay
-  - my   # mya_Mymr Burmese
-  - nl   # nld_Latn Dutch
-  - nn   # nno_Latn Norwegian Nynorsk
-  - nb   # nob_Latn Norwegian Bokmål
-  - ne   # npi_Deva Nepali
-  - nso  # nso_Latn Northern Sotho
-  - nus  # nus_Latn Nuer
-  - ny   # nya_Latn Nyanja
-  - oc   # oci_Latn Occitan
-  - gaz  # gaz_Latn West Central Oromo
-  - ory  # ory_Orya Odia
-  - pag  # pag_Latn Pangasinan
-  - pa   # pan_Guru Eastern Panjabi
-  - pap  # pap_Latn Papiamento
-  - pl   # pol_Latn Polish
-  - pt   # por_Latn Portuguese
-  - prs  # prs_Arab Dari
-  - pbt  # pbt_Arab Southern Pashto
-  - qu   # quy_Latn Ayacucho Quechua
-  - ro   # ron_Latn Romanian
-  - rn   # run_Latn Rundi
-  - ru   # rus_Cyrl Russian
-  - sg   # sag_Latn Sango
-  - sa   # san_Deva Sanskrit
-  - sat  # sat_Beng ?
-  - scn  # scn_Latn Sicilian
-  - shn  # shn_Mymr Shan
-  - si   # sin_Sinh Sinhala
-  - sk   # slk_Latn Slovak
-  - sl   # slv_Latn Slovenian
-  - sm   # smo_Latn Samoan
-  - sn   # sna_Latn Shona
-  - sd   # snd_Arab Sindhi
-  - so   # som_Latn Somali
-  - st   # sot_Latn Southern Sotho
-  - es   # spa_Latn Spanish
-  - als  # als_Latn Tosk Albanian
-  - sc   # srd_Latn Sardinian
-  - sr   # srp_Cyrl Serbian
-  - ss   # ssw_Latn Swati
-  - su   # sun_Latn Sundanese
-  - sv   # swe_Latn Swedish
-  - sw   # swh_Latn Swahili
-  - szl  # szl_Latn Silesian
-  - ta   # tam_Taml Tamil
-  - tt   # tat_Cyrl Tatar
-  - te   # tel_Telu Telugu
-  - tg   # tgk_Cyrl Tajik
-  - tl   # tgl_Latn Tagalog
-  - th   # tha_Thai Thai
-  - ti   # tir_Ethi Tigrinya
-  - taq  # taq_Latn Tamasheq (Latin script)
-  - taq  # taq_Tfng Tamasheq (Tifinagh script)
-  - tpi  # tpi_Latn Tok Pisin
-  - tn   # tsn_Latn Tswana
-  - ts   # tso_Latn Tsonga
-  - tk   # tuk_Latn Turkmen
-  - tum  # tum_Latn Tumbuka
-  - tr   # tur_Latn Turkish
-  - tw   # twi_Latn Twi
-  - tzm  # tzm_Tfng Central Atlas Tamazight
-  - ug   # uig_Arab Uyghur
-  - uk   # ukr_Cyrl Ukrainian
-  - umb  # umb_Latn Umbundu
-  - ur   # urd_Arab Urdu
-  - uz   # uzn_Latn Northern Uzbek
-  - vec  # vec_Latn Venetian
-  - vi   # vie_Latn Vietnamese
-  - war  # war_Latn Waray
-  - wo   # wol_Latn Wolof
-  - xh   # xho_Latn Xhosa
-  - yi   # ydd_Hebr Eastern Yiddish
-  - yo   # yor_Latn Yoruba
-  - yue  # yue_Hant Yue Chinese
-  - zh   # zho_Hans Chinese (Simplified)
-  - zh   # zho_Hant Chinese (Traditional)
-  - zu   # zul_Latn Zulu
+- ace
+- ace
+- acm
+- acq
+- aeb
+- af
+- ajp
+- ak
+- am
+- apc
+- ar
+- ars
+- ary
+- arz
+- as
+- ast
+- awa
+- ay
+- azb
+- azj
+- ba
+- bm
+- ban
+- be
+- bem
+- bn
+- bho
+- bjn
+- bjn
+- bo
+- bs
+- bug
+- bg
+- ca
+- ceb
+- cs
+- cjk
+- ckb
+- crh
+- cy
+- da
+- de
+- dik
+- dyu
+- dz
+- el
+- en
+- eo
+- et
+- eu
+- ee
+- fo
+- fa
+- fj
+- fi
+- fon
+- fr
+- fur
+- ff
+- gd
+- ga
+- gl
+- gn
+- gu
+- ht
+- ha
+- he
+- hi
+- hne
+- hr
+- hu
+- hy
+- ig
+- ilo
+- id
+- is
+- it
+- jv
+- ja
+- kab
+- kac
+- kam
+- kn
+- ks
+- ks
+- ka
+- kr
+- kr
+- kk
+- kbp
+- kea
+- km
+- ki
+- rw
+- ky
+- kmb
+- kg
+- ko
+- kmr
+- lo
+- lv
+- lij
+- li
+- ln
+- lt
+- lmo
+- ltg
+- lb
+- lua
+- lg
+- luo
+- lus
+- mag
+- mai
+- ml
+- mr
+- min
+- mk
+- plt
+- mt
+- mni
+- mn
+- mos
+- mi
+- ms
+- my
+- nl
+- nn
+- nb
+- ne
+- nso
+- nus
+- ny
+- oc
+- gaz
+- ory
+- pag
+- pa
+- pap
+- pl
+- pt
+- prs
+- pbt
+- qu
+- ro
+- rn
+- ru
+- sg
+- sa
+- sat
+- scn
+- shn
+- si
+- sk
+- sl
+- sm
+- sn
+- sd
+- so
+- st
+- es
+- als
+- sc
+- sr
+- ss
+- su
+- sv
+- sw
+- szl
+- ta
+- tt
+- te
+- tg
+- tl
+- th
+- ti
+- taq
+- taq
+- tpi
+- tn
+- ts
+- tk
+- tum
+- tr
+- tw
+- tzm
+- ug
+- uk
+- umb
+- ur
+- uz
+- vec
+- vi
+- war
+- wo
+- xh
+- yi
+- yo
+- yue
+- zh
+- zh
+- zu
 language_details: >-
   ace_Arab, ace_Latn, acm_Arab, acq_Arab, aeb_Arab, afr_Latn, ajp_Arab,
   aka_Latn, amh_Ethi, apc_Arab, arb_Arab, ars_Arab, ary_Arab, arz_Arab,
@@ -233,16 +233,18 @@ language_details: >-
   tur_Latn, twi_Latn, tzm_Tfng, uig_Arab, ukr_Cyrl, umb_Latn, urd_Arab,
   uzn_Latn, vec_Latn, vie_Latn, war_Latn, wol_Latn, xho_Latn, ydd_Hebr,
   yor_Latn, yue_Hant, zho_Hans, zho_Hant, zul_Latn
+pipeline_tag: sentence-similarity
 ---
-This is a port of the multilingual text encoder from https://huggingface.co/facebook/SONAR to `transformers` format from `fairseq2`.
+This is a port of the multilingual SONAR text encoder (https://huggingface.co/facebook/SONAR) to the `transformers` format from `fairseq2`.
 
-It supports the same 202 languages as [NLLB-200](https://huggingface.co/facebook/nllb-200-distilled-600M) 
+Its embeddings are expected be equal to those the official implementation (https://github.com/facebookresearch/SONAR), but the latter stays the source of truth.
+
+The encoder supports the same 202 languages as [NLLB-200](https://huggingface.co/facebook/nllb-200-distilled-600M) 
 (see also [the source model card](https://github.com/facebookresearch/SONAR/blob/main/sonar/store/cards/text_sonar_basic_encoder.yaml#L14)
 and [FLORES-200 lang code mapping](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200)). 
 
-For advanced examples of usage, please take a look at https://github.com/facebookresearch/SONAR.
 
-How to use: 
+How to compute embeddings: 
 ```Python
 # !pip install transformers sentencepiece -q
 
@@ -272,4 +274,6 @@ print(embs.shape)
 print(embs)
 # tensor([[-0.0053,  0.0020, -0.0006,  ...,  0.0094, -0.0009,  0.0070],
 #         [-0.0003, -0.0071,  0.0076,  ...,  0.0055,  0.0022, -0.0083]])
-```
+```
+
+For advanced examples of usage, please take a look at the readme in https://github.com/facebookresearch/SONAR.