add files

added_tokens.json

@@ -0,0 +1 @@
+{"<el>": 250017, "</de>": 250055, "</es>": 250065, "<tl>": 250008, "<es>": 250026, "<id>": 250005, "<it>": 250024, "<ms>": 250007, "</zh>": 250074, "</sw>": 250072, "<ar>": 250002, "</fr>": 250062, "<fi>": 250039, "<bg>": 250027, "<hi>": 250019, "</ko>": 250070, "<he>": 250003, "</et>": 250077, "<th>": 250032, "</te>": 250051, "</he>": 250042, "</jv>": 250045, "<zh>": 250035, "</id>": 250044, "<hu>": 250040, "<jv>": 250006, "<de>": 250016, "<ru>": 250028, "<my>": 250034, "<ko>": 250031, "<ur>": 250021, "</ja>": 250068, "<ka>": 250030, "</nl>": 250053, "<bn>": 250018, "</bg>": 250066, "<sw>": 250033, "</ms>": 250046, "<ml>": 250010, "</en>": 250054, "<vi>": 250004, "</th>": 250071, "</af>": 250052, "</fi>": 250078, "<ja>": 250029, "</mr>": 250059, "</ru>": 250067, "<af>": 250013, "</ka>": 250069, "<eu>": 250009, "</tr>": 250076, "<te>": 250012, "<tr>": 250037, "<ta>": 250011, "</pt>": 250064, "</ml>": 250049, "</hu>": 250079, "<nl>": 250014, "</it>": 250063, "<mr>": 250020, "</kk>": 250075, "</tl>": 250047, "</vi>": 250043, "</my>": 250073, "</hi>": 250058, "<fr>": 250023, "</eu>": 250048, "</ur>": 250060, "<kk>": 250036, "</bn>": 250057, "</ar>": 250041, "</ta>": 250050, "<fa>": 250022, "</el>": 250056, "<et>": 250038, "</fa>": 250061, "<en>": 250015, "<pt>": 250025}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "xlm-roberta-base",
+  "architectures": [
+    "XLMRobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.15.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250080
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:441dd5e5704786dc16c46ad8605be8157e62296f9045e75f695cfd8ca2d5fba7
+size 1113511570

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["<ar>", "<he>", "<vi>", "<id>", "<jv>", "<ms>", "<tl>", "<eu>", "<ml>", "<ta>", "<te>", "<af>", "<nl>", "<en>", "<de>", "<el>", "<bn>", "<hi>", "<mr>", "<ur>", "<fa>", "<fr>", "<it>", "<pt>", "<es>", "<bg>", "<ru>", "<ja>", "<ka>", "<ko>", "<th>", "<sw>", "<my>", "<zh>", "<kk>", "<tr>", "<et>", "<fi>", "<hu>", "</ar>", "</he>", "</vi>", "</id>", "</jv>", "</ms>", "</tl>", "</eu>", "</ml>", "</ta>", "</te>", "</af>", "</nl>", "</en>", "</de>", "</el>", "</bn>", "</hi>", "</mr>", "</ur>", "</fa>", "</fr>", "</it>", "</pt>", "</es>", "</bg>", "</ru>", "</ja>", "</ka>", "</ko>", "</th>", "</sw>", "</my>", "</zh>", "</kk>", "</tr>", "</et>", "</fi>", "</hu>"]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "cls_token": "<s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "xlm-roberta-base", "tokenizer_class": "XLMRobertaTokenizer"}