Initial commit

README.md

@@ -0,0 +1,28 @@
+---
+language: is
+widget:
+- text: Má bjóða þér <mask> í kvöld?
+- text: Forseti <mask> er ágæt.
+- text: Súpan var <mask> á bragðið.
+tags:
+- roberta
+- icelandic
+- masked-lm
+- pytorch
+license: gpl-3.0
+---
+
+# IceBERT
+
+IceBERT was trained with fairseq using the RoBERTa-base model. The training data used is shown in the table below.
+
+| Dataset                                              | Size    | Tokens |
+|------------------------------------------------------|---------|--------|
+| Icelandic Gigaword Corpus v20.05 (IGC)               | 8.2 GB  | 1,388M |
+| Icelandic Common Crawl Corpus (IC3)                  | 4.9 GB  | 824M   |
+| Greynir News articles                                | 456 MB  | 76M    |
+| Icelandic Sagas                                      | 9 MB    | 1.7M   |
+| Open Icelandic e-books (Rafbókavefurinn)             | 14 MB   | 2.6M   |
+| Data from the medical library of Landspitali         | 33 MB   | 5.2M   |
+| Student theses from Icelandic universities (Skemman) | 2.2 GB  | 367M   |
+| Total                                                | 15.8 GB | 2,664M |

config.json

@@ -0,0 +1,22 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "type_vocab_size": 1,
+  "vocab_size": 50000
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6a596f01b857cbcdf749b91a5746fddcfa11f41042717d4e23fc3a471012b62
+size 651861218

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}}