Trained model

README.md

@@ -0,0 +1,42 @@
+---
+language: sv
+widget:
+- text: "Det var en gång"
+---
+
+# Nordic GPT2--wikipedia
+A Nordic GPT2 style model trained using Flax CLM pipeline on the Nordic parts
+part of the wiki40b dataset.
+
+https://huggingface.co/datasets/wiki40b
+
+## Data cleaning and preprocessing
+The data was cleaned and preprocessed using the following script. Make sure to install depencies for beam_runner to make the dataset work.
+
+```python
+from datasets import load_dataset
+def load_and_clean_wiki():
+    dataset = load_dataset('wiki40b', 'da', beam_runner='DirectRunner', split="train")
+    #dataset = load_dataset('wiki40b', 'sv', beam_runner='DirectRunner')
+    dataset = dataset.remove_columns(['wikidata_id', 'version_id'])
+    filtered_dataset = dataset.map(filter_wikipedia)
+    # filtered_dataset[:3]
+    # print(filtered_dataset[:3])
+    return filtered_dataset
+
+def filter_wikipedia(batch):
+    batch["text"] = " ".join(batch["text"].split("\n_START_SECTION_\n"))
+    batch["text"] = " ".join(batch["text"].split("\n_START_ARTICLE_\n"))
+    batch["text"] = " ".join(batch["text"].split("\n_START_ARTICLE_\n"))
+    batch["text"] = " ".join(batch["text"].split("\n_START_PARAGRAPH_\n"))
+    batch["text"] = " ".join(batch["text"].split("_NEWLINE_"))
+    batch["text"] = " ".join(batch["text"].split("\xa0"))
+    return batch
+```
+
+## Training script
+The following training script was used to train the model.
+```bash
+./run_clm_flax.py     --output_dir="${MODEL_DIR}"     --model_type="gpt2"     --config_name="${MODEL_DIR}"     --tokenizer_name="${MODEL_DIR}"     --dataset_name="wiki40b"     --dataset_config_name="da"     --do_train --do_eval     --block_size="512"     --per_device_train_batch_size="64"     --per_device_eval_batch_size="64"     --learning_rate="5e-3" --warmup_steps="1000"     --adam_beta1="0.9" --adam_beta2="0.98" --weight_decay="0.01"     --overwrite_output_dir     --num_train_epochs="20"     --logging_steps="500"     --save_steps="1000"     --eval_steps="2500"     --push_to_hub
+```
+

added_tokens.json

@@ -0,0 +1 @@
+{"<|endoftext|>": 50265}

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": ".",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -30,6 +31,7 @@
       "max_length": 50
     }
   },
+  "torch_dtype": "float32",
   "transformers_version": "4.9.0.dev0",
   "use_cache": true,
   "vocab_size": 50257

evaluate.py

@@ -0,0 +1,10 @@
+from transformers import GPT2Tokenizer, GPT2Model, FlaxGPT2LMHeadModel, GPT2LMHeadModel, pipeline, set_seed
+ 
+tokenizer = GPT2Tokenizer.from_pretrained("flax-community/dansk-gpt-wiki")
+model = GPT2LMHeadModel.from_pretrained("flax-community/dansk-gpt-wiki")
+
+
+generator = pipeline('text-generation', model=model, tokenizer=tokenizer)
+set_seed(42)
+result = generator("Jeg elsker livet", max_length=150, num_return_sequences=5)
+print(result)

events.out.tfevents.1626215918.t1v-n-fa4889f7-w-0.111814.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cae02b91942cdea2386a3e40840b7a82be5d34869cda0a680f9c65862b39fd8
-size 441341
+oid sha256:caa96f036315b3b0e97359554d139bde12a9d85853d6c7c0a7fa45ab3dac4c9d
+size 2502509

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b37e16c6cabb3f3fd1d54f0377c3c1f4ef4bce7f27ad14ca90a617c7d59334c
+oid sha256:42ad41602565ae10d10d8856d5ad96a4a09ab2ddeb3ebd3817363c0ae9acef1a
 size 497764120

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64335acf6fedf3435fa4e2a674a813ab8d26e8306485254b730c903befe54ebf
+size 510401385

save_model.py

@@ -0,0 +1,8 @@
+from transformers import AutoTokenizer, GPT2LMHeadModel
+'''
+This is a script to convert the Jax model and the tokenizer to Pytorch model
+'''
+model = GPT2LMHeadModel.from_pretrained(".", from_flax=True)
+model.save_pretrained(".")
+tokenizer = AutoTokenizer.from_pretrained(".")
+tokenizer.save_pretrained(".")

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "special_tokens_map_file": null, "name_or_path": ".", "tokenizer_class": "GPT2Tokenizer"}