Uploading testfile

README.md

@@ -0,0 +1,71 @@
+---
+tags:
+- generated_from_trainer
+datasets:
+- ottomoritz/TriboliumCastaneum
+model-index:
+- name: your_output_dir
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# your_output_dir
+
+This model is a fine-tuned version of [](https://huggingface.co/) on the ottomoritz/TriboliumCastaneum dataset.
+It achieves the following results on the evaluation set:
+- Loss: nan
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.001
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: constant_with_warmup
+- lr_scheduler_warmup_steps: 1000
+- training_steps: 120000
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch  | Step   | Validation Loss |
+|:-------------:|:------:|:------:|:---------------:|
+| 1.217         | 0.0833 | 10000  | nan             |
+| 1.1568        | 0.1667 | 20000  | nan             |
+| 1.1522        | 0.25   | 30000  | nan             |
+| 1.1443        | 0.3333 | 40000  | nan             |
+| 1.1404        | 0.4167 | 50000  | nan             |
+| 1.1329        | 0.5    | 60000  | nan             |
+| 1.1323        | 0.5833 | 70000  | nan             |
+| 1.1292        | 0.6667 | 80000  | nan             |
+| 1.1264        | 0.75   | 90000  | nan             |
+| 1.1312        | 0.8333 | 100000 | nan             |
+| 1.1305        | 0.9167 | 110000 | nan             |
+| 1.1285        | 1.0    | 120000 | nan             |
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.2
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 1.0,
+    "eval_loss": NaN,
+    "eval_runtime": 571.893,
+    "eval_samples_per_second": 241.171,
+    "eval_steps_per_second": 60.293,
+    "perplexity": NaN,
+    "total_flos": 9.714412005674189e+16,
+    "train_loss": 1.1434922200520834,
+    "train_runtime": 17035.2815,
+    "train_samples_per_second": 28.177,
+    "train_steps_per_second": 7.044
+}

checkpoint-10000/config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "ConvNetForMaskedLM"
+  ],
+  "aux_features_vocab_size": 5,
+  "dilation_base": 2,
+  "dilation_cycle": 6,
+  "dilation_double_every": 1,
+  "dilation_max": 32,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "kernel_size": 9,
+  "model_type": "ConvNet",
+  "n_aux_features": 0,
+  "n_layers": 25,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.2",
+  "vocab_size": 7
+}

checkpoint-10000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99c73170894e6f908793e7d8092ac085344046391e384a526242a89c2b1274f5
+size 263540548

checkpoint-10000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:867b4b6c09b5edc24d783718bd9aa13d088eaceb78d285e0c4529e6711ea98f2
+size 527212602

checkpoint-10000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec915ecc453dbbfdd0ab75a49b4e4d24df74862ad77a9f68dc03bfa3b2a11f70
+size 14244

checkpoint-10000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:651c17881739fb1e81c6ac9a2b03392313a3e3f597c8803b041c9a31b8c0d76c
+size 1064

checkpoint-10000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-10000/tokenizer.json

@@ -0,0 +1,62 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Lowercase"
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[MASK]": 1,
+      "[UNK]": 2,
+      "a": 3,
+      "c": 4,
+      "g": 5,
+      "t": 6
+    },
+    "merges": []
+  }
+}

checkpoint-10000/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}

checkpoint-10000/trainer_state.json

@@ -0,0 +1,48 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.08333333333333333,
+  "eval_steps": 10000,
+  "global_step": 10000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08333333333333333,
+      "grad_norm": 0.6651873588562012,
+      "learning_rate": 0.001,
+      "loss": 1.217,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 874.824,
+      "eval_samples_per_second": 157.659,
+      "eval_steps_per_second": 39.415,
+      "step": 10000
+    }
+  ],
+  "logging_steps": 10000,
+  "max_steps": 120000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8095343338061824.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-10000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e24d240c0212150205bb434684f1d705979477cac1d7cdb25ba76821568db2
+size 5112

checkpoint-100000/config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "ConvNetForMaskedLM"
+  ],
+  "aux_features_vocab_size": 5,
+  "dilation_base": 2,
+  "dilation_cycle": 6,
+  "dilation_double_every": 1,
+  "dilation_max": 32,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "kernel_size": 9,
+  "model_type": "ConvNet",
+  "n_aux_features": 0,
+  "n_layers": 25,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.2",
+  "vocab_size": 7
+}

checkpoint-100000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d97c12a35b705d932e7aa1aeff8c96c09c4774c700a49fadc0dd9772b8eeea7f
+size 263540548

checkpoint-100000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f82bda1bf25f895adbe9376bae9508b34618f4b525151e86e7bf99f6c259698
+size 527212602

checkpoint-100000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56038c7844026d477cf6aa7ee2ca7b5dfd712a7bb2ab5d1bd89d0285c1140839
+size 14244

checkpoint-100000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0f585e27de5d2f3265c6a9c63d374ed84caea636d087de5c766dcffdefe1484
+size 1064

checkpoint-100000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-100000/tokenizer.json

@@ -0,0 +1,62 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Lowercase"
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[MASK]": 1,
+      "[UNK]": 2,
+      "a": 3,
+      "c": 4,
+      "g": 5,
+      "t": 6
+    },
+    "merges": []
+  }
+}

checkpoint-100000/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}

checkpoint-100000/trainer_state.json

@@ -0,0 +1,183 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8333333333333334,
+  "eval_steps": 10000,
+  "global_step": 100000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08333333333333333,
+      "grad_norm": 0.6651873588562012,
+      "learning_rate": 0.001,
+      "loss": 1.217,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 874.824,
+      "eval_samples_per_second": 157.659,
+      "eval_steps_per_second": 39.415,
+      "step": 10000
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "grad_norm": 0.3876087963581085,
+      "learning_rate": 0.001,
+      "loss": 1.1568,
+      "step": 20000
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 774.5634,
+      "eval_samples_per_second": 178.067,
+      "eval_steps_per_second": 44.517,
+      "step": 20000
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.2238207757472992,
+      "learning_rate": 0.001,
+      "loss": 1.1522,
+      "step": 30000
+    },
+    {
+      "epoch": 0.25,
+      "eval_loss": NaN,
+      "eval_runtime": 739.1972,
+      "eval_samples_per_second": 186.586,
+      "eval_steps_per_second": 46.647,
+      "step": 30000
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 0.1445535570383072,
+      "learning_rate": 0.001,
+      "loss": 1.1443,
+      "step": 40000
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 500.0215,
+      "eval_samples_per_second": 275.836,
+      "eval_steps_per_second": 68.959,
+      "step": 40000
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "grad_norm": 0.17618992924690247,
+      "learning_rate": 0.001,
+      "loss": 1.1404,
+      "step": 50000
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "eval_loss": NaN,
+      "eval_runtime": 589.6171,
+      "eval_samples_per_second": 233.921,
+      "eval_steps_per_second": 58.48,
+      "step": 50000
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.12298904359340668,
+      "learning_rate": 0.001,
+      "loss": 1.1329,
+      "step": 60000
+    },
+    {
+      "epoch": 0.5,
+      "eval_loss": NaN,
+      "eval_runtime": 599.1773,
+      "eval_samples_per_second": 230.189,
+      "eval_steps_per_second": 57.547,
+      "step": 60000
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "grad_norm": 0.21368645131587982,
+      "learning_rate": 0.001,
+      "loss": 1.1323,
+      "step": 70000
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "eval_loss": NaN,
+      "eval_runtime": 526.8173,
+      "eval_samples_per_second": 261.806,
+      "eval_steps_per_second": 65.452,
+      "step": 70000
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.15141108632087708,
+      "learning_rate": 0.001,
+      "loss": 1.1292,
+      "step": 80000
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 542.434,
+      "eval_samples_per_second": 254.269,
+      "eval_steps_per_second": 63.567,
+      "step": 80000
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.15512333810329437,
+      "learning_rate": 0.001,
+      "loss": 1.1264,
+      "step": 90000
+    },
+    {
+      "epoch": 0.75,
+      "eval_loss": NaN,
+      "eval_runtime": 700.6971,
+      "eval_samples_per_second": 196.838,
+      "eval_steps_per_second": 49.21,
+      "step": 90000
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.15970458090305328,
+      "learning_rate": 0.001,
+      "loss": 1.1312,
+      "step": 100000
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "eval_loss": NaN,
+      "eval_runtime": 774.1988,
+      "eval_samples_per_second": 178.151,
+      "eval_steps_per_second": 44.538,
+      "step": 100000
+    }
+  ],
+  "logging_steps": 10000,
+  "max_steps": 120000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.095343338061824e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-100000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e24d240c0212150205bb434684f1d705979477cac1d7cdb25ba76821568db2
+size 5112

checkpoint-110000/config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "ConvNetForMaskedLM"
+  ],
+  "aux_features_vocab_size": 5,
+  "dilation_base": 2,
+  "dilation_cycle": 6,
+  "dilation_double_every": 1,
+  "dilation_max": 32,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "kernel_size": 9,
+  "model_type": "ConvNet",
+  "n_aux_features": 0,
+  "n_layers": 25,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.2",
+  "vocab_size": 7
+}

checkpoint-110000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f709e4d698ae62bb811225a5e87798d09f5a6579d137e29ddf0d1efa2d2d033
+size 263540548

checkpoint-110000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24df7f0d093ad6bd5e95f5b40158b8c0365ad766813a200557716da11b5ba7f6
+size 527212602

checkpoint-110000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bc5a17f58f700ac501af8e40f71184ae5e7a36fe3304bb4dac612aebb94a26c
+size 14244

checkpoint-110000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9df09116e63bfdc0e09627da3aa2680596ae9b928f01740c0acedf00513cf2cb
+size 1064

checkpoint-110000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-110000/tokenizer.json

@@ -0,0 +1,62 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Lowercase"
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[MASK]": 1,
+      "[UNK]": 2,
+      "a": 3,
+      "c": 4,
+      "g": 5,
+      "t": 6
+    },
+    "merges": []
+  }
+}

checkpoint-110000/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}

checkpoint-110000/trainer_state.json

@@ -0,0 +1,198 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9166666666666666,
+  "eval_steps": 10000,
+  "global_step": 110000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08333333333333333,
+      "grad_norm": 0.6651873588562012,
+      "learning_rate": 0.001,
+      "loss": 1.217,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 874.824,
+      "eval_samples_per_second": 157.659,
+      "eval_steps_per_second": 39.415,
+      "step": 10000
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "grad_norm": 0.3876087963581085,
+      "learning_rate": 0.001,
+      "loss": 1.1568,
+      "step": 20000
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 774.5634,
+      "eval_samples_per_second": 178.067,
+      "eval_steps_per_second": 44.517,
+      "step": 20000
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.2238207757472992,
+      "learning_rate": 0.001,
+      "loss": 1.1522,
+      "step": 30000
+    },
+    {
+      "epoch": 0.25,
+      "eval_loss": NaN,
+      "eval_runtime": 739.1972,
+      "eval_samples_per_second": 186.586,
+      "eval_steps_per_second": 46.647,
+      "step": 30000
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 0.1445535570383072,
+      "learning_rate": 0.001,
+      "loss": 1.1443,
+      "step": 40000
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 500.0215,
+      "eval_samples_per_second": 275.836,
+      "eval_steps_per_second": 68.959,
+      "step": 40000
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "grad_norm": 0.17618992924690247,
+      "learning_rate": 0.001,
+      "loss": 1.1404,
+      "step": 50000
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "eval_loss": NaN,
+      "eval_runtime": 589.6171,
+      "eval_samples_per_second": 233.921,
+      "eval_steps_per_second": 58.48,
+      "step": 50000
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.12298904359340668,
+      "learning_rate": 0.001,
+      "loss": 1.1329,
+      "step": 60000
+    },
+    {
+      "epoch": 0.5,
+      "eval_loss": NaN,
+      "eval_runtime": 599.1773,
+      "eval_samples_per_second": 230.189,
+      "eval_steps_per_second": 57.547,
+      "step": 60000
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "grad_norm": 0.21368645131587982,
+      "learning_rate": 0.001,
+      "loss": 1.1323,
+      "step": 70000
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "eval_loss": NaN,
+      "eval_runtime": 526.8173,
+      "eval_samples_per_second": 261.806,
+      "eval_steps_per_second": 65.452,
+      "step": 70000
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.15141108632087708,
+      "learning_rate": 0.001,
+      "loss": 1.1292,
+      "step": 80000
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 542.434,
+      "eval_samples_per_second": 254.269,
+      "eval_steps_per_second": 63.567,
+      "step": 80000
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.15512333810329437,
+      "learning_rate": 0.001,
+      "loss": 1.1264,
+      "step": 90000
+    },
+    {
+      "epoch": 0.75,
+      "eval_loss": NaN,
+      "eval_runtime": 700.6971,
+      "eval_samples_per_second": 196.838,
+      "eval_steps_per_second": 49.21,
+      "step": 90000
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.15970458090305328,
+      "learning_rate": 0.001,
+      "loss": 1.1312,
+      "step": 100000
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "eval_loss": NaN,
+      "eval_runtime": 774.1988,
+      "eval_samples_per_second": 178.151,
+      "eval_steps_per_second": 44.538,
+      "step": 100000
+    },
+    {
+      "epoch": 0.9166666666666666,
+      "grad_norm": 0.16049639880657196,
+      "learning_rate": 0.001,
+      "loss": 1.1305,
+      "step": 110000
+    },
+    {
+      "epoch": 0.9166666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 612.5819,
+      "eval_samples_per_second": 225.152,
+      "eval_steps_per_second": 56.288,
+      "step": 110000
+    }
+  ],
+  "logging_steps": 10000,
+  "max_steps": 120000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.904877671868006e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-110000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e24d240c0212150205bb434684f1d705979477cac1d7cdb25ba76821568db2
+size 5112

checkpoint-120000/config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "ConvNetForMaskedLM"
+  ],
+  "aux_features_vocab_size": 5,
+  "dilation_base": 2,
+  "dilation_cycle": 6,
+  "dilation_double_every": 1,
+  "dilation_max": 32,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "kernel_size": 9,
+  "model_type": "ConvNet",
+  "n_aux_features": 0,
+  "n_layers": 25,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.2",
+  "vocab_size": 7
+}

checkpoint-120000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bde649b0d713fd5b2ba7286dc1ab9bacff3d4bf38d38661784348d23d47a014
+size 263540548

checkpoint-120000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03cf05ae3b962c46c40d8346a7cb1ddb98c00fff5ea3197f4630bb73e953f9f
+size 527212602

checkpoint-120000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab2c92317db8387da7d95090bd49efb7856eaec0bb349c768b2e290d8b50f843
+size 14244

checkpoint-120000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972940722fecab610fecaa9267d56b9e65a388a1d5ec43b18bbdde9fa7a69235
+size 1064

checkpoint-120000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-120000/tokenizer.json

@@ -0,0 +1,62 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Lowercase"
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[MASK]": 1,
+      "[UNK]": 2,
+      "a": 3,
+      "c": 4,
+      "g": 5,
+      "t": 6
+    },
+    "merges": []
+  }
+}

checkpoint-120000/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}

checkpoint-120000/trainer_state.json

@@ -0,0 +1,213 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 10000,
+  "global_step": 120000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08333333333333333,
+      "grad_norm": 0.6651873588562012,
+      "learning_rate": 0.001,
+      "loss": 1.217,
+      "step": 10000
+    },
+    {
+      "epoch": 0.08333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 874.824,
+      "eval_samples_per_second": 157.659,
+      "eval_steps_per_second": 39.415,
+      "step": 10000
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "grad_norm": 0.3876087963581085,
+      "learning_rate": 0.001,
+      "loss": 1.1568,
+      "step": 20000
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 774.5634,
+      "eval_samples_per_second": 178.067,
+      "eval_steps_per_second": 44.517,
+      "step": 20000
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.2238207757472992,
+      "learning_rate": 0.001,
+      "loss": 1.1522,
+      "step": 30000
+    },
+    {
+      "epoch": 0.25,
+      "eval_loss": NaN,
+      "eval_runtime": 739.1972,
+      "eval_samples_per_second": 186.586,
+      "eval_steps_per_second": 46.647,
+      "step": 30000
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 0.1445535570383072,
+      "learning_rate": 0.001,
+      "loss": 1.1443,
+      "step": 40000
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "eval_loss": NaN,
+      "eval_runtime": 500.0215,
+      "eval_samples_per_second": 275.836,
+      "eval_steps_per_second": 68.959,
+      "step": 40000
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "grad_norm": 0.17618992924690247,
+      "learning_rate": 0.001,
+      "loss": 1.1404,
+      "step": 50000
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "eval_loss": NaN,
+      "eval_runtime": 589.6171,
+      "eval_samples_per_second": 233.921,
+      "eval_steps_per_second": 58.48,
+      "step": 50000
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.12298904359340668,
+      "learning_rate": 0.001,
+      "loss": 1.1329,
+      "step": 60000
+    },
+    {
+      "epoch": 0.5,
+      "eval_loss": NaN,
+      "eval_runtime": 599.1773,
+      "eval_samples_per_second": 230.189,
+      "eval_steps_per_second": 57.547,
+      "step": 60000
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "grad_norm": 0.21368645131587982,
+      "learning_rate": 0.001,
+      "loss": 1.1323,
+      "step": 70000
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "eval_loss": NaN,
+      "eval_runtime": 526.8173,
+      "eval_samples_per_second": 261.806,
+      "eval_steps_per_second": 65.452,
+      "step": 70000
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.15141108632087708,
+      "learning_rate": 0.001,
+      "loss": 1.1292,
+      "step": 80000
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 542.434,
+      "eval_samples_per_second": 254.269,
+      "eval_steps_per_second": 63.567,
+      "step": 80000
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.15512333810329437,
+      "learning_rate": 0.001,
+      "loss": 1.1264,
+      "step": 90000
+    },
+    {
+      "epoch": 0.75,
+      "eval_loss": NaN,
+      "eval_runtime": 700.6971,
+      "eval_samples_per_second": 196.838,
+      "eval_steps_per_second": 49.21,
+      "step": 90000
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "grad_norm": 0.15970458090305328,
+      "learning_rate": 0.001,
+      "loss": 1.1312,
+      "step": 100000
+    },
+    {
+      "epoch": 0.8333333333333334,
+      "eval_loss": NaN,
+      "eval_runtime": 774.1988,
+      "eval_samples_per_second": 178.151,
+      "eval_steps_per_second": 44.538,
+      "step": 100000
+    },
+    {
+      "epoch": 0.9166666666666666,
+      "grad_norm": 0.16049639880657196,
+      "learning_rate": 0.001,
+      "loss": 1.1305,
+      "step": 110000
+    },
+    {
+      "epoch": 0.9166666666666666,
+      "eval_loss": NaN,
+      "eval_runtime": 612.5819,
+      "eval_samples_per_second": 225.152,
+      "eval_steps_per_second": 56.288,
+      "step": 110000
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.1032409593462944,
+      "learning_rate": 0.001,
+      "loss": 1.1285,
+      "step": 120000
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": NaN,
+      "eval_runtime": 585.9083,
+      "eval_samples_per_second": 235.402,
+      "eval_steps_per_second": 58.851,
+      "step": 120000
+    }
+  ],
+  "logging_steps": 10000,
+  "max_steps": 120000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.714412005674189e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-120000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e24d240c0212150205bb434684f1d705979477cac1d7cdb25ba76821568db2
+size 5112

checkpoint-20000/config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "ConvNetForMaskedLM"
+  ],
+  "aux_features_vocab_size": 5,
+  "dilation_base": 2,
+  "dilation_cycle": 6,
+  "dilation_double_every": 1,
+  "dilation_max": 32,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "kernel_size": 9,
+  "model_type": "ConvNet",
+  "n_aux_features": 0,
+  "n_layers": 25,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.2",
+  "vocab_size": 7
+}

checkpoint-20000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ad9ac6b3b5978abc8064224397508f994dff694d460ac0f949031324fe8bae8
+size 263540548

checkpoint-20000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88fc9044e7c2e1856c407f01030ffb844c63770bd0511b2868f0c08482a2cdb2
+size 527212602

checkpoint-20000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7efde149035b4a6a5c64f8ee01eaa17edfa91c1e5a7933194a15152b3e7090e2
+size 14244

checkpoint-20000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b907d95b8c893b8c166b45f79e5cbc1c7d4f6eac179082acaa5709176eb0786
+size 1064

checkpoint-20000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-20000/tokenizer.json

@@ -0,0 +1,62 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Lowercase"
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[MASK]": 1,
+      "[UNK]": 2,
+      "a": 3,
+      "c": 4,
+      "g": 5,
+      "t": 6
+    },
+    "merges": []
+  }
+}

checkpoint-20000/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}