TinyStories-3M-val-Hebrew

README.md

@@ -0,0 +1,51 @@
+---
+tags:
+- generated_from_trainer
+model-index:
+- name: TinyStories-3M-val-Hebrew
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# TinyStories-3M-val-Hebrew
+
+<!--This model is a fine-tuned version of [./TinyStories-3M-val-Hebrew](https://huggingface.co/./TinyStories-3M-val-Hebrew) on an unknown dataset.-->
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0004
+- train_batch_size: 24
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine_with_restarts
+- lr_scheduler_warmup_steps: 500
+- num_epochs: 300.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.31.0.dev0
+- Pytorch 2.0.0
+- Datasets 2.13.1
+- Tokenizers 0.13.3

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|pad|>": 50258,
+  "<|startoftext|>": 50257
+}

config.json

@@ -0,0 +1,51 @@
+{
+  "_name_or_path": "./TinyStories-3M-val-Hebrew",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPTNeoForCausalLM"
+  ],
+  "attention_dropout": 0,
+  "attention_layers": [
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local",
+    "global",
+    "local"
+  ],
+  "attention_types": [
+    [
+      [
+        "global",
+        "local"
+      ],
+      4
+    ]
+  ],
+  "bos_token_id": 50256,
+  "classifier_dropout": 0.1,
+  "embed_dropout": 0,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neo",
+  "num_heads": 16,
+  "num_layers": 8,
+  "resid_dropout": 0,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50259,
+  "window_size": 256
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.31.0.dev0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4695360bd64459039470e08baac4c0438a8e4e6efdee037ab05204cce4123c2
+size 66709565

runs/Jun27_20-10-05_DESKTOP-7NAF1LL/events.out.tfevents.1687885819.DESKTOP-7NAF1LL.13904.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e4ecc276e72888272879211a2f2ce35ac17592cf6712b22e9390ba66d0bf5b5
+size 6903

runs/Jun27_20-28-01_DESKTOP-7NAF1LL/events.out.tfevents.1687886890.DESKTOP-7NAF1LL.4064.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66a84c7b9eac98b3a0381d97480a76e44b487b3ebcaf7d26810d488bc4f73bb3
+size 7996

runs/Jun27_21-16-53_DESKTOP-7NAF1LL/events.out.tfevents.1687889832.DESKTOP-7NAF1LL.11060.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b72dce80eb843f269d56ccb5bb6d609a49f41dee58141ecf499e0e68eee41850
+size 4544

runs/Jun27_21-20-43_DESKTOP-7NAF1LL/events.out.tfevents.1687890064.DESKTOP-7NAF1LL.13404.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bc457a2db9d48ad45a07b326131fd49586f01868f4e545e66a6f12e4775f2d8
+size 4815

runs/Jun27_21-28-23_DESKTOP-7NAF1LL/events.out.tfevents.1687890508.DESKTOP-7NAF1LL.3328.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:850ec3747ee6106a887c7850d619e8f55c55ac05488d44ae7769e28073ec4316
+size 4503

runs/Jun27_21-32-29_DESKTOP-7NAF1LL/events.out.tfevents.1687890754.DESKTOP-7NAF1LL.1068.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4979c9025c9e085f86257d5106ea3c71d64ded1494882fa16ce6233fb251388
+size 27487

runs/Jun28_07-13-15_DESKTOP-7NAF1LL/events.out.tfevents.1687925598.DESKTOP-7NAF1LL.12756.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2b3d1ce943396cd9f37d336fc95db5f6f789e2df143cc1a6a6382d2e4a5ee48
+size 11326

special_tokens_map.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|pad|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unknown_token": "<|unknown|>"
+}