Add files using upload-large-folder tool

attention_2_only_emb/final_model/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_1000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_1000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 1000,
+    "update_step": 1000,
+    "tokens_seen": 639025152,
+    "tokens_seen_before": 638386176,
+    "update_time": 2.876735210418701,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_10000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_10000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 10000,
+    "update_step": 10000,
+    "tokens_seen": 6389809152,
+    "tokens_seen_before": 6389170176,
+    "update_time": 2.879143476486206,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_11000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_11000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 11000,
+    "update_step": 11000,
+    "tokens_seen": 7028785152,
+    "tokens_seen_before": 7028146176,
+    "update_time": 2.8783798217773438,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_12000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_12000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 12000,
+    "update_step": 12000,
+    "tokens_seen": 7667761152,
+    "tokens_seen_before": 7667122176,
+    "update_time": 2.8790109157562256,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_13000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_13000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 13000,
+    "update_step": 13000,
+    "tokens_seen": 8306737152,
+    "tokens_seen_before": 8306098176,
+    "update_time": 2.877460241317749,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_14000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_14000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 14000,
+    "update_step": 14000,
+    "tokens_seen": 8945713152,
+    "tokens_seen_before": 8945074176,
+    "update_time": 2.8788270950317383,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_15000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_15000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 15000,
+    "update_step": 15000,
+    "tokens_seen": 9584689152,
+    "tokens_seen_before": 9584050176,
+    "update_time": 2.877842426300049,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_2000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_2000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 2000,
+    "update_step": 2000,
+    "tokens_seen": 1278001152,
+    "tokens_seen_before": 1277362176,
+    "update_time": 2.8781590461730957,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_3000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_3000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 3000,
+    "update_step": 3000,
+    "tokens_seen": 1916977152,
+    "tokens_seen_before": 1916338176,
+    "update_time": 2.876859426498413,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_4000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_4000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 4000,
+    "update_step": 4000,
+    "tokens_seen": 2555953152,
+    "tokens_seen_before": 2555314176,
+    "update_time": 2.8781702518463135,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_5000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_5000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 5000,
+    "update_step": 5000,
+    "tokens_seen": 3194929152,
+    "tokens_seen_before": 3194290176,
+    "update_time": 2.8779265880584717,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_6000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_6000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 6000,
+    "update_step": 6000,
+    "tokens_seen": 3833905152,
+    "tokens_seen_before": 3833266176,
+    "update_time": 2.880322217941284,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_7000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_7000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 7000,
+    "update_step": 7000,
+    "tokens_seen": 4472881152,
+    "tokens_seen_before": 4472242176,
+    "update_time": 2.8802781105041504,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_8000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_8000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 8000,
+    "update_step": 8000,
+    "tokens_seen": 5111857152,
+    "tokens_seen_before": 5111218176,
+    "update_time": 2.8798861503601074,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/model_9000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

attention_2_only_emb/model_9000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 9000,
+    "update_step": 9000,
+    "tokens_seen": 5750833152,
+    "tokens_seen_before": 5750194176,
+    "update_time": 2.881277322769165,
+    "wandb_id": "sgqffduo"
+}

attention_2_only_emb/training_config.yaml

@@ -0,0 +1,50 @@
+adam_beta1: 0.9
+adam_beta2: 0.95
+adjust_step: 0
+autoresume: false
+batch_size: 6
+clip_grad_norm: 1.0
+comment: null
+cycle_length: null
+dtype: bfloat16
+emb_freeze: null
+eval_dataset_path: /work01/yanokazuki/fineweb/valid_data_gpt2/
+eval_every: 1000
+first_attention: false
+first_attention_2: true
+first_attention_resume: false
+first_layer: false
+first_layer_2: false
+gradient_accumulation: 13
+keep_checkpoints: null
+layer_freeze: null
+layer_freeze_2: false
+load_optimizer_state_on_resume: true
+lr: 0.0004
+max_length: 1024
+max_train_tokens: null
+min_lr_ratio: 0.1
+model_config: model_config/478m.json
+model_name_or_path: null
+model_revision: null
+num_training_steps: 15000
+optimizer: Adam
+restart_warmup_steps: null
+resume_from: null
+run_name: attention_2_only_emb
+save_dir: checkpoints/attention_2_only_emb
+save_every: 1000
+scheduler: cosine
+seed: 0
+shuffle: true
+skip_batches: !!set {}
+tags:
+- 396m-for-680m
+total_batch_size: 624
+train_dataset_path: /work01/yanokazuki/fineweb/train_data_gpt2/
+training_config: training_config/two_stage/attention_2_only_emb.yaml
+wandb_watch: true
+warmed_up_model: null
+warmup_steps: 1500
+weight_decay: 0.0
+workers: 8

baseline/model_15000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a68abbc43af60bd4c7f2d95c72f10740acbbb95df43e3fd9ba6ae48d8c02ccc
+size 2533545094

bigram_2_full/model_1000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_10000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_10000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 10000,
+    "update_step": 10000,
+    "tokens_seen": 6389809152,
+    "tokens_seen_before": 6389170176,
+    "update_time": 3.281773805618286,
+    "wandb_id": "7nopmkvs"
+}

bigram_2_full/model_12000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_12000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 12000,
+    "update_step": 12000,
+    "tokens_seen": 7667761152,
+    "tokens_seen_before": 7667122176,
+    "update_time": 3.285931348800659,
+    "wandb_id": "7nopmkvs"
+}

bigram_2_full/model_14000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_15000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_15000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 15000,
+    "update_step": 15000,
+    "tokens_seen": 9584689152,
+    "tokens_seen_before": 9584050176,
+    "update_time": 3.2829833030700684,
+    "wandb_id": "7nopmkvs"
+}

bigram_2_full/model_6000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_6000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 6000,
+    "update_step": 6000,
+    "tokens_seen": 3833905152,
+    "tokens_seen_before": 3833266176,
+    "update_time": 3.283099412918091,
+    "wandb_id": "7nopmkvs"
+}

bigram_2_full/model_7000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_7000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 7000,
+    "update_step": 7000,
+    "tokens_seen": 4472881152,
+    "tokens_seen_before": 4472242176,
+    "update_time": 3.2802579402923584,
+    "wandb_id": "7nopmkvs"
+}

bigram_2_full/model_8000/model_config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

bigram_2_full/model_8000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 8000,
+    "update_step": 8000,
+    "tokens_seen": 5111857152,
+    "tokens_seen_before": 5111218176,
+    "update_time": 3.285356044769287,
+    "wandb_id": "7nopmkvs"
+}

first_layer_1/final_model/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

first_layer_1/model_1000/config.json

@@ -0,0 +1,19 @@
+{
+    "architectures": [
+        "LLaMAForCausalLM"
+    ],
+    "bos_token_id": 50256,
+    "eos_token_id": 50256,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "intermediate_size": 5376,
+    "initializer_range": 0.02,
+    "max_sequence_length": 1024,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 14,
+    "rms_norm_eps": 1e-05,
+    "transformers_version": "4.28.1",
+    "use_cache": true,
+    "vocab_size": 50257
+}

first_layer_1/model_1000/training_state.json

@@ -0,0 +1,8 @@
+{
+    "global_step": 1000,
+    "update_step": 1000,
+    "tokens_seen": 639025152,
+    "tokens_seen_before": 638386176,
+    "update_time": 0.9770157337188721,
+    "wandb_id": "krzb2185"
+}