gate_OutEffHop_opt-125m

all_results.json

@@ -0,0 +1 @@
+{"perplexity": 15.706686202168713, "model.decoder.final_layer_norm": 90.96276274729331, "model.decoder.layers.0": 0.4204075758269325, "model.decoder.layers.1": 0.4908598484700949, "model.decoder.layers.2": 0.5028251055120726, "model.decoder.layers.3": 0.691462532509645, "model.decoder.layers.4": 0.8262411430478096, "model.decoder.layers.5": 1.0756556153220116, "model.decoder.layers.6": 1.366795804172793, "model.decoder.layers.7": 1.5691746961852069, "model.decoder.layers.8": 2.1472622930385104, "model.decoder.layers.9": 3.6863469854770314, "model.decoder.layers.10": 6.168255855968336, "model.decoder.layers.11": 7.57078872866513, "model.decoder.layers.0.fc2": 0.1039871887996039, "model.decoder.layers.1.fc2": 0.05608035337503627, "model.decoder.layers.2.fc2": 0.03349900133207189, "model.decoder.layers.3.fc2": 0.06225394718614727, "model.decoder.layers.4.fc2": 0.06453566397655025, "model.decoder.layers.5.fc2": 0.07603821023602894, "model.decoder.layers.6.fc2": 0.11477861616001905, "model.decoder.layers.7.fc2": 0.16301063338992816, "model.decoder.layers.8.fc2": 0.2619496168986678, "model.decoder.layers.9.fc2": 0.414940554808086, "model.decoder.layers.10.fc2": 0.4336998329416942, "model.decoder.layers.11.fc2": 0.28299739546155805, "model.decoder.layers.0.final_layer_norm": 0.45984292787960546, "model.decoder.layers.1.final_layer_norm": 0.4209096071819691, "model.decoder.layers.2.final_layer_norm": 0.5994071354647184, "model.decoder.layers.3.final_layer_norm": 0.46284601323922103, "model.decoder.layers.4.final_layer_norm": 0.5087763514933797, "model.decoder.layers.5.final_layer_norm": 0.5348684589040256, "model.decoder.layers.6.final_layer_norm": 0.5873251914226254, "model.decoder.layers.7.final_layer_norm": 0.655958993094573, "model.decoder.layers.8.final_layer_norm": 0.814051132275788, "model.decoder.layers.9.final_layer_norm": 0.9984980142119612, "model.decoder.layers.10.final_layer_norm": 1.5374041555680895, "model.decoder.layers.11.final_layer_norm": 1.5156749826808207, "model.decoder.layers.0.self_attn.out_proj": 0.2508772022529335, "model.decoder.layers.1.self_attn.out_proj": 0.24341993913267063, "model.decoder.layers.2.self_attn.out_proj": 0.07271888871446504, "model.decoder.layers.3.self_attn.out_proj": 0.1664157649731667, "model.decoder.layers.4.self_attn.out_proj": 0.1435992467264257, "model.decoder.layers.5.self_attn.out_proj": 0.12288297165847165, "model.decoder.layers.6.self_attn.out_proj": 0.20423455739299612, "model.decoder.layers.7.self_attn.out_proj": 0.2692112508296348, "model.decoder.layers.8.self_attn.out_proj": 0.4702520902364016, "model.decoder.layers.9.self_attn.out_proj": 0.7821362368018402, "model.decoder.layers.10.self_attn.out_proj": 1.6762599524511592, "model.decoder.layers.11.self_attn.out_proj": 2.955877756794139, "model.decoder.layers.0.self_attn_layer_norm": 4.502585898500163, "model.decoder.layers.1.self_attn_layer_norm": 4.977919626135461, "model.decoder.layers.2.self_attn_layer_norm": 6.711245270361387, "model.decoder.layers.3.self_attn_layer_norm": 9.585455415352463, "model.decoder.layers.4.self_attn_layer_norm": 7.619464027974392, "model.decoder.layers.5.self_attn_layer_norm": 6.711193338558368, "model.decoder.layers.6.self_attn_layer_norm": 7.448162097836592, "model.decoder.layers.7.self_attn_layer_norm": 6.470245250931973, "model.decoder.layers.8.self_attn_layer_norm": 5.9982524145438045, "model.decoder.layers.9.self_attn_layer_norm": 5.239535203123835, "model.decoder.layers.10.self_attn_layer_norm": 5.41346654508519, "model.decoder.layers.11.self_attn_layer_norm": 6.829138810687183, "max_inf_norm": 90.96276274729331, "max_ffn_inf_norm": 0.4336998329416942, "max_layer_inf_norm": 7.57078872866513, "avg_kurtosis": 11.205696386600962, "max_kurtosis": 111.01246542132782, "max_kurtosis_layers": 31.2915544352952}

checkpoints/checkpoint_125000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:289ecb4897346feccbe75a39ed0b39742316a84bb45976e8cd0c98e415cf72cc
+size 496331808

checkpoints/checkpoint_125000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e08a15572caae4409868d713f8a73b206a090646d407c5adcb12a028b7400dbd
+size 992954750

checkpoints/checkpoint_125000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a62622f9ba131fe899729c87384413488db7506aa92557f39afda370a66d40dd
+size 14540

checkpoints/checkpoint_125000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6ed5313f128b91696fef11f6607d81abc7c1b8bb42c881f65d15ac0585ccc4f
+size 988

checkpoints/checkpoint_125000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d37a131294d0b27a11c8580fc6096736b28a4ddd0f827583794798cd35582761
+size 1064

config.json

@@ -0,0 +1,28 @@
+{
+  "_remove_final_layer_norm": false,
+  "activation_function": "relu",
+  "architectures": [
+    "OPTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "do_layer_norm_before": true,
+  "dropout": 0.1,
+  "enable_bias": true,
+  "eos_token_id": 2,
+  "ffn_dim": 3072,
+  "hidden_size": 768,
+  "init_std": 0.006,
+  "layer_norm_elementwise_affine": true,
+  "layerdrop": 0.0,
+  "max_position_embeddings": 512,
+  "model_type": "opt",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "use_cache": true,
+  "vocab_size": 50272,
+  "word_embed_proj_dim": 768
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.31.0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cefdad9675d141f479cec20bebfaee1eb5e8d5a267cff30fa7bdc5455f12b8a
+size 496440090

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}