| { | |
| "name": "InterpBench", | |
| "version": "1.0.0", | |
| "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.", | |
| "license": "https://creativecommons.org/licenses/by/4.0/", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench", | |
| "cases": [ | |
| { | |
| "case_id": "11", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11", | |
| "task_description": "Counts the number of words in a sequence based on their length.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "no", | |
| "oCLrZaW", | |
| "poiVg" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 12, | |
| "n_ctx": 10, | |
| "d_head": 3, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 48, | |
| "act_fn": "gelu", | |
| "d_vocab": 10, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.1460593486680443, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 5, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 3456, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 500, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 1.0, | |
| "lr_scheduler": "" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" | |
| }, | |
| { | |
| "case_id": "13", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13", | |
| "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.", | |
| "vocab": [ | |
| 0, | |
| 1, | |
| 2 | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 20, | |
| "n_ctx": 10, | |
| "d_head": 5, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 80, | |
| "act_fn": "gelu", | |
| "d_vocab": 5, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "bidirectional", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.1460593486680443, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 3, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 9600, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 500, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 1.0, | |
| "lr_scheduler": "" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" | |
| }, | |
| { | |
| "case_id": "18", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18", | |
| "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.", | |
| "vocab": [ | |
| "a", | |
| "b", | |
| "c", | |
| "d", | |
| "e" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 26, | |
| "n_ctx": 10, | |
| "d_head": 6, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 104, | |
| "act_fn": "gelu", | |
| "d_vocab": 7, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "bidirectional", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.12344267996967354, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 3, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 15808, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.001, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 1.0, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" | |
| }, | |
| { | |
| "case_id": "19", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19", | |
| "task_description": "Removes consecutive duplicate tokens from a sequence.", | |
| "vocab": [ | |
| "a", | |
| "b", | |
| "c" | |
| ], | |
| "max_seq_len": 15, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 32, | |
| "n_ctx": 15, | |
| "d_head": 8, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 128, | |
| "act_fn": "gelu", | |
| "d_vocab": 5, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.15689290811054724, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 3, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 24576, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.001, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" | |
| }, | |
| { | |
| "case_id": "20", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20", | |
| "task_description": "Detect spam messages based on appearance of spam keywords.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "click", | |
| "no", | |
| "now", | |
| "oCLrZaW", | |
| "offer", | |
| "poiVg", | |
| "spam" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 13, | |
| "n_ctx": 10, | |
| "d_head": 3, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 52, | |
| "act_fn": "gelu", | |
| "d_vocab": 14, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.16, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 2, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 3952, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" | |
| }, | |
| { | |
| "case_id": "21", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21", | |
| "task_description": "Extract unique tokens from a string", | |
| "vocab": [ | |
| "a", | |
| "b", | |
| "c" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 4, | |
| "d_model": 50, | |
| "n_ctx": 10, | |
| "d_head": 12, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 200, | |
| "act_fn": "gelu", | |
| "d_vocab": 5, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.09847319278346618, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 3, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 118400, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.0005, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.5, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" | |
| }, | |
| { | |
| "case_id": "26", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/26", | |
| "task_description": "Creates a cascading effect by repeating each token in sequence incrementally.", | |
| "vocab": [ | |
| "a", | |
| "b", | |
| "c" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 21, | |
| "n_ctx": 10, | |
| "d_head": 5, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 84, | |
| "act_fn": "gelu", | |
| "d_vocab": 5, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.12344267996967354, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 27, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 10416, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/edges.pkl" | |
| }, | |
| { | |
| "case_id": "29", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/29", | |
| "task_description": "Creates abbreviations for each token in the sequence.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "no", | |
| "oCLrZaW", | |
| "poiVg" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 13, | |
| "n_ctx": 10, | |
| "d_head": 3, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 52, | |
| "act_fn": "gelu", | |
| "d_vocab": 10, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.1539600717839002, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 8, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 3952, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/edges.pkl" | |
| }, | |
| { | |
| "case_id": "3", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3", | |
| "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.", | |
| "vocab": [ | |
| "a", | |
| "b", | |
| "c", | |
| "x" | |
| ], | |
| "max_seq_len": 5, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 12, | |
| "n_ctx": 5, | |
| "d_head": 3, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 48, | |
| "act_fn": "gelu", | |
| "d_vocab": 6, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.22188007849009167, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 1, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 3456, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 10.0, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict", | |
| "same_size": false | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" | |
| }, | |
| { | |
| "case_id": "33", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33", | |
| "task_description": "Checks if each token's length is odd or even.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "no", | |
| "oCLrZaW", | |
| "poiVg" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 4, | |
| "n_ctx": 10, | |
| "d_head": 1, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 16, | |
| "act_fn": "gelu", | |
| "d_vocab": 10, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.17457431218879393, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 2, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 384, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.001, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" | |
| }, | |
| { | |
| "case_id": "34", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34", | |
| "task_description": "Calculate the ratio of vowels to consonants in each word.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "no", | |
| "oCLrZaW", | |
| "poiVg" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 16, | |
| "n_ctx": 10, | |
| "d_head": 4, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 64, | |
| "act_fn": "gelu", | |
| "d_vocab": 10, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.16329931618554522, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 5, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 6144, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 1.0, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" | |
| }, | |
| { | |
| "case_id": "35", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35", | |
| "task_description": "Alternates capitalization of each character in words.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "no", | |
| "oCLrZaW", | |
| "poiVg" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 9, | |
| "n_ctx": 10, | |
| "d_head": 2, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 36, | |
| "act_fn": "gelu", | |
| "d_vocab": 10, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.1539600717839002, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 8, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 1872, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 1.0, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" | |
| }, | |
| { | |
| "case_id": "36", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36", | |
| "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.", | |
| "vocab": [ | |
| "\ud83d\udcd8", | |
| "\ud83d\ude0a", | |
| "\ud83d\ude22" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 6, | |
| "n_ctx": 10, | |
| "d_head": 1, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 24, | |
| "act_fn": "gelu", | |
| "d_vocab": 5, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.19402850002906638, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 3, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 768, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 1.0, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" | |
| }, | |
| { | |
| "case_id": "37", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37", | |
| "task_description": "Reverses each word in the sequence except for specified exclusions.", | |
| "vocab": [ | |
| "J", | |
| "LB", | |
| "TPSI", | |
| "V", | |
| "b", | |
| "no", | |
| "oCLrZaW", | |
| "poiVg" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 12, | |
| "n_ctx": 10, | |
| "d_head": 3, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 48, | |
| "act_fn": "gelu", | |
| "d_vocab": 10, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.1539600717839002, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 8, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 3456, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 1.0, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" | |
| }, | |
| { | |
| "case_id": "4", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4", | |
| "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.", | |
| "vocab": [ | |
| "(", | |
| ")", | |
| "a", | |
| "b", | |
| "c" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 20, | |
| "n_ctx": 10, | |
| "d_head": 5, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 80, | |
| "act_fn": "gelu", | |
| "d_vocab": 7, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.17056057308448835, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 1, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 9600, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.001, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" | |
| }, | |
| { | |
| "case_id": "7", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/7", | |
| "task_description": "Returns the number of times each token occurs in the input.", | |
| "vocab": [ | |
| "a", | |
| "b", | |
| "c" | |
| ], | |
| "max_seq_len": 10, | |
| "min_seq_len": 4, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 2, | |
| "d_model": 17, | |
| "n_ctx": 10, | |
| "d_head": 4, | |
| "model_name": "custom", | |
| "n_heads": 4, | |
| "d_mlp": 68, | |
| "act_fn": "gelu", | |
| "d_vocab": 5, | |
| "eps": 1e-05, | |
| "use_attn_result": true, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": true, | |
| "use_hook_mlp_in": true, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": null, | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": null, | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": null, | |
| "n_devices": 1, | |
| "attention_dir": "bidirectional", | |
| "attn_only": false, | |
| "seed": 0, | |
| "initializer_range": 0.15689290811054724, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 10, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 6800, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": null, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model_cfg.pkl", | |
| "training_args": { | |
| "atol": 0.05, | |
| "lr": 0.01, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.5, | |
| "epochs": 2000, | |
| "act_fn": "gelu", | |
| "clip_grad_norm": 0.1, | |
| "lr_scheduler": "", | |
| "model_pair": "strict", | |
| "same_size": false, | |
| "seed": 1234, | |
| "batch_size": 256, | |
| "include_mlp": false, | |
| "next_token": false, | |
| "detach_while_caching": true | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/edges.pkl" | |
| }, | |
| { | |
| "case_id": "ioi", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi", | |
| "task_description": "Indirect Object Identification (IOI) task.", | |
| "max_seq_len": 16, | |
| "min_seq_len": 16, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 6, | |
| "d_model": 64, | |
| "n_ctx": 1024, | |
| "d_head": 16, | |
| "model_name": "gpt2", | |
| "n_heads": 4, | |
| "d_mlp": 3072, | |
| "act_fn": "gelu_new", | |
| "d_vocab": 50257, | |
| "eps": 1e-05, | |
| "use_attn_result": false, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": false, | |
| "use_hook_mlp_in": false, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": "GPT2LMHeadModel", | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": "gpt2", | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": "LNPre", | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": null, | |
| "initializer_range": 0.02886751345948129, | |
| "init_weights": false, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 50257, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 84934656, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": false, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_cfg.pkl", | |
| "training_args": { | |
| "next_token": false, | |
| "non_ioi_thresh": 0.65, | |
| "use_per_token_check": false, | |
| "batch_size": 512, | |
| "num_workers": 0, | |
| "early_stop": true, | |
| "lr_scheduler": null, | |
| "scheduler_val_metric": [ | |
| "val/accuracy", | |
| "val/IIA" | |
| ], | |
| "scheduler_mode": "max", | |
| "scheduler_kwargs": {}, | |
| "clip_grad_norm": 1.0, | |
| "seed": 0, | |
| "lr": 0.001, | |
| "detach_while_caching": true, | |
| "optimizer_kwargs": { | |
| "betas": [ | |
| 0.9, | |
| 0.9 | |
| ] | |
| }, | |
| "atol": 0.05, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "val_IIA_sampling": "random", | |
| "use_all_tokens_for_behavior": false, | |
| "strict_weight": 0.4, | |
| "siit_sampling": "individual" | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/edges.pkl" | |
| }, | |
| { | |
| "case_id": "ioi_next_token", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token", | |
| "task_description": "Indirect Object Identification (IOI) task, trained using next token prediction.", | |
| "max_seq_len": 16, | |
| "min_seq_len": 16, | |
| "files": [ | |
| { | |
| "file_name": "edges.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/edges.pkl" | |
| }, | |
| { | |
| "file_name": "ll_model.pth", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model.pth" | |
| }, | |
| { | |
| "file_name": "ll_model_cfg.pkl", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_cfg.pkl" | |
| }, | |
| { | |
| "file_name": "meta.json", | |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/meta.json" | |
| } | |
| ], | |
| "transformer_cfg": { | |
| "n_layers": 6, | |
| "d_model": 64, | |
| "n_ctx": 1024, | |
| "d_head": 16, | |
| "model_name": "gpt2", | |
| "n_heads": 4, | |
| "d_mlp": 3072, | |
| "act_fn": "gelu_new", | |
| "d_vocab": 50257, | |
| "eps": 1e-05, | |
| "use_attn_result": false, | |
| "use_attn_scale": true, | |
| "use_split_qkv_input": false, | |
| "use_hook_mlp_in": false, | |
| "use_attn_in": false, | |
| "use_local_attn": false, | |
| "original_architecture": "GPT2LMHeadModel", | |
| "from_checkpoint": false, | |
| "checkpoint_index": null, | |
| "checkpoint_label_type": null, | |
| "checkpoint_value": null, | |
| "tokenizer_name": "gpt2", | |
| "window_size": null, | |
| "attn_types": null, | |
| "init_mode": "gpt2", | |
| "normalization_type": "LNPre", | |
| "n_devices": 1, | |
| "attention_dir": "causal", | |
| "attn_only": false, | |
| "seed": null, | |
| "initializer_range": 0.02886751345948129, | |
| "init_weights": true, | |
| "scale_attn_by_inverse_layer_idx": false, | |
| "positional_embedding_type": "standard", | |
| "final_rms": false, | |
| "d_vocab_out": 50257, | |
| "parallel_attn_mlp": false, | |
| "rotary_dim": null, | |
| "n_params": 2457600, | |
| "use_hook_tokens": false, | |
| "gated_mlp": false, | |
| "default_prepend_bos": true, | |
| "dtype": "torch.float32", | |
| "tokenizer_prepends_bos": false, | |
| "n_key_value_heads": null, | |
| "post_embedding_ln": false, | |
| "rotary_base": 10000, | |
| "trust_remote_code": false, | |
| "rotary_adjacent_pairs": false | |
| }, | |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_cfg.pkl", | |
| "training_args": { | |
| "next_token": true, | |
| "non_ioi_thresh": 0.65, | |
| "use_per_token_check": false, | |
| "batch_size": 256, | |
| "lr": 0.001, | |
| "num_workers": 0, | |
| "early_stop": true, | |
| "lr_scheduler": null, | |
| "scheduler_val_metric": [ | |
| "val/accuracy", | |
| "val/IIA" | |
| ], | |
| "scheduler_mode": "max", | |
| "clip_grad_norm": 1.0, | |
| "atol": 0.05, | |
| "use_single_loss": false, | |
| "iit_weight": 1.0, | |
| "behavior_weight": 1.0, | |
| "strict_weight": 0.4 | |
| }, | |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/meta.json", | |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model.pth", | |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/edges.pkl" | |
| } | |
| ] | |
| } |