{ "name": "InterpBench", "version": "1.0.0", "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.", "license": "https://creativecommons.org/licenses/by/4.0/", "url": "https://huggingface.co/cybershiptrooper/InterpBench", "cases": [ { "case_id": "11", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11", "task_description": "Counts the number of words in a sequence based on their length.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1460593486680443, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 5, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" }, { "case_id": "13", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13", "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.", "vocab": [ 0, 1, 2 ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.1460593486680443, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" }, { "case_id": "18", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18", "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.", "vocab": [ "a", "b", "c", "d", "e" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 26, "n_ctx": 10, "d_head": 6, "model_name": "custom", "n_heads": 4, "d_mlp": 104, "act_fn": "gelu", "d_vocab": 7, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.12344267996967354, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 15808, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" }, { "case_id": "19", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19", "task_description": "Removes consecutive duplicate tokens from a sequence.", "vocab": [ "a", "b", "c" ], "max_seq_len": 15, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 32, "n_ctx": 15, "d_head": 8, "model_name": "custom", "n_heads": 4, "d_mlp": 128, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.15689290811054724, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 24576, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" }, { "case_id": "20", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20", "task_description": "Detect spam messages based on appearance of spam keywords.", "vocab": [ "J", "LB", "TPSI", "V", "b", "click", "no", "now", "oCLrZaW", "offer", "poiVg", "spam" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 13, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 52, "act_fn": "gelu", "d_vocab": 14, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.16, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3952, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" }, { "case_id": "21", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21", "task_description": "Extract unique tokens from a string", "vocab": [ "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta.json" } ], "transformer_cfg": { "n_layers": 4, "d_model": 50, "n_ctx": 10, "d_head": 12, "model_name": "custom", "n_heads": 4, "d_mlp": 200, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.09847319278346618, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 118400, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.0005, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.5, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" }, { "case_id": "26", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/26", "task_description": "Creates a cascading effect by repeating each token in sequence incrementally.", "vocab": [ "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 21, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 84, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.12344267996967354, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 27, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 10416, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/edges.pkl" }, { "case_id": "29", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/29", "task_description": "Creates abbreviations for each token in the sequence.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 13, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 52, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3952, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/edges.pkl" }, { "case_id": "3", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3", "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.", "vocab": [ "a", "b", "c", "x" ], "max_seq_len": 5, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 5, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 6, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.22188007849009167, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 1, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 10.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict", "same_size": false }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" }, { "case_id": "33", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33", "task_description": "Checks if each token's length is odd or even.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.17457431218879393, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" }, { "case_id": "34", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34", "task_description": "Calculate the ratio of vowels to consonants in each word.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 16, "n_ctx": 10, "d_head": 4, "model_name": "custom", "n_heads": 4, "d_mlp": 64, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.16329931618554522, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 5, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 6144, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" }, { "case_id": "35", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35", "task_description": "Alternates capitalization of each character in words.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 9, "n_ctx": 10, "d_head": 2, "model_name": "custom", "n_heads": 4, "d_mlp": 36, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 1872, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" }, { "case_id": "36", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36", "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.", "vocab": [ "\ud83d\udcd8", "\ud83d\ude0a", "\ud83d\ude22" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 6, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 24, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.19402850002906638, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 768, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" }, { "case_id": "37", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37", "task_description": "Reverses each word in the sequence except for specified exclusions.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" }, { "case_id": "4", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4", "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.", "vocab": [ "(", ")", "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 7, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.17056057308448835, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 1, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" }, { "case_id": "7", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/7", "task_description": "Returns the number of times each token occurs in the input.", "vocab": [ "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/meta.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 17, "n_ctx": 10, "d_head": 4, "model_name": "custom", "n_heads": 4, "d_mlp": 68, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.15689290811054724, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 10, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 6800, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model_cfg.pkl", "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.5, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "", "model_pair": "strict", "same_size": false, "seed": 1234, "batch_size": 256, "include_mlp": false, "next_token": false, "detach_while_caching": true }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/edges.pkl" }, { "case_id": "ioi", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi", "task_description": "Indirect Object Identification (IOI) task.", "max_seq_len": 16, "min_seq_len": 16, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/meta.json" } ], "transformer_cfg": { "n_layers": 6, "d_model": 64, "n_ctx": 1024, "d_head": 16, "model_name": "gpt2", "n_heads": 4, "d_mlp": 3072, "act_fn": "gelu_new", "d_vocab": 50257, "eps": 1e-05, "use_attn_result": false, "use_attn_scale": true, "use_split_qkv_input": false, "use_hook_mlp_in": false, "use_attn_in": false, "use_local_attn": false, "original_architecture": "GPT2LMHeadModel", "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": "gpt2", "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": "LNPre", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": null, "initializer_range": 0.02886751345948129, "init_weights": false, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 50257, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 84934656, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": false, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_cfg.pkl", "training_args": { "next_token": false, "non_ioi_thresh": 0.65, "use_per_token_check": false, "batch_size": 512, "num_workers": 0, "early_stop": true, "lr_scheduler": null, "scheduler_val_metric": [ "val/accuracy", "val/IIA" ], "scheduler_mode": "max", "scheduler_kwargs": {}, "clip_grad_norm": 1.0, "seed": 0, "lr": 0.001, "detach_while_caching": true, "optimizer_kwargs": { "betas": [ 0.9, 0.9 ] }, "atol": 0.05, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "val_IIA_sampling": "random", "use_all_tokens_for_behavior": false, "strict_weight": 0.4, "siit_sampling": "individual" }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/edges.pkl" }, { "case_id": "ioi_next_token", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token", "task_description": "Indirect Object Identification (IOI) task, trained using next token prediction.", "max_seq_len": 16, "min_seq_len": 16, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/edges.pkl" }, { "file_name": "ll_model.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model.pth" }, { "file_name": "ll_model_cfg.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_cfg.pkl" }, { "file_name": "meta.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/meta.json" } ], "transformer_cfg": { "n_layers": 6, "d_model": 64, "n_ctx": 1024, "d_head": 16, "model_name": "gpt2", "n_heads": 4, "d_mlp": 3072, "act_fn": "gelu_new", "d_vocab": 50257, "eps": 1e-05, "use_attn_result": false, "use_attn_scale": true, "use_split_qkv_input": false, "use_hook_mlp_in": false, "use_attn_in": false, "use_local_attn": false, "original_architecture": "GPT2LMHeadModel", "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": "gpt2", "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": "LNPre", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": null, "initializer_range": 0.02886751345948129, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 50257, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 2457600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": false, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_cfg.pkl", "training_args": { "next_token": true, "non_ioi_thresh": 0.65, "use_per_token_check": false, "batch_size": 256, "lr": 0.001, "num_workers": 0, "early_stop": true, "lr_scheduler": null, "scheduler_val_metric": [ "val/accuracy", "val/IIA" ], "scheduler_mode": "max", "clip_grad_norm": 1.0, "atol": 0.05, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4 }, "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/meta.json", "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model.pth", "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/edges.pkl" } ] }