Upload MMaDA-VLA model

args.json

@@ -0,0 +1,57 @@
+{
+    "pretrained_mmada": "/home/MMaDA-8B-Base",
+    "pretrained_mmadavla": null,
+    "action_chunk_size": 5,
+    "data_paths": [
+        "/home/mmadavla_arxiv_ver/roboturk_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/iamlab_cmu_pickup_insert_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/dlr_edan_shared_control_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/droid_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/berkeley_fanuc_manipulation_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/austin_buds_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/language_table_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/kuka_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/taco_play_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/austin_sirius_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/goal.parquet",
+        "/home/mmadavla_arxiv_ver/bc_z_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/stanford_hydra_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/berkeley_autolab_ur5_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/90.parquet",
+        "/home/mmadavla_arxiv_ver/10.parquet",
+        "/home/mmadavla_arxiv_ver/furniture_bench_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/calvin.parquet",
+        "/home/mmadavla_arxiv_ver/utaustin_mutex_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/fmb_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/toto_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/austin_sailor_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/jaco_play_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/object.parquet",
+        "/home/mmadavla_arxiv_ver/cmu_stretch_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/berkeley_cable_routing_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/spatial.parquet",
+        "/home/mmadavla_arxiv_ver/viola_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/fractal20220817_data_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/nyu_franka_play_dataset_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/bridgev2_lerobot.parquet",
+        "/home/mmadavla_arxiv_ver/ucsd_kitchen_dataset_lerobot.parquet"
+    ],
+    "cache_dir": "/home/cache_dir",
+    "max_text_len": 128,
+    "output_dir": "./ckpt-arxiv-ver/MMaDA-VLA/pretrained/lr1e-4",
+    "seed": 509,
+    "batch_size_per_gpu": 10,
+    "mixed_precision": "bf16",
+    "learning_rate": 0.0001,
+    "weight_decay": 0.01,
+    "beta": [
+        0.9,
+        0.999
+    ],
+    "warmup_ratio": 0.01,
+    "num_train_epochs": 1,
+    "save_epoch": 1,
+    "max_grad_norm": 1.0,
+    "min_masking_rate": 0.0,
+    "mask_contiguous_region_prob": null
+}

config.json

@@ -0,0 +1,60 @@
+{
+  "action_codebook_size": 256,
+  "action_num_vq_tokens": 35,
+  "activation_type": "silu",
+  "alibi": false,
+  "alibi_bias_max": 8.0,
+  "architectures": [
+    "MMaDAVLAModelLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_layer_norm": false,
+  "attention_layer_norm_with_affine": true,
+  "bias_for_layer_norm": false,
+  "block_group_size": 1,
+  "block_type": "llama",
+  "d_model": 4096,
+  "embedding_dropout": 0.0,
+  "embedding_size": 134912,
+  "eos_token_id": 126081,
+  "flash_attention": false,
+  "include_bias": false,
+  "include_qkv_bias": false,
+  "init_cutoff_factor": null,
+  "init_device": "meta",
+  "init_fn": "mitchell",
+  "init_std": 0.02,
+  "input_emb_norm": false,
+  "layer_norm_type": "rms",
+  "layer_norm_with_affine": true,
+  "llm_vocab_size": 126464,
+  "mask_token_id": 126336,
+  "max_sequence_length": 4096,
+  "mlp_hidden_size": 12288,
+  "mlp_ratio": 4,
+  "model_type": "mmadavla",
+  "multi_query_attention": null,
+  "n_heads": 32,
+  "n_kv_heads": 32,
+  "n_layers": 32,
+  "new_vocab_size": 134912,
+  "num_new_special_tokens": 0,
+  "pad_token_id": 126081,
+  "precision": "amp_bf16",
+  "pretrained_model_path": "/data_storage/shared/pretrained_models/LLaDA-8B-Instruct",
+  "residual_dropout": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_full_precision": true,
+  "rope_theta": 500000.0,
+  "scale_logits": false,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.3",
+  "use_cache": false,
+  "vision_codebook_size": 8192,
+  "vision_num_vq_tokens": 256,
+  "vocab_size": 134912,
+  "w_clip_vit": false,
+  "weight_tying": false
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 126080,
+  "eos_token_id": 126081,
+  "transformers_version": "4.52.3"
+}

mask_rates.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc8ffa9ea9eaf07b5698632a4028164481a77be6bd8df21e1a9699cfef8b3e69
+size 762128

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea1492f671296e2b59855c704e8543b27cf64316af34a4c54248ac71cda3127b
+size 4930569720

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ae0c69e73d2d8a4d7dae16bca044a57417a9f880ad40f10a339601d53bbda1c
+size 4932693832

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2bc0927549625ab21ddb28a9312c3dcf6872ad8f7b1c2a22d0cc403ead60a8c
+size 4999819560

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25af5db4f4915f4e1351def358d83ca91f142264d750db2d21ee6124dc00bd9c
+size 1306526080

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16169574400
+  },
+  "weight_map": {
+    "model.transformer.blocks.0.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.0.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.1.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.10.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.10.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.11.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.12.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.13.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.14.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.15.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.16.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.17.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.18.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.19.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.2.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.2.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.20.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.20.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.20.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.21.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.22.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.23.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.24.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.25.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.26.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.27.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.28.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.29.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.3.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.3.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.30.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.ff_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.30.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.attn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.attn_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.ff_norm.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.ff_out.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.ff_proj.weight": "model-00004-of-00004.safetensors",
+    "model.transformer.blocks.31.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.31.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.transformer.blocks.31.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.transformer.blocks.4.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.4.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.5.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.6.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.7.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.attn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.attn_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.ff_norm.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.ff_out.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.ff_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.8.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.8.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.blocks.9.attn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.attn_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.ff_norm.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.ff_out.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.ff_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.blocks.9.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.transformer.ff_out.weight": "model-00004-of-00004.safetensors",
+    "model.transformer.ln_f.weight": "model-00001-of-00004.safetensors",
+    "model.transformer.wte.weight": "model-00001-of-00004.safetensors"
+  }
+}

total_losses.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddcdf9245aac803510e0fb2d4ab60965088caa48934e967387dc68ac59730729
+size 762128