Initial model commit

config.yml

@@ -0,0 +1,229 @@
+{
+    "version": ".unknown.",
+    "runner": {
+        "runner_type": "pdsh",
+        "hostsfile": null,
+        "hosts": null,
+        "master_port": 29500,
+        "master_addr": null,
+        "script": "src/scaling/transformer/train.py",
+        "default_gpu_count": 8,
+        "docker_config": {
+            "docker_container": null,
+            "docker_sudo": false,
+            "docker_mounts": [
+                [
+                    "/mnt/",
+                    "/mnt/"
+                ]
+            ]
+        },
+        "use_determined": true
+    },
+    "logger": {
+        "log_level": "info",
+        "log_dir": null,
+        "metrics_ranks": null,
+        "use_wandb": true,
+        "wandb_ranks": null,
+        "wandb_host": "https://api.wandb.ai",
+        "wandb_team": "aleph-alpha",
+        "wandb_project": "ng-semantic-embedding",
+        "wandb_group": "pharia_7B_embedding_all_source_256-2024-10-01-19-28-52",
+        "wandb_api_key": "d80060c84c496915fcad15283809dbde65087d73",
+        "use_tensorboard": false,
+        "tensorboard_ranks": null,
+        "determined_metrics_ranks": null
+    },
+    "topology": {
+        "global_rank": 0,
+        "world_size": 256,
+        "local_slot": 0,
+        "model_parallel_size": 1,
+        "pipe_parallel_size": 1,
+        "data_parallel_size": 256,
+        "global_batch_size": 2048,
+        "micro_batch_size": 8,
+        "gradient_accumulation_steps": 1,
+        "pipe_partition_method": "balanced",
+        "pipe_partition_overwrite": null,
+        "activation_checkpointing_type": "every_layer",
+        "sequence_parallel": false
+    },
+    "optimizer": {
+        "method": "adamw",
+        "beta1": 0.9,
+        "beta2": 0.95,
+        "eps": 1e-15,
+        "gradient_clipping": 0.0,
+        "allreduce_bucket_size": 500000000,
+        "loss_scaler": {
+            "enable": false,
+            "initial_scale": 4294967296.0,
+            "window": 1000,
+            "hysteresis": 2.0,
+            "consecutive_hysteresis": false,
+            "min_scale": 1.0,
+            "factor": 2.0
+        },
+        "zero": true,
+        "zero_save_static": false,
+        "debug_log": false
+    },
+    "learning_rate_scheduler": {
+        "learning_rate": 2e-05,
+        "learning_rate_minimum": 0.0,
+        "learning_rate_decay_style": "linear",
+        "learning_rate_decay_iters": 938,
+        "learning_rate_warmup_steps": 50
+    },
+    "embedding_learning_rate_scheduler": {
+        "learning_rate": 0.0,
+        "learning_rate_minimum": 0.0,
+        "learning_rate_decay_style": "cosine",
+        "learning_rate_decay_iters": 0,
+        "learning_rate_warmup_steps": 0
+    },
+    "training": {
+        "weight_decay": 0.0001,
+        "finetune": true,
+        "finetunable_parameters": [
+            "embed"
+        ],
+        "parameters_exclude": [],
+        "use_separate_lr_on_embeddings": false,
+        "use_deterministic_torch_algorithms": false,
+        "loss_function_config": {
+            "loss_type": "contrastive_loss",
+            "number_of_hard_negatives": 1,
+            "use_instructions": true,
+            "query_side_only": false,
+            "scale": 50,
+            "log_verbose_metrics": true
+        }
+    },
+    "trainer": {
+        "save_dir": "checkpoints",
+        "save_interval": 938,
+        "load_dir": "/scratch/samuel/Pharia-1-LLM-7B-control",
+        "train_iterations": 938,
+        "assert_checkpoint_loaded": true,
+        "load_optimizer_states": false,
+        "delete_past_optimizer_states": true,
+        "load_context": false,
+        "allowed_missing_keys_in_checkpoint": [
+            "embed"
+        ],
+        "allowed_unexpected_keys_in_checkpoint": null,
+        "ignore_keys_in_checkpoint": null,
+        "merge_lora_after_loading_checkpoint": false,
+        "seed": 42,
+        "dataloader_num_workers": 0,
+        "dataloader_pin_memory": true,
+        "dataloader_prefetch_factor": null,
+        "eval_iterations": 1,
+        "eval_interval": null,
+        "separate_file_for_parameters": [
+            "adapter_embed"
+        ]
+    },
+    "profiler": {
+        "profile_steps": 0,
+        "profile_start_at_step": 0,
+        "profiler_output": null
+    },
+    "transformer_architecture": {
+        "vocab_size": 128000,
+        "vocab_file": "/scratch/samuel/Pharia-1-LLM-7B-control/vocab.json",
+        "hidden_size": 4608,
+        "num_layers": 27,
+        "num_attention_heads": 36,
+        "num_local_attention_heads": 0,
+        "local_attention_window_size": null,
+        "rotary_embedding_base": 1000000,
+        "rotary_percentage": 1.0,
+        "sequence_length": 2048,
+        "norm_type": "layernorm",
+        "relative_position_embedding_type": "rotary_complex",
+        "mlp_type": "default",
+        "mlp_factor": 4.0,
+        "attention_bias": true,
+        "attention_qkv_in_one": false,
+        "attention_num_kv_heads": 4,
+        "attention_use_matmul": false,
+        "mlp_bias": true,
+        "key_query_norm": false,
+        "weight_tying": false,
+        "masked_softmax": {
+            "kernel": "torch",
+            "softmax_in_fp32": true,
+            "scale": 1.0,
+            "deterministic_flash_attn_bwd": false
+        },
+        "layernorm": {
+            "optimization_type": "torch",
+            "layernorm_epsilon": 1e-05
+        },
+        "precision": "bfloat16",
+        "dropout_embedding": 0.0,
+        "dropout_attention_probs": 0.0,
+        "dropout_after_attention": 0.0,
+        "dropout_after_mlp": 0.0,
+        "bitfit_bias_config": null,
+        "finetunable_token_ids": [],
+        "image_encoder": false,
+        "dropout_image_encoder": 0.0,
+        "softprompt_config": null,
+        "adapter_config": {
+            "name": "embed",
+            "attention_downsampling_factor": 0.25,
+            "mlp_downsampling_factor": 0.25,
+            "init_std": 1e-05,
+            "version": ".unknown."
+        },
+        "lora_config": null,
+        "embedding_head_config": {
+            "name": "pooling_only",
+            "proj_layers": [
+                256
+            ],
+            "pooling": "weighted_mean"
+        },
+        "lm_head": false,
+        "causal": true
+    },
+    "data": {
+        "legacy_dataset": false,
+        "load_mmap_index_to_memory": false,
+        "use_mmap": false,
+        "load_data_item_mmap_index_to_memory": false,
+        "finetuning_dataset": false,
+        "finetuning_chat_dataset": false,
+        "finetuning_dataset_memory_map": false,
+        "embedding_dataset": true,
+        "embedding_dataset_memory_map": false,
+        "data_prefixes": [
+            "/scratch/samuel/all_source_data_templated/allnli_eng.jsonl",
+            "/scratch/samuel/all_source_data_templated/hotpot_with_bm25_negatives.jsonl",
+            "/scratch/samuel/all_source_data_templated/medi_1.jsonl",
+            "/scratch/samuel/all_source_data_templated/msmarco.jsonl"
+        ],
+        "validation_data_prefixes": null,
+        "blended_dataset": {
+            "weight_by_num_documents": true,
+            "weighted_sampler_alpha": 1.0,
+            "weights": null,
+            "weight_examples_proportional": false,
+            "ep_maximum": null,
+            "ep_temperature": 1.0,
+            "minimum_dataset_size": 0,
+            "cache_directory": "/scratch/samuel/blended_dataset_cache",
+            "shuffle_dataset_indices": false,
+            "load_dataset_indices_to_memory": false
+        },
+        "only_full_sequences": false,
+        "allow_incomplete_sequences_every_n": 0
+    },
+    "determined_experiment_id": 2965,
+    "determined_trial_id": 86040
+}

model_state_layer_0_EmbeddingInput.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3722b0053cc854fcf13733cf9d10317d041d9aae5b65643f697974cc2882efb3
+size 1179649443

model_state_layer_10_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef63496e306c9f3e0112742300dd47b51c31fd2226fd3d7aa5c2adc1a5ffb265
+size 434219902

model_state_layer_10_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3071455dec4665113216f294ea7cd1af7196380dda917a59c3efb0285b7e7c1f
+size 42469872

model_state_layer_11_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ae21f20ea4e1492bdf2369f7ef91dda55015f81ab4eab9604a9362086d12cb
+size 434219902

model_state_layer_11_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55da45ccc291c5dd372ad890c4d050f1d3cbb7f8408a284bbc70ec8128988f11
+size 42469872

model_state_layer_12_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd902b46a5a353687a2f31e754164e3db18fe72c769a69ed13284bbff224f455
+size 434219902

model_state_layer_12_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db75b1193e64fe400550066600e0cac3444a7ccada4a8a44c2d13a3c084c1fed
+size 42469872

model_state_layer_13_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:664c1b719da7b9218debdc2e9794798ca9d0f22568b138996e182229d79c3444
+size 434219902

model_state_layer_13_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97f60798bafe4179fda66410bc6ea4d926b4bc10be86c565187a505387ecc9ec
+size 42469872

model_state_layer_14_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef77d20a502e6c8d8908cbd7cb222d04ccea69002e78042a57ac11a8a0b4362
+size 434219902

model_state_layer_14_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2416a2626b40f412a01b9177bbc4b295856663430abbf4c1f3770f2bb2c33170
+size 42469872

model_state_layer_15_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7de81299e97471d03af358c730a40238867eb6032c93c4cebcba714f2db5d47
+size 434219902

model_state_layer_15_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c68832f2ed909e3663b9317413b6f08e37e3326553b63b4d87b5f99b25e05e
+size 42469872

model_state_layer_16_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e866357f6d06e90c4a4785e7b0cc2056b50361904168d54fb6fc314ef9f4639b
+size 434219902

model_state_layer_16_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00f14bd76f5e18b3d790ad6b15b49273dadad3b6682cdb39c1690505e01a8b6f
+size 42469872

model_state_layer_17_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b4c380ed92cd055a067919127a7365bcd66f18fa8dfa8d38653018ac0f597c
+size 434219902

model_state_layer_17_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c6fd7770f7910dfb57aee214b30d8be581c1621c83eabbdab0ae76421d244ac
+size 42469872

model_state_layer_18_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c34494e568a417b9adae97acb4d7c76012a9903a3e3d596e68d0020f1af6d2b0
+size 434219902

model_state_layer_18_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d8a4cdc5f694d1607a2c0824f49a5a43e36917ba31e7cbdbaecd17179fa85e9
+size 42469872

model_state_layer_19_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2807c758304a165e6140b8942fa9eab390814f81c26fd327628822ce004ebfe7
+size 434219902

model_state_layer_19_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88ff78dc09729fe2a641b665b06bbe51537bf12f702269759abbc94335c66a30
+size 42469872

model_state_layer_1_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c2679a5ca251090f3a75c87b7add5f42e975a213ec8aaee90bde27da688a3d1
+size 434219882

model_state_layer_1_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e21ae05b1e2d5e791b3aaeacca8f47668b2f93ff4c1c046a765be91aae26f95
+size 42469864

model_state_layer_20_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:197bec6a16feca3ac18653521b975673a56a071bf59ae3a9bfce737648bda775
+size 434219902

model_state_layer_20_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85b5a318c6ecabdf8019452d2d74e50f9ae4ad99ef1d034a180133503afcd190
+size 42469872

model_state_layer_21_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31eb0f91b2b326f7b869a2dc5dae07be2a6831efb72d3193d1d6d457a2d6fcff
+size 434219902

model_state_layer_21_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02d4de53679a0261925cc092fa0be06ba6d663735f037cbbb8f506a61baa88c2
+size 42469872

model_state_layer_22_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8deac3ee07d0285db53ce915d9315491287708edec05a9f16e17e3b0c2395ceb
+size 434219902

model_state_layer_22_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b85af03d8a9ed9c6bf66619563e5cd26b8894b30a0301cb8cd0e45df52a7a0f
+size 42469872

model_state_layer_23_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a4de46c226702e34dbf4d5cd8ead1d366f7cd94b5c5a0f9481dc0f5d502214
+size 434219902

model_state_layer_23_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb864841645a7ae04dc3867d1fe26a307be28da55a787ccb1c8ab044352c13fd
+size 42469872

model_state_layer_24_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:290bde2e41ce42a45bef1ae6977cbc4f8add6ca43f1493fedeac5d17b7497248
+size 434219902

model_state_layer_24_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a671aed521d6783c25d80f43a9530599d2a15d48f6f2d98b3eceba8294a94c77
+size 42469872

model_state_layer_25_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45968daef6d99223322950b1251b22b8462b8d95fd355edb67421feaef60fc49
+size 434219902

model_state_layer_25_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:487b308e165857cdbd6e52deb2df0239372dad547befb07bdd0f7ff7662d0314
+size 42469872

model_state_layer_26_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47274e293886f267b6ef434f55b4087c3400132b40e33ea732762fdcf544648d
+size 434219902

model_state_layer_26_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ac9f37f0eb3b2aa48338ae92482917cede328234aac94cb5acd4e6e0535826a
+size 42469872

model_state_layer_27_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f25827475fe28d36d12002c77f4e337f01af1d38524d74163c804f387ebcad4
+size 434219902

model_state_layer_27_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8619d0cd488dc2f267dc84bf9a85094894e93e6067b623223442bc706ed88f52
+size 42469872

model_state_layer_28_LayerNormWrapper.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cd1372d2ae21d66d644fb98394d0868cc9aeff84ccc20619a9bbf95eeebcd62
+size 20172

model_state_layer_29_TransformerEmbeddingHead.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84863b3c4b23eb9d18224b2c4275f2998fd48901ce5361bfe4cd5618598c2386
+size 2360794

model_state_layer_2_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15decebd9fe0f57b1d69766411ab106d37f354b7dc1b6b280d48baca79626756
+size 434219882

model_state_layer_2_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cecf3037992e2b5d443fae5d312368032dc3e3191bf4c466c69cd90380b4bc0
+size 42469864

model_state_layer_3_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c9465e9dcb05d29f2c9e320c53e486825e05341a6c5097fcefa86ca1384f495
+size 434219882

model_state_layer_3_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe555db4a265531edfd537af40a1c62de8a0b363aeb310af820aedc8eac1a3c
+size 42469864

model_state_layer_4_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f8449ff2999f09a5ae6a621104c5d8929bb9f6969315cf4210bfe327dd0bc72
+size 434219882

model_state_layer_4_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36242c30fb06de70fe7a674fbf2b8aef6291f837847e90b365d58f9314ca2345
+size 42469864

model_state_layer_5_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a4b77dab79cedd475c6335c22188eefd71bcb07cf39a11d8f80deb7b678f2ac
+size 434219882

model_state_layer_5_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25e9e8c698d343cfd55e21e113ade823149883e63fe38b0c550e061c28acff54
+size 42469864