Synchronizing local compiler cache.

neuronxcc-2.12.68.0+4480452af/0_REGISTRY/0.0.19.dev0/training/llama/NousResearch/Llama-2-7b-hf/a79789557d36431a64ce.json

@@ -0,0 +1 @@
+{"vocab_size": 32000, "max_position_embeddings": 4096, "hidden_size": 4096, "intermediate_size": 11008, "num_hidden_layers": 4, "num_attention_heads": 32, "num_key_value_heads": 32, "hidden_act": "silu", "initializer_range": 0.02, "rms_norm_eps": 1e-05, "pretraining_tp": 1, "use_cache": true, "rope_theta": 10000.0, "rope_scaling": null, "attention_bias": false, "attention_dropout": 0.0, "torch_dtype": "float16", "tie_word_embeddings": false, "architectures": ["LlamaForCausalLM"], "bos_token_id": 1, "pad_token_id": 0, "eos_token_id": 2, "model_type": "llama", "neuron": {"model_class": "LlamaForCausalLM", "precision": "bfloat16", "num_neuron_cores_per_node": 32, "compiler_version": "2.12.68.0+4480452af", "tensor_parallel_size": 8, "pipeline_parallel_size": 1, "training": true, "input_specs": {"input_ids": [1, 4096], "attention_mask": [1, 4096], "labels": [1, 4096]}}}

neuronxcc-2.12.68.0+4480452af/MODULE_11259215915185906806+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe17255dfcd4bcad5ab454eff4fc41d9662632213cb5f72b7d20bcc8e3fabeb
+size 716

neuronxcc-2.12.68.0+4480452af/MODULE_1140062030530484659+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2b0a3678652641996f00a6eb1e6f9b6d2fa22ab9d71875d3d19ce7d52d08de
+size 370

neuronxcc-2.12.68.0+4480452af/MODULE_11465265256744440066+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1f847bf6d3927f7f9e117a7defb8d5ef10c9e7dac2e0f85b8a363be0bf7a538
+size 2123

neuronxcc-2.12.68.0+4480452af/MODULE_12742115949388795772+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b5bd2b0d64212c95c821b5dde60a8d110368df819b795cc37c25dcc86cfda19
+size 3755

neuronxcc-2.12.68.0+4480452af/MODULE_12805077405077525634+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c17d3b21fb996991edb027240a477bb927d004407922ce9d9f4bb1ab73597047
+size 519

neuronxcc-2.12.68.0+4480452af/MODULE_12931649185302938647+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f602781b334f0dda4740cc975f930ba99d062a886b8d1d5ccd440a82d34d676c
+size 383

neuronxcc-2.12.68.0+4480452af/MODULE_13182116114780172099+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d919419f34bce4c1b9e29a036ef68ad57c1d21ffd98997ee716ddb73900066a2
+size 1991

neuronxcc-2.12.68.0+4480452af/MODULE_13182116114780172099+d41d8cd9/model.log

@@ -0,0 +1 @@
+Failed compilation with ['neuronx-cc', '--target=trn1', 'compile', '--framework', 'XLA', '/tmp/ubuntu/neuroncc_compile_workdir/ab44af52-f5af-4277-87f9-676ef58cadb6/model.MODULE_13182116114780172099+d41d8cd9.hlo.pb', '--output', '/tmp/ubuntu/neuroncc_compile_workdir/ab44af52-f5af-4277-87f9-676ef58cadb6/model.MODULE_13182116114780172099+d41d8cd9.neff', '--verbose=35']: 2024-02-21T22:24:14Z [TEN404] Internal tensorizer error - Please open a support ticket at https://github.com/aws-neuron/aws-neuron-sdk/issues/new

neuronxcc-2.12.68.0+4480452af/MODULE_13182116114780172099+d41d8cd9/model.neff

@@ -0,0 +1 @@
+ERROR

neuronxcc-2.12.68.0+4480452af/MODULE_13768167953909147434+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ed29eafc4216e771a545796b414cf9be73d349942278b65c26eb971c882254e
+size 495

neuronxcc-2.12.68.0+4480452af/MODULE_1384246517362840472+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:297549782234547dc5eef9c8f2648584c26491c058d44f3f46f73a09f45c5942
+size 3438

neuronxcc-2.12.68.0+4480452af/MODULE_14243915418106942438+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce0d680fec13f461e411ba96f48bb86089dfa4f6f4589e77a67a9396335858a8
+size 370

neuronxcc-2.12.68.0+4480452af/MODULE_15512879877928675536+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57eab738a213035620e227e61bb8bc139fb7bf971f49c51980343d59ab2c8d7b
+size 1729

neuronxcc-2.12.68.0+4480452af/MODULE_17349209021482049450+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12ef8e77d95b98cafb5353f2d91ba3dee4a79b064123c394a71e622226f8e29a
+size 1356

neuronxcc-2.12.68.0+4480452af/MODULE_17537820016390692363+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d86414330ccff923123f0b4723afaf2ec333d879ae6cb9eb1d64cb24941f23b
+size 1794

neuronxcc-2.12.68.0+4480452af/MODULE_17537820016390692363+d41d8cd9/model.log

@@ -0,0 +1 @@
+Failed compilation with ['neuronx-cc', '--target=trn1', 'compile', '--framework', 'XLA', '/tmp/ubuntu/neuroncc_compile_workdir/28aeedc6-550a-4bf5-a1a4-ba7aa5c5176e/model.MODULE_17537820016390692363+d41d8cd9.hlo.pb', '--output', '/tmp/ubuntu/neuroncc_compile_workdir/28aeedc6-550a-4bf5-a1a4-ba7aa5c5176e/model.MODULE_17537820016390692363+d41d8cd9.neff', '--verbose=35']: 2024-02-21T22:25:30Z [TEN404] Internal tensorizer error - Please open a support ticket at https://github.com/aws-neuron/aws-neuron-sdk/issues/new

neuronxcc-2.12.68.0+4480452af/MODULE_17537820016390692363+d41d8cd9/model.neff

@@ -0,0 +1 @@
+ERROR

neuronxcc-2.12.68.0+4480452af/MODULE_18030807366965164841+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:869050743b6a3021e69e07681602c015a24f93be9b0331408aef0b9b321b2c79
+size 534

neuronxcc-2.12.68.0+4480452af/MODULE_18338046637025348618+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cefc0db532bb9b38f7be12d6b72161745f9d2382bdcc9dbba9e557b6d5e2d3cb
+size 519

neuronxcc-2.12.68.0+4480452af/MODULE_18423395511079979584+d41d8cd9/model.hlo.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4fb68720e3568d62948b94f23e42f66b47c4c999c1a43a3213264c491d9ee88
+size 679