df6f621a01d4c181ee97365da9654c0277ec8587d5fe1eafb33a62f30ee98ddb

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/tmpu4qxllsm",
+  "_name_or_path": "/tmp/tmpetnsv95z",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -11,7 +11,7 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 8192,
+  "max_position_embeddings": 524288,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
@@ -27,7 +27,7 @@
   },
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
-  "rope_theta": 500000.0,
+  "rope_theta": 2000000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
   "transformers_version": "4.40.0",

results.json

@@ -1,30 +1,32 @@
 {
     "base_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "base_current_gpu_total_memory": 40339.3125,
-    "base_token_generation_latency_sync": 54.04956970214844,
-    "base_token_generation_latency_async": 54.09220550209284,
-    "base_token_generation_throughput_sync": 0.01850153489677552,
-    "base_token_generation_throughput_async": 0.018486951876297775,
+    "base_perplexity": 13.74486255645752,
+    "base_token_generation_latency_sync": 37.35860481262207,
+    "base_token_generation_latency_async": 37.739571928977966,
+    "base_token_generation_throughput_sync": 0.026767594909276098,
+    "base_token_generation_throughput_async": 0.026497385870775065,
     "base_token_generation_CO2_emissions": null,
     "base_token_generation_energy_consumption": null,
-    "base_inference_latency_sync": 53.23417663574219,
-    "base_inference_latency_async": 52.190279960632324,
-    "base_inference_throughput_sync": 0.018784924708097873,
-    "base_inference_throughput_async": 0.01916065598334231,
+    "base_inference_latency_sync": 122.0685821533203,
+    "base_inference_latency_async": 43.52927207946777,
+    "base_inference_throughput_sync": 0.008192116123246046,
+    "base_inference_throughput_async": 0.022973046693139806,
     "base_inference_CO2_emissions": null,
     "base_inference_energy_consumption": null,
     "smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "smashed_current_gpu_total_memory": 40339.3125,
-    "smashed_token_generation_latency_sync": 40.57583160400391,
-    "smashed_token_generation_latency_async": 40.6801862642169,
-    "smashed_token_generation_throughput_sync": 0.024645212691126284,
-    "smashed_token_generation_throughput_async": 0.024581991672924564,
+    "smashed_perplexity": 14.672628402709961,
+    "smashed_token_generation_latency_sync": 43.5498722076416,
+    "smashed_token_generation_latency_async": 43.305750377476215,
+    "smashed_token_generation_throughput_sync": 0.02296217989417044,
+    "smashed_token_generation_throughput_async": 0.02309162158104783,
     "smashed_token_generation_CO2_emissions": null,
     "smashed_token_generation_energy_consumption": null,
-    "smashed_inference_latency_sync": 51.8927360534668,
-    "smashed_inference_latency_async": 41.21739864349365,
-    "smashed_inference_throughput_sync": 0.019270519846355125,
-    "smashed_inference_throughput_async": 0.02426159905552056,
+    "smashed_inference_latency_sync": 164.25011291503907,
+    "smashed_inference_latency_async": 69.53554153442383,
+    "smashed_inference_throughput_sync": 0.006088275875446525,
+    "smashed_inference_throughput_async": 0.014381134854683576,
     "smashed_inference_CO2_emissions": null,
     "smashed_inference_energy_consumption": null
 }

smash_config.json

@@ -14,7 +14,7 @@
         "controlnet": "None",
         "unet_dim": 4,
         "device": "cuda",
-        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsa_klkits",
+        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsgmyikxqu",
         "batch_size": 1,
         "model_name": "cognitivecomputations/dolphin-2.9-llama3-8b-256k",
         "task": "text_text_generation",

tokenizer_config.json

@@ -2069,6 +2069,7 @@
   "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|im_end|>",
+  "legacy": false,
   "model_input_names": [
     "input_ids",
     "attention_mask"