Upload folder using huggingface_hub

README.md

@@ -1,5 +1,6 @@
 ---
 thumbnail: "https://assets-global.website-files.com/646b351987a8d8ce158d1940/64ec9e96b4334c0e1ac41504_Logo%20with%20white%20text.svg"
+base_model: gradientai/Llama-3-8B-Instruct-262k
 metrics:
 - memory_disk
 - memory_inference
@@ -59,9 +60,9 @@ You can run the smashed model with these steps:
 2. Load & run the model.
     ```python 
    from transformers import AutoModelForCausalLM, AutoTokenizer
-   
+   from awq import AutoAWQForCausalLM
 
-   model = AutoModelForCausalLM.from_pretrained("PrunaAI/gradientai-Llama-3-8B-Instruct-262k-AWQ-4bit-smashed", trust_remote_code=True, device_map='auto')
+   model = AutoAWQForCausalLM.from_quantized("PrunaAI/gradientai-Llama-3-8B-Instruct-262k-AWQ-4bit-smashed", trust_remote_code=True, device_map='auto')
    tokenizer = AutoTokenizer.from_pretrained("gradientai/Llama-3-8B-Instruct-262k")
     
    input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/tmpv9irb50c",
+  "_name_or_path": "/tmp/tmp4pr25txe",
   "architectures": [
     "LlamaForCausalLM"
   ],

results.json

@@ -1,26 +1,30 @@
 {
     "base_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "base_current_gpu_total_memory": 40339.3125,
-    "base_token_generation_latency_sync": 105.3484390258789,
-    "base_token_generation_latency_async": 104.91261184215546,
-    "base_token_generation_throughput_sync": 0.009492309608444691,
-    "base_token_generation_throughput_async": 0.009531742489687833,
-    "base_token_generation_CO2_emissions": 2.412649618989479e-05,
-    "base_token_generation_energy_consumption": 0.008419673305905371,
-    "base_inference_latency_sync": 86.63613433837891,
-    "base_inference_latency_async": 88.81196975708008,
-    "base_inference_throughput_sync": 0.01154252792598354,
-    "base_inference_throughput_async": 0.01125974350906996,
+    "base_token_generation_latency_sync": 53.581108474731444,
+    "base_token_generation_latency_async": 53.849875181913376,
+    "base_token_generation_throughput_sync": 0.018663294367483915,
+    "base_token_generation_throughput_async": 0.01857014517901559,
+    "base_token_generation_CO2_emissions": null,
+    "base_token_generation_energy_consumption": null,
+    "base_inference_latency_sync": 51.79688911437988,
+    "base_inference_latency_async": 51.439499855041504,
+    "base_inference_throughput_sync": 0.019306178751232753,
+    "base_inference_throughput_async": 0.019440313432635206,
+    "base_inference_CO2_emissions": null,
+    "base_inference_energy_consumption": null,
     "smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "smashed_current_gpu_total_memory": 40339.3125,
-    "smashed_token_generation_latency_sync": 57.02575531005859,
-    "smashed_token_generation_latency_async": 55.63120283186436,
-    "smashed_token_generation_throughput_sync": 0.017535936079457998,
-    "smashed_token_generation_throughput_async": 0.017975523610775167,
-    "smashed_token_generation_CO2_emissions": 1.8021672618399667e-05,
-    "smashed_token_generation_energy_consumption": 0.004920462357914908,
-    "smashed_inference_latency_sync": 59.50504913330078,
-    "smashed_inference_latency_async": 57.179927825927734,
-    "smashed_inference_throughput_sync": 0.016805296601971387,
-    "smashed_inference_throughput_async": 0.01748865446357837
+    "smashed_token_generation_latency_sync": 40.933387756347656,
+    "smashed_token_generation_latency_async": 41.39378983527422,
+    "smashed_token_generation_throughput_sync": 0.024429934945829818,
+    "smashed_token_generation_throughput_async": 0.024158213200083406,
+    "smashed_token_generation_CO2_emissions": null,
+    "smashed_token_generation_energy_consumption": null,
+    "smashed_inference_latency_sync": 52.317081451416016,
+    "smashed_inference_latency_async": 39.870476722717285,
+    "smashed_inference_throughput_sync": 0.019114216088843655,
+    "smashed_inference_throughput_async": 0.02508121502922043,
+    "smashed_inference_CO2_emissions": null,
+    "smashed_inference_energy_consumption": null
 }

smash_config.json

@@ -2,19 +2,19 @@
     "api_key": null,
     "verify_url": "http://johnrachwan.pythonanywhere.com",
     "smash_config": {
-        "pruners": "[]",
+        "pruners": "None",
         "pruning_ratio": 0.0,
-        "factorizers": "[]",
+        "factorizers": "None",
         "quantizers": "['awq']",
         "weight_quantization_bits": 4,
-        "output_deviation": 0.01,
-        "compilers": "[]",
+        "output_deviation": 0.005,
+        "compilers": "None",
         "static_batch": true,
         "static_shape": true,
         "controlnet": "None",
         "unet_dim": 4,
         "device": "cuda",
-        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelswbwdrhux",
+        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsnqte778j",
         "batch_size": 1,
         "model_name": "gradientai/Llama-3-8B-Instruct-262k",
         "task": "text_text_generation",