Upload folder using huggingface_hub (#2)

- 496ea46e041fc7d353b3ff37808a913445f5cb1bcc6a400ec7b5f44e173d1dc2 (6fcde3866a07ebbb15e62d04045aa93f0fcd035c)
- 17a3b7ac4467af3ace2cba5f78e634773978e56c358e00cadabc16e6364dcd73 (257abbb9a40316f1e95d74b5f4c770167a98650d)

README.md

@@ -60,9 +60,9 @@ You can run the smashed model with these steps:
 2. Load & run the model.
     ```python 
    from transformers import AutoModelForCausalLM, AutoTokenizer
-   
+   from awq import AutoAWQForCausalLM
 
-   model = AutoModelForCausalLM.from_pretrained("PrunaAI/meta-llama-Meta-Llama-3-8B-AWQ-4bit-smashed", trust_remote_code=True, device_map='auto')
+   model = AutoAWQForCausalLM.from_quantized("PrunaAI/meta-llama-Meta-Llama-3-8B-AWQ-4bit-smashed", trust_remote_code=True, device_map='auto')
    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
     
    input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/tmpdz8e_z82",
+  "_name_or_path": "/tmp/tmpbck3xy20",
   "architectures": [
     "LlamaForCausalLM"
   ],

results.json

@@ -1,30 +1,30 @@
 {
     "base_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "base_current_gpu_total_memory": 40339.3125,
-    "base_token_generation_latency_sync": 85.63984069824218,
-    "base_token_generation_latency_async": 86.27743311226368,
-    "base_token_generation_throughput_sync": 0.011676808268753886,
-    "base_token_generation_throughput_async": 0.011590516360156497,
-    "base_token_generation_CO2_emissions": 4.384232136591548e-05,
-    "base_token_generation_energy_consumption": 0.015093384395993267,
-    "base_inference_latency_sync": 83.11347198486328,
-    "base_inference_latency_async": 81.85713291168213,
-    "base_inference_throughput_sync": 0.012031743784956079,
-    "base_inference_throughput_async": 0.012216406370828147,
-    "base_inference_CO2_emissions": 4.367591428358372e-05,
-    "base_inference_energy_consumption": 0.00012165686007698748,
+    "base_token_generation_latency_sync": 56.77331275939942,
+    "base_token_generation_latency_async": 57.322778180241585,
+    "base_token_generation_throughput_sync": 0.017613909624015017,
+    "base_token_generation_throughput_async": 0.01744507212919221,
+    "base_token_generation_CO2_emissions": null,
+    "base_token_generation_energy_consumption": null,
+    "base_inference_latency_sync": 54.84390411376953,
+    "base_inference_latency_async": 53.76389026641846,
+    "base_inference_throughput_sync": 0.018233566996353427,
+    "base_inference_throughput_async": 0.018599844524729483,
+    "base_inference_CO2_emissions": null,
+    "base_inference_energy_consumption": null,
     "smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "smashed_current_gpu_total_memory": 40339.3125,
-    "smashed_token_generation_latency_sync": 53.57108993530274,
-    "smashed_token_generation_latency_async": 53.81308030337095,
-    "smashed_token_generation_throughput_sync": 0.018666784663289283,
-    "smashed_token_generation_throughput_async": 0.01858284257958298,
-    "smashed_token_generation_CO2_emissions": 3.582011233414709e-05,
-    "smashed_token_generation_energy_consumption": 0.009884854558870261,
-    "smashed_inference_latency_sync": 56.26316757202149,
-    "smashed_inference_latency_async": 55.486488342285156,
-    "smashed_inference_throughput_sync": 0.017773617148730877,
-    "smashed_inference_throughput_async": 0.01802240563200176,
-    "smashed_inference_CO2_emissions": 3.7329031799201206e-05,
-    "smashed_inference_energy_consumption": 9.846113068875813e-05
+    "smashed_token_generation_latency_sync": 43.27337074279785,
+    "smashed_token_generation_latency_async": 43.26997306197882,
+    "smashed_token_generation_throughput_sync": 0.02310890006566992,
+    "smashed_token_generation_throughput_async": 0.023110714641944086,
+    "smashed_token_generation_CO2_emissions": null,
+    "smashed_token_generation_energy_consumption": null,
+    "smashed_inference_latency_sync": 51.69879035949707,
+    "smashed_inference_latency_async": 43.543338775634766,
+    "smashed_inference_throughput_sync": 0.019342812337509556,
+    "smashed_inference_throughput_async": 0.02296562523955014,
+    "smashed_inference_CO2_emissions": null,
+    "smashed_inference_energy_consumption": null
 }

smash_config.json

@@ -2,19 +2,19 @@
     "api_key": null,
     "verify_url": "http://johnrachwan.pythonanywhere.com",
     "smash_config": {
-        "pruners": "[]",
+        "pruners": "None",
         "pruning_ratio": 0.0,
-        "factorizers": "[]",
+        "factorizers": "None",
         "quantizers": "['awq']",
         "weight_quantization_bits": 4,
-        "output_deviation": 0.01,
-        "compilers": "[]",
+        "output_deviation": 0.005,
+        "compilers": "None",
         "static_batch": true,
         "static_shape": true,
         "controlnet": "None",
         "unet_dim": 4,
         "device": "cuda",
-        "cache_dir": "/ceph/hdd/staff/charpent/.cache/models9k4p46gn",
+        "cache_dir": "/ceph/hdd/staff/charpent/.cache/models1tymb0wo",
         "batch_size": 1,
         "model_name": "meta-llama/Meta-Llama-3-8B",
         "task": "text_text_generation",