{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d00601c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a8d52aa0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:numexpr.utils:Note: NumExpr detected 24 cores but \"NUMEXPR_MAX_THREADS\" not set, so enforcing safe limit of 8.\n",
      "INFO:numexpr.utils:NumExpr defaulting to 8 threads.\n",
      "INFO:datasets:PyTorch version 2.2.1+cu118 available.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from datasets import load_dataset\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ab513a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from awq import AutoAWQForCausalLM\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "model_path = 'llava-hf/llava-v1.6-vicuna-13b-hf'\n",
    "quant_path = './llava-v1.6-vicuna-13b-hf-awq'\n",
    "quant_config = { \"zero_point\": True, \"q_group_size\": 128, \"w_bit\": 4, \"version\": \"GEMM\" }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "41d1869f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/main/config.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140170094142960 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/e5a59fb69b73666871f8db1853e3bb796aa2b340.lock\n",
      "DEBUG:filelock:Lock 140170094142960 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/e5a59fb69b73666871f8db1853e3bb796aa2b340.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/main/config.json HTTP/1.1\" 200 1341\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "303d81955f424d5487d1fd5af1bbba12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/1.34k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140170094142960 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/e5a59fb69b73666871f8db1853e3bb796aa2b340.lock\n",
      "DEBUG:filelock:Lock 140170094142960 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/e5a59fb69b73666871f8db1853e3bb796aa2b340.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/models/llava-hf/llava-v1.6-vicuna-13b-hf/revision/main HTTP/1.1\" 200 1746\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7493b0fcf384434889a7f08f0b636f2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 17 files:   0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model-00004-of-00006.safetensors HTTP/1.1\" 302 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459220896 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/5c897a21e3a58a1e9e2a6b8f846d6af561da1e18e83ddd5f084b43b9f59c001a.lock\n",
      "DEBUG:filelock:Lock 140174459220896 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/5c897a21e3a58a1e9e2a6b8f846d6af561da1e18e83ddd5f084b43b9f59c001a.lock\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/generation_config.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140169840967104 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/15f9d0696a9bcf281dee46eab23dd82ac4e75f4a.lock\n",
      "DEBUG:filelock:Lock 140169840967104 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/15f9d0696a9bcf281dee46eab23dd82ac4e75f4a.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/generation_config.json HTTP/1.1\" 200 159\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/added_tokens.json HTTP/1.1\" 200 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "994b5ae1a7b44169acc1b252c77eae70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/159 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to acquire lock 140174459217392 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/93a7c1986e488383273279c4e698a8484b6973a5.lock\n",
      "DEBUG:filelock:Attempting to release lock 140169840967104 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/15f9d0696a9bcf281dee46eab23dd82ac4e75f4a.lock\n",
      "DEBUG:filelock:Lock 140169840967104 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/15f9d0696a9bcf281dee46eab23dd82ac4e75f4a.lock\n",
      "DEBUG:filelock:Lock 140174459217392 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/93a7c1986e488383273279c4e698a8484b6973a5.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model-00005-of-00006.safetensors HTTP/1.1\" 302 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/added_tokens.json HTTP/1.1\" 200 23\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/.gitattributes HTTP/1.1\" 200 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model-00001-of-00006.safetensors HTTP/1.1\" 302 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459212160 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/388813b227c1a35a05bac248932c99af9e7b0b1842a7bb9f45a829275f7bcb98.lock\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbbd8a21f6aa49c993e8fab19ecf3f64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "added_tokens.json:   0%|          | 0.00/23.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to acquire lock 140169840963888 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a6344aac8c09253b3b630fb776ae94478aa0275b.lock\n",
      "DEBUG:filelock:Lock 140174459212160 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/388813b227c1a35a05bac248932c99af9e7b0b1842a7bb9f45a829275f7bcb98.lock\n",
      "DEBUG:filelock:Attempting to acquire lock 140169840964704 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/764c9e65681b29cd662a83ea56e72f208c7f2f1c51cd5ea6bd5fb34f068d9178.lock\n",
      "DEBUG:filelock:Lock 140169840963888 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a6344aac8c09253b3b630fb776ae94478aa0275b.lock\n",
      "DEBUG:filelock:Attempting to release lock 140174459217392 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/93a7c1986e488383273279c4e698a8484b6973a5.lock\n",
      "DEBUG:filelock:Lock 140169840964704 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/764c9e65681b29cd662a83ea56e72f208c7f2f1c51cd5ea6bd5fb34f068d9178.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model-00003-of-00006.safetensors HTTP/1.1\" 302 0\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:filelock:Lock 140174459217392 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/93a7c1986e488383273279c4e698a8484b6973a5.lock\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459217296 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/26bccbc7080ad02bd9952fcdd5dd0cf2892f40e1320afcc7f867d66eb78e8f3e.lock\n",
      "DEBUG:filelock:Lock 140174459217296 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/26bccbc7080ad02bd9952fcdd5dd0cf2892f40e1320afcc7f867d66eb78e8f3e.lock\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/.gitattributes HTTP/1.1\" 200 1519\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3d12ebc28624ecf8998eaf5cfb75632",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       ".gitattributes:   0%|          | 0.00/1.52k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140169840963888 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a6344aac8c09253b3b630fb776ae94478aa0275b.lock\n",
      "DEBUG:filelock:Lock 140169840963888 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a6344aac8c09253b3b630fb776ae94478aa0275b.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model-00006-of-00006.safetensors HTTP/1.1\" 302 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/README.md HTTP/1.1\" 200 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model-00002-of-00006.safetensors HTTP/1.1\" 302 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459214512 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/3d8438370e503c447b91194e3f62a502bd6e9c2be6d4c58c38490ccaa560a4f2.lock\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459212736 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/df42e3dcadf1a732564f1bebd081f2bdea04e400.lock\n",
      "DEBUG:filelock:Lock 140174459214512 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/3d8438370e503c447b91194e3f62a502bd6e9c2be6d4c58c38490ccaa560a4f2.lock\n",
      "DEBUG:filelock:Lock 140174459212736 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/df42e3dcadf1a732564f1bebd081f2bdea04e400.lock\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459212544 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a1ab626c3c647e770bc60202c7c123897865447b14af9072a7e901dbcf6ad90e.lock\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:filelock:Lock 140174459212544 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a1ab626c3c647e770bc60202c7c123897865447b14af9072a7e901dbcf6ad90e.lock\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model.safetensors.index.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140169840964224 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/f1db3deb1511e5d4a98077bd9d4a0227a9612e95.lock\n",
      "DEBUG:filelock:Lock 140169840964224 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/f1db3deb1511e5d4a98077bd9d4a0227a9612e95.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/README.md HTTP/1.1\" 200 4098\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/model.safetensors.index.json HTTP/1.1\" 200 77209\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa45d8b8698d419b8cc3361ef96fc47d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/4.10k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25fa91381bc94efa88873bd7fb9e877b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors.index.json:   0%|          | 0.00/77.2k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140174459212736 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/df42e3dcadf1a732564f1bebd081f2bdea04e400.lock\n",
      "DEBUG:filelock:Lock 140174459212736 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/df42e3dcadf1a732564f1bebd081f2bdea04e400.lock\n",
      "DEBUG:filelock:Attempting to release lock 140169840964224 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/f1db3deb1511e5d4a98077bd9d4a0227a9612e95.lock\n",
      "DEBUG:filelock:Lock 140169840964224 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/f1db3deb1511e5d4a98077bd9d4a0227a9612e95.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/special_tokens_map.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459208320 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/8bedc05a6476080d7f473a9da72394f2cee48340.lock\n",
      "DEBUG:filelock:Lock 140174459208320 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/8bedc05a6476080d7f473a9da72394f2cee48340.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/preprocessor_config.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140169840955584 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/685e3802faf03f8b84afec3d1529a233c4fc9bde.lock\n",
      "DEBUG:filelock:Lock 140169840955584 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/685e3802faf03f8b84afec3d1529a233c4fc9bde.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/special_tokens_map.json HTTP/1.1\" 200 552\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/preprocessor_config.json HTTP/1.1\" 200 754\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a70c352fe15545b0948d05172ea56efd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/552 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140174459208320 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/8bedc05a6476080d7f473a9da72394f2cee48340.lock\n",
      "DEBUG:filelock:Lock 140174459208320 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/8bedc05a6476080d7f473a9da72394f2cee48340.lock\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0109d413f856486ab8e1f9deac50f43e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "preprocessor_config.json:   0%|          | 0.00/754 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140169840955584 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/685e3802faf03f8b84afec3d1529a233c4fc9bde.lock\n",
      "DEBUG:filelock:Lock 140169840955584 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/685e3802faf03f8b84afec3d1529a233c4fc9bde.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/tokenizer.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140169840957264 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/2f7e455a9786b471644a266b48a533b9137a18ee.lock\n",
      "DEBUG:filelock:Lock 140169840957264 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/2f7e455a9786b471644a266b48a533b9137a18ee.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/tokenizer.model HTTP/1.1\" 302 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140169840953760 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347.lock\n",
      "DEBUG:filelock:Lock 140169840953760 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347.lock\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): cdn-lfs-us-1.huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/tokenizer.json HTTP/1.1\" 200 1843105\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47a6e46102ea46c597c369ec3f7e7cb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.84M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140169840957264 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/2f7e455a9786b471644a266b48a533b9137a18ee.lock\n",
      "DEBUG:filelock:Lock 140169840957264 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/2f7e455a9786b471644a266b48a533b9137a18ee.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/tokenizer_config.json HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140174459211296 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/41a9683d8ae9e0e14407623eb44c8f4aaca07205.lock\n",
      "DEBUG:filelock:Lock 140174459211296 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/41a9683d8ae9e0e14407623eb44c8f4aaca07205.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/f898b9922e5f24439021033da2c2a0237339f2b8/tokenizer_config.json HTTP/1.1\" 200 1180\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4206c59d50d847babab4adbf68f3e2b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/1.18k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140174459211296 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/41a9683d8ae9e0e14407623eb44c8f4aaca07205.lock\n",
      "DEBUG:filelock:Lock 140174459211296 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/41a9683d8ae9e0e14407623eb44c8f4aaca07205.lock\n",
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/5c897a21e3a58a1e9e2a6b8f846d6af561da1e18e83ddd5f084b43b9f59c001a?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00004-of-00006.safetensors%3B+filename%3D%22model-00004-of-00006.safetensors%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvNWM4OTdhMjFlM2E1OGExZTllMmE2YjhmODQ2ZDZhZjU2MWRhMWUxOGU4M2RkZDVmMDg0YjQzYjlmNTljMDAxYT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=Qq-9PQ~o6CfNb8U2JwREv8EHF4eeOzZBnBgza3wisXJSKgv9SiCeL4IojOQyGBh1~6maZbb19oDRUqsb3XV1Jbuxs8yCuKae9ZWF3CDT7SyQwFWeMM0-k-F-WzUWUnobVIN3C9Uirum~t6iXbWxvWslVz0haHkS59C9JLpwijSVb5ue~lO0va2Sa-5DPzvGmvkI21Yjqu67m2BVdGAuT3bvhUkfnPkXs1Rd467h6ypuFYsPmtytGAPfgwuEZE9nU6guS2lPBoJhAoNeAiYp~Re7BXwarUIf8lYHp4dWKn0CcH8Jd2ND7XIZCi9Uio5obNh7z1ntCPxDEMNkfiC556A__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 4933723208\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ece0c3806ae247428206f158e6c06fba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00004-of-00006.safetensors:   0%|          | 0.00/4.93G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/388813b227c1a35a05bac248932c99af9e7b0b1842a7bb9f45a829275f7bcb98?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00005-of-00006.safetensors%3B+filename%3D%22model-00005-of-00006.safetensors%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvMzg4ODEzYjIyN2MxYTM1YTA1YmFjMjQ4OTMyYzk5YWY5ZTdiMGIxODQyYTdiYjlmNDVhODI5Mjc1ZjdiY2I5OD9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=WWsfzQ72sH6Ct9fhliS0zS8EguVP41H-gxEBdt5kXhjG4UFaxYFh-1WVV77ZfY2VQDUQT65UnLfCGB5AZOr7FA6Bv0GSzbWaY2IGuZ6y6ZOpDhzbcsmGeCA6ziGIZEaDKIwWTBA9uSXeEpks3gY3t6NbCswLm8GHvCsAVyDnSj6DM-u6B~UOgd1t1AVEn9B~jbC1~acUrqlvu5mAxdINPcNCuNwAfp8p6umDJ-E2Y1csOeiyej2mZnNFwhCXyXeC8bk1p9MDzzKYJUQKQcdj47M10pmmt7l1wETCL2Cb9ZO1QUf77f8~862dTcq1DdvPjwWzuyeusvV6ug~bTl5m3A__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 4933723208\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b44f92c3391341fc905783b82d475f3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00005-of-00006.safetensors:   0%|          | 0.00/4.93G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/764c9e65681b29cd662a83ea56e72f208c7f2f1c51cd5ea6bd5fb34f068d9178?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00001-of-00006.safetensors%3B+filename%3D%22model-00001-of-00006.safetensors%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvNzY0YzllNjU2ODFiMjljZDY2MmE4M2VhNTZlNzJmMjA4YzdmMmYxYzUxY2Q1ZWE2YmQ1ZmIzNGYwNjhkOTE3OD9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=EYilR3B002Xm5TUMFEUqFH4kJlGSONA7DivrO1jFivHmaypKu-3KCCWTYGkKQjsZkkMeyVUNFdi07ay95WDqxFu9XxlDc-hc1aHREkXo6~YAWh3pUkxMozUTEYaaAVpadD6ih0Tb1XlNHyvxv15cXNMCx-WZbFiC-N0svIeeEURjy-Q3jjRUK7f4st7ro3HUUOjyEHDjB7LdmHzbxMX0RSSepEtl2SSeEZJlv6t12ru1wHBumrJx7RZIKU1cJOz~WWbUNiylC1JpHuxCA8mBCeVraOMViNldXwS39iUzE1niUn3SZ~o2KqdDYwPpHsX0lEe3R91BG1Z1fVVwSTfVtA__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 4962097328\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a993c7abf98f49dfbd75b2a782d5be0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00006.safetensors:   0%|          | 0.00/4.96G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/26bccbc7080ad02bd9952fcdd5dd0cf2892f40e1320afcc7f867d66eb78e8f3e?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00003-of-00006.safetensors%3B+filename%3D%22model-00003-of-00006.safetensors%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvMjZiY2NiYzcwODBhZDAyYmQ5OTUyZmNkZDVkZDBjZjI4OTJmNDBlMTMyMGFmY2M3Zjg2N2Q2NmViNzhlOGYzZT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=UBc~yiOgf-dnHgi3o-TjuOHAZR1XLSs11N43WVKTFk31QjnIHvZufCMxcwwCXhLAWsNh2bNbt56AeWrDunJGagSVPXEj-hXUE~np8rRBagg8RKe0cKu2RJYXdbueM6hz332CHqwysvP8lVxKoGkFNKU8TdBKdh7ODJXSXsC7B76y3HHiYWTe5MxfF9feabF64fwDGKGBs~sX5810luPXM~mqGAUASmrUTNbs2Opc2uYRGlQFr2ftMfv7jIWm1mHQbAHcuDzjJT1~T6BaqsJYKNuBz0cbqIi1neGW8KFbyyfL5gzmIvz3nNhqO9ILd-KtVFIoXmC6HP3z1CEjFOsb-A__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 4881273536\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "931ef785cbd74bbb9cd7e91b646cb593",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00003-of-00006.safetensors:   0%|          | 0.00/4.88G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/3d8438370e503c447b91194e3f62a502bd6e9c2be6d4c58c38490ccaa560a4f2?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00006-of-00006.safetensors%3B+filename%3D%22model-00006-of-00006.safetensors%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvM2Q4NDM4MzcwZTUwM2M0NDdiOTExOTRlM2Y2MmE1MDJiZDZlOWMyYmU2ZDRjNThjMzg0OTBjY2FhNTYwYTRmMj9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=NE~or8J~sSc-jGv7JGPNmEUITEPuyJtnfA7lQNLfdbWksqUREEH4KYgm7mNQ9VuoqoakFGVTMTLTDDIVIJ3ifMsrk168rA5O8Q5ro872R02~5OjN-lmknyGrFjQGPyS5zuQ0BrbSafTXXzOr7BQYdRdS2UphZCV3cnn0m6BEu0F5Ygabay9JArgbj72J3taOWTnoYq1kezk1Jz1a3Fl5xzLfUfpoxZNw~uBt-hhVNxUKHkr6zVE~08G~hpUf-keE5QX3Ak4NF1xK0-8PzcRLzShlA-T8Q4SRTbovslLNaQ0vU8F5Xiqm7K4scdbm1iyUEhEnvnnydZWxBQFRdybYmg__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 2021860512\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc54db062d814b55aa8cd2f8b9240433",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00006-of-00006.safetensors:   0%|          | 0.00/2.02G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27tokenizer.model%3B+filename%3D%22tokenizer.model%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvOWU1NTZhZmQ0NDIxM2I2YmQxYmUyYjg1MGViYmJkOThmNTQ4MTQzN2E4MDIxYWZhZjU4ZWU3ZmIxODE4ZDM0Nz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=dTXk9aNH5FWwH8qqhBY90YSgUCVgMRgP7wOtK5gp76kHXv4M8U6HYAodFRlLE02NBK3soj~0Ac4u7xb0n0JhbNR5nmTPX6WCy3eMUcQ-h5uBFjsbYH0BESjnYt6PF46AqyXavXUQxPBaSsD8aN1Zdvrpdv9BloGPuK8fN~3uSSVTRuxa4PoZlsFyExJcZzzuj69Xd554AY1vwO0XnzYFteSOc7lHmQ2eGjP4YFdxu77VrOxpAGbgSfwQjN7SVwWnAeV~GIBNuLDZT0bPzfQh5YVICnl8jZkJ384LBMBZ5CIQ1grKKkUULwFJy5fQQ5ONnjuTPNiKiYe5BLfCFHUz5w__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 499723\n",
      "DEBUG:urllib3.connectionpool:https://cdn-lfs-us-1.huggingface.co:443 \"GET /repos/91/45/9145bb55c3b64c665f14518a07e65d992404a96d7aca1ff6b04e84363895966b/a1ab626c3c647e770bc60202c7c123897865447b14af9072a7e901dbcf6ad90e?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27model-00002-of-00006.safetensors%3B+filename%3D%22model-00002-of-00006.safetensors%22%3B&Expires=1717154564&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNzE1NDU2NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zLzkxLzQ1LzkxNDViYjU1YzNiNjRjNjY1ZjE0NTE4YTA3ZTY1ZDk5MjQwNGE5NmQ3YWNhMWZmNmIwNGU4NDM2Mzg5NTk2NmIvYTFhYjYyNmMzYzY0N2U3NzBiYzYwMjAyYzdjMTIzODk3ODY1NDQ3YjE0YWY5MDcyYTdlOTAxZGJjZjZhZDkwZT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSoifV19&Signature=W5dUzU-FGKFr1LwHP50fWkswcClUesy~8kgqCKDP4kvZELpGGxjfkoTyN94owAmkSdL4xWEHP5aphGvViJGVqyFNSy3myNutCT6IJuGf0xHylZPfWaDBFjc9NzXvOUF4KzHHdVHjxq-C-reRxsexa~KQdzPAGDocMaZKs7QpzsudRhMz~XS8dI2BiKFrvdu3z7pVO3ZN6Yt4bcyNS5cmClbJHkVBHqc~SISvnQCBTnJCJdnRcFlW9-gPJHjshYNmmLv9Anjz1CfIfyO--5l1bNrrjYdww4nrhaYEiLtqVgRjb7II~HDoS~CvL~v6-kpkXEGW~PHG1M3gTjbNJnvjQw__&Key-Pair-Id=KCD77M1F0VK2B HTTP/1.1\" 200 4970423200\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8d5da62a6974d1ea054ca165988279f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "399c5b9b382c4a2d9736bd2076c0adc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00006.safetensors:   0%|          | 0.00/4.97G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140169840953760 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347.lock\n",
      "DEBUG:filelock:Lock 140169840953760 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347.lock\n",
      "DEBUG:filelock:Attempting to release lock 140174459214512 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/3d8438370e503c447b91194e3f62a502bd6e9c2be6d4c58c38490ccaa560a4f2.lock\n",
      "DEBUG:filelock:Lock 140174459214512 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/3d8438370e503c447b91194e3f62a502bd6e9c2be6d4c58c38490ccaa560a4f2.lock\n",
      "DEBUG:filelock:Attempting to release lock 140174459217296 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/26bccbc7080ad02bd9952fcdd5dd0cf2892f40e1320afcc7f867d66eb78e8f3e.lock\n",
      "DEBUG:filelock:Lock 140174459217296 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/26bccbc7080ad02bd9952fcdd5dd0cf2892f40e1320afcc7f867d66eb78e8f3e.lock\n",
      "DEBUG:filelock:Attempting to release lock 140169840964704 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/764c9e65681b29cd662a83ea56e72f208c7f2f1c51cd5ea6bd5fb34f068d9178.lock\n",
      "DEBUG:filelock:Lock 140169840964704 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/764c9e65681b29cd662a83ea56e72f208c7f2f1c51cd5ea6bd5fb34f068d9178.lock\n",
      "DEBUG:filelock:Attempting to release lock 140174459212160 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/388813b227c1a35a05bac248932c99af9e7b0b1842a7bb9f45a829275f7bcb98.lock\n",
      "DEBUG:filelock:Lock 140174459212160 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/388813b227c1a35a05bac248932c99af9e7b0b1842a7bb9f45a829275f7bcb98.lock\n",
      "DEBUG:filelock:Attempting to release lock 140174459212544 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a1ab626c3c647e770bc60202c7c123897865447b14af9072a7e901dbcf6ad90e.lock\n",
      "DEBUG:filelock:Lock 140174459212544 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/a1ab626c3c647e770bc60202c7c123897865447b14af9072a7e901dbcf6ad90e.lock\n",
      "DEBUG:filelock:Attempting to release lock 140174459220896 on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/5c897a21e3a58a1e9e2a6b8f846d6af561da1e18e83ddd5f084b43b9f59c001a.lock\n",
      "DEBUG:filelock:Lock 140174459220896 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--llava-hf--llava-v1.6-vicuna-13b-hf/5c897a21e3a58a1e9e2a6b8f846d6af561da1e18e83ddd5f084b43b9f59c001a.lock\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b60b916c8eb478f947872f4ae677038",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n"
     ]
    }
   ],
   "source": [
    "model = AutoAWQForCausalLM.from_pretrained(\n",
    "    model_path, torch_dtype = torch.bfloat16,\n",
    ")\n",
    "_ = model.cuda()\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "23d8a658",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_wikitext():\n",
    "    data = load_dataset('wikitext', 'wikitext-2-raw-v1', split=\"train\")\n",
    "    return [text for text in data[\"text\"] if text.strip() != '' and len(text.split(' ')) > 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5dcf2167",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/datasets/wikitext HTTP/1.1\" 200 4846\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): s3.amazonaws.com:443\n",
      "DEBUG:urllib3.connectionpool:https://s3.amazonaws.com:443 \"HEAD /datasets.huggingface.co/datasets/datasets/wikitext/wikitext.py HTTP/1.1\" 200 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/datasets/wikitext HTTP/1.1\" 200 4846\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /datasets/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/README.md HTTP/1.1\" 200 0\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /datasets/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/.huggingface.yaml HTTP/1.1\" 404 0\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): datasets-server.huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://datasets-server.huggingface.co:443 \"GET /info?dataset=wikitext HTTP/1.1\" 200 None\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/datasets/wikitext/revision/b08601e04326c79dfdd32d625aee71d232d685c3 HTTP/1.1\" 200 4846\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/datasets/wikitext/tree/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-103-raw-v1?recursive=False&expand=False HTTP/1.1\" 200 1017\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/datasets/wikitext/revision/b08601e04326c79dfdd32d625aee71d232d685c3 HTTP/1.1\" 200 4846\n",
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /datasets/wikitext/resolve/b08601e04326c79dfdd32d625aee71d232d685c3/dataset_infos.json HTTP/1.1\" 404 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /api/datasets/wikitext/tree/b08601e04326c79dfdd32d625aee71d232d685c3/wikitext-2-raw-v1?recursive=False&expand=False HTTP/1.1\" 200 751\n",
      "DEBUG:filelock:Attempting to acquire lock 140170024048176 on /home/ubuntu/.cache/huggingface/datasets/_home_ubuntu_.cache_huggingface_datasets_wikitext_wikitext-2-raw-v1_0.0.0_b08601e04326c79dfdd32d625aee71d232d685c3.lock\n",
      "DEBUG:filelock:Lock 140170024048176 acquired on /home/ubuntu/.cache/huggingface/datasets/_home_ubuntu_.cache_huggingface_datasets_wikitext_wikitext-2-raw-v1_0.0.0_b08601e04326c79dfdd32d625aee71d232d685c3.lock\n",
      "DEBUG:fsspec.local:open file: /home/ubuntu/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3/dataset_info.json\n",
      "DEBUG:filelock:Attempting to release lock 140170024048176 on /home/ubuntu/.cache/huggingface/datasets/_home_ubuntu_.cache_huggingface_datasets_wikitext_wikitext-2-raw-v1_0.0.0_b08601e04326c79dfdd32d625aee71d232d685c3.lock\n",
      "DEBUG:filelock:Lock 140170024048176 released on /home/ubuntu/.cache/huggingface/datasets/_home_ubuntu_.cache_huggingface_datasets_wikitext_wikitext-2-raw-v1_0.0.0_b08601e04326c79dfdd32d625aee71d232d685c3.lock\n",
      "DEBUG:filelock:Attempting to acquire lock 140174458159984 on /home/ubuntu/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3_builder.lock\n",
      "DEBUG:filelock:Lock 140174458159984 acquired on /home/ubuntu/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3_builder.lock\n",
      "DEBUG:fsspec.local:open file: /home/ubuntu/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3/dataset_info.json\n",
      "DEBUG:filelock:Attempting to release lock 140174458159984 on /home/ubuntu/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3_builder.lock\n",
      "DEBUG:filelock:Lock 140174458159984 released on /home/ubuntu/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/0.0.0/b08601e04326c79dfdd32d625aee71d232d685c3_builder.lock\n",
      "DEBUG:root: * Split into 47 blocks\n",
      "AWQ: 100%|██████████| 40/40 [16:43<00:00, 25.09s/it]\n"
     ]
    }
   ],
   "source": [
    "model.quantize(tokenizer, quant_config=quant_config, calib_data=load_wikitext())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fa16f58f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-05-28 11:40:44,136] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    }
   ],
   "source": [
    "model.save_quantized(quant_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7f8083da",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./llava-v1.6-vicuna-13b-hf-awq/tokenizer_config.json',\n",
       " './llava-v1.6-vicuna-13b-hf-awq/special_tokens_map.json',\n",
       " './llava-v1.6-vicuna-13b-hf-awq/tokenizer.model',\n",
       " './llava-v1.6-vicuna-13b-hf-awq/added_tokens.json',\n",
       " './llava-v1.6-vicuna-13b-hf-awq/tokenizer.json')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(quant_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "840e775b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:Resetting dropped connection: huggingface.co\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/repos/create HTTP/1.1\" 200 153\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /mesolitica/llava-v1.6-vicuna-13b-hf-awq/resolve/main/README.md HTTP/1.1\" 404 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/validate-yaml HTTP/1.1\" 200 27\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/models/mesolitica/llava-v1.6-vicuna-13b-hf-awq/preupload/main HTTP/1.1\" 200 442\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /mesolitica/llava-v1.6-vicuna-13b-hf-awq.git/info/lfs/objects/batch HTTP/1.1\" 200 917\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78149f8211c0456bada0b8e7a7b06038",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-hub-lfs-us-east-1.s3-accelerate.amazonaws.com:443\n",
      "DEBUG:urllib3.connectionpool:https://hf-hub-lfs-us-east-1.s3-accelerate.amazonaws.com:443 \"PUT /repos/b7/15/b7155eea874601de0be8432515ad12e9fc7b86e8ab0f07307755692939ba69f2/9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2JU7TKAQFN2FTF47%2F20240528%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240528T114649Z&X-Amz-Expires=900&X-Amz-Signature=a17ad20b8ce11ee575c93f1431f7f8f61067d460e93d51d1f55557933c01cbd4&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject HTTP/1.1\" 200 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /mesolitica/llava-v1.6-vicuna-13b-hf-awq.git/info/lfs/objects/verify HTTP/1.1\" 200 2\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/models/mesolitica/llava-v1.6-vicuna-13b-hf-awq/commit/main HTTP/1.1\" 200 212\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/llava-v1.6-vicuna-13b-hf-awq/commit/9de778ddf8b3518f4b70b1cd5365eb71dfbed1ee', commit_message='Upload tokenizer', commit_description='', oid='9de778ddf8b3518f4b70b1cd5365eb71dfbed1ee', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('mesolitica/llava-v1.6-vicuna-13b-hf-awq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1af2adcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /llava-hf/llava-v1.6-vicuna-13b-hf/resolve/main/config.json HTTP/1.1\" 200 0\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/repos/create HTTP/1.1\" 409 118\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /mesolitica/llava-v1.6-vicuna-13b-hf-awq/resolve/main/README.md HTTP/1.1\" 200 0\n",
      "DEBUG:filelock:Attempting to acquire lock 140168314889280 on /home/ubuntu/.cache/huggingface/hub/.locks/models--mesolitica--llava-v1.6-vicuna-13b-hf-awq/bc5f30d6632ac0efdc7be2e9095e9e9579af2e33.lock\n",
      "DEBUG:filelock:Lock 140168314889280 acquired on /home/ubuntu/.cache/huggingface/hub/.locks/models--mesolitica--llava-v1.6-vicuna-13b-hf-awq/bc5f30d6632ac0efdc7be2e9095e9e9579af2e33.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"GET /mesolitica/llava-v1.6-vicuna-13b-hf-awq/resolve/main/README.md HTTP/1.1\" 200 5174\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73816ecd346045c4964ad02403ca0fd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/5.17k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:filelock:Attempting to release lock 140168314889280 on /home/ubuntu/.cache/huggingface/hub/.locks/models--mesolitica--llava-v1.6-vicuna-13b-hf-awq/bc5f30d6632ac0efdc7be2e9095e9e9579af2e33.lock\n",
      "DEBUG:filelock:Lock 140168314889280 released on /home/ubuntu/.cache/huggingface/hub/.locks/models--mesolitica--llava-v1.6-vicuna-13b-hf-awq/bc5f30d6632ac0efdc7be2e9095e9e9579af2e33.lock\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/validate-yaml HTTP/1.1\" 200 27\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/models/mesolitica/llava-v1.6-vicuna-13b-hf-awq/preupload/main HTTP/1.1\" 200 143\n",
      "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"POST /api/models/mesolitica/llava-v1.6-vicuna-13b-hf-awq/commit/main HTTP/1.1\" 200 212\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/llava-v1.6-vicuna-13b-hf-awq/commit/15adcf73216c79dba946bb8bd1f1a48229881505', commit_message='Upload config', commit_description='', oid='15adcf73216c79dba946bb8bd1f1a48229881505', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoConfig, AwqConfig\n",
    "\n",
    "quantization_config = AwqConfig(\n",
    "    bits=quant_config['w_bit'],\n",
    "    group_size=quant_config['q_group_size'],\n",
    "    zero_point=quant_config['zero_point'],\n",
    "    backend='autoawq',\n",
    "    version=quant_config['version'].lower(),\n",
    ")\n",
    "\n",
    "config = AutoConfig.from_pretrained(model_path)\n",
    "config.quantization_config = quantization_config\n",
    "\n",
    "config.push_to_hub('mesolitica/llava-v1.6-vicuna-13b-hf-awq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4546e2f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import HfApi\n",
    "\n",
    "api = HfApi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4d6dc901",
   "metadata": {},
   "outputs": [],
   "source": [
    "api.upload_folder(\n",
    "    folder_path=quant_path,\n",
    "    repo_id='mesolitica/llava-v1.6-vicuna-13b-hf-awq',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d91bd622",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}