Created using Colaboratory

🤖_AutoQuantize_(GGUF,_AWQ,_EXL2,_GPTQ).ipynb

@@ -0,0 +1,316 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/almutareb/InnovationPathfinderAI/blob/main/%F0%9F%A4%96_AutoQuantize_(GGUF%2C_AWQ%2C_EXL2%2C_GPTQ).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title # 🤖 AutoQuantize\n",
+        "\n",
+        "# @markdown 🔮 Created by [@zainulabideen](https://huggingface.co/abideen).\n",
+        "\n",
+        "# @markdown Please add HF token to the secrets tab in Google Colab before.\n",
+        "\n",
+        "# @markdown Quantization formats supported: `GGUF`, `AWQ`, `EXL2`, `GPTQ`\n",
+        "\n",
+        "# @markdown ---\n",
+        "\n",
+        "\n",
+        "# @markdown ### 🤗 Hugging Face Hub\n",
+        "\n",
+        "MODEL_ID = \"abideen/Heimer-dpo-TinyLlama-1.1B\" # @param {type:\"string\"}\n",
+        "MODEL_NAME = MODEL_ID.split('/')[-1]\n",
+        "\n",
+        "# Download model\n",
+        "!git lfs install\n",
+        "!git clone https://huggingface.co/{MODEL_ID}\n",
+        "\n",
+        "username = \"abideen\" # @param {type:\"string\"}\n",
+        "token = \"\" # @param {type:\"string\"}\n",
+        "!pip install -q huggingface_hub\n",
+        "from huggingface_hub import create_repo, HfApi\n",
+        "from google.colab import userdata, runtime"
+      ],
+      "metadata": {
+        "id": "fD24jJxq7t3k",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title # 🛸 GGUF\n",
+        "# @markdown ### ✨ Quantization parameters\n",
+        "\n",
+        "QUANTIZATION_FORMAT = \"q4_k_m\" # @param {type:\"string\"}\n",
+        "QUANTIZATION_METHODS = QUANTIZATION_FORMAT.replace(\" \", \"\").split(\",\")\n",
+        "# Install llama.cpp\n",
+        "!git clone https://github.com/ggerganov/llama.cpp\n",
+        "!cd llama.cpp && git pull && make clean && LLAMA_CUBLAS=1 make\n",
+        "!pip install -r llama.cpp/requirements.txt\n",
+        "\n",
+        "# Convert to fp16\n",
+        "fp16 = f\"{MODEL_NAME}/{MODEL_NAME.lower()}.fp16.bin\"\n",
+        "!python llama.cpp/convert.py {MODEL_NAME} --outtype f16 --outfile {fp16}\n",
+        "\n",
+        "# Quantize the model for each method in the QUANTIZATION_METHODS list\n",
+        "for method in QUANTIZATION_METHODS:\n",
+        "    qtype = f\"{MODEL_NAME}/{MODEL_NAME.lower()}.{method.upper()}.gguf\"\n",
+        "    !./llama.cpp/quantize {fp16} {qtype} {method}\n",
+        "\n",
+        "# Defined in the secrets tab in Google Colab\n",
+        "hf_token = userdata.get(token)\n",
+        "api = HfApi()\n",
+        "\n",
+        "# Create empty repo\n",
+        "create_repo(\n",
+        "    repo_id = f\"{username}/{MODEL_NAME}-GGUF\",\n",
+        "    repo_type=\"model\",\n",
+        "    exist_ok=True,\n",
+        "    token=hf_token\n",
+        ")\n",
+        "\n",
+        "# Upload gguf files\n",
+        "api.upload_folder(\n",
+        "    folder_path=MODEL_NAME,\n",
+        "    repo_id=f\"{username}/{MODEL_NAME}-GGUF\",\n",
+        "    allow_patterns=[\"*.gguf\",\"$.md\"],\n",
+        "    token=hf_token\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "NL0yGhbe3EFk",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title # 🏛️ AWQ\n",
+        "# @markdown ### ✨ Quantization parameters\n",
+        "\n",
+        "Q_GROUP_SIZE = 128 # @param {type:\"integer\"}\n",
+        "ZERO_POINT = True # @param {text:\"boolean\"}\n",
+        "W_BIT = 4 # @param {type:\"integer\"}\n",
+        "VERSION = \"GEMM\" # @param {type:\"string\"}\n",
+        "SAFETENSORS = True # @param {text:\"boolean\"}\n",
+        "\n",
+        "# Install AutoAWQ\n",
+        "!git clone https://github.com/casper-hansen/AutoAWQ\n",
+        "%cd AutoAWQ\n",
+        "!pip install -e .\n",
+        "!pip install git+https://github.com/huggingface/transformers\n",
+        "!pip install zstandard\n",
+        "\n",
+        "from awq import AutoAWQForCausalLM\n",
+        "from transformers import AutoTokenizer\n",
+        "\n",
+        "\n",
+        "quant_path = MODEL_NAME + \"-awq\"\n",
+        "quant_config = { \"zero_point\": ZERO_POINT, \"q_group_size\": Q_GROUP_SIZE, \"w_bit\": W_BIT, \"version\": VERSION }\n",
+        "\n",
+        "# Load model\n",
+        "PATH = \"/content/\" + MODEL_NAME\n",
+        "model = AutoAWQForCausalLM.from_pretrained(PATH, safetensors=SAFETENSORS)\n",
+        "tokenizer = AutoTokenizer.from_pretrained(PATH, trust_remote_code=True)\n",
+        "\n",
+        "# Quantize\n",
+        "model.quantize(tokenizer, quant_config=quant_config)\n",
+        "\n",
+        "# Save quantized model\n",
+        "model.save_quantized(quant_path)\n",
+        "tokenizer.save_pretrained(quant_path)\n",
+        "\n",
+        "# Defined in the secrets tab in Google Colab\n",
+        "hf_token = userdata.get(token)\n",
+        "api = HfApi()\n",
+        "\n",
+        "# Create empty repo\n",
+        "create_repo(\n",
+        "    repo_id = f\"{username}/{MODEL_NAME}-AWQ\",\n",
+        "    repo_type=\"model\",\n",
+        "    exist_ok=True,\n",
+        "    token=hf_token\n",
+        ")\n",
+        "\n",
+        "# Upload awq files\n",
+        "api.upload_folder(\n",
+        "    folder_path=quant_path,\n",
+        "    repo_id=f\"{username}/{MODEL_NAME}-AWQ\",\n",
+        "    token=hf_token\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "MyyUO2Fj3WHt",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title # 🔬 EXL2\n",
+        "# @markdown ### ✨ Quantization parameters\n",
+        "\n",
+        "BPW = 5.0 # @param {type:\"number\"}\n",
+        "\n",
+        "# Install ExLLamaV2\n",
+        "!git clone https://github.com/turboderp/exllamav2\n",
+        "!pip install -e exllamav2\n",
+        "\n",
+        "!mv {MODEL_NAME} base_model\n",
+        "!rm base_mode/*.bin\n",
+        "\n",
+        "# Download dataset\n",
+        "!wget https://huggingface.co/datasets/wikitext/resolve/9a9e482b5987f9d25b3a9b2883fc6cc9fd8071b3/wikitext-103-v1/wikitext-test.parquet\n",
+        "\n",
+        "# Quantize model\n",
+        "!mkdir quant\n",
+        "!python exllamav2/convert.py \\\n",
+        "    -i base_model \\\n",
+        "    -o quant \\\n",
+        "    -c wikitext-test.parquet \\\n",
+        "    -b {BPW}\n",
+        "\n",
+        "# Copy files\n",
+        "!rm -rf quant/out_tensor\n",
+        "!rsync -av --exclude='*.safetensors' --exclude='.*' ./base_model/ ./quant/\n",
+        "\n",
+        "# Defined in the secrets tab in Google Colab\n",
+        "hf_token = userdata.get(token)\n",
+        "api = HfApi()\n",
+        "\n",
+        "# Create empty repo\n",
+        "create_repo(\n",
+        "    repo_id = f\"{username}/{MODEL_NAME}-{BPW:.1f}bpw-exl2\",\n",
+        "    repo_type=\"model\",\n",
+        "    exist_ok=True,\n",
+        "    token=hf_token\n",
+        ")\n",
+        "\n",
+        "# Upload exl2 files\n",
+        "api.upload_folder(\n",
+        "    folder_path=quant,\n",
+        "    repo_id=f\"{username}/{MODEL_NAME}-{BPW:.1f}bpw-exl2\",\n",
+        "    token=hf_token\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "ZC9Nsr9u5WhN",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title # 📝 GPTQ\n",
+        "# @markdown ### ✨ Quantization parameters\n",
+        "\n",
+        "BITS = 4 # @param {type:\"integer\"}\n",
+        "GROUP_SIZE = 128 # @param {type:\"integer\"}\n",
+        "DAMP_PERCENT = 0.01 # @param {type:\"number\"}\n",
+        "\n",
+        "!BUILD_CUDA_EXT=0 pip install -q auto-gptq transformers\n",
+        "import random\n",
+        "from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig\n",
+        "from datasets import load_dataset\n",
+        "import torch\n",
+        "from transformers import AutoTokenizer\n",
+        "out_dir = MODEL_ID + \"-GPTQ\"\n",
+        "\n",
+        "# Load quantize config, model and tokenizer\n",
+        "quantize_config = BaseQuantizeConfig(\n",
+        "    bits=BITS,\n",
+        "    group_size=GROUP_SIZE,\n",
+        "    damp_percent=DAMP_PERCENT,\n",
+        "    desc_act=False,\n",
+        ")\n",
+        "PATH = \"/content/\" + MODEL_NAME\n",
+        "model = AutoGPTQForCausalLM.from_pretrained(PATH, quantize_config)\n",
+        "tokenizer = AutoTokenizer.from_pretrained(PATH)\n",
+        "\n",
+        "# Load data and tokenize examples\n",
+        "n_samples = 1024\n",
+        "data = load_dataset(\"allenai/c4\", data_files=\"en/c4-train.00001-of-01024.json.gz\", split=f\"train[:{n_samples*5}]\")\n",
+        "tokenized_data = tokenizer(\"\\n\\n\".join(data['text']), return_tensors='pt')\n",
+        "\n",
+        "# Format tokenized examples\n",
+        "examples_ids = []\n",
+        "for _ in range(n_samples):\n",
+        "    i = random.randint(0, tokenized_data.input_ids.shape[1] - tokenizer.model_max_length - 1)\n",
+        "    j = i + tokenizer.model_max_length\n",
+        "    input_ids = tokenized_data.input_ids[:, i:j]\n",
+        "    attention_mask = torch.ones_like(input_ids)\n",
+        "    examples_ids.append({'input_ids': input_ids, 'attention_mask': attention_mask})\n",
+        "\n",
+        "# Quantize with GPTQ\n",
+        "model.quantize(\n",
+        "    examples_ids,\n",
+        "    batch_size=1,\n",
+        "    use_triton=True,\n",
+        ")\n",
+        "\n",
+        "# Save model and tokenizer\n",
+        "model.save_quantized(out_dir, use_safetensors=True)\n",
+        "tokenizer.save_pretrained(out_dir)\n",
+        "\n",
+        "# Defined in the secrets tab in Google Colab\n",
+        "hf_token = userdata.get(token)\n",
+        "api = HfApi()\n",
+        "\n",
+        "# Create empty repo\n",
+        "create_repo(\n",
+        "    repo_id = f\"{username}/{MODEL_NAME}-GPTQ\",\n",
+        "    repo_type=\"model\",\n",
+        "    exist_ok=True,\n",
+        "    token=hf_token\n",
+        ")\n",
+        "\n",
+        "# Upload gptq files\n",
+        "api.upload_folder(\n",
+        "    folder_path=out_dir,\n",
+        "    repo_id=f\"{username}/{MODEL_NAME}-GPTQ\",\n",
+        "    token=hf_token\n",
+        ")\n"
+      ],
+      "metadata": {
+        "id": "OE_R3AXG5Y-F",
+        "cellView": "form"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}