Solshine
/

Llama-3-1-big-thoughtful-passthrough-merge-2

+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gdf-gWhrxVLc"
+      },
+      "source": [
+        "(Loading can take a long time. Please be patient.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "RPA4EuwJHudN"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Installing Requirements\n",
+        "%%capture\n",
+        "!pip install gradio\n",
+        "# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n",
+        "!pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
+        "\n",
+        "# We have to check which Torch version for Xformers (2.3 -> 0.0.27)\n",
+        "from torch import __version__; from packaging.version import Version as V\n",
+        "xformers = \"xformers==0.0.27\" if V(__version__) < V(\"2.4.0\") else \"xformers\"\n",
+        "!pip install --no-deps {xformers} trl peft accelerate bitsandbytes triton\n",
+        "\n",
+        "from unsloth import FastLanguageModel\n",
+        "import torch\n",
+        "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n",
+        "dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n",
+        "load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.\n",
+        "\n",
+        "# 4bit pre quantized models we support for 4x faster downloading + no OOMs.\n",
+        "fourbit_models = [\n",
+        "    #\"unsloth/Meta-Llama-3.1-8B-bnb-4bit\",      # Llama-3.1 15 trillion tokens model 2x faster!\n",
+        "    \"Solshine/mergekit-passthrough-aogjzpb\",\n",
+        "    #\"unsloth/Meta-Llama-3.1-70B-bnb-4bit\",\n",
+        "    #\"unsloth/Meta-Llama-3.1-405B-bnb-4bit\",    # We also uploaded 4bit for 405b!\n",
+        "    #\"unsloth/Mistral-Nemo-Base-2407-bnb-4bit\", # New Mistral 12b 2x faster!\n",
+        "    #\"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit\",\n",
+        "    #\"unsloth/mistral-7b-v0.3-bnb-4bit\",        # Mistral v3 2x faster!\n",
+        "    #\"unsloth/mistral-7b-instruct-v0.3-bnb-4bit\",\n",
+        "    #\"unsloth/Phi-3.5-mini-instruct\",           # Phi-3.5 2x faster!\n",
+        "    #\"unsloth/Phi-3-medium-4k-instruct\",\n",
+        "    #\"unsloth/gemma-2-9b-bnb-4bit\",\n",
+        "    #\"unsloth/gemma-2-27b-bnb-4bit\",            # Gemma 2x faster!\n",
+        "] # More models at https://huggingface.co/unsloth\n",
+        "\n",
+        "model, tokenizer = FastLanguageModel.from_pretrained(\n",
+        "    model_name = \"Solshine/mergekit-passthrough-aogjzpb\",\n",
+        "    max_seq_length = max_seq_length,\n",
+        "    dtype = dtype,\n",
+        "    load_in_4bit = load_in_4bit,\n",
+        "    token = \"XXXXX\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n",
+        "\n",
+        ")\n",
+        "\n",
+        "alpaca_prompt = \"\"\"Below is an instruction that describes a task. Reason through the query inside <thinking> tags, and then provide your final response inside <output> tags. If you detect that you made a mistake in your reasoning at any point, correct yourself inside <reflection> tags. Write a response that appropriately completes the request.\n",
+        "\n",
+        "### Instruction:\n",
+        "{}\n",
+        "\n",
+        "### Response:\n",
+        "{}\"\"\"\n",
+        "\n",
+        "EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN\n",
+        "def formatting_prompts_func(examples):\n",
+        "    instructions = examples[\"prompt\"]\n",
+        "    outputs = examples[\"response\"]\n",
+        "    texts = []\n",
+        "    for instruction, output in zip(instructions, outputs):\n",
+        "        # Must add EOS_TOKEN, otherwise your generation will go on forever!\n",
+        "        text = alpaca_prompt.format(instruction, output) + EOS_TOKEN\n",
+        "        texts.append(text)\n",
+        "    return {\"text\": texts}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {
+        "id": "Rh4hKp8AglIY"
+      },
+      "outputs": [],
+      "source": [
+        "# Ensure model is prepared for inference\n",
+        "model = FastLanguageModel.for_inference(model)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 610
+        },
+        "id": "u9SeBZ6WoHD0",
+        "outputId": "8950b091-4b4d-4353-a6aa-ed0699cd7955"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
+            "Running on public URL: https://88a900ce9e5af38193.gradio.live\n",
+            "\n",
+            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
+          ]
+        },
+        {
+          "data": {
+            "text/html": [
+              "<div><iframe src=\"https://88a900ce9e5af38193.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+            ],
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "# @title Create Gradio User Inferface\n",
+        "import gradio as gr\n",
+        "\n",
+        "MASTER_PROMPT = \"\"\"You are an assistant who answers all questions by first thinking internally using thinking tags, then reflects on your thinking, then answers, and then reflects on your answer offering any needed corrections. You are concise and accurate.\"\"\"\n",
+        "\n",
+        "# Add this function for the chatbot\n",
+        "def chatbot(message, history):\n",
+        "    # Combine history and master prompt and new message\n",
+        "    full_prompt = MASTER_PROMPT + \"\\n\\n\" + \"\\n\".join([f\"Human: {h[0]}\\nAI: {h[1]}\" for h in history])\n",
+        "    full_prompt += f\"\\nHuman: {message}\\nAI:\"\n",
+        "\n",
+        "    inputs = tokenizer([alpaca_prompt.format(full_prompt, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
+        "\n",
+        "    output = model.generate(**inputs, max_new_tokens=1000, temperature=0.7)\n",
+        "    response = tokenizer.decode(output[0], skip_special_tokens=True)\n",
+        "\n",
+        "    # Extract the AI's response\n",
+        "    ai_response = response.split(\"AI:\")[-1].strip()\n",
+        "\n",
+        "    return ai_response\n",
+        "\n",
+        "# Create the Gradio interface\n",
+        "iface = gr.ChatInterface(\n",
+        "    chatbot,\n",
+        "    title=\"AI Assistant\",\n",
+        "    description=\"Ask questions, get reflections.\",\n",
+        "    theme=\"soft\",\n",
+        "    examples=[\n",
+        "        \"How do I make fertilizer for tomatoes in Oregon?\",\n",
+        "        \"What is Korean Natural Farming?\",\n",
+        "        \"Can you explain the philosophy behind quantum physics?\",\n",
+        "        \"Please explain quantum entanglement in simple terms.\",\n",
+        "    ],\n",
+        ")\n",
+        "\n",
+        "# Launch the interface\n",
+        "iface.launch(share=True, debug=True)\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}