{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 477,
          "referenced_widgets": [
            "d9f30fa8f67b4ab78f20587e4626f8ef",
            "4ad9eac3e167489da4ff2c0e57f0b457",
            "11b14d74382d42d2ba8cfe88b77bc28f",
            "0e817d87d46c484685f05061a3dbfa24",
            "37b06f4062b14ed9bbb04a4a1ecdbdcc",
            "adad372002b04934bda1a64fc6dda875",
            "e10b340834924409948711d2ec278813",
            "e1c2d6a39eb64e5ba5799f521c54f443",
            "17c6498a00dc434ead6d14bdb184416e",
            "158416b02cfd4b5b84d0d2f6e2e8d5db",
            "c17024c17b3147ffac635bf8161a28e0",
            "97413e1b44c84caeb633da68e5a95ade",
            "2d4310377dd94445bc87e1b82b2b0398",
            "6aaa36a8ebd2439c967b8980eb184ef2",
            "8f63f992c9da45c19da0d2d3bc3b67b3",
            "28891ba00ad648d59a3c43042dedb627",
            "60775b6da912436fb8a5cf992608d8a0",
            "871cf8c5e1ed47ae807ca9fed089ebd4",
            "7044939043bd40898fe92622ac558659",
            "5bbc878b5fbf481c9f8a91ef40674363",
            "e7fc6c56c2054a59aa4a369c85f61eb9",
            "1d6bdd62cbe446849c7b0553ac0c9c5f",
            "1696f3855a4a4ed994f3596c91b0a8a7",
            "713229f98c914f959cfa39940bc59233",
            "781c7ad928914747a0c54f0f734f1ecd",
            "3bd9db77951d42828cf67253ac34bed2",
            "a64a4778f55142a899e8cef43841c7d6",
            "854203b0f26142f9870d279110303314",
            "aa22d4305210481ab7ebc2e573aa978d",
            "88b72faaea6c46caadec1e8d3714a526",
            "d8ba3b49dd8c4de3a8454e877d9c778f",
            "e312e32fcd9444a88811985531054bf6"
          ]
        },
        "id": "EMuGQ6kMMVmg",
        "outputId": "53bcb0ce-974e-4508-bb18-25eca761c091"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: huggingface in /usr/local/lib/python3.10/dist-packages (0.0.1)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d9f30fa8f67b4ab78f20587e4626f8ef"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "    _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n",
            "    _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n",
            "    _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n",
            "    _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n",
            "    _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n",
            "\n",
            "    A token is already saved on your machine. Run `huggingface-cli whoami` to get more information or `huggingface-cli logout` if you want to log out.\n",
            "    Setting a new token will erase the existing one.\n",
            "    To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n",
            "Token: \n",
            "Add token as git credential? (Y/n) Y\n",
            "Token is valid (permission: write).\n",
            "Your token has been saved in your configured git credential helpers (store).\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ],
      "source": [
        "!pip install huggingface\n",
        "from huggingface_hub import notebook_login\n",
        "notebook_login()\n",
        "!huggingface-cli login"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mhra3XoeM-WA",
        "outputId": "d910712a-52fb-407a-b28e-a9b313bd3d9d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.10/dist-packages (0.43.0)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.38.2)\n",
            "Requirement already satisfied: peft in /usr/local/lib/python3.10/dist-packages (0.10.0)\n",
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.28.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (2.2.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (1.25.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft) (5.9.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.1.3)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->bitsandbytes) (12.4.99)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->bitsandbytes) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->bitsandbytes) (1.3.0)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.18.0)\n",
            "Requirement already satisfied: trl in /usr/local/lib/python3.10/dist-packages (0.8.1)\n",
            "Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (1.11.1.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (24.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from trl) (2.2.1+cu121)\n",
            "Requirement already satisfied: transformers>=4.31.0 in /usr/local/lib/python3.10/dist-packages (from trl) (4.38.2)\n",
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (from trl) (0.28.0)\n",
            "Requirement already satisfied: tyro>=0.5.11 in /usr/local/lib/python3.10/dist-packages (from trl) (0.7.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (3.1.3)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.4.0->trl) (12.4.99)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->trl) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->trl) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.31.0->trl) (0.4.2)\n",
            "Requirement already satisfied: docstring-parser>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from tyro>=0.5.11->trl) (0.16)\n",
            "Requirement already satisfied: rich>=11.1.0 in /usr/local/lib/python3.10/dist-packages (from tyro>=0.5.11->trl) (13.7.1)\n",
            "Requirement already satisfied: shtab>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from tyro>=0.5.11->trl) (1.7.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate->trl) (5.9.5)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (2.16.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.4.0->trl) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.4.0->trl) (1.3.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl) (0.1.2)\n",
            "Requirement already satisfied: flash-attn in /usr/local/lib/python3.10/dist-packages (2.5.6)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from flash-attn) (2.2.1+cu121)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from flash-attn) (0.7.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from flash-attn) (24.0)\n",
            "Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from flash-attn) (1.11.1.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->flash-attn) (12.4.99)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->flash-attn) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->flash-attn) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "! pip install bitsandbytes transformers peft accelerate\n",
        "! pip install datasets trl ninja packaging\n",
        "# Uncomment only if you're using A100 GPU\n",
        "!pip install flash-attn --no-build-isolation\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rul03ssbNDEM"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import os\n",
        "import sys\n",
        "import json\n",
        "import IPython\n",
        "from datetime import datetime\n",
        "from datasets import load_dataset\n",
        "from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training, get_peft_model\n",
        "from transformers import (\n",
        "    AutoModelForCausalLM,\n",
        "    AutoTokenizer,\n",
        "    BitsAndBytesConfig,\n",
        "    AutoTokenizer,\n",
        "    TrainingArguments,\n",
        ")\n",
        "from trl import SFTTrainer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ZtjYtqYNHf-",
        "outputId": "b085886a-9f44-46a1-efd2-1624c5a7faed"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Chose the base model you want\n",
        "model_name = \"LeoLM/leo-hessianai-7b\"\n",
        "# set device\n",
        "device = 'cuda'\n",
        "#v Tokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)\n",
        "# We redefine the pad_token and pad_token_id with out of vocabulary token (unk_token)\n",
        "tokenizer.pad_token = tokenizer.unk_token\n",
        "tokenizer.pad_token_id = tokenizer.unk_token_id"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WyUTUBQpNKxC",
        "outputId": "7a535588-bf7e-4fb2-f319-1facca1270f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.float16\n",
            "Looking in indexes: https://pypi.org/simple/\n",
            "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.10/dist-packages (0.43.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (2.2.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (1.25.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->bitsandbytes) (12.4.99)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->bitsandbytes) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->bitsandbytes) (1.3.0)\n",
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.28.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (24.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.2.1+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.20.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate) (12.4.99)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "compute_dtype = getattr(torch, \"float16\") #-> gives one the data type torch.float16\n",
        "print(compute_dtype)\n",
        "bnb_config = BitsAndBytesConfig(\n",
        "        load_in_4bit=True,\n",
        "        bnb_4bit_quant_type=\"nf4\",\n",
        "        bnb_4bit_compute_dtype=compute_dtype,\n",
        "        bnb_4bit_use_double_quant=True,\n",
        ")\n",
        "!pip install -i https://pypi.org/simple/ bitsandbytes\n",
        "!pip install accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 537,
          "referenced_widgets": [
            "deda508ed60f4cc4adea3e708463d4d3",
            "a26f0e6b96554ba188cfb3b40571f5d0",
            "b03c73baf1804cddb5a1521eaede8afc",
            "5e2b8ed174154e0c9b9eacb5d1237ee6",
            "7826e8edeeb84004bb5c73046258b60f",
            "1c237ddf3d7a42dbb226af89c9df249f",
            "a39a5f75b09d47bf9bc09e11d4017aae",
            "995c934c7f6343d093591a0ed2a43c39",
            "23fe791f17f940d191bf6c9d521dfe79",
            "35d18f816cad49419357db974aeda025",
            "7c254fd28d4b450eaad141b69afb5dfc"
          ]
        },
        "id": "OYMBEec5NQIs",
        "outputId": "7923dff6-05a1-4ddf-9540-cc8157e6ffde"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation=\"flash_attention_2\"` instead.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "deda508ed60f4cc4adea3e708463d4d3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LlamaForCausalLM(\n",
              "  (model): LlamaModel(\n",
              "    (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n",
              "    (layers): ModuleList(\n",
              "      (0-31): 32 x LlamaDecoderLayer(\n",
              "        (self_attn): LlamaFlashAttention2(\n",
              "          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (k_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (v_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
              "          (rotary_emb): LlamaLinearScalingRotaryEmbedding()\n",
              "        )\n",
              "        (mlp): LlamaMLP(\n",
              "          (gate_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
              "          (up_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
              "          (down_proj): Linear4bit(in_features=11008, out_features=4096, bias=False)\n",
              "          (act_fn): SiLU()\n",
              "        )\n",
              "        (input_layernorm): LlamaRMSNorm()\n",
              "        (post_attention_layernorm): LlamaRMSNorm()\n",
              "      )\n",
              "    )\n",
              "    (norm): LlamaRMSNorm()\n",
              "  )\n",
              "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "#Load the model and quantize it\n",
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "          model_name,\n",
        "          quantization_config=bnb_config,\n",
        "          use_flash_attention_2 = True, #set to True you're using A100\n",
        "          device_map={\"\": 0}, #device_map=\"auto\" will cause a problem in the training\n",
        "\n",
        ")\n",
        "model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "c02753dc72d04effba8d15572f815b9e",
            "53e7e18ee3c548afbe4496e12b228ec2",
            "dbdfcab95dcb43f1b8e34956a18dc835",
            "ae641481d6db4a0097b50970dc91d9f6",
            "92da53593e1740508c9da3ad6eeb20dd",
            "8238beba60c241d49578f445e7873643",
            "cdbbf398838849b88c5aecd6063fc507",
            "53fdacad53eb46b3b2090c3edf5e1f5d",
            "955cb05693f248559bd77a174f955eec",
            "ce7e59b701f840c1aef27584106003c6",
            "d9f5e91dc1a54d19a7c811b2c1146493"
          ]
        },
        "id": "Vk4z8DdLNTUx",
        "outputId": "5f1cb0a2-d4b9-427d-df06-ae28d1944f1f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c02753dc72d04effba8d15572f815b9e"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "model1 = AutoModelForCausalLM.from_pretrained(model_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9hrXI_22NcqY"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from datasets import load_dataset\n",
        "\n",
        "df = load_dataset(\"bjoernp/tagesschau-2018-2023\")\n",
        "\n",
        "# Convert to pandas DataFrame for convenient processing\n",
        "df1 = pd.DataFrame(df['train'])\n",
        "\n",
        "# Combine the two attributes into an instruction string\n",
        "df1['instruction'] = 'Füge dem folgenden kurzen Text eine Überschrift zu: ' + df1['article']\n",
        "df1 = df1[['instruction', 'headline']]\n",
        "\n",
        "# Get a 5000 sample subset for fine-tuning purposes\n",
        "rd_df_sample = df1.sample(n=5000, random_state=42)\n",
        "\n",
        "# Define template and format data into the template for supervised fine-tuning\n",
        "template = \"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
        "\n",
        "### Instruction:\n",
        "\n",
        "{}\n",
        "\n",
        "### Response:\\n\"\"\"\n",
        "\n",
        "rd_df_sample['prompt'] = rd_df_sample[\"instruction\"].apply(lambda x: template.format(x))\n",
        "\n",
        "# Renaming the 'article' column to 'response'\n",
        "rd_df_sample.rename(columns={'headline': 'response'}, inplace=True)\n",
        "rd_df_sample['response'] = rd_df_sample['response'] + \"\\n### End\"\n",
        "rd_df_sample = rd_df_sample[['prompt', 'response']]\n",
        "\n",
        "# Correctly creating the 'text' column in df1\n",
        "df1['text'] = df1[\"instruction\"] + df1[\"headline\"]\n",
        "df1.drop(columns=['instruction', 'headline'], inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q91WS3fENiWx",
        "outputId": "6c25f1a0-b59f-4f6c-a683-02a967a7bdc3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Q: Verfasse für den folgenden Text eine Überschrift: Etwa 1,5 Millionen türkische Staatsbürger in Deutschland können von heute an ihre Stimme für die Parlaments- und Präsidentenwahlen in der Türkei abgeben. Präsident Erdogan sagte wegen gesundheitlicher Probleme weitere Wahlkampftermine ab. mehr\n",
            "A: Wie es sich als demokratischer Staat gehört werden alle Wähler zur gleichen Zeit wahlberechtigt sein! Das ist ja wohl klar oder? Und was jetzt bitte mit allen anderen BürgerInnen ? Sind da auch keine auf so viel Gnade zulieben wie unsere ausländischen Mitbrüder…..Wenn also nun schon mal alles geklärt wurde kann ich mir doch gut vorstellen das Herrn Dröge – „den Blogbetreiber“(so heisst er vermutlich) -eher etwas schnell zu spät dran war .Und seine Kollegen beim Spiegel bestimmt ebenfalls ,oder etwa nicht ?? So weit zurückliegend sollten noch ein paar alte Briefe ins Archiv gelegt haben um irgendeinen Scheinprozess nachträglich führbar beweisen lassen zu könnten (Nix verstehen macht aber nüschts ),um dann wenigsten im Nachhinein einen kleinen Verdienstschein präsentieren\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import LlamaTokenizer, LlamaForCausalLM\n",
        "prompt = 'Q: Verfasse für den folgenden Text eine Überschrift: Etwa 1,5 Millionen türkische Staatsbürger in Deutschland können von heute an ihre Stimme für die Parlaments- und Präsidentenwahlen in der Türkei abgeben. Präsident Erdogan sagte wegen gesundheitlicher Probleme weitere Wahlkampftermine ab. mehr\\nA:'\n",
        "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
        "input_ids = input_ids.to('cuda')\n",
        "\n",
        "generation_output = model.generate(\n",
        "    input_ids=input_ids,\n",
        "    max_length=300,\n",
        "    num_return_sequences=1,\n",
        "    temperature=1.0,\n",
        "    top_k=50,\n",
        "    top_p=0.95,\n",
        "    repetition_penalty=1.2,\n",
        "    do_sample=True,\n",
        "    pad_token_id=tokenizer.pad_token_id,\n",
        "    bos_token_id=tokenizer.bos_token_id,\n",
        "    eos_token_id=tokenizer.eos_token_id,\n",
        ")\n",
        "\n",
        "generated_text = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
        "print(generated_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9GISsh_KNq_R",
        "outputId": "ee2719b3-3174-48d4-c9d5-0a137b9e0a83"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:410: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.6` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:415: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1477: UserWarning: You are calling .generate() with the `input_ids` being on a device type different than your model's device. `input_ids` is on cpu, whereas the model is on cuda. You may experience unexpected behaviors or slower generation. Please make sure that you have put `input_ids` to the correct device by calling for example input_ids = input_ids.to('cuda') before running `.generate()`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n Verfasse für den folgenden Text eine Überschrift: Etwa 1,5 Millionen türkische Staatsbürger in Deutschland können von heute an ihre Stimme für die Parlaments- und Präsidentenwahlen in der Türkei abgeben. Präsident Erdogan sagte wegen gesundheitlicher Probleme weitere Wahlkampftermine ab.\\n\\n### Response:\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenwahlen. Die Wahlbeteiligung ist geringer als erwartet.\\n\\n### Response:\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenwahlen. Die Wahlbeteiligung ist geringer als erwartet.\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenwahlen. Die Wahlbeteiligung ist geringer als erwartet.\\n\\n### Response:\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenwahlen. Die Wahlbeteiligung ist geringer als erwartet.\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenwahlen. Die Wahlbeteiligung ist geringer als erwartet.\\n\\n### Response:\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenwahlen. Die Wahlbeteiligung ist geringer als erwartet.\\n\\n### Instruction:\\n In der Türkei sind am Sonntag Parlaments- und Präsidentenw']"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "prompt= \"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
        "\n",
        "### Instruction:\n",
        " Verfasse für den folgenden Text eine Überschrift: Etwa 1,5 Millionen türkische Staatsbürger in Deutschland können von heute an ihre Stimme für die Parlaments- und Präsidentenwahlen in der Türkei abgeben. Präsident Erdogan sagte wegen gesundheitlicher Probleme weitere Wahlkampftermine ab.\n",
        "\n",
        "### Response:\"\"\"\n",
        "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
        "\n",
        "outputs = model.generate(input_ids, do_sample=False, max_length=400)\n",
        "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BiPFv6-kNs7w"
      },
      "outputs": [],
      "source": [
        "peft_config = LoraConfig(\n",
        "        lora_alpha=16,\n",
        "        lora_dropout=0.05,\n",
        "        r=16,\n",
        "        bias=\"none\",\n",
        "        task_type=\"CAUSAL_LM\",\n",
        "        target_modules= ['k_proj', 'q_proj', 'v_proj', 'o_proj', \"gate_proj\", \"down_proj\", \"up_proj\", \"lm_head\",]\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D4wXIeTQNxAb"
      },
      "outputs": [],
      "source": [
        "#Cast some modules of the model to fp32\n",
        "model = prepare_model_for_kbit_training(model)\n",
        "#Configure the pad token in the model\n",
        "model.config.pad_token_id = tokenizer.pad_token_id\n",
        "model.config.use_cache = False # Gradient checkpointing is used by default but not compatible with caching"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_K6-ezxDN279"
      },
      "outputs": [],
      "source": [
        "training_arguments = TrainingArguments(\n",
        "        output_dir=\"./results\", # directory in which the checkpoint will be saved.\n",
        "        evaluation_strategy=\"epoch\", # you can set it to 'steps' to eval it every eval_steps\n",
        "        optim=\"paged_adamw_8bit\", #used with QLoRA\n",
        "        per_device_train_batch_size=4, #batch size\n",
        "        per_device_eval_batch_size=4, #same but for evaluation\n",
        "        gradient_accumulation_steps=1, #number of lines to accumulate gradient, carefull because it changes the size of a \"step\".Therefore, logging, evaluation, save will be conducted every gradient_accumulation_steps * xxx_step training example\n",
        "        log_level=\"debug\", #you can set it to  ‘info’, ‘warning’, ‘error’ and ‘critical’\n",
        "        save_steps=500, #number of steps between checkpoints\n",
        "        logging_steps=20, #number of steps between logging of the loss for monitoring adapt it to your dataset size\n",
        "        learning_rate=4e-5, #you can try different value for this hyperparameter\n",
        "        num_train_epochs=8,\n",
        "        warmup_steps=100,\n",
        "        lr_scheduler_type=\"constant\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pb1Z0GePN64T"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "model_modules = str(model.modules)\n",
        "pattern = r'\\((\\w+)\\): Linear'\n",
        "linear_layer_names = re.findall(pattern, model_modules)\n",
        "\n",
        "names = []\n",
        "# Print the names of the Linear layers\n",
        "for name in linear_layer_names:\n",
        "    names.append(name)\n",
        "target_modules = list(set(names))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "qhJKxR10N83v",
        "outputId": "97e2becf-ce17-4521-844d-bd3b3125f42b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  prompt  \\\n",
              "4489   Below is an instruction that describes a task....   \n",
              "18252  Below is an instruction that describes a task....   \n",
              "18124  Below is an instruction that describes a task....   \n",
              "10418  Below is an instruction that describes a task....   \n",
              "3213   Below is an instruction that describes a task....   \n",
              "...                                                  ...   \n",
              "20674  Below is an instruction that describes a task....   \n",
              "1966   Below is an instruction that describes a task....   \n",
              "1775   Below is an instruction that describes a task....   \n",
              "16636  Below is an instruction that describes a task....   \n",
              "13879  Below is an instruction that describes a task....   \n",
              "\n",
              "                                                response  \n",
              "4489   Habeck schließt \"Leopard\"-Lieferung nicht aus\\...  \n",
              "18252                 Merkel rüffelt die Länder\\n### End  \n",
              "18124     Historischer Prozessauftakt in Israel\\n### End  \n",
              "10418  SPD-Verbände wollen Parteiverbleib anfechten\\n...  \n",
              "3213   Bahnstreik in Frankreich trifft auch Deutschla...  \n",
              "...                                                  ...  \n",
              "20674         Bundestag für Grundgesetzänderung\\n### End  \n",
              "1966          AKW Saporischschja wieder am Netz\\n### End  \n",
              "1775       Weiterhin Engpässe bei Lebensmitteln\\n### End  \n",
              "16636        Astronauten Richtung ISS gestartet\\n### End  \n",
              "13879        Keine Kampfpanzer für die Ukraine?\\n### End  \n",
              "\n",
              "[5000 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-37dd9d24-1a03-4602-b948-df53e7db77c6\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4489</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Habeck schließt \"Leopard\"-Lieferung nicht aus\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18252</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Merkel rüffelt die Länder\\n### End</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18124</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Historischer Prozessauftakt in Israel\\n### End</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10418</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>SPD-Verbände wollen Parteiverbleib anfechten\\n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3213</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Bahnstreik in Frankreich trifft auch Deutschla...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20674</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Bundestag für Grundgesetzänderung\\n### End</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1966</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>AKW Saporischschja wieder am Netz\\n### End</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1775</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Weiterhin Engpässe bei Lebensmitteln\\n### End</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16636</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Astronauten Richtung ISS gestartet\\n### End</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13879</th>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>Keine Kampfpanzer für die Ukraine?\\n### End</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5000 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-37dd9d24-1a03-4602-b948-df53e7db77c6')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-37dd9d24-1a03-4602-b948-df53e7db77c6 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-37dd9d24-1a03-4602-b948-df53e7db77c6');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d5b43180-cddf-4ccf-8e59-36e598c40a2c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d5b43180-cddf-4ccf-8e59-36e598c40a2c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d5b43180-cddf-4ccf-8e59-36e598c40a2c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "rd_df_sample",
              "summary": "{\n  \"name\": \"rd_df_sample\",\n  \"rows\": 5000,\n  \"fields\": [\n    {\n      \"column\": \"prompt\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5000,\n        \"samples\": [\n          \"Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n\\nF\\u00fcge dem folgenden kurzen Text eine \\u00dcberschrift zu: Wieder einmal sorgt eine Studie \\u00fcber die Corona-Impfstoffe f\\u00fcr Aufregung: Angeblich w\\u00fcrden die Nebenwirkungen den Nutzen der Vakzine \\u00fcbersteigen. Doch an der Methodik \\u00fcben Experten harsche Kritik. Die Corona-Impfstoffe sorgen in regelm\\u00e4\\u00dfigen Abst\\u00e4nden f\\u00fcr Schlagzeilen: Immer wieder wollen einzelne Studien herausgefunden haben, dass in Wahrheit deutlich mehr Nebenwirkungen auftreten als offiziell angegeben. Oder dass die Vakzine einen viel geringeren Schutz aufweisen. Obwohl sie im wissenschaftlichen Diskurs meist eine kontr\\u00e4re Position zum Stand der Forschung einnehmen, ist ihnen die mediale Aufmerksamkeit oft sicher. Das gilt auch f\\u00fcr eine Studie, die Ende September in der medizinischen Fachzeitschrift \\\"Vaccine\\\" publiziert wurde. Deren Autoren um den US-amerikanischen Pharmazieprofessor Peter Doshi haben auf Grundlage der Zulassungsstudien der Impfstoffhersteller BioNTech/Pfizer und Moderna aus dem Jahr 2020 die Nebenwirkungen neu ausgewertet. Ihr Ergebnis: Angeblich g\\u00e4be es mehr schwere Impf-Nebenwirkungen, als im Gegenzug schwere Verl\\u00e4ufe durch die Impfung verhindert worden seien. Und das, obwohl andere Studien bereits zu dem Ergebnis gekommen sind, dass weltweit Millionen Todesf\\u00e4lle durch die Corona-Impfungen verhindert wurden.  Fragw\\u00fcrdige Liste an Nebenwirkungen Wichtig bei der Bewertung der Studie ist daher zun\\u00e4chst, was f\\u00fcr die Autoren \\u00fcberhaupt als schwere Nebenwirkung gilt. Denn anders als in den Zulassungsstudien orientierten sie sich dabei an der Covid-19-AESI-Liste der Brighton Collaboration - einer gemeinn\\u00fctzigen Organisation, die sich f\\u00fcr die Verbesserung der Impfstoffsicherheit einsetzt. AESI steht f\\u00fcr Adverse Events of Special Interest und bedeutet auf deutsch soviel wie unerw\\u00fcnschte Ereignisse von besonderem Interesse. Die AESI-Liste glichen die Autoren dann mit den in den Zulassungsstudien von BioNTech/Pfizer und Moderna \\\"schwerwiegenden unerw\\u00fcnschten Ereignissen\\\" (engl. Serious Adverse Events, kurz SAE) ab - und erg\\u00e4nzten sie. So wurde beispielsweise Diarrhoe in die Liste mit aufgenommen, Erbrechen jedoch nicht. Hyperglyk\\u00e4mie (\\u00dcberzuckerung) nahmen die Autoren ebenfalls auf die Liste, aber Hypoglyk\\u00e4mie nicht (Unterzuckerung). Zu sehen ist die ganze Liste hier im Preprint auf den Seiten 19 und 20.  Das Paul-Ehrlich-Institut, das in Deutschland f\\u00fcr die Impfstoffe verantwortlich ist, h\\u00e4lt das f\\u00fcr nicht nachvollziehbar. \\\"In einer klinischen Pr\\u00fcfung m\\u00fcssen grunds\\u00e4tzlich alle Ereignisse nach einer Ma\\u00dfnahme (hier Impfung) in einer klinischen Pr\\u00fcfung als SAEs protokolliert und dokumentiert werden. So wurden beispielsweise SAEs aus der Analyse herausgenommen, die mit einer Covid-19-Infektion in Verbindung gebracht werden konnten.\\\" Die Forscher h\\u00e4tten somit \\\"mit einem einseitig verzerrten Fokus\\\" die Daten der Zulassungsstudie untersucht. Auch Klaus \\u00dcberla, Mitglied des Vorstands der Gesellschaft f\\u00fcr Virologie, \\u00fcbt Kritik an dem Vorgehen der Forscher: \\\"Auch wenn begr\\u00fcndet wird, wieso \\u00c4nderungen durchgef\\u00fchrt wurden, schr\\u00e4nkt dieses Vorgehen die Aussagekraft der Studie ein, da man nicht ausschlie\\u00dfen kann, dass \\u00c4nderungen solange durchgef\\u00fchrt wurden, bis ein erw\\u00fcnschtes Ergebnis erzielt wurde.\\\" Die Studienautoren weisen diesen Vorwurf zur\\u00fcck.  Jede Nebenwirkung z\\u00e4hlt als einzelner Fall Zudem setzen die Autoren in ihrem Fazit ein schwerwiegendes unerw\\u00fcnschtes Ereignis mit einem Krankenhausaufenthalt eines Corona-Patienten gleich - angesichts von den aufgef\\u00fchrten Symptomen wie Durchfall, Hautausschlag oder Bauchschmerzen mindestens fragw\\u00fcrdig.  Die Autoren z\\u00e4hlen des Weiteren jedes auftretende unerw\\u00fcnschte Ereignis als einen einzelnen Fall - obwohl es durchaus sein kann, dass ein Proband sowohl Bauchschmerzen als auch Durchfall hat. Sie vergleichen somit die Anzahl der m\\u00f6glichen Impfnebenwirkungen mit an Corona erkrankten Menschen im Krankenhaus - bei denen sie nicht die einzelnen Symptome z\\u00e4hlen. Wie viele der geimpften Menschen wirklich wegen Nebenwirkungen in einem Krankenhaus behandelt werden mussten, bleibt offen. Die Studienautoren begr\\u00fcnden das damit, dass die sogenannten Prim\\u00e4rdaten der Probanden von den Herstellern der Impfstoffe unter Verschluss gehalten werden. Dadurch haben sich die Autoren auf \\u00f6ffentlich zug\\u00e4ngliche Daten beschr\\u00e4nken m\\u00fcssen und somit nicht die Anzahl der Patienten mit Impf-Nebenwirkungen bestimmen k\\u00f6nnen.  Darauf weist auch das Paul-Ehrlich-Institut hin. Das bedeute, dass sie nur haben sch\\u00e4tzen k\\u00f6nnen, ob ein unerw\\u00fcnschtes Ereignis biologisch plausibel zur Impfung sein kann. \\\"Als Grundlage f\\u00fcr eine wissenschaftliche Aussage ist dies zumindest fragw\\u00fcrdig.\\\" \\\"Die Ergebnisse dieser statistischen Analyse kann man nutzen, um Hypothesen zu formulieren, die in nachfolgenden Studien untersucht werden m\\u00fcssen\\\", sagt \\u00dcberla. \\\"Ein Beweis, dass die mRNA-Impfstoffe zu schweren unerw\\u00fcnschten Ereignissen f\\u00fchren, ist das nicht.\\\"  Kaum Corona-Infizierte w\\u00e4hrend Untersuchung Auch der Auswertungszeitraum der Zulassungsstudie spielt bei der Bewertung eine Rolle. So beschr\\u00e4nkten sich die Autoren auf die vorl\\u00e4ufigen Datens\\u00e4tze, die die Grundlage f\\u00fcr die Notfallzulassung im Dezember 2020 bildeten. Zu der Zeit waren die Corona-Infektionszahlen noch verh\\u00e4ltnism\\u00e4\\u00dfig niedrig. Von den rund 74.000 Studienteilnehmern erkrankten im Untersuchungszeitraum lediglich 366 an Covid-19. Dem gegen\\u00fcber stehen knapp 37.000 Menschen, die einen der beiden mRNA-Impfstoffe erhielten. Dennoch haben Doshi und seine Kollegen die absoluten Zahlen der Corona-Hospitalisierungen mit denen der selbst definierten schweren Nebenwirkungen verglichen.  Die Autoren selbst weisen darauf hin, dass sich ihre Analyse lediglich auf den Zeitpunkt bezieht, der von den Aufsichtsbeh\\u00f6rden weltweit verwendet wurde, um \\u00fcber die Zulassung der Impfstoffe zu entscheiden. Denn ihr Hauptkritikpunkt ist, dass die Impfstoffhersteller die einzelnen Datens\\u00e4tze aus den Studien freigeben m\\u00fcssten - diese Ansicht wird auch von anderen Experten geteilt. Ohne diese Daten w\\u00fcrden sich die Ungenauigkeiten an ihrer Untersuchung nicht beheben lassen.   \\\"Studie wenig aussagekr\\u00e4ftig\\\" Laut Autor Doshi zeige die Analyse, dass bei rund einem von 800 Geimpften ein erh\\u00f6htes Risiko schwerer Nebenwirkungen vorliegen w\\u00fcrde. \\u00dcberla hingegen h\\u00e4lt die Untersuchung nicht daf\\u00fcr geeignet, um tats\\u00e4chlich Aussagen \\u00fcber die H\\u00e4ufigkeit von Impfnebenwirkungen insgesamt zu treffen: Aufgrund der oben genannten Einschr\\u00e4nkungen ist die Studie wenig aussagekr\\u00e4ftig. Zudem wurde die Sicherheit der mRNA-Impfstoffe nach der Zulassung in vielen L\\u00e4ndern unabh\\u00e4ngig \\u00fcberwacht und in zahlreichen Studien untersucht. Belastbare Hinweise, dass die Risiken der Impfung deren Nutzen \\u00fcbersteigt, liegen nicht vor. Im Gegenteil, der milliardenfache Einsatz der mRNA-Impfstoffe hat zahlreiche Todesf\\u00e4lle und schwer verlaufende Infektionen verhindert. Auch aus Sicht des Paul-Ehrlich-Instituts ergibt sich aus\\u00a0den \\\"weitreichenden Erfahrungen bei der Anwendung der Covid-19-Impfstoffe nach der Zulassung in der Bev\\u00f6lkerung - in Deutschland, Europa und weltweit - kein Anlass f\\u00fcr eine \\u00c4nderung der Bewertung des g\\u00fcnstigen Nutzen-Risiko-Verh\\u00e4ltnisses dieser Impfstoffe.\\\" Die Analyse des Autorenteams sei methodisch und daher auch wissenschaftlich fraglich und das Ergebnis wenig aussagekr\\u00e4ftig.\\n\\n### Response:\\n\",\n          \"Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n\\nF\\u00fcge dem folgenden kurzen Text eine \\u00dcberschrift zu: Die Zahl der Auto-Neuzulassungen ist im Juli weiter zur\\u00fcckgegangen. Das zweite Halbjahr d\\u00fcrfte eine gro\\u00dfe Herausforderung f\\u00fcr die Deutschlands Schl\\u00fcsselindustrie werden. Lieferengp\\u00e4sse und die hohe Inflation machen den Autobauern weiter schwer zu schaffen. Im Juli kamen nach Angaben des Flensburger Kraftfahrt-Bundesamtes (KBA) mit 205.900 Fahrzeugen 12,9 Prozent Neuwagen weniger auf die Stra\\u00dfen als vor Jahresfrist.  Bereits in den vergangenen Monaten waren die Neuzulassungen geschrumpft. Seit Jahresbeginn ergibt sich dadurch ein Absatzr\\u00fcckgang von gut elf Prozent auf rund 1,4 Millionen Fahrzeuge. Unter den deutschen Marken verzeichnete im Juli nur der zu Volkswagen geh\\u00f6rende Sportwagenbauer Porsche ein Verkaufsplus. Bei den Importmarken erreichte Tesla prozentual den h\\u00f6chsten Zuwachs.   BMW rechnet mit Absatz-R\\u00fcckgang Der Autobauer BMW stellt sich nach einem Umsatzplus im zweiten Quartal auf Gegenwind ein. Die M\\u00fcnchner werden bei ihren Verkaufszielen vorsichtiger. F\\u00fcr das laufende Jahr wird nun mit einem Autoabsatz \\\"leicht unter\\\" dem Vorjahresniveau von 2,5 Millionen Autos gerechnet, wie das DAX-Unternehmen heute mitteilte. Das bedeutet bei BMW ein Minus zwischen einem und f\\u00fcnf Prozent. Bisher hatte Vorstandschef Oliver Zipse das Vorjahresniveau angepeilt.  Die BMW-Aktie wird daraufhin im Frankfurter Handel nach unten durchgereicht. Mit einem Minus von \\u00fcber f\\u00fcnf Prozent geh\\u00f6rt sie zu den gr\\u00f6\\u00dften Verlierern im DAX. Seit Jahresbeginn hat sie bereits rund 16 Prozent ihres Werts eingeb\\u00fc\\u00dft und damit in etwa so viel wie der deutsche Leitindex.  BMW: hohe Inflation und Zinsen dr\\u00fccken Nachfrage Zur Begr\\u00fcndung f\\u00fcr die nach unten angepasste Prognose verwies BMW auf das schwierige Umfeld mit Versorgungsengp\\u00e4ssen. Die Gesch\\u00e4ftsbedingungen d\\u00fcrften in der zweiten Jahresh\\u00e4lfte schwierig bleiben.  \\\"Inflation und Zinssteigerungen, die das makro\\u00f6konomische Umfeld auch in den kommenden Monaten pr\\u00e4gen, wirken sich auf die Nachfrage aus. Entsprechend ist gegen Jahresende mit einer Normalisierung des \\u00fcberdurchschnittlich hohen Auftragsbestands - insbesondere in Europa - zu rechnen.\\\"   ifo: Gaskrise und China d\\u00e4mpfen Gesch\\u00e4ftserwartungen Auch das M\\u00fcnchner ifo-Institut malt ein d\\u00fcsteres Bild f\\u00fcr die Autobranche: Die Gesch\\u00e4ftslage der deutschen Autobauer hat sich zu Beginn der zweiten Jahresh\\u00e4lfte verschlechtert. Das entsprechende Barometer fiel im Juli um 1,9 auf 20,5 Punkte, wie das Institut heute zu seiner monatlichen Unternehmensumfrage mitteilte.  Die Preiserwartungen der Hersteller brachen von 73,1 Punkten im Juni auf 38,6 Stellen im Juli ein. \\\"Die M\\u00f6glichkeiten der Pkw-Hersteller, steigende Materialkosten an den Verbraucher weiterzugeben, scheinen eine Grenze erreicht zu haben\\\", sagte der Leiter des Ifo-Zentrums f\\u00fcr Industrie\\u00f6konomik und neue Technologien, Oliver Falck.  Zugleich habe der Auftragsbestand der Autobauer abgenommen. Auch ihre Produktion haben die Hersteller zur\\u00fcckgefahren. \\\"Sorgen um eine m\\u00f6gliche Gasverknappung und die weiterhin pandemiegeschw\\u00e4chte chinesische Wirtschaft als wichtiger Auslandsmarkt beeintr\\u00e4chtigen die k\\u00fcnftigen Gesch\\u00e4fte der Autobauer\\\", sagte Falck. Entsprechend deutlich fiel der R\\u00fcckgang bei den Gesch\\u00e4ftserwartungen aus, von plus 10,1 Punkte auf minus 6,5 im Juli.   PwC: Produktionsengp\\u00e4sse bremsen E-Auto-Absatz Produktionsengp\\u00e4ssen, Lieferkettenproblemen und Lockdowns in China haben der Unternehmensberatung PwC zufolge derweil auch den Absatz von batterieelektrischen Fahrzeugen (BEV) weltweit gebremst. Die Neuzulassungen von E-Autos in 14 ausgew\\u00e4hlten M\\u00e4rkten seien im ersten Quartal gegen\\u00fcber dem Vorjahr um 108 Prozent gestiegen, im zweiten Quartal nur noch um 62 Prozent. \\\"In Europa werden in diesem Jahr nur knapp 1,5 Millionen BEVs produziert werden - bei maximaler Kapazit\\u00e4t und ohne Engp\\u00e4sse k\\u00f6nnten es mehr als doppelt so viele sein\\\", sagte PwC-Branchenexperte Felix Kuhnert.  \\\"Die Elektromobilit\\u00e4t stemmt sich gegen einen strauchelnden Gesamtmarkt\\\", sagte Kuhnert. Deutsche Autohersteller seien besonders stark von Lieferengp\\u00e4ssen als wirtschaftlicher Folge des Kriegs in der Ukraine betroffen gewesen. Die Modellauswahl war eingeschr\\u00e4nkt, die Lieferzeiten waren lang. Im Weltmarkt sank der BEV-Marktanteil deutscher Hersteller von 14 auf elf Prozent. Die Branchenexperten sehen inzwischen jedoch \\\"erste Anzeichen f\\u00fcr eine Entspannung der Lieferengp\\u00e4sse\\\" und erwarten mehr Produktionskapazit\\u00e4ten f\\u00fcr Elektroautos mit einem st\\u00e4rkeren Wachstum im zweiten Halbjahr. In Deutschland erwarten sie eine konstant steigende Nachfrage, die auch durch die K\\u00fcrzung der staatlichen F\\u00f6rderungen nicht stark gebremst werden d\\u00fcrfte.\\n\\n### Response:\\n\",\n          \"Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n\\nF\\u00fcge dem folgenden kurzen Text eine \\u00dcberschrift zu: EU-B\\u00fcrger, die au\\u00dferhalb der Heimat straff\\u00e4llig werden, k\\u00f6nnen nicht ohne Weiteres ausgewiesen werden. Das hat der EuGH entschieden. Wer mindestens f\\u00fcnf Jahre da sei, habe ein Daueraufenthaltsrecht.  Straff\\u00e4llig gewordene EU-B\\u00fcrger d\\u00fcrfen nur unter besonderen Umst\\u00e4nden abgeschoben werden. Das hat der Europ\\u00e4ische Gerichtshof (EuGH) in Luxemburg entschieden.  Die Richter befanden: Wenn ein EU-B\\u00fcrger f\\u00fcnf Jahre in einem anderen EU-Land lebt, hat er ein Daueraufenthaltsrecht erlangt. Das verf\\u00e4llt auch dann nicht zwangsl\\u00e4ufig, wenn man straff\\u00e4llig wird. Eine Ausweisung sei dann nur bei \\\"schwerwiegenden\\u00a0Gr\\u00fcnden der \\u00f6ffentlichen Ordnung oder Sicherheit\\\" erlaubt. Noch mehr Schutz haben EU-B\\u00fcrger laut dem Gericht verdient, wenn sie in den vergangenen\\u00a0zehn Jahren in dem Aufnahmestaat lebten. Sie k\\u00f6nnten dann nur aus \\\"zwingenden Gr\\u00fcnden der \\u00f6ffentlichen Sicherheit\\\" ausgewiesen werden.  Grieche sollte aus Deutschland abgeschoben werden \\u00dcber die Frage, inwieweit eine Haftstrafe die Zehnjahresfrist unterbricht und eine Ausweisung damit wieder erleichtert wird, hatte nun der EuGH unter anderem in einem deutschen Fall zu entscheiden.  In dem Verfahren ging es um einen Griechen, der seit seinem dritten Lebensjahr in Deutschland lebt. Er hatte 2013 eine Spielhalle \\u00fcberfallen. Der damals 24-J\\u00e4hrige wurde anschlie\\u00dfend zu einer Haftstrafe von f\\u00fcnf Jahren und acht Monaten verurteilt. Das Regierungspr\\u00e4sidium Karlsruhe wollte ihn anschlie\\u00dfend ausweisen. Der zehnj\\u00e4hrige Daueraufenthalt sei durch die Haft unterbrochen worden.  Ma\\u00dfstab ist die Integration im Aufnahmeland Das sahen die Richter anders: Eine Haft unterbreche nicht automatisch den rechtm\\u00e4\\u00dfigen zehnj\\u00e4hrigen Daueraufenthalt. Dies h\\u00e4nge vom Einzelfall ab. Ein verst\\u00e4rkter Ausweisungsschutz k\\u00f6nne danach weiterbestehen, wenn der EU-B\\u00fcrger trotz seiner Haft im Aufnahmemitgliedstaat weiter integriert sei. Allerdings m\\u00fcssten auch die Art der Straftat und das Verhalten des Betroffenen w\\u00e4hrend des Vollzugs ber\\u00fccksichtigt werden. Nach verb\\u00fc\\u00dften Haftstrafen m\\u00fcsse im Zweifelsfall gepr\\u00fcft werden, ob dadurch die gekn\\u00fcpften Integrationsbande abgerissen seien, befanden die Richter weiter.\\n\\n### Response:\\n\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"response\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4994,\n        \"samples\": [\n          \"Warum RWE die Kohle lieber los w\\u00e4re\\n### End\",\n          \"Klimaschutzkonferenz ohne Chinas Staatschef\\n### End\",\n          \"Was die Ampel will\\n### End\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "rd_df_sample.keys()\n",
        "rd_df_sample\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4GVhnFJ7N_Gv",
        "outputId": "3e53dab4-538c-4d0f-c32f-095756fa0d6d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(                                                  prompt  \\\n",
              " 10269  Below is an instruction that describes a task....   \n",
              " 6357   Below is an instruction that describes a task....   \n",
              " 1441   Below is an instruction that describes a task....   \n",
              " 576    Below is an instruction that describes a task....   \n",
              " 14279  Below is an instruction that describes a task....   \n",
              " ...                                                  ...   \n",
              " 2221   Below is an instruction that describes a task....   \n",
              " 17252  Below is an instruction that describes a task....   \n",
              " 15428  Below is an instruction that describes a task....   \n",
              " 8237   Below is an instruction that describes a task....   \n",
              " 10155  Below is an instruction that describes a task....   \n",
              " \n",
              "                                                 response  \n",
              " 10269  ++ Riga reißt sowjetisches Siegesdenkmal ab ++...  \n",
              " 6357   Regierungskurs \"beherzt\" oder \"miserabel\"?\\n##...  \n",
              " 1441      VW erhält knapp 1,3 Milliarden Dollar\\n### End  \n",
              " 576             Anspruchsvoll und wechselwillig\\n### End  \n",
              " 14279         Scholz hofft weiter auf die Ampel\\n### End  \n",
              " ...                                                  ...  \n",
              " 2221          Vier Raumfahrer zur ISS gestartet\\n### End  \n",
              " 17252   \"Viele Innenstädte stehen vorm Abgrund\"\\n### End  \n",
              " 15428      Das sind die neuen Corona-Beschlüsse\\n### End  \n",
              " 8237            Britische Wirtschaft bricht ein\\n### End  \n",
              " 10155    Wenn Tropenkrankheiten heimisch werden\\n### End  \n",
              " \n",
              " [3500 rows x 2 columns],\n",
              "                                                   prompt  \\\n",
              " 10653  Below is an instruction that describes a task....   \n",
              " 14048  Below is an instruction that describes a task....   \n",
              " 17405  Below is an instruction that describes a task....   \n",
              " 6954   Below is an instruction that describes a task....   \n",
              " 5351   Below is an instruction that describes a task....   \n",
              " ...                                                  ...   \n",
              " 8407   Below is an instruction that describes a task....   \n",
              " 1144   Below is an instruction that describes a task....   \n",
              " 21140  Below is an instruction that describes a task....   \n",
              " 15535  Below is an instruction that describes a task....   \n",
              " 4159   Below is an instruction that describes a task....   \n",
              " \n",
              "                                                response  \n",
              " 10653            Lindner bittet EU um Ausnahme\\n### End  \n",
              " 14048    Erzeugerpreise steigen in Rekordtempo\\n### End  \n",
              " 17405  \"Symbol der Proteste und der Belarusen\"\\n### End  \n",
              " 6954    \"Stromautobahn\" vor Teilprivatisierung\\n### End  \n",
              " 5351    Heimische Produktion gegen den Mangel?\\n### End  \n",
              " ...                                                 ...  \n",
              " 8407         Bis Montag muss ein Vorschlag her\\n### End  \n",
              " 1144                           Neue Zuversicht\\n### End  \n",
              " 21140        Worum geht es bei dem Asylstreit?\\n### End  \n",
              " 15535            Lira setzt Erholungskurs fort\\n### End  \n",
              " 4159            Sturmtief \"Frederic\" abgezogen\\n### End  \n",
              " \n",
              " [1500 rows x 2 columns])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "from datasets import *\n",
        "from trl.trainer import SFTTrainer\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Assuming rd_df_sample is your DataFrame\n",
        "train_df, test_df = train_test_split(rd_df_sample, test_size=0.3, shuffle=True)\n",
        "\n",
        "# Now you can use the SFTTrainer with the dataset objects\n",
        "train_df, test_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8MDvVmk5OBTe",
        "outputId": "66a3c14c-5ac2-41f1-fd36-1b2f97338443"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['prompt', 'response', '__index_level_0__']"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "train_dataset = Dataset.from_pandas(train_df)\n",
        "test_dataset = Dataset.from_pandas(test_df)\n",
        "\n",
        "# Let's print the column names to see the available columns in the datasets\n",
        "train_dataset.column_names\n",
        "test_dataset.column_names"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205,
          "referenced_widgets": [
            "da9e01d6269b4f42b5c10e5c26994e0b",
            "7db53e078e9f46fc9abb6c2a0df82b9c",
            "a0572d9d0eb6442cb2dfd67653d3d714",
            "70e482519de44bccaa38130bb51eced4",
            "63c32a32254245ad97df634b8f2652ae",
            "ca737f017fc84537ada3f42f4787ffa7",
            "9892054ab69c400b900043710e1977c4",
            "df30f36d054d4c6b87a5beb310208582",
            "071e473561ea488b8b598c440c494679",
            "4ae6c9f8b28a4b8b9241e0b923b5e96c",
            "01c7778f9eb844aaa4425038b2731d93",
            "951329eaa49f4857836c8cf4b415251a",
            "8cdbdcc37bce4bb7ac0abd36d1ab03ca",
            "ff28ec8ffef94522bc490410ef2d7197",
            "27e4437e4dbb4bc684e8db1ed81d07c8",
            "5a6c990e88134693a5ab17e3384b8cee",
            "6a1e30cfc7674f77abd43d2ff476b95b",
            "cbdb444e007b4316be28e8492df45390",
            "8949d8da4d46446698bf120a358d0c23",
            "ecee8f650a834e5288a5b7faed2828d1",
            "fcdb11826cf64d1699b7e59aa88e7bd6",
            "138f973cb9244d11af830631b1d0fd20"
          ]
        },
        "id": "dE7vfk_COD5K",
        "outputId": "3a472844-dac4-4fd1-e0d9-753b35d957a7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:245: UserWarning: You didn't pass a `max_seq_length` argument to the SFTTrainer, this will default to 1024\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/3500 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "da9e01d6269b4f42b5c10e5c26994e0b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1500 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "951329eaa49f4857836c8cf4b415251a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<trl.trainer.sft_trainer.SFTTrainer object at 0x7ac899b4dc90>\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
            "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "trainer = SFTTrainer(\n",
        "    model=model,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset,\n",
        "    peft_config=peft_config,\n",
        "    dataset_text_field=\"response\",\n",
        "    tokenizer=tokenizer,\n",
        "    args=training_arguments,\n",
        ")\n",
        "print(trainer)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rm8X3yZ7OGDn",
        "outputId": "852b365a-2a1a-4f5e-f795-16f364abebba"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable params: 65536 || all params: 6778970112 || trainable%: 0.0009667545204837156\n",
            "trainable params: 131072 || all params: 6778970112 || trainable%: 0.0019335090409674312\n",
            "trainable params: 196608 || all params: 6778970112 || trainable%: 0.002900263561451147\n",
            "trainable params: 262144 || all params: 6778970112 || trainable%: 0.0038670180819348624\n",
            "trainable params: 327680 || all params: 6778970112 || trainable%: 0.004833772602418578\n",
            "trainable params: 393216 || all params: 6778970112 || trainable%: 0.005800527122902294\n",
            "trainable params: 458752 || all params: 6778970112 || trainable%: 0.006767281643386009\n",
            "trainable params: 524288 || all params: 6778970112 || trainable%: 0.007734036163869725\n",
            "trainable params: 589824 || all params: 6778970112 || trainable%: 0.00870079068435344\n",
            "trainable params: 765952 || all params: 6778970112 || trainable%: 0.011298943458153427\n",
            "trainable params: 831488 || all params: 6778970112 || trainable%: 0.012265697978637143\n",
            "trainable params: 1007616 || all params: 6778970112 || trainable%: 0.014863850752437128\n",
            "trainable params: 1183744 || all params: 6778970112 || trainable%: 0.017462003526237113\n",
            "trainable params: 1249280 || all params: 6778970112 || trainable%: 0.01842875804672083\n",
            "trainable params: 1314816 || all params: 6778970112 || trainable%: 0.019395512567204544\n",
            "trainable params: 1380352 || all params: 6778970112 || trainable%: 0.02036226708768826\n",
            "trainable params: 1445888 || all params: 6778970112 || trainable%: 0.021329021608171975\n",
            "trainable params: 1511424 || all params: 6778970112 || trainable%: 0.02229577612865569\n",
            "trainable params: 1576960 || all params: 6778970112 || trainable%: 0.02326253064913941\n",
            "trainable params: 1642496 || all params: 6778970112 || trainable%: 0.024229285169623124\n",
            "trainable params: 1708032 || all params: 6778970112 || trainable%: 0.02519603969010684\n",
            "trainable params: 1773568 || all params: 6778970112 || trainable%: 0.026162794210590555\n",
            "trainable params: 1839104 || all params: 6778970112 || trainable%: 0.02712954873107427\n",
            "trainable params: 2015232 || all params: 6778970112 || trainable%: 0.029727701504874256\n",
            "trainable params: 2080768 || all params: 6778970112 || trainable%: 0.03069445602535797\n",
            "trainable params: 2256896 || all params: 6778970112 || trainable%: 0.033292608799157956\n",
            "trainable params: 2433024 || all params: 6778970112 || trainable%: 0.035890761572957945\n",
            "trainable params: 2498560 || all params: 6778970112 || trainable%: 0.03685751609344166\n",
            "trainable params: 2564096 || all params: 6778970112 || trainable%: 0.037824270613925376\n",
            "trainable params: 2629632 || all params: 6778970112 || trainable%: 0.03879102513440909\n",
            "trainable params: 2695168 || all params: 6778970112 || trainable%: 0.03975777965489281\n",
            "trainable params: 2760704 || all params: 6778970112 || trainable%: 0.04072453417537652\n",
            "trainable params: 2826240 || all params: 6778970112 || trainable%: 0.04169128869586024\n",
            "trainable params: 2891776 || all params: 6778970112 || trainable%: 0.04265804321634395\n",
            "trainable params: 2957312 || all params: 6778970112 || trainable%: 0.04362479773682767\n",
            "trainable params: 3022848 || all params: 6778970112 || trainable%: 0.04459155225731138\n",
            "trainable params: 3088384 || all params: 6778970112 || trainable%: 0.0455583067777951\n",
            "trainable params: 3264512 || all params: 6778970112 || trainable%: 0.04815645955159509\n",
            "trainable params: 3330048 || all params: 6778970112 || trainable%: 0.0491232140720788\n",
            "trainable params: 3506176 || all params: 6778970112 || trainable%: 0.05172136684587879\n",
            "trainable params: 3682304 || all params: 6778970112 || trainable%: 0.05431951961967877\n",
            "trainable params: 3747840 || all params: 6778970112 || trainable%: 0.05528627414016249\n",
            "trainable params: 3813376 || all params: 6778970112 || trainable%: 0.0562530286606462\n",
            "trainable params: 3878912 || all params: 6778970112 || trainable%: 0.05721978318112992\n",
            "trainable params: 3944448 || all params: 6778970112 || trainable%: 0.05818653770161363\n",
            "trainable params: 4009984 || all params: 6778970112 || trainable%: 0.05915329222209735\n",
            "trainable params: 4075520 || all params: 6778970112 || trainable%: 0.06012004674258106\n",
            "trainable params: 4141056 || all params: 6778970112 || trainable%: 0.06108680126306478\n",
            "trainable params: 4206592 || all params: 6778970112 || trainable%: 0.06205355578354849\n",
            "trainable params: 4272128 || all params: 6778970112 || trainable%: 0.06302031030403221\n",
            "trainable params: 4337664 || all params: 6778970112 || trainable%: 0.06398706482451592\n",
            "trainable params: 4513792 || all params: 6778970112 || trainable%: 0.06658521759831591\n",
            "trainable params: 4579328 || all params: 6778970112 || trainable%: 0.06755197211879962\n",
            "trainable params: 4755456 || all params: 6778970112 || trainable%: 0.07015012489259961\n",
            "trainable params: 4931584 || all params: 6778970112 || trainable%: 0.0727482776663996\n",
            "trainable params: 4997120 || all params: 6778970112 || trainable%: 0.07371503218688331\n",
            "trainable params: 5062656 || all params: 6778970112 || trainable%: 0.07468178670736703\n",
            "trainable params: 5128192 || all params: 6778970112 || trainable%: 0.07564854122785075\n",
            "trainable params: 5193728 || all params: 6778970112 || trainable%: 0.07661529574833446\n",
            "trainable params: 5259264 || all params: 6778970112 || trainable%: 0.07758205026881818\n",
            "trainable params: 5324800 || all params: 6778970112 || trainable%: 0.07854880478930189\n",
            "trainable params: 5390336 || all params: 6778970112 || trainable%: 0.07951555930978561\n",
            "trainable params: 5455872 || all params: 6778970112 || trainable%: 0.08048231383026933\n",
            "trainable params: 5521408 || all params: 6778970112 || trainable%: 0.08144906835075304\n",
            "trainable params: 5586944 || all params: 6778970112 || trainable%: 0.08241582287123676\n",
            "trainable params: 5763072 || all params: 6778970112 || trainable%: 0.08501397564503674\n",
            "trainable params: 5828608 || all params: 6778970112 || trainable%: 0.08598073016552046\n",
            "trainable params: 6004736 || all params: 6778970112 || trainable%: 0.08857888293932044\n",
            "trainable params: 6180864 || all params: 6778970112 || trainable%: 0.09117703571312043\n",
            "trainable params: 6246400 || all params: 6778970112 || trainable%: 0.09214379023360414\n",
            "trainable params: 6311936 || all params: 6778970112 || trainable%: 0.09311054475408787\n",
            "trainable params: 6377472 || all params: 6778970112 || trainable%: 0.09407729927457158\n",
            "trainable params: 6443008 || all params: 6778970112 || trainable%: 0.09504405379505529\n",
            "trainable params: 6508544 || all params: 6778970112 || trainable%: 0.09601080831553901\n",
            "trainable params: 6574080 || all params: 6778970112 || trainable%: 0.09697756283602273\n",
            "trainable params: 6639616 || all params: 6778970112 || trainable%: 0.09794431735650644\n",
            "trainable params: 6705152 || all params: 6778970112 || trainable%: 0.09891107187699015\n",
            "trainable params: 6770688 || all params: 6778970112 || trainable%: 0.09987782639747388\n",
            "trainable params: 6836224 || all params: 6778970112 || trainable%: 0.10084458091795759\n",
            "trainable params: 7012352 || all params: 6778970112 || trainable%: 0.10344273369175758\n",
            "trainable params: 7077888 || all params: 6778970112 || trainable%: 0.10440948821224129\n",
            "trainable params: 7254016 || all params: 6778970112 || trainable%: 0.10700764098604128\n",
            "trainable params: 7430144 || all params: 6778970112 || trainable%: 0.10960579375984125\n",
            "trainable params: 7495680 || all params: 6778970112 || trainable%: 0.11057254828032498\n",
            "trainable params: 7561216 || all params: 6778970112 || trainable%: 0.11153930280080869\n",
            "trainable params: 7626752 || all params: 6778970112 || trainable%: 0.1125060573212924\n",
            "trainable params: 7692288 || all params: 6778970112 || trainable%: 0.11347281184177613\n",
            "trainable params: 7757824 || all params: 6778970112 || trainable%: 0.11443956636225984\n",
            "trainable params: 7823360 || all params: 6778970112 || trainable%: 0.11540632088274355\n",
            "trainable params: 7888896 || all params: 6778970112 || trainable%: 0.11637307540322726\n",
            "trainable params: 7954432 || all params: 6778970112 || trainable%: 0.11733982992371099\n",
            "trainable params: 8019968 || all params: 6778970112 || trainable%: 0.1183065844441947\n",
            "trainable params: 8085504 || all params: 6778970112 || trainable%: 0.11927333896467841\n",
            "trainable params: 8261632 || all params: 6778970112 || trainable%: 0.1218714917384784\n",
            "trainable params: 8327168 || all params: 6778970112 || trainable%: 0.12283824625896211\n",
            "trainable params: 8503296 || all params: 6778970112 || trainable%: 0.1254363990327621\n",
            "trainable params: 8679424 || all params: 6778970112 || trainable%: 0.1280345518065621\n",
            "trainable params: 8744960 || all params: 6778970112 || trainable%: 0.12900130632704582\n",
            "trainable params: 8810496 || all params: 6778970112 || trainable%: 0.12996806084752952\n",
            "trainable params: 8876032 || all params: 6778970112 || trainable%: 0.13093481536801324\n",
            "trainable params: 8941568 || all params: 6778970112 || trainable%: 0.13190156988849694\n",
            "trainable params: 9007104 || all params: 6778970112 || trainable%: 0.13286832440898066\n",
            "trainable params: 9072640 || all params: 6778970112 || trainable%: 0.1338350789294644\n",
            "trainable params: 9138176 || all params: 6778970112 || trainable%: 0.1348018334499481\n",
            "trainable params: 9203712 || all params: 6778970112 || trainable%: 0.13576858797043181\n",
            "trainable params: 9269248 || all params: 6778970112 || trainable%: 0.13673534249091554\n",
            "trainable params: 9334784 || all params: 6778970112 || trainable%: 0.13770209701139924\n",
            "trainable params: 9510912 || all params: 6778970112 || trainable%: 0.14030024978519923\n",
            "trainable params: 9576448 || all params: 6778970112 || trainable%: 0.14126700430568295\n",
            "trainable params: 9752576 || all params: 6778970112 || trainable%: 0.14386515707948294\n",
            "trainable params: 9928704 || all params: 6778970112 || trainable%: 0.14646330985328293\n",
            "trainable params: 9994240 || all params: 6778970112 || trainable%: 0.14743006437376663\n",
            "trainable params: 10059776 || all params: 6778970112 || trainable%: 0.14839681889425035\n",
            "trainable params: 10125312 || all params: 6778970112 || trainable%: 0.14936357341473405\n",
            "trainable params: 10190848 || all params: 6778970112 || trainable%: 0.15033032793521778\n",
            "trainable params: 10256384 || all params: 6778970112 || trainable%: 0.1512970824557015\n",
            "trainable params: 10321920 || all params: 6778970112 || trainable%: 0.1522638369761852\n",
            "trainable params: 10387456 || all params: 6778970112 || trainable%: 0.15323059149666893\n",
            "trainable params: 10452992 || all params: 6778970112 || trainable%: 0.15419734601715265\n",
            "trainable params: 10518528 || all params: 6778970112 || trainable%: 0.15516410053763635\n",
            "trainable params: 10584064 || all params: 6778970112 || trainable%: 0.15613085505812008\n",
            "trainable params: 10760192 || all params: 6778970112 || trainable%: 0.15872900783192007\n",
            "trainable params: 10825728 || all params: 6778970112 || trainable%: 0.15969576235240376\n",
            "trainable params: 11001856 || all params: 6778970112 || trainable%: 0.16229391512620375\n",
            "trainable params: 11177984 || all params: 6778970112 || trainable%: 0.16489206790000374\n",
            "trainable params: 11243520 || all params: 6778970112 || trainable%: 0.16585882242048747\n",
            "trainable params: 11309056 || all params: 6778970112 || trainable%: 0.16682557694097117\n",
            "trainable params: 11374592 || all params: 6778970112 || trainable%: 0.1677923314614549\n",
            "trainable params: 11440128 || all params: 6778970112 || trainable%: 0.16875908598193862\n",
            "trainable params: 11505664 || all params: 6778970112 || trainable%: 0.16972584050242231\n",
            "trainable params: 11571200 || all params: 6778970112 || trainable%: 0.17069259502290604\n",
            "trainable params: 11636736 || all params: 6778970112 || trainable%: 0.17165934954338977\n",
            "trainable params: 11702272 || all params: 6778970112 || trainable%: 0.17262610406387346\n",
            "trainable params: 11767808 || all params: 6778970112 || trainable%: 0.1735928585843572\n",
            "trainable params: 11833344 || all params: 6778970112 || trainable%: 0.17455961310484092\n",
            "trainable params: 12009472 || all params: 6778970112 || trainable%: 0.17715776587864088\n",
            "trainable params: 12075008 || all params: 6778970112 || trainable%: 0.1781245203991246\n",
            "trainable params: 12251136 || all params: 6778970112 || trainable%: 0.1807226731729246\n",
            "trainable params: 12427264 || all params: 6778970112 || trainable%: 0.18332082594672458\n",
            "trainable params: 12492800 || all params: 6778970112 || trainable%: 0.18428758046720828\n",
            "trainable params: 12558336 || all params: 6778970112 || trainable%: 0.185254334987692\n",
            "trainable params: 12623872 || all params: 6778970112 || trainable%: 0.18622108950817573\n",
            "trainable params: 12689408 || all params: 6778970112 || trainable%: 0.18718784402865943\n",
            "trainable params: 12754944 || all params: 6778970112 || trainable%: 0.18815459854914315\n",
            "trainable params: 12820480 || all params: 6778970112 || trainable%: 0.18912135306962688\n",
            "trainable params: 12886016 || all params: 6778970112 || trainable%: 0.19008810759011058\n",
            "trainable params: 12951552 || all params: 6778970112 || trainable%: 0.1910548621105943\n",
            "trainable params: 13017088 || all params: 6778970112 || trainable%: 0.19202161663107803\n",
            "trainable params: 13082624 || all params: 6778970112 || trainable%: 0.19298837115156173\n",
            "trainable params: 13258752 || all params: 6778970112 || trainable%: 0.19558652392536172\n",
            "trainable params: 13324288 || all params: 6778970112 || trainable%: 0.19655327844584544\n",
            "trainable params: 13500416 || all params: 6778970112 || trainable%: 0.19915143121964543\n",
            "trainable params: 13676544 || all params: 6778970112 || trainable%: 0.2017495839934454\n",
            "trainable params: 13742080 || all params: 6778970112 || trainable%: 0.20271633851392912\n",
            "trainable params: 13807616 || all params: 6778970112 || trainable%: 0.20368309303441284\n",
            "trainable params: 13873152 || all params: 6778970112 || trainable%: 0.20464984755489654\n",
            "trainable params: 13938688 || all params: 6778970112 || trainable%: 0.20561660207538027\n",
            "trainable params: 14004224 || all params: 6778970112 || trainable%: 0.206583356595864\n",
            "trainable params: 14069760 || all params: 6778970112 || trainable%: 0.2075501111163477\n",
            "trainable params: 14135296 || all params: 6778970112 || trainable%: 0.20851686563683142\n",
            "trainable params: 14200832 || all params: 6778970112 || trainable%: 0.20948362015731514\n",
            "trainable params: 14266368 || all params: 6778970112 || trainable%: 0.21045037467779884\n",
            "trainable params: 14331904 || all params: 6778970112 || trainable%: 0.21141712919828257\n",
            "trainable params: 14508032 || all params: 6778970112 || trainable%: 0.21401528197208255\n",
            "trainable params: 14573568 || all params: 6778970112 || trainable%: 0.21498203649256625\n",
            "trainable params: 14749696 || all params: 6778970112 || trainable%: 0.21758018926636624\n",
            "trainable params: 14925824 || all params: 6778970112 || trainable%: 0.22017834204016623\n",
            "trainable params: 14991360 || all params: 6778970112 || trainable%: 0.22114509656064996\n",
            "trainable params: 15056896 || all params: 6778970112 || trainable%: 0.22211185108113365\n",
            "trainable params: 15122432 || all params: 6778970112 || trainable%: 0.22307860560161738\n",
            "trainable params: 15187968 || all params: 6778970112 || trainable%: 0.2240453601221011\n",
            "trainable params: 15253504 || all params: 6778970112 || trainable%: 0.2250121146425848\n",
            "trainable params: 15319040 || all params: 6778970112 || trainable%: 0.22597886916306853\n",
            "trainable params: 15384576 || all params: 6778970112 || trainable%: 0.22694562368355226\n",
            "trainable params: 15450112 || all params: 6778970112 || trainable%: 0.22791237820403595\n",
            "trainable params: 15515648 || all params: 6778970112 || trainable%: 0.22887913272451968\n",
            "trainable params: 15581184 || all params: 6778970112 || trainable%: 0.22984588724500338\n",
            "trainable params: 15757312 || all params: 6778970112 || trainable%: 0.23244404001880337\n",
            "trainable params: 15822848 || all params: 6778970112 || trainable%: 0.2334107945392871\n",
            "trainable params: 15998976 || all params: 6778970112 || trainable%: 0.23600894731308708\n",
            "trainable params: 16175104 || all params: 6778970112 || trainable%: 0.23860710008688707\n",
            "trainable params: 16240640 || all params: 6778970112 || trainable%: 0.23957385460737077\n",
            "trainable params: 16306176 || all params: 6778970112 || trainable%: 0.2405406091278545\n",
            "trainable params: 16371712 || all params: 6778970112 || trainable%: 0.24150736364833822\n",
            "trainable params: 16437248 || all params: 6778970112 || trainable%: 0.24247411816882192\n",
            "trainable params: 16502784 || all params: 6778970112 || trainable%: 0.24344087268930564\n",
            "trainable params: 16568320 || all params: 6778970112 || trainable%: 0.24440762720978937\n",
            "trainable params: 16633856 || all params: 6778970112 || trainable%: 0.24537438173027307\n",
            "trainable params: 16699392 || all params: 6778970112 || trainable%: 0.2463411362507568\n",
            "trainable params: 16764928 || all params: 6778970112 || trainable%: 0.2473078907712405\n",
            "trainable params: 16830464 || all params: 6778970112 || trainable%: 0.24827464529172422\n",
            "trainable params: 17006592 || all params: 6778970112 || trainable%: 0.2508727980655242\n",
            "trainable params: 17072128 || all params: 6778970112 || trainable%: 0.25183955258600793\n",
            "trainable params: 17248256 || all params: 6778970112 || trainable%: 0.2544377053598079\n",
            "trainable params: 17424384 || all params: 6778970112 || trainable%: 0.2570358581336079\n",
            "trainable params: 17489920 || all params: 6778970112 || trainable%: 0.25800261265409163\n",
            "trainable params: 17555456 || all params: 6778970112 || trainable%: 0.2589693671745753\n",
            "trainable params: 17620992 || all params: 6778970112 || trainable%: 0.25993612169505903\n",
            "trainable params: 17686528 || all params: 6778970112 || trainable%: 0.26090287621554276\n",
            "trainable params: 17752064 || all params: 6778970112 || trainable%: 0.2618696307360265\n",
            "trainable params: 17817600 || all params: 6778970112 || trainable%: 0.2628363852565102\n",
            "trainable params: 17883136 || all params: 6778970112 || trainable%: 0.2638031397769939\n",
            "trainable params: 17948672 || all params: 6778970112 || trainable%: 0.2647698942974776\n",
            "trainable params: 18014208 || all params: 6778970112 || trainable%: 0.26573664881796133\n",
            "trainable params: 18079744 || all params: 6778970112 || trainable%: 0.26670340333844506\n",
            "trainable params: 18255872 || all params: 6778970112 || trainable%: 0.26930155611224504\n",
            "trainable params: 18321408 || all params: 6778970112 || trainable%: 0.27026831063272877\n",
            "trainable params: 18497536 || all params: 6778970112 || trainable%: 0.27286646340652876\n",
            "trainable params: 18673664 || all params: 6778970112 || trainable%: 0.27546461618032875\n",
            "trainable params: 18739200 || all params: 6778970112 || trainable%: 0.2764313707008124\n",
            "trainable params: 18804736 || all params: 6778970112 || trainable%: 0.27739812522129614\n",
            "trainable params: 18870272 || all params: 6778970112 || trainable%: 0.27836487974177987\n",
            "trainable params: 18935808 || all params: 6778970112 || trainable%: 0.2793316342622636\n",
            "trainable params: 19001344 || all params: 6778970112 || trainable%: 0.2802983887827473\n",
            "trainable params: 19066880 || all params: 6778970112 || trainable%: 0.281265143303231\n",
            "trainable params: 19132416 || all params: 6778970112 || trainable%: 0.2822318978237147\n",
            "trainable params: 19197952 || all params: 6778970112 || trainable%: 0.28319865234419844\n",
            "trainable params: 19263488 || all params: 6778970112 || trainable%: 0.28416540686468217\n",
            "trainable params: 19329024 || all params: 6778970112 || trainable%: 0.2851321613851659\n",
            "trainable params: 19505152 || all params: 6778970112 || trainable%: 0.2877303141589659\n",
            "trainable params: 19570688 || all params: 6778970112 || trainable%: 0.28869706867944955\n",
            "trainable params: 19746816 || all params: 6778970112 || trainable%: 0.29129522145324954\n",
            "trainable params: 19922944 || all params: 6778970112 || trainable%: 0.29389337422704953\n",
            "trainable params: 19988480 || all params: 6778970112 || trainable%: 0.29486012874753326\n",
            "trainable params: 20054016 || all params: 6778970112 || trainable%: 0.295826883268017\n",
            "trainable params: 20119552 || all params: 6778970112 || trainable%: 0.2967936377885007\n",
            "trainable params: 20185088 || all params: 6778970112 || trainable%: 0.29776039230898443\n",
            "trainable params: 20250624 || all params: 6778970112 || trainable%: 0.2987271468294681\n",
            "trainable params: 20316160 || all params: 6778970112 || trainable%: 0.29969390134995183\n",
            "trainable params: 20381696 || all params: 6778970112 || trainable%: 0.30066065587043556\n",
            "trainable params: 20447232 || all params: 6778970112 || trainable%: 0.3016274103909193\n",
            "trainable params: 20512768 || all params: 6778970112 || trainable%: 0.302594164911403\n",
            "trainable params: 20578304 || all params: 6778970112 || trainable%: 0.30356091943188673\n",
            "trainable params: 20754432 || all params: 6778970112 || trainable%: 0.30615907220568667\n",
            "trainable params: 20819968 || all params: 6778970112 || trainable%: 0.3071258267261704\n",
            "trainable params: 20996096 || all params: 6778970112 || trainable%: 0.3097239794999704\n",
            "trainable params: 21172224 || all params: 6778970112 || trainable%: 0.31232213227377037\n",
            "trainable params: 21237760 || all params: 6778970112 || trainable%: 0.3132888867942541\n",
            "trainable params: 21303296 || all params: 6778970112 || trainable%: 0.3142556413147378\n",
            "trainable params: 21368832 || all params: 6778970112 || trainable%: 0.31522239583522155\n",
            "trainable params: 21434368 || all params: 6778970112 || trainable%: 0.3161891503557052\n",
            "trainable params: 21499904 || all params: 6778970112 || trainable%: 0.31715590487618894\n",
            "trainable params: 21565440 || all params: 6778970112 || trainable%: 0.31812265939667267\n",
            "trainable params: 21630976 || all params: 6778970112 || trainable%: 0.3190894139171564\n",
            "trainable params: 21696512 || all params: 6778970112 || trainable%: 0.3200561684376401\n",
            "trainable params: 21762048 || all params: 6778970112 || trainable%: 0.32102292295812385\n",
            "trainable params: 21827584 || all params: 6778970112 || trainable%: 0.3219896774786075\n",
            "trainable params: 22003712 || all params: 6778970112 || trainable%: 0.3245878302524075\n",
            "trainable params: 22069248 || all params: 6778970112 || trainable%: 0.32555458477289123\n",
            "trainable params: 22245376 || all params: 6778970112 || trainable%: 0.3281527375466912\n",
            "trainable params: 22421504 || all params: 6778970112 || trainable%: 0.3307508903204912\n",
            "trainable params: 22487040 || all params: 6778970112 || trainable%: 0.33171764484097493\n",
            "trainable params: 22552576 || all params: 6778970112 || trainable%: 0.33268439936145866\n",
            "trainable params: 22618112 || all params: 6778970112 || trainable%: 0.33365115388194233\n",
            "trainable params: 22683648 || all params: 6778970112 || trainable%: 0.33461790840242606\n",
            "trainable params: 22749184 || all params: 6778970112 || trainable%: 0.3355846629229098\n",
            "trainable params: 22814720 || all params: 6778970112 || trainable%: 0.3365514174433935\n",
            "trainable params: 22880256 || all params: 6778970112 || trainable%: 0.33751817196387723\n",
            "trainable params: 22945792 || all params: 6778970112 || trainable%: 0.33848492648436096\n",
            "trainable params: 23011328 || all params: 6778970112 || trainable%: 0.33945168100484463\n",
            "trainable params: 23076864 || all params: 6778970112 || trainable%: 0.34041843552532836\n",
            "trainable params: 23252992 || all params: 6778970112 || trainable%: 0.34301658829912834\n",
            "trainable params: 23318528 || all params: 6778970112 || trainable%: 0.34398334281961207\n",
            "trainable params: 23494656 || all params: 6778970112 || trainable%: 0.34658149559341206\n",
            "trainable params: 23670784 || all params: 6778970112 || trainable%: 0.34917964836721205\n",
            "trainable params: 23736320 || all params: 6778970112 || trainable%: 0.3501464028876958\n",
            "trainable params: 23801856 || all params: 6778970112 || trainable%: 0.35111315740817944\n",
            "trainable params: 23867392 || all params: 6778970112 || trainable%: 0.35207991192866317\n",
            "trainable params: 23932928 || all params: 6778970112 || trainable%: 0.3530466664491469\n",
            "trainable params: 23998464 || all params: 6778970112 || trainable%: 0.3540134209696306\n",
            "trainable params: 24064000 || all params: 6778970112 || trainable%: 0.35498017549011435\n",
            "trainable params: 24129536 || all params: 6778970112 || trainable%: 0.3559469300105981\n",
            "trainable params: 24195072 || all params: 6778970112 || trainable%: 0.35691368453108174\n",
            "trainable params: 24260608 || all params: 6778970112 || trainable%: 0.35788043905156547\n",
            "trainable params: 24326144 || all params: 6778970112 || trainable%: 0.3588471935720492\n",
            "trainable params: 24502272 || all params: 6778970112 || trainable%: 0.3614453463458492\n",
            "trainable params: 24567808 || all params: 6778970112 || trainable%: 0.3624121008663329\n",
            "trainable params: 24743936 || all params: 6778970112 || trainable%: 0.3650102536401329\n",
            "trainable params: 24920064 || all params: 6778970112 || trainable%: 0.3676084064139329\n",
            "trainable params: 24985600 || all params: 6778970112 || trainable%: 0.36857516093441656\n",
            "trainable params: 25051136 || all params: 6778970112 || trainable%: 0.3695419154549003\n",
            "trainable params: 25116672 || all params: 6778970112 || trainable%: 0.370508669975384\n",
            "trainable params: 25182208 || all params: 6778970112 || trainable%: 0.37147542449586773\n",
            "trainable params: 25247744 || all params: 6778970112 || trainable%: 0.37244217901635146\n",
            "trainable params: 25313280 || all params: 6778970112 || trainable%: 0.3734089335368352\n",
            "trainable params: 25378816 || all params: 6778970112 || trainable%: 0.37437568805731886\n",
            "trainable params: 25444352 || all params: 6778970112 || trainable%: 0.3753424425778026\n",
            "trainable params: 25509888 || all params: 6778970112 || trainable%: 0.3763091970982863\n",
            "trainable params: 25575424 || all params: 6778970112 || trainable%: 0.37727595161877003\n",
            "trainable params: 25751552 || all params: 6778970112 || trainable%: 0.37987410439257\n",
            "trainable params: 25817088 || all params: 6778970112 || trainable%: 0.38084085891305375\n",
            "trainable params: 25993216 || all params: 6778970112 || trainable%: 0.38343901168685374\n",
            "trainable params: 26169344 || all params: 6778970112 || trainable%: 0.38603716446065367\n",
            "trainable params: 26234880 || all params: 6778970112 || trainable%: 0.3870039189811374\n",
            "trainable params: 26300416 || all params: 6778970112 || trainable%: 0.3879706735016211\n",
            "trainable params: 26365952 || all params: 6778970112 || trainable%: 0.38893742802210485\n",
            "trainable params: 26431488 || all params: 6778970112 || trainable%: 0.3899041825425886\n",
            "trainable params: 26497024 || all params: 6778970112 || trainable%: 0.3908709370630723\n",
            "trainable params: 26562560 || all params: 6778970112 || trainable%: 0.39183769158355597\n",
            "trainable params: 26628096 || all params: 6778970112 || trainable%: 0.3928044461040397\n",
            "trainable params: 26693632 || all params: 6778970112 || trainable%: 0.3937712006245234\n",
            "trainable params: 26759168 || all params: 6778970112 || trainable%: 0.39473795514500715\n",
            "trainable params: 26824704 || all params: 6778970112 || trainable%: 0.3957047096654909\n",
            "trainable params: 27000832 || all params: 6778970112 || trainable%: 0.39830286243929086\n",
            "trainable params: 27066368 || all params: 6778970112 || trainable%: 0.39926961695977453\n",
            "trainable params: 27242496 || all params: 6778970112 || trainable%: 0.4018677697335745\n",
            "trainable params: 27418624 || all params: 6778970112 || trainable%: 0.4044659225073745\n",
            "trainable params: 27484160 || all params: 6778970112 || trainable%: 0.40543267702785823\n",
            "trainable params: 27549696 || all params: 6778970112 || trainable%: 0.40639943154834196\n",
            "trainable params: 27615232 || all params: 6778970112 || trainable%: 0.4073661860688257\n",
            "trainable params: 27680768 || all params: 6778970112 || trainable%: 0.4083329405893094\n",
            "trainable params: 27746304 || all params: 6778970112 || trainable%: 0.4092996951097931\n",
            "trainable params: 27811840 || all params: 6778970112 || trainable%: 0.4102664496302768\n",
            "trainable params: 27877376 || all params: 6778970112 || trainable%: 0.41123320415076053\n",
            "trainable params: 27942912 || all params: 6778970112 || trainable%: 0.41219995867124426\n",
            "trainable params: 28008448 || all params: 6778970112 || trainable%: 0.413166713191728\n",
            "trainable params: 28073984 || all params: 6778970112 || trainable%: 0.41413346771221166\n",
            "trainable params: 28250112 || all params: 6778970112 || trainable%: 0.41673162048601164\n",
            "trainable params: 28315648 || all params: 6778970112 || trainable%: 0.41769837500649537\n",
            "trainable params: 28491776 || all params: 6778970112 || trainable%: 0.42029652778029536\n",
            "trainable params: 28667904 || all params: 6778970112 || trainable%: 0.42289468055409535\n",
            "trainable params: 28733440 || all params: 6778970112 || trainable%: 0.4238614350745791\n",
            "trainable params: 28798976 || all params: 6778970112 || trainable%: 0.4248281895950628\n",
            "trainable params: 28864512 || all params: 6778970112 || trainable%: 0.4257949441155465\n",
            "trainable params: 28930048 || all params: 6778970112 || trainable%: 0.4267616986360302\n",
            "trainable params: 28995584 || all params: 6778970112 || trainable%: 0.4277284531565139\n",
            "trainable params: 29061120 || all params: 6778970112 || trainable%: 0.42869520767699765\n",
            "trainable params: 29126656 || all params: 6778970112 || trainable%: 0.4296619621974814\n",
            "trainable params: 29192192 || all params: 6778970112 || trainable%: 0.4306287167179651\n",
            "trainable params: 29257728 || all params: 6778970112 || trainable%: 0.43159547123844877\n",
            "trainable params: 29323264 || all params: 6778970112 || trainable%: 0.4325622257589325\n",
            "trainable params: 29499392 || all params: 6778970112 || trainable%: 0.4351603785327325\n",
            "trainable params: 29564928 || all params: 6778970112 || trainable%: 0.4361271330532162\n",
            "trainable params: 29741056 || all params: 6778970112 || trainable%: 0.4387252858270162\n",
            "trainable params: 29917184 || all params: 6778970112 || trainable%: 0.4413234386008162\n",
            "trainable params: 29982720 || all params: 6778970112 || trainable%: 0.4422901931212999\n",
            "trainable params: 30048256 || all params: 6778970112 || trainable%: 0.44325694764178364\n",
            "trainable params: 30113792 || all params: 6778970112 || trainable%: 0.4442237021622673\n",
            "trainable params: 30179328 || all params: 6778970112 || trainable%: 0.44519045668275103\n",
            "trainable params: 30244864 || all params: 6778970112 || trainable%: 0.44615721120323476\n",
            "trainable params: 30310400 || all params: 6778970112 || trainable%: 0.4471239657237185\n",
            "trainable params: 30375936 || all params: 6778970112 || trainable%: 0.4480907202442022\n",
            "trainable params: 30441472 || all params: 6778970112 || trainable%: 0.4490574747646859\n",
            "trainable params: 30507008 || all params: 6778970112 || trainable%: 0.4500242292851696\n",
            "trainable params: 30572544 || all params: 6778970112 || trainable%: 0.45099098380565333\n",
            "trainable params: 30748672 || all params: 6778970112 || trainable%: 0.4535891365794533\n",
            "trainable params: 30814208 || all params: 6778970112 || trainable%: 0.45455589109993705\n",
            "trainable params: 30990336 || all params: 6778970112 || trainable%: 0.45715404387373704\n",
            "trainable params: 31166464 || all params: 6778970112 || trainable%: 0.459752196647537\n",
            "trainable params: 31232000 || all params: 6778970112 || trainable%: 0.46071895116802075\n",
            "trainable params: 31297536 || all params: 6778970112 || trainable%: 0.4616857056885044\n",
            "trainable params: 31363072 || all params: 6778970112 || trainable%: 0.46265246020898815\n",
            "trainable params: 31428608 || all params: 6778970112 || trainable%: 0.4636192147294719\n",
            "trainable params: 31494144 || all params: 6778970112 || trainable%: 0.4645859692499556\n",
            "trainable params: 31559680 || all params: 6778970112 || trainable%: 0.4655527237704393\n",
            "trainable params: 31625216 || all params: 6778970112 || trainable%: 0.466519478290923\n",
            "trainable params: 31690752 || all params: 6778970112 || trainable%: 0.4674862328114067\n",
            "trainable params: 31756288 || all params: 6778970112 || trainable%: 0.46845298733189045\n",
            "trainable params: 31821824 || all params: 6778970112 || trainable%: 0.4694197418523742\n",
            "trainable params: 31997952 || all params: 6778970112 || trainable%: 0.47201789462617416\n",
            "trainable params: 32063488 || all params: 6778970112 || trainable%: 0.4729846491466579\n",
            "trainable params: 32239616 || all params: 6778970112 || trainable%: 0.4755828019204579\n",
            "trainable params: 32415744 || all params: 6778970112 || trainable%: 0.47818095469425786\n",
            "trainable params: 32481280 || all params: 6778970112 || trainable%: 0.47914770921474154\n",
            "trainable params: 32546816 || all params: 6778970112 || trainable%: 0.48011446373522526\n",
            "trainable params: 32612352 || all params: 6778970112 || trainable%: 0.481081218255709\n",
            "trainable params: 32677888 || all params: 6778970112 || trainable%: 0.4820479727761927\n",
            "trainable params: 32743424 || all params: 6778970112 || trainable%: 0.48301472729667644\n",
            "trainable params: 32808960 || all params: 6778970112 || trainable%: 0.4839814818171601\n",
            "trainable params: 32874496 || all params: 6778970112 || trainable%: 0.48494823633764383\n",
            "trainable params: 32940032 || all params: 6778970112 || trainable%: 0.48591499085812756\n",
            "trainable params: 33005568 || all params: 6778970112 || trainable%: 0.4868817453786113\n",
            "trainable params: 33071104 || all params: 6778970112 || trainable%: 0.487848499899095\n",
            "trainable params: 33247232 || all params: 6778970112 || trainable%: 0.490446652672895\n",
            "trainable params: 33312768 || all params: 6778970112 || trainable%: 0.49141340719337867\n",
            "trainable params: 33488896 || all params: 6778970112 || trainable%: 0.49401155996717866\n",
            "trainable params: 33665024 || all params: 6778970112 || trainable%: 0.49660971274097865\n",
            "trainable params: 33730560 || all params: 6778970112 || trainable%: 0.4975764672614624\n",
            "trainable params: 33796096 || all params: 6778970112 || trainable%: 0.4985432217819461\n",
            "trainable params: 33861632 || all params: 6778970112 || trainable%: 0.4995099763024298\n",
            "trainable params: 33927168 || all params: 6778970112 || trainable%: 0.5004767308229136\n",
            "trainable params: 33992704 || all params: 6778970112 || trainable%: 0.5014434853433972\n",
            "trainable params: 34058240 || all params: 6778970112 || trainable%: 0.502410239863881\n",
            "trainable params: 34123776 || all params: 6778970112 || trainable%: 0.5033769943843647\n",
            "trainable params: 34189312 || all params: 6778970112 || trainable%: 0.5043437489048483\n",
            "trainable params: 34254848 || all params: 6778970112 || trainable%: 0.5053105034253321\n",
            "trainable params: 34320384 || all params: 6778970112 || trainable%: 0.5062772579458158\n",
            "trainable params: 34496512 || all params: 6778970112 || trainable%: 0.5088754107196158\n",
            "trainable params: 34562048 || all params: 6778970112 || trainable%: 0.5098421652400995\n",
            "trainable params: 34738176 || all params: 6778970112 || trainable%: 0.5124403180138996\n",
            "trainable params: 34914304 || all params: 6778970112 || trainable%: 0.5150384707876995\n",
            "trainable params: 34979840 || all params: 6778970112 || trainable%: 0.5160052253081833\n",
            "trainable params: 35045376 || all params: 6778970112 || trainable%: 0.5169719798286669\n",
            "trainable params: 35110912 || all params: 6778970112 || trainable%: 0.5179387343491506\n",
            "trainable params: 35176448 || all params: 6778970112 || trainable%: 0.5189054888696344\n",
            "trainable params: 35241984 || all params: 6778970112 || trainable%: 0.5198722433901181\n",
            "trainable params: 35307520 || all params: 6778970112 || trainable%: 0.5208389979106018\n",
            "trainable params: 35373056 || all params: 6778970112 || trainable%: 0.5218057524310855\n",
            "trainable params: 35438592 || all params: 6778970112 || trainable%: 0.5227725069515692\n",
            "trainable params: 35504128 || all params: 6778970112 || trainable%: 0.523739261472053\n",
            "trainable params: 35569664 || all params: 6778970112 || trainable%: 0.5247060159925366\n",
            "trainable params: 35745792 || all params: 6778970112 || trainable%: 0.5273041687663367\n",
            "trainable params: 35811328 || all params: 6778970112 || trainable%: 0.5282709232868203\n",
            "trainable params: 35987456 || all params: 6778970112 || trainable%: 0.5308690760606204\n",
            "trainable params: 36163584 || all params: 6778970112 || trainable%: 0.5334672288344203\n",
            "trainable params: 36229120 || all params: 6778970112 || trainable%: 0.534433983354904\n",
            "trainable params: 36294656 || all params: 6778970112 || trainable%: 0.5354007378753878\n",
            "trainable params: 36360192 || all params: 6778970112 || trainable%: 0.5363674923958714\n",
            "trainable params: 36425728 || all params: 6778970112 || trainable%: 0.5373342469163552\n",
            "trainable params: 36491264 || all params: 6778970112 || trainable%: 0.5383010014368389\n",
            "trainable params: 36556800 || all params: 6778970112 || trainable%: 0.5392677559573226\n",
            "trainable params: 36622336 || all params: 6778970112 || trainable%: 0.5402345104778064\n",
            "trainable params: 36687872 || all params: 6778970112 || trainable%: 0.54120126499829\n",
            "trainable params: 36753408 || all params: 6778970112 || trainable%: 0.5421680195187738\n",
            "trainable params: 36818944 || all params: 6778970112 || trainable%: 0.5431347740392575\n",
            "trainable params: 36995072 || all params: 6778970112 || trainable%: 0.5457329268130575\n",
            "trainable params: 37060608 || all params: 6778970112 || trainable%: 0.5466996813335412\n",
            "trainable params: 37236736 || all params: 6778970112 || trainable%: 0.5492978341073411\n",
            "trainable params: 37412864 || all params: 6778970112 || trainable%: 0.5518959868811412\n",
            "trainable params: 37478400 || all params: 6778970112 || trainable%: 0.5528627414016248\n",
            "trainable params: 37543936 || all params: 6778970112 || trainable%: 0.5538294959221086\n",
            "trainable params: 37609472 || all params: 6778970112 || trainable%: 0.5547962504425923\n",
            "trainable params: 37675008 || all params: 6778970112 || trainable%: 0.5557630049630761\n",
            "trainable params: 37740544 || all params: 6778970112 || trainable%: 0.5567297594835597\n",
            "trainable params: 37806080 || all params: 6778970112 || trainable%: 0.5576965140040434\n",
            "trainable params: 37871616 || all params: 6778970112 || trainable%: 0.5586632685245272\n",
            "trainable params: 37937152 || all params: 6778970112 || trainable%: 0.5596300230450109\n",
            "trainable params: 38002688 || all params: 6778970112 || trainable%: 0.5605967775654946\n",
            "trainable params: 38068224 || all params: 6778970112 || trainable%: 0.5615635320859783\n",
            "trainable params: 38244352 || all params: 6778970112 || trainable%: 0.5641616848597782\n",
            "trainable params: 38309888 || all params: 6778970112 || trainable%: 0.565128439380262\n",
            "trainable params: 38486016 || all params: 6778970112 || trainable%: 0.567726592154062\n",
            "trainable params: 38662144 || all params: 6778970112 || trainable%: 0.570324744927862\n",
            "trainable params: 38727680 || all params: 6778970112 || trainable%: 0.5712914994483457\n",
            "trainable params: 38793216 || all params: 6778970112 || trainable%: 0.5722582539688295\n",
            "trainable params: 38858752 || all params: 6778970112 || trainable%: 0.5732250084893131\n",
            "trainable params: 38924288 || all params: 6778970112 || trainable%: 0.5741917630097968\n",
            "trainable params: 38989824 || all params: 6778970112 || trainable%: 0.5751585175302806\n",
            "trainable params: 39055360 || all params: 6778970112 || trainable%: 0.5761252720507642\n",
            "trainable params: 39120896 || all params: 6778970112 || trainable%: 0.577092026571248\n",
            "trainable params: 39186432 || all params: 6778970112 || trainable%: 0.5780587810917317\n",
            "trainable params: 39251968 || all params: 6778970112 || trainable%: 0.5790255356122155\n",
            "trainable params: 39317504 || all params: 6778970112 || trainable%: 0.5799922901326992\n",
            "trainable params: 39493632 || all params: 6778970112 || trainable%: 0.5825904429064991\n",
            "trainable params: 39559168 || all params: 6778970112 || trainable%: 0.5835571974269829\n",
            "trainable params: 39735296 || all params: 6778970112 || trainable%: 0.5861553502007828\n",
            "trainable params: 39911424 || all params: 6778970112 || trainable%: 0.5887535029745828\n",
            "trainable params: 39976960 || all params: 6778970112 || trainable%: 0.5897202574950665\n",
            "trainable params: 40042496 || all params: 6778970112 || trainable%: 0.5906870120155503\n",
            "trainable params: 40554496 || all params: 6778970112 || trainable%: 0.5982397817068292\n"
          ]
        }
      ],
      "source": [
        "def thisismyfunction(model):\n",
        "  trainable_params = 0\n",
        "  all_param = model.num_parameters()\n",
        "  for _, param in model.named_parameters():\n",
        "    if param.requires_grad:\n",
        "      trainable_params += param.numel()\n",
        "\n",
        "      print(f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\")\n",
        "thisismyfunction(model)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "LWKyACbXOIIR",
        "outputId": "23385a8b-118a-4e8d-8a8d-e32f9d188000"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in torch.float16.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='750' max='375' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [375/375 11:20]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Currently training with a batch size of: 4\n",
            "***** Running training *****\n",
            "  Num examples = 3,500\n",
            "  Num Epochs = 8\n",
            "  Instantaneous batch size per device = 4\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 4\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 7,000\n",
            "  Number of trainable parameters = 40,554,496\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='6126' max='7000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [6126/7000 1:10:54 < 10:07, 1.44 it/s, Epoch 7/8]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.830200</td>\n",
              "      <td>1.825582</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.614800</td>\n",
              "      <td>1.873953</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.243500</td>\n",
              "      <td>2.081712</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.025600</td>\n",
              "      <td>2.314584</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.857700</td>\n",
              "      <td>2.510728</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.763700</td>\n",
              "      <td>2.722790</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "    <div>\n",
              "      \n",
              "      <progress value='72' max='375' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 72/375 00:12 < 00:53, 5.66 it/s]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/tmp-checkpoint-500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-500/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-1000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-1000/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-1500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-1500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-1500/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-2000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-2000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-2000/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-2500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-2500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-2500/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-3000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-3000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-3000/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-3500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-3500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-3500/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-4000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-4000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-4000/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-4500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-4500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-4500/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-5000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-5000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-5000/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-5500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-5500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-5500/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-6000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-6000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-6000/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='7000' max='7000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [7000/7000 1:22:23, Epoch 8/8]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.830200</td>\n",
              "      <td>1.825582</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.614800</td>\n",
              "      <td>1.873953</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.243500</td>\n",
              "      <td>2.081712</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.025600</td>\n",
              "      <td>2.314584</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.857700</td>\n",
              "      <td>2.510728</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.763700</td>\n",
              "      <td>2.722790</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.770500</td>\n",
              "      <td>2.693078</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.732700</td>\n",
              "      <td>2.761926</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./results/tmp-checkpoint-6500\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-6500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-6500/special_tokens_map.json\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "Saving model checkpoint to ./results/tmp-checkpoint-7000\n",
            "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:139: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n",
            "  warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n",
            "tokenizer config file saved in ./results/tmp-checkpoint-7000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/tmp-checkpoint-7000/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1500\n",
            "  Batch size = 4\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-f2bfc55e0870>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdrive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/gdrive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36mmount\u001b[0;34m(mountpoint, force_remount, timeout_ms, readonly)\u001b[0m\n\u001b[1;32m     98\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmountpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforce_remount\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m120000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreadonly\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m   \u001b[0;34m\"\"\"Mount your Google Drive at the specified mountpoint path.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m   return _mount(\n\u001b[0m\u001b[1;32m    101\u001b[0m       \u001b[0mmountpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m       \u001b[0mforce_remount\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mforce_remount\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36m_mount\u001b[0;34m(mountpoint, force_remount, timeout_ms, ephemeral, readonly)\u001b[0m\n\u001b[1;32m    131\u001b[0m   )\n\u001b[1;32m    132\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mephemeral\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 133\u001b[0;31m     _message.blocking_request(\n\u001b[0m\u001b[1;32m    134\u001b[0m         \u001b[0;34m'request_auth'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'authType'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'dfs_ephemeral'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_sec\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    135\u001b[0m     )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mblocking_request\u001b[0;34m(request_type, request, timeout_sec, parent)\u001b[0m\n\u001b[1;32m    174\u001b[0m       \u001b[0mrequest_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexpect_reply\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m   )\n\u001b[0;32m--> 176\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mread_reply_from_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_sec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mread_reply_from_input\u001b[0;34m(message_id, timeout_sec)\u001b[0m\n\u001b[1;32m     94\u001b[0m     \u001b[0mreply\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_read_next_input_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreply\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_NOT_READY\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreply\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m       \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.025\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m       \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     if (\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "trainer.evaluate()\n",
        "# Launch the training\n",
        "trainer.train()\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_model = 'LeoLMfinetuning_Tagesschau_update'\n",
        "newmodel = trainer.model.save_pretrained(new_model)\n",
        "base_model = AutoModelForCausalLM.from_pretrained(model_name)\n",
        "peft_model = PeftModel.from_pretrained(base_model, new_model)\n",
        "merged_model = peft_model.merge_and_unload()\n",
        "output_merged_dir = \"/content/results\""
      ],
      "metadata": {
        "id": "bOyFMFyAyNvO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "outputId": "9cd6c852-eb80-4b8d-82ce-be84a6cbd60a"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'trainer' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-48d20764ff9f>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mnew_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'LeoLMfinetuning_Tagesschau_update'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mnewmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mbase_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoModelForCausalLM\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mpeft_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPeftModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mmerged_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpeft_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge_and_unload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'trainer' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xcDwpHJ5OXPI"
      },
      "outputs": [],
      "source": [
        "#trainer.evaluate()\n",
        "eval_prompt = \"\"\"Instruction: Vefasse eine Überschrift für den folgenden Text: Das Geschäft ist besiegelt: Der Heizungsbauer Viessmann verkauft seine Klimatechniksparte für zwölf Milliarden Euro an den US-Konzern Carrier Global. Wirtschaftsminister Habeck will die Übernahme prüfen. Der hessische Heizungsbauer Viessmann verkauft seine Klimasparte einschließlich der lukrativen Wärmepumpen an den US-Konkurrenten Carrier Global. Dieser bezifferte den Preis auf zwölf Milliarden Euro. Die verbleibende Viessmann-Gruppe erhält 80 Prozent des Kaufpreises in bar, die restlichen 20 Prozent als Aktienpaket. Dadurch wird die Viessmann-Gruppe einer der größten Anteilseigner des US-Konzerns. Das Geschäft soll bis zum Ende des Jahres abgeschlossen sein. Der Kaufpreis entspreche dem 13-fachen des für 2023 erwarteten operativen Ergebnisses (Ebitda), teilte Carrier in der Nacht auf Mittwoch mit. Langfristige Garantien für Mitarbeiter Beide Seiten hätten sich auf langfristige Garantien geeinigt, teilte Viessmann mit. So seien betriebsbedingte Kündigungen für drei Jahre ausgeschlossen, wichtige Standorte für fünf Jahre gesichert und Allendorf an der Eder für zehn Jahre als Hauptsitz gesetzt. An die Mitarbeiter der Sparte sollen 106 Millionen Euro als Sonderprämie \"für 106 Erfolgsjahre\" ausgeschüttet werden. Carrier setzt auf Siegeszug der Wärmepumpe Mit dem Verkauf entstehe ein \"zukunftssicherer globaler Klima-Champion\", erklärte Konzernchef Max Viessmann, der in den Verwaltungsrat von Carrier einzieht. \"Wir können die weltweite Energiewende nur dann erfolgreich meistern, wenn Unternehmen global denken, handeln und zusammenarbeiten.\" Carrier-Chef David Gittin bezeichnete die Akquisition als \"spielverändernde Gelegenheit\". Die Viessmann-Klimasparte mit 11.000 Beschäftigten sei entscheidend für die europäische Energiewende. Carrier setzt mit der Übernahme vor allem auf den Siegeszug der Wärmepumpe: Der Markt in Europa werde sich bis 2027 auf 15 Milliarden Euro verdreifachen. Guter Marktzugang über Installateure Dabei will das US-Unternehmen künftig auch vom Marktzugang über 75.000 Installateure in 25 Ländern profitieren, die Viessmann-Produkte in die Haushalte bringen könnten. Das ist ein großer Vorteil gegenüber den asiatischen Anbietern, die in der Massenproduktion von Klimaanlagen führend sind, welche mit Wärmepumpen in weiten Teilen bauähnlich sind. Bekannte asiatische Anbieter sind Daikin, Mitsubishi (beide Japan), Midea (China) oder Samsung (Korea). Doch etwa in Deutschland fehlt ihnen bislang noch der Marktzugang über die Installateure. Zwei Unternehmen mit langer Tradition Viessmann ist neben Bosch (Buderus) und Vaillant einer der größten Heizungshersteller in Deutschland. Der Geschäftsbereich Klimalösungen steht für 85 Prozent der Umsätze, die 2022 auf den Rekordwert von rund vier Milliarden Euro angestiegen waren. Das 1917 aus einer Schlosserei gegründete Unternehmen gehört zu den bekanntesten deutschen Heizungsbauern und zählte bislang zu den Gewinnern der Klimawende insbesondere im Gebäudebereich. Das Unternehmen Carrier aus dem US-Staat Florida gilt als Erfinder der modernen Klimaanlage und wurde 1902 gegründet. Der Konzern beschäftigt 52.000 Menschen und erlöste im vergangenen Jahr 20,4 Milliarden Dollar. 60 Prozent des Umsatzes entfielen auf Nord- und Südamerika. Deal nicht unumstritten Das Geschäft zwischen Viessmann und Carrier wird von Politikern und Ökonomen hierzulande nicht nur positiv gesehen. Einige kritische Stimmen warnen, dass Deutschland nach dem Niedergang der Solarenergiebranche nun die nächste Zukunftstechnologie zu verlieren drohe. Bundeswirtschaftsminister Robert Habeck will den milliardenschweren Verkauf unter die Lupe nehmen. \"Wir werden uns das Vorhaben im Rahmen der vorgesehenen Prüfschritte anschauen und sind im Gespräch mit dem Verkäufer und dem Investor, damit das Projekt unserer Wirtschaft und dem Standort Deutschland dient\", erklärte der Grünen-Politiker. Wichtig sei, \"dass die Vorteile unserer Energiepolitik und Gewinne, die damit erwirtschaftet werden, auch weiter dem Standort Deutschland zugutekommen\". Darauf werde die Regierung achten.\"\"\"\n",
        "\n",
        "# import random\n",
        "model_input = tokenizer(eval_prompt, return_tensors=\"pt\").to(\"cuda\")\n",
        "\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "    print(tokenizer.decode(model.generate(**model_input, max_new_tokens=256, pad_token_id=2)[0], skip_special_tokens=True))\n",
        "model.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vVMIol9dObdG"
      },
      "outputs": [],
      "source": [
        "from tensorboard import notebook\n",
        "log_dir = \"results/runs\"\n",
        "notebook.start(\"--logdir {} --port 4000\".format(log_dir))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4VDPEmc0OqSz"
      },
      "outputs": [],
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\"\n",
        "!pip install huggingface_hub\n",
        "\n",
        "from huggingface_hub import notebook_login\n",
        "notebook_login()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "GH_RTCQM5hoU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "d5426ced6adf451dab2ab39ea0691167",
            "19c8f95f381746d1815315d2c523ec00",
            "1278d3e6f855405f9971629e690bfaf7",
            "8af33168a35540efb64ea1e9e729d5c5",
            "501a88b92aa34b9ba9d3e91edacf0f34",
            "b22cbec8c4834483a6011bec9293a9f0",
            "a9018df68d6344b4bbe6fdf3a44f250c",
            "7c28c28dcae1497285569d5734ee1662",
            "ed5d62fdfa7b4f28924f66ecbdb7e40c",
            "a0f9ec67dddf48118a55a487cdef9914",
            "9dd9d89df803446881cc052961aef9bd"
          ]
        },
        "outputId": "03c9013c-83f3-4263-87a8-6a1533fd382b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--LeoLM--leo-hessianai-7b/snapshots/88c5ac07006ea8f1b5d10aa4f03f0d624dd27e56/config.json\n",
            "Model config LlamaConfig {\n",
            "  \"_name_or_path\": \"LeoLM/leo-hessianai-7b\",\n",
            "  \"architectures\": [\n",
            "    \"LlamaForCausalLM\"\n",
            "  ],\n",
            "  \"attention_bias\": false,\n",
            "  \"attention_dropout\": 0.0,\n",
            "  \"auto_map\": {\n",
            "    \"AutoModelForCausalLM\": \"LeoLM/leo-hessianai-7b--modeling_flash_llama.LlamaForCausalLM\"\n",
            "  },\n",
            "  \"bos_token_id\": 1,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"silu\",\n",
            "  \"hidden_size\": 4096,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 11008,\n",
            "  \"max_position_embeddings\": 8192,\n",
            "  \"model_type\": \"llama\",\n",
            "  \"num_attention_heads\": 32,\n",
            "  \"num_hidden_layers\": 32,\n",
            "  \"num_key_value_heads\": 32,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"pretraining_tp\": 1,\n",
            "  \"rms_norm_eps\": 1e-05,\n",
            "  \"rope_scaling\": {\n",
            "    \"factor\": 2.0,\n",
            "    \"type\": \"linear\"\n",
            "  },\n",
            "  \"rope_theta\": 10000.0,\n",
            "  \"tie_word_embeddings\": false,\n",
            "  \"torch_dtype\": \"float16\",\n",
            "  \"transformers_version\": \"4.38.2\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--LeoLM--leo-hessianai-7b/snapshots/88c5ac07006ea8f1b5d10aa4f03f0d624dd27e56/pytorch_model.bin.index.json\n",
            "Generate config GenerationConfig {\n",
            "  \"bos_token_id\": 1,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"pad_token_id\": 0\n",
            "}\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d5426ced6adf451dab2ab39ea0691167"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint weights were used when initializing LlamaForCausalLM.\n",
            "\n",
            "All the weights of LlamaForCausalLM were initialized from the model checkpoint at LeoLM/leo-hessianai-7b.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.\n",
            "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--LeoLM--leo-hessianai-7b/snapshots/88c5ac07006ea8f1b5d10aa4f03f0d624dd27e56/generation_config.json\n",
            "Generate config GenerationConfig {\n",
            "  \"bos_token_id\": 1,\n",
            "  \"do_sample\": true,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"max_length\": 4096,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"temperature\": 0.6,\n",
            "  \"top_p\": 0.9\n",
            "}\n",
            "\n",
            "Could not locate the tokenizer configuration file, will try to use the model config instead.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "OSError",
          "evalue": "/content/LeoLMfinetuning_Tagesschau_update does not appear to have a file named config.json. Checkout 'https://huggingface.co//content/LeoLMfinetuning_Tagesschau_update/None' for available files.",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-c4fced432df4>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoModelForCausalLM\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/LeoLMfinetuning_Tagesschau_update'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/LeoLMfinetuning_Tagesschau_update'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mconfig_tokenizer_class\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    781\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPretrainedConfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 782\u001b[0;31m                 config = AutoConfig.from_pretrained(\n\u001b[0m\u001b[1;32m    783\u001b[0m                     \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrust_remote_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrust_remote_code\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    784\u001b[0m                 )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/auto/configuration_auto.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m   1109\u001b[0m         \u001b[0mcode_revision\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"code_revision\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1110\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1111\u001b[0;31m         \u001b[0mconfig_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munused_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPretrainedConfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_config_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1112\u001b[0m         \u001b[0mhas_remote_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"auto_map\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"AutoConfig\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"auto_map\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1113\u001b[0m         \u001b[0mhas_local_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"model_type\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"model_type\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mCONFIG_MAPPING\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/configuration_utils.py\u001b[0m in \u001b[0;36mget_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    631\u001b[0m         \u001b[0moriginal_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m         \u001b[0;31m# Get config dict associated with the base config file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m         \u001b[0mconfig_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_config_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    634\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m\"_commit_hash\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m             \u001b[0moriginal_kwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_commit_hash\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_commit_hash\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/configuration_utils.py\u001b[0m in \u001b[0;36m_get_config_dict\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    686\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m                 \u001b[0;31m# Load from local folder or from cache or download from model Hub and cache\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m                 resolved_config_file = cached_file(\n\u001b[0m\u001b[1;32m    689\u001b[0m                     \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m                     \u001b[0mconfiguration_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/utils/hub.py\u001b[0m in \u001b[0;36mcached_file\u001b[0;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[1;32m    367\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresolved_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0m_raise_exceptions_for_missing_entries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 369\u001b[0;31m                 raise EnvironmentError(\n\u001b[0m\u001b[1;32m    370\u001b[0m                     \u001b[0;34mf\"{path_or_repo_id} does not appear to have a file named {full_filename}. Checkout \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m                     \u001b[0;34mf\"'https://huggingface.co/{path_or_repo_id}/{revision}' for available files.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mOSError\u001b[0m: /content/LeoLMfinetuning_Tagesschau_update does not appear to have a file named config.json. Checkout 'https://huggingface.co//content/LeoLMfinetuning_Tagesschau_update/None' for available files."
          ]
        }
      ],
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "model = AutoModelForCausalLM.from_pretrained('/content/results')\n",
        "tokenizer = AutoTokenizer.from_pretrained('/content/results')\n",
        "print(tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QqXcov7b53KW"
      },
      "outputs": [],
      "source": [
        "directory_path = '/content/LEO_german_finetuned_Tagesschau'\n",
        "files = os.listdir(directory_path)\n",
        "print(\"Files in the directory:\", files)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "newmodel.push_to_hub(\"Kamilatr/Ueberschriftengenerator_LEOLM_update\")"
      ],
      "metadata": {
        "id": "MjiURIRR0TSU"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d9f30fa8f67b4ab78f20587e4626f8ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e7fc6c56c2054a59aa4a369c85f61eb9",
              "IPY_MODEL_1d6bdd62cbe446849c7b0553ac0c9c5f",
              "IPY_MODEL_1696f3855a4a4ed994f3596c91b0a8a7",
              "IPY_MODEL_713229f98c914f959cfa39940bc59233"
            ],
            "layout": "IPY_MODEL_e10b340834924409948711d2ec278813"
          }
        },
        "4ad9eac3e167489da4ff2c0e57f0b457": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e1c2d6a39eb64e5ba5799f521c54f443",
            "placeholder": "​",
            "style": "IPY_MODEL_17c6498a00dc434ead6d14bdb184416e",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "11b14d74382d42d2ba8cfe88b77bc28f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_158416b02cfd4b5b84d0d2f6e2e8d5db",
            "placeholder": "​",
            "style": "IPY_MODEL_c17024c17b3147ffac635bf8161a28e0",
            "value": ""
          }
        },
        "0e817d87d46c484685f05061a3dbfa24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_97413e1b44c84caeb633da68e5a95ade",
            "style": "IPY_MODEL_2d4310377dd94445bc87e1b82b2b0398",
            "value": true
          }
        },
        "37b06f4062b14ed9bbb04a4a1ecdbdcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_6aaa36a8ebd2439c967b8980eb184ef2",
            "style": "IPY_MODEL_8f63f992c9da45c19da0d2d3bc3b67b3",
            "tooltip": ""
          }
        },
        "adad372002b04934bda1a64fc6dda875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28891ba00ad648d59a3c43042dedb627",
            "placeholder": "​",
            "style": "IPY_MODEL_60775b6da912436fb8a5cf992608d8a0",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "e10b340834924409948711d2ec278813": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "e1c2d6a39eb64e5ba5799f521c54f443": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "17c6498a00dc434ead6d14bdb184416e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "158416b02cfd4b5b84d0d2f6e2e8d5db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c17024c17b3147ffac635bf8161a28e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "97413e1b44c84caeb633da68e5a95ade": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d4310377dd94445bc87e1b82b2b0398": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6aaa36a8ebd2439c967b8980eb184ef2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f63f992c9da45c19da0d2d3bc3b67b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "28891ba00ad648d59a3c43042dedb627": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60775b6da912436fb8a5cf992608d8a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "871cf8c5e1ed47ae807ca9fed089ebd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7044939043bd40898fe92622ac558659",
            "placeholder": "​",
            "style": "IPY_MODEL_5bbc878b5fbf481c9f8a91ef40674363",
            "value": "Connecting..."
          }
        },
        "7044939043bd40898fe92622ac558659": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5bbc878b5fbf481c9f8a91ef40674363": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e7fc6c56c2054a59aa4a369c85f61eb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_781c7ad928914747a0c54f0f734f1ecd",
            "placeholder": "​",
            "style": "IPY_MODEL_3bd9db77951d42828cf67253ac34bed2",
            "value": "Token is valid (permission: write)."
          }
        },
        "1d6bdd62cbe446849c7b0553ac0c9c5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a64a4778f55142a899e8cef43841c7d6",
            "placeholder": "​",
            "style": "IPY_MODEL_854203b0f26142f9870d279110303314",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "1696f3855a4a4ed994f3596c91b0a8a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aa22d4305210481ab7ebc2e573aa978d",
            "placeholder": "​",
            "style": "IPY_MODEL_88b72faaea6c46caadec1e8d3714a526",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "713229f98c914f959cfa39940bc59233": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d8ba3b49dd8c4de3a8454e877d9c778f",
            "placeholder": "​",
            "style": "IPY_MODEL_e312e32fcd9444a88811985531054bf6",
            "value": "Login successful"
          }
        },
        "781c7ad928914747a0c54f0f734f1ecd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bd9db77951d42828cf67253ac34bed2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a64a4778f55142a899e8cef43841c7d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "854203b0f26142f9870d279110303314": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aa22d4305210481ab7ebc2e573aa978d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88b72faaea6c46caadec1e8d3714a526": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d8ba3b49dd8c4de3a8454e877d9c778f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e312e32fcd9444a88811985531054bf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "deda508ed60f4cc4adea3e708463d4d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a26f0e6b96554ba188cfb3b40571f5d0",
              "IPY_MODEL_b03c73baf1804cddb5a1521eaede8afc",
              "IPY_MODEL_5e2b8ed174154e0c9b9eacb5d1237ee6"
            ],
            "layout": "IPY_MODEL_7826e8edeeb84004bb5c73046258b60f"
          }
        },
        "a26f0e6b96554ba188cfb3b40571f5d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c237ddf3d7a42dbb226af89c9df249f",
            "placeholder": "​",
            "style": "IPY_MODEL_a39a5f75b09d47bf9bc09e11d4017aae",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "b03c73baf1804cddb5a1521eaede8afc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_995c934c7f6343d093591a0ed2a43c39",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_23fe791f17f940d191bf6c9d521dfe79",
            "value": 2
          }
        },
        "5e2b8ed174154e0c9b9eacb5d1237ee6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35d18f816cad49419357db974aeda025",
            "placeholder": "​",
            "style": "IPY_MODEL_7c254fd28d4b450eaad141b69afb5dfc",
            "value": " 2/2 [00:04&lt;00:00,  2.21s/it]"
          }
        },
        "7826e8edeeb84004bb5c73046258b60f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c237ddf3d7a42dbb226af89c9df249f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a39a5f75b09d47bf9bc09e11d4017aae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "995c934c7f6343d093591a0ed2a43c39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23fe791f17f940d191bf6c9d521dfe79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "35d18f816cad49419357db974aeda025": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c254fd28d4b450eaad141b69afb5dfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c02753dc72d04effba8d15572f815b9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_53e7e18ee3c548afbe4496e12b228ec2",
              "IPY_MODEL_dbdfcab95dcb43f1b8e34956a18dc835",
              "IPY_MODEL_ae641481d6db4a0097b50970dc91d9f6"
            ],
            "layout": "IPY_MODEL_92da53593e1740508c9da3ad6eeb20dd"
          }
        },
        "53e7e18ee3c548afbe4496e12b228ec2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8238beba60c241d49578f445e7873643",
            "placeholder": "​",
            "style": "IPY_MODEL_cdbbf398838849b88c5aecd6063fc507",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "dbdfcab95dcb43f1b8e34956a18dc835": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_53fdacad53eb46b3b2090c3edf5e1f5d",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_955cb05693f248559bd77a174f955eec",
            "value": 2
          }
        },
        "ae641481d6db4a0097b50970dc91d9f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce7e59b701f840c1aef27584106003c6",
            "placeholder": "​",
            "style": "IPY_MODEL_d9f5e91dc1a54d19a7c811b2c1146493",
            "value": " 2/2 [00:04&lt;00:00,  1.94s/it]"
          }
        },
        "92da53593e1740508c9da3ad6eeb20dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8238beba60c241d49578f445e7873643": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdbbf398838849b88c5aecd6063fc507": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53fdacad53eb46b3b2090c3edf5e1f5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "955cb05693f248559bd77a174f955eec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ce7e59b701f840c1aef27584106003c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9f5e91dc1a54d19a7c811b2c1146493": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "da9e01d6269b4f42b5c10e5c26994e0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7db53e078e9f46fc9abb6c2a0df82b9c",
              "IPY_MODEL_a0572d9d0eb6442cb2dfd67653d3d714",
              "IPY_MODEL_70e482519de44bccaa38130bb51eced4"
            ],
            "layout": "IPY_MODEL_63c32a32254245ad97df634b8f2652ae"
          }
        },
        "7db53e078e9f46fc9abb6c2a0df82b9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca737f017fc84537ada3f42f4787ffa7",
            "placeholder": "​",
            "style": "IPY_MODEL_9892054ab69c400b900043710e1977c4",
            "value": "Map: 100%"
          }
        },
        "a0572d9d0eb6442cb2dfd67653d3d714": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df30f36d054d4c6b87a5beb310208582",
            "max": 3500,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_071e473561ea488b8b598c440c494679",
            "value": 3500
          }
        },
        "70e482519de44bccaa38130bb51eced4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ae6c9f8b28a4b8b9241e0b923b5e96c",
            "placeholder": "​",
            "style": "IPY_MODEL_01c7778f9eb844aaa4425038b2731d93",
            "value": " 3500/3500 [00:00&lt;00:00, 38517.11 examples/s]"
          }
        },
        "63c32a32254245ad97df634b8f2652ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca737f017fc84537ada3f42f4787ffa7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9892054ab69c400b900043710e1977c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "df30f36d054d4c6b87a5beb310208582": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "071e473561ea488b8b598c440c494679": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4ae6c9f8b28a4b8b9241e0b923b5e96c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01c7778f9eb844aaa4425038b2731d93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "951329eaa49f4857836c8cf4b415251a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8cdbdcc37bce4bb7ac0abd36d1ab03ca",
              "IPY_MODEL_ff28ec8ffef94522bc490410ef2d7197",
              "IPY_MODEL_27e4437e4dbb4bc684e8db1ed81d07c8"
            ],
            "layout": "IPY_MODEL_5a6c990e88134693a5ab17e3384b8cee"
          }
        },
        "8cdbdcc37bce4bb7ac0abd36d1ab03ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a1e30cfc7674f77abd43d2ff476b95b",
            "placeholder": "​",
            "style": "IPY_MODEL_cbdb444e007b4316be28e8492df45390",
            "value": "Map: 100%"
          }
        },
        "ff28ec8ffef94522bc490410ef2d7197": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8949d8da4d46446698bf120a358d0c23",
            "max": 1500,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ecee8f650a834e5288a5b7faed2828d1",
            "value": 1500
          }
        },
        "27e4437e4dbb4bc684e8db1ed81d07c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcdb11826cf64d1699b7e59aa88e7bd6",
            "placeholder": "​",
            "style": "IPY_MODEL_138f973cb9244d11af830631b1d0fd20",
            "value": " 1500/1500 [00:00&lt;00:00, 31486.88 examples/s]"
          }
        },
        "5a6c990e88134693a5ab17e3384b8cee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a1e30cfc7674f77abd43d2ff476b95b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbdb444e007b4316be28e8492df45390": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8949d8da4d46446698bf120a358d0c23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ecee8f650a834e5288a5b7faed2828d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fcdb11826cf64d1699b7e59aa88e7bd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "138f973cb9244d11af830631b1d0fd20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5426ced6adf451dab2ab39ea0691167": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_19c8f95f381746d1815315d2c523ec00",
              "IPY_MODEL_1278d3e6f855405f9971629e690bfaf7",
              "IPY_MODEL_8af33168a35540efb64ea1e9e729d5c5"
            ],
            "layout": "IPY_MODEL_501a88b92aa34b9ba9d3e91edacf0f34"
          }
        },
        "19c8f95f381746d1815315d2c523ec00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b22cbec8c4834483a6011bec9293a9f0",
            "placeholder": "​",
            "style": "IPY_MODEL_a9018df68d6344b4bbe6fdf3a44f250c",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "1278d3e6f855405f9971629e690bfaf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c28c28dcae1497285569d5734ee1662",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ed5d62fdfa7b4f28924f66ecbdb7e40c",
            "value": 2
          }
        },
        "8af33168a35540efb64ea1e9e729d5c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a0f9ec67dddf48118a55a487cdef9914",
            "placeholder": "​",
            "style": "IPY_MODEL_9dd9d89df803446881cc052961aef9bd",
            "value": " 2/2 [00:15&lt;00:00,  7.60s/it]"
          }
        },
        "501a88b92aa34b9ba9d3e91edacf0f34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b22cbec8c4834483a6011bec9293a9f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9018df68d6344b4bbe6fdf3a44f250c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c28c28dcae1497285569d5734ee1662": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed5d62fdfa7b4f28924f66ecbdb7e40c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a0f9ec67dddf48118a55a487cdef9914": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9dd9d89df803446881cc052961aef9bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}