{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UQF7nAH1syz4"
   },
   "source": [
    "Based on [alpaca lora](https://github.com/tloen/alpaca-lora/blob/main/finetune.py)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T11:49:03.267959Z",
     "iopub.status.busy": "2023-07-18T11:49:03.267686Z",
     "iopub.status.idle": "2023-07-18T11:51:38.082879Z",
     "shell.execute_reply": "2023-07-18T11:51:38.082079Z",
     "shell.execute_reply.started": "2023-07-18T11:49:03.267936Z"
    }
   },
   "outputs": [],
   "source": [
    "# !apt update\n",
    "# !apt upgrade -y cuda-nvcc-12-0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T11:51:38.084514Z",
     "iopub.status.busy": "2023-07-18T11:51:38.084291Z",
     "iopub.status.idle": "2023-07-18T11:51:40.683284Z",
     "shell.execute_reply": "2023-07-18T11:51:40.682579Z",
     "shell.execute_reply.started": "2023-07-18T11:51:38.084488Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "print(\"Torch Version: \" + torch.__version__ + \"\\n\")\n",
    "\n",
    "# !nvcc --version\n",
    "\n",
    "# !nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2023-07-18T11:51:40.684754Z",
     "iopub.status.busy": "2023-07-18T11:51:40.684408Z",
     "iopub.status.idle": "2023-07-18T11:55:27.915935Z",
     "shell.execute_reply": "2023-07-18T11:55:27.915414Z",
     "shell.execute_reply.started": "2023-07-18T11:51:40.684734Z"
    },
    "id": "RXurA0q5jtaf",
    "outputId": "93942094-5399-4f21-9660-fbfd344598ee"
   },
   "outputs": [],
   "source": [
    "# !pip install -U cuda-python\n",
    "# !pip3 install -U torch torchvision torchaudio #--index-url https://download.pytorch.org/whl/cu118\n",
    "\n",
    "# # Paperspace\n",
    "# !git clone https://github.com/timdettmers/bitsandbytes.git\n",
    "# !cd bitsandbytes && CUDA_VERSION=116 make cuda11x && python setup.py install\n",
    "# !cp /notebooks/bitsandbytes/bitsandbytes/libbitsandbytes_cuda116.so /usr/lib/python3.9/\n",
    "# !pip install -U bitsandbytes\n",
    "\n",
    "# # Google Colab\n",
    "# #!pip install -U git+https://github.com/TimDettmers/bitsandbytes\n",
    "\n",
    "# !pip install -U git+https://github.com/huggingface/transformers.git\n",
    "# !pip install -U git+https://github.com/huggingface/peft.git\n",
    "# !pip install -U datasets accelerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T01:41:01.400102Z",
     "iopub.status.busy": "2023-07-18T01:41:01.399691Z",
     "iopub.status.idle": "2023-07-18T01:41:10.921838Z",
     "shell.execute_reply": "2023-07-18T01:41:10.920427Z",
     "shell.execute_reply.started": "2023-07-18T01:41:01.400069Z"
    }
   },
   "outputs": [],
   "source": [
    "#!find / -name bitsandbytes\n",
    "\n",
    "#!find / -name libbitsandbytes_cuda116.so\n",
    "\n",
    "#!cp /notebooks/bitsandbytes/bitsandbytes/libbitsandbytes_cuda116.so /usr/lib/python3.9/\n",
    "\n",
    "#!ls /usr/lib/python3.9/\n",
    "\n",
    "#!python -m bitsandbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "execution": {
     "iopub.execute_input": "2023-07-18T11:55:56.450496Z",
     "iopub.status.busy": "2023-07-18T11:55:56.449938Z",
     "iopub.status.idle": "2023-07-18T11:56:04.138278Z",
     "shell.execute_reply": "2023-07-18T11:56:04.137468Z",
     "shell.execute_reply.started": "2023-07-18T11:55:56.450472Z"
    },
    "id": "fhmLLJD0lM5S",
    "outputId": "d1ef4a8d-156c-4e0c-b92c-2d499e8ad4ed"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# To choose a specific GPU:\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import bitsandbytes as bnb\n",
    "from datasets import load_dataset\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoConfig, LlamaForCausalLM, LlamaTokenizer, AutoModelForCausalLM\n",
    "from peft import prepare_model_for_kbit_training, prepare_model_for_int8_training, LoraConfig, get_peft_model\n",
    "from peft.peft_model import PeftModel\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T11:56:06.901323Z",
     "iopub.status.busy": "2023-07-18T11:56:06.900588Z",
     "iopub.status.idle": "2023-07-18T11:56:06.904953Z",
     "shell.execute_reply": "2023-07-18T11:56:06.904367Z",
     "shell.execute_reply.started": "2023-07-18T11:56:06.901289Z"
    },
    "id": "XnTp0gOUlOCU"
   },
   "outputs": [],
   "source": [
    "MICRO_BATCH_SIZE = 6  # this could actually be 5 but i like powers of 2\n",
    "BATCH_SIZE = 128\n",
    "GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE\n",
    "EPOCHS = 2\n",
    "LEARNING_RATE = 3e-4  # the Karpathy constant\n",
    "CUTOFF_LEN = 256\n",
    "LORA_R = 8\n",
    "LORA_ALPHA = 16\n",
    "LORA_DROPOUT = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "f3fb12d97dee43aaa51784182caed544",
      "03717756681149898e8c3d40cbc16d10",
      "d718c5abc21b4394a4314fdc289d830d",
      "ee8532b0710541abb5f208e654b6d55a",
      "af52a9b6412e46c6b82acb9afabd12d9",
      "109d32c6d765417daf49b425b7ccee68",
      "56217a5954624e6f8742ba914183cb9e",
      "42a0b19d55b24e4ea5ce7894e4b8df50",
      "4348c4efdd4d4deb945bb2838c24cd83",
      "8f860f6f760e49b889fc450c53e49ac5",
      "43c5c35af8f44f1f98def57ea60a9615"
     ]
    },
    "execution": {
     "iopub.execute_input": "2023-07-18T11:56:08.096435Z",
     "iopub.status.busy": "2023-07-18T11:56:08.095783Z",
     "iopub.status.idle": "2023-07-18T12:05:29.403294Z",
     "shell.execute_reply": "2023-07-18T12:05:29.402714Z",
     "shell.execute_reply.started": "2023-07-18T11:56:08.096410Z"
    },
    "id": "vdQfvhHo0afo",
    "outputId": "71768c73-92d7-4430-cd75-3b262d86fc9b"
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "\n",
    "model = '''openlm-research/open_llama_3b_v2'''\n",
    "\"\"\"VMware/open-llama-13b-open-instruct\"\"\"\n",
    "use_fast_tokenizer=False\n",
    "# snapshot_download(repo_id=model)\n",
    "\n",
    "# LlamaTokenizer, is faster, if model is Llama\n",
    "# tokenizer = LlamaTokenizer.from_pretrained(model, use_fast=use_fast_tokenizer)\n",
    "# For other models:\n",
    "tokenizer = AutoTokenizer.from_pretrained(model, use_fast=use_fast_tokenizer)\n",
    "\n",
    "# model = LlamaForCausalLM.from_pretrained(model, load_in_8bit=True, low_cpu_mem_usage=True, device_map='auto', torch_dtype=torch.float16)\n",
    "# For other models:\n",
    "model = AutoModelForCausalLM.from_pretrained(model, load_in_8bit=True, low_cpu_mem_usage=True, device_map='auto', torch_dtype=torch.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T12:05:29.404911Z",
     "iopub.status.busy": "2023-07-18T12:05:29.404341Z",
     "iopub.status.idle": "2023-07-18T12:05:43.916286Z",
     "shell.execute_reply": "2023-07-18T12:05:43.915849Z",
     "shell.execute_reply.started": "2023-07-18T12:05:29.404890Z"
    },
    "id": "Xkb9pQTflS-b"
   },
   "outputs": [],
   "source": [
    "model = prepare_model_for_int8_training(model)\n",
    "\n",
    "config = LoraConfig(\n",
    "    r=LORA_R,\n",
    "    lora_alpha=LORA_ALPHA,\n",
    "    target_modules=[\"q_proj\", \"v_proj\"],\n",
    "    lora_dropout=LORA_DROPOUT,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "model = get_peft_model(model, config)\n",
    "# model = PeftModel.from_pretrained(model, \"open-llama-3bv2-lora-cabra-adapter-120steps\", config=config)\n",
    "tokenizer.pad_token_id = 0  # unk. we want this to be different from the eos token\n",
    "data = load_dataset(\"json\", data_files=\"https://huggingface.co/datasets/Gustrd/dolly-15k-libretranslate-pt/resolve/main/dolly-15k-libretranslate-pt.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T12:05:43.917470Z",
     "iopub.status.busy": "2023-07-18T12:05:43.916924Z",
     "iopub.status.idle": "2023-07-18T12:05:43.929680Z",
     "shell.execute_reply": "2023-07-18T12:05:43.929220Z",
     "shell.execute_reply.started": "2023-07-18T12:05:43.917450Z"
    },
    "id": "ad0PFPPmFRMv"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "# Create a slice of the dataset to handle time constraints\n",
    "# Calculate the number of rows to select for 1/2 of the data\n",
    "dataSliceNumber = 1\n",
    "num_rows = math.ceil(len(data['train']) // (1/dataSliceNumber))\n",
    "data['train'] = data['train'].shuffle().select(range(num_rows))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T12:05:43.931063Z",
     "iopub.status.busy": "2023-07-18T12:05:43.930761Z",
     "iopub.status.idle": "2023-07-18T12:05:43.935033Z",
     "shell.execute_reply": "2023-07-18T12:05:43.934613Z",
     "shell.execute_reply.started": "2023-07-18T12:05:43.931044Z"
    },
    "id": "_VCfL3BhlV_x"
   },
   "outputs": [],
   "source": [
    "def generate_prompt(data_point):\n",
    "    # desculpe o desastre de formatação, preciso ser rápido\n",
    "    if data_point[\"context\"]:\n",
    "        return f\"\"\"Abaixo está uma instrução que descreve uma tarefa, juntamente com uma entrada que fornece mais contexto. Escreva uma resposta que complete adequadamente o pedido.\n",
    "### Instrução:\n",
    "{data_point[\"instruction\"]}\n",
    "### Entrada:\n",
    "{data_point[\"context\"]}\n",
    "### Resposta:\n",
    "{data_point[\"response\"]}\"\"\"\n",
    "    else:\n",
    "        return f\"\"\"Abaixo está uma instrução que descreve uma tarefa. Escreva uma resposta que complete adequadamente o pedido.\n",
    "### Instrução:\n",
    "{data_point[\"instruction\"]}\n",
    "### Resposta:\n",
    "{data_point[\"response\"]}\"\"\"\n",
    "\n",
    "def tokenize(prompt):\n",
    "    # there's probably a way to do this with the tokenizer settings\n",
    "    # but again, gotta move fast\n",
    "    result = tokenizer(\n",
    "        prompt,\n",
    "        truncation=True,\n",
    "        max_length=CUTOFF_LEN + 1,\n",
    "        padding=\"max_length\",\n",
    "    )\n",
    "    return {\n",
    "        \"input_ids\": result[\"input_ids\"][:-1],\n",
    "        \"attention_mask\": result[\"attention_mask\"][:-1],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T12:05:43.936195Z",
     "iopub.status.busy": "2023-07-18T12:05:43.935598Z",
     "iopub.status.idle": "2023-07-18T12:06:03.137794Z",
     "shell.execute_reply": "2023-07-18T12:06:03.136986Z",
     "shell.execute_reply.started": "2023-07-18T12:05:43.936177Z"
    },
    "id": "81oSm3GL9z72"
   },
   "outputs": [],
   "source": [
    "data = data.shuffle().map(lambda x: tokenize(generate_prompt(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T12:06:03.139179Z",
     "iopub.status.busy": "2023-07-18T12:06:03.138947Z",
     "iopub.status.idle": "2023-07-18T12:06:03.292197Z",
     "shell.execute_reply": "2023-07-18T12:06:03.291371Z",
     "shell.execute_reply.started": "2023-07-18T12:06:03.139159Z"
    },
    "id": "INGJJZ6dkpJu"
   },
   "outputs": [],
   "source": [
    "trainer = transformers.Trainer(\n",
    "    model=model,\n",
    "    train_dataset=data[\"train\"],\n",
    "    args=transformers.TrainingArguments(\n",
    "        per_device_train_batch_size=MICRO_BATCH_SIZE,\n",
    "        gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,\n",
    "        warmup_steps=100,\n",
    "        num_train_epochs=EPOCHS,\n",
    "        learning_rate=LEARNING_RATE,\n",
    "        fp16=True,\n",
    "        logging_steps=20,\n",
    "        output_dir=\"lora-cabra-3Bv2\",\n",
    "        save_total_limit=4, \n",
    "        save_steps=20\n",
    "    ),\n",
    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
    ")\n",
    "model.config.use_cache = False\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T12:06:03.293628Z",
     "iopub.status.busy": "2023-07-18T12:06:03.293172Z",
     "iopub.status.idle": "2023-07-18T19:30:15.565676Z",
     "shell.execute_reply": "2023-07-18T19:30:15.564948Z",
     "shell.execute_reply.started": "2023-07-18T12:06:03.293609Z"
    },
    "id": "XrdM-8F8_59v"
   },
   "outputs": [],
   "source": [
    "\n",
    "trainer.train(resume_from_checkpoint=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T19:30:15.567903Z",
     "iopub.status.busy": "2023-07-18T19:30:15.567445Z",
     "iopub.status.idle": "2023-07-18T19:30:15.617103Z",
     "shell.execute_reply": "2023-07-18T19:30:15.616512Z",
     "shell.execute_reply.started": "2023-07-18T19:30:15.567879Z"
    },
    "id": "3JY4QEi7lXyY"
   },
   "outputs": [],
   "source": [
    "model.save_pretrained(\"open-llama-3bv2-lora-cabra-adapter-140steps\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-18T19:45:30.280575Z",
     "iopub.status.busy": "2023-07-18T19:45:30.280294Z",
     "iopub.status.idle": "2023-07-18T19:45:31.880370Z",
     "shell.execute_reply": "2023-07-18T19:45:31.879581Z",
     "shell.execute_reply.started": "2023-07-18T19:45:30.280556Z"
    }
   },
   "outputs": [],
   "source": [
    "# !tar -czvf open-llama-13b-lora-cabra-adapter.tar.gz ./open-llama-13b-lora-cabra-adapter"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuClass": "premium",
   "gpuType": "T4",
   "machine_shape": "hm",
   "provenance": []
  },
  "gpuClass": "premium",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "03717756681149898e8c3d40cbc16d10": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_109d32c6d765417daf49b425b7ccee68",
      "placeholder": "​",
      "style": "IPY_MODEL_56217a5954624e6f8742ba914183cb9e",
      "value": "Loading checkpoint shards:   0%"
     }
    },
    "109d32c6d765417daf49b425b7ccee68": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "42a0b19d55b24e4ea5ce7894e4b8df50": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4348c4efdd4d4deb945bb2838c24cd83": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "43c5c35af8f44f1f98def57ea60a9615": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "56217a5954624e6f8742ba914183cb9e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8f860f6f760e49b889fc450c53e49ac5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "af52a9b6412e46c6b82acb9afabd12d9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d718c5abc21b4394a4314fdc289d830d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_42a0b19d55b24e4ea5ce7894e4b8df50",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_4348c4efdd4d4deb945bb2838c24cd83",
      "value": 0
     }
    },
    "ee8532b0710541abb5f208e654b6d55a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8f860f6f760e49b889fc450c53e49ac5",
      "placeholder": "​",
      "style": "IPY_MODEL_43c5c35af8f44f1f98def57ea60a9615",
      "value": " 0/2 [00:00<?, ?it/s]"
     }
    },
    "f3fb12d97dee43aaa51784182caed544": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_03717756681149898e8c3d40cbc16d10",
       "IPY_MODEL_d718c5abc21b4394a4314fdc289d830d",
       "IPY_MODEL_ee8532b0710541abb5f208e654b6d55a"
      ],
      "layout": "IPY_MODEL_af52a9b6412e46c6b82acb9afabd12d9"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}