Upload 2 files

dnagpt/class6.ipynb

@@ -0,0 +1,501 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "29888eda-2755-4add-af9c-8927afb07db4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, AutoModel\n",
+    "tokenizer = AutoTokenizer.from_pretrained('dnagpt/human_gpt2-v1')\n",
+    "tokenizer.pad_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "75e3778d-9cbb-48c4-8203-0f71f485a49d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "full_model = AutoModel.from_pretrained('dnagpt/human_gpt2-v1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "be845310-8215-4434-bb92-d44c343f274e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "transformers.models.gpt2.modeling_gpt2\n"
+     ]
+    }
+   ],
+   "source": [
+    "gena_module_name = full_model.__class__.__module__\n",
+    "print(gena_module_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bb9ee647-cac2-4475-ac44-2f11aef99735",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "import importlib\n",
+    "myclass = importlib.import_module(gena_module_name)\n",
+    "#dir(myclass)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "62ebfcc0-5d46-4c3b-b462-a3842a985e9b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "transformers.models.gpt2.modeling_gpt2.GPT2ForSequenceClassification"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cls = getattr(importlib.import_module(gena_module_name), 'GPT2ForSequenceClassification')\n",
+    "cls"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "4523e9b6-3613-41e4-b81d-c139d044e70c",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of GPT2ForSequenceClassification were not initialized from the model checkpoint at dnagpt/human_gpt2-v1 and are newly initialized: ['score.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "model = cls.from_pretrained('dnagpt/human_gpt2-v1', num_labels=2)\n",
+    "model.config.pad_token_id = model.config.eos_token_id"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "bd024199-16e6-4eb1-b404-c49dc535469b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "# load ~11k samples from promoters prediction dataset\n",
+    "dataset = load_dataset(\"yurakuratov/example_promoters_2k\")['train'].train_test_split(test_size=0.1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "a3a5213d-66f0-4d17-876c-de4426145412",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['sequence', 'promoter_presence'],\n",
+       "        num_rows: 10656\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['sequence', 'promoter_presence'],\n",
+       "        num_rows: 1184\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "36a23870-1f56-466e-b2bb-c5047072b8f0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'sequence': 'TTGATATGCCCGCAATAAATGTTAGCCCTTCCTCTTTACAAACAATTATTATTTCATTAAGACTTACTCCTGGGAAGAAGTCACACTTTTCTGTGTGCTAACATCATGTACATTCTGATTCTCTGTCCTGTGTGCCATTCCATCATGCTCTCTACCAGCAAAACGCCACCAGTCTAAGGAGGAGACAGAACCGCATAAAAATAGTCACACAGTCTTCAGAGCTGCACAGGGAGTGGCACAGGGGCTTCACAGGCCTTGGGAAGTTTGTAAAGTGAGGGAGAAGCACTTTACTGTCGCCAGCTCCATTGAAAACAGATTGTAGCAAATGACTTCCCAGGAGCTCCTCCTCTCACCTTGGTGAATGTTGGGAGAGAATGAAGACTGCGGCTGAGAGGCAGCGCCTTGAGCCTTGAACGGAAAGAGGGACAGGGCCTGGGGGCTGTCACTGAGGTGCAGAAATGGGTTTTGGGCTGGGCATCAGACAACCCCAGATCCCAACACTGGCTCAGCCAATTCCTGGCTGTGTGACATGGAGCCATTACTGTACTTTCCTTGCCAGGCCTCCATTTTTCCTCTCTCTAGAATGGGGAGGCTGGTAATACCCACTTTGGAGGGCAGTGTGAGGACCGAAGGGGGCTGTTAAGTGCTCGGCGCGCAGCGAGCACTCAGGACGGGTGGCAGCGGTCACCTTTGTCGCCACCGTCCAAGCCTTTGCAGAGGCTCAGGGCCGAGCCCCGGACATGCGCATGCGCACGCGCACCCGGCCGGCCAGGTGAGAAGGAGGCGGCCGCCTCGCACCTGGGCTTTAACCCTTCCCAGTCACGGGAGTGGGCGGAGTGCACGCAGGGACGGAGTCGGGGGCGAGCAGCGAGAGAGGCCGGGCTTGACCCTTGAGGCGGGGTGGGCAGCGTCCGGGGAGCGCGCCGAGCCGCTTCTAGCCTGGGGTCCGCGCGCGCCGCGGGGAGGAGGGGGAGTGCCGAGGGGAGCGAAGTCTCGCGAGATCGCGCGGCGGCGGCGGGAGCGGCGGCGGCGGCGGCCGGGGAGGTGAGCGGCGGGCGGTGGCGGCCGTTGGGGGCTGAGGCCGGGTGAGAGCGGCCGAGACCGAGGGCTGGGTGGGGGAGCGGGCCTGGTGAACATCCCGCGTCCCAGGCAGCCATCCCTGTTCGTCCCCGCAGCAGCTGCGGCGCCGTTTGGTCCGCACCGTCCCCGGGACTCGCGCGGGGCGGGGCGGCCGGGGCCGCGGGCCGGTGCCTCCTTCAACCGGTCCTCGCGCCCGGGCGCGACCCCGGGGCCGCCTCAGTGTCCATCTCGACTGCAGAGTTGGCCCTGCTGACAGTGGCAGCGGCGTTTATGGAGCCTCTGCTGTGTGCCGGGCACTTAAGTTGCGCGCTTGCACGCCGGGGCTGCACAGAGAGGTGGAGCGCCCGGGCCAGAGCCACACTGCGAGGAGGCGGTGGGAGGGGACCGGGACCCCTGCGCCCCCTTCCTCTTCCCACCTCCCACTCCCCTCAGCCCTTTTCTCCTGGGGTCCCGACCAGGCTTCAGAGGGGGTGTCTGGGAGCGCCCTGAAGTTGGATGCAGCTTGGTATATGGGTTTCTTGGGAGAGAGTTTTTCCAGGGCTTTTGTCTCGTTCCTAAAAGGGTCCCCGATCCCCAAGTGGTCAAGAACCATTGTTTGGCCCTGCTTGGAAGCAGAGGTGGGCTGAGAGCAGCGGGCGTGTGCTGGTAGTGGGGTGTCAGGCCCAATGAAGTGGAATATTGGAGCCCAGGGCTTGCCTTAGGCAAAGGAAACTCTTCTCTAAAAGTTCGATTTTTGTTTCAGCACGCGATTCCACTCTTGGGAACAGTGGTTCTCAACCTCGTCAGAATCAAGGTCCCTTCTTCCCCTTTGCAGCAAACATGTATAAAGCCCTCTTTACTACCCTGAAATGAAATTCATAAACAATGTGACCTCCCTACACACATCCTTGAAAAGCAGTATGATGGTTTGATTGAAGCGT',\n",
+       " 'promoter_presence': 1}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset['train'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "19e22500-c1e0-444e-9b5c-2a1a2af81997",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "# base pairs:  2000\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('# base pairs: ', len(dataset['train'][0]['sequence']))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "16db0a5d-7aec-4d59-b44b-d22ff43045fd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tokens:  TTGATATG CCCGC AATAAATG TTAGCCC TTCCTC TTTACAAAC AATTATT ATTTCATT AAGAC TTAC TCCTGGG AAGAAGTC ACAC TTTTC TGTGTGC TAAC ATCATG TAC ATTCTG ATTC TCTGTCC TGTGTGCC ATTCC ATCATGC TCTC TACCAGC AAAAC GCC ACCAGTC TAAGG AGGAGAC AGAACC GC ATAAAA ATAGTC ACAC AGTCTTC AGAGC TGCAC AGGG AGTGGCAC AGGGGC TTCAC AGGCC TTGGGAAG TTTGTAA AGTGAGGG AGAAGC ACTTTAC TGTCGCC AGCTCC ATTGAAAAC AGATTG TAGCAAATG ACTTCCC AGGAGC TCCTCC TCTCACC TTGGTG AATG TTGGGAG AGAATGAAG ACTGC GGC TGAG AGGCAGC GCC TTGAGCC TTGAAC GG AAAG AGGGAC AGGGCC TGGGGGC TGTCAC TGAGG TGCAGAA ATGGG TTTTGGGC TGGGC ATCAGAC AACCCC AGATCCC AACACTGGC TCAGCC AATTCC TGGC TGTGTGAC ATGGAGCC ATTACTG TACTTTCC TTGCC AGGCC TCCATT TTTCC TCTCTC TAGAA TGGGG AGGCTGG TAA TACCC ACTTTGG AGGGC AGTGTG AGGACCG AAGG GGGCTG TTAAG TGCTC GGC GCGC AGCG AGCACTC AGGAC GGGTGGC AGCGG TCACC TTTGTC GCC ACCGTCC AAGCC TTTGC AGAGGC TC AGGGCCG AGCCCC GGAC ATGCGC ATGCGC ACGC GCACCCGGCC GGCC AGGTG AGAAGG AGGC GGCCGCC TCGC ACCTGGGC TTTAACCC TTCCC AGTCAC GGG AGTGGGC GG AGTGCAC GCAGGG ACGG AGTCGG GGGCG AGCAGCG AGAGAGGCC GGGC TTGACCC TTGAGGC GGGG TGGGC AGCGTCC GGGG AGCGC GCCG AGCCGC TTC TAGCC TGGGGTCC GCGC GCGCC GCGGGG AGGAGGGGG AGTGCCG AGGGG AGCG AAG TCTCGC GAG ATCGC GC GGCGGC GGC GGGAGC GGCGGCGGCGGC GGCC GGGG AGGTG AGCGGC GGGCGG TGGC GGCCG TTGGGGGC TG AGGCCGGG TGAGAGC GGCC GAG ACCG AGGGCTGGG TGGGGG AGC GGGCC TGGTGAAC ATCCC GCG TCCCAGGC AGCC ATCCCTG TTCG TCCCCGC AGCAGC TGCGGC GCCG TTTGG TCCGC ACCG TCCCC GGG ACTCGC GCGGGGC GGGGC GGCC GGGGCC GC GGGCC GGTGCC TCCTTC AACCGG TCCTC GCGCCC GGGC GCG ACCCC GGGGCC GCC TCAGTG TCC ATCTCG ACTGC AGAG TTGGCCC TGCTGAC AGTGGC AGC GGCG TTTATGG AGCCTCTGC TGTGTGCC GGGC ACTT AAGTTGC GCGC TTGCAC GCC GGGGC TGCAC AGAG AGGTGG AGCGCCC GGGCC AGAGCC ACAC TGCG AGGAGGCGG TGGGAGGGG ACCGGG ACCCCTGC GCCCCC TTCCTC TTCCCACC TCCCAC TCCCC TCAGCCC TTTTCTCC TGGGG TCCCG ACC AGGCTTC AGAGGGGG TGTCTGGG AGCGCCC TGAAG TTGGATGC AGCTTGG TATATGGG TTTC TTGGG AGAG AGTTTT TCCAGGGC TTTTG TCTCG TTCCTAA AAGGG TCCCCG ATCCCC AAGTGG TCAAG AACCATTG TTTGGCCC TGCTTGG AAGCAGAGG TGGGCTGAG AGCAGC GGGC GTGTGC TGGTAG TGGGG TGTC AGGCCC AATGAAG TGGAA TATTGG AGCCC AGGGC TTGCC TTAGGC AAAGG AAACTC TTCTC TAAAAG TTCG ATTTTTG TTTC AGCAC GCG ATTCCACTC TTGGGAAC AGTGG TTCTCAACC TCG TCAGAA TCAAGG TCCCTTC TTCCCC TTTGC AGCAAAC ATG TATAAAG CCCTC TTTAC TACCC TGAAATGAA ATTCATAA ACAATGTG ACCTCCC TACACAC ATCCTTG AAAAGC AGTATG ATGG TTTGATTG AAGCG T\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('tokens: ', ' '.join(tokenizer.tokenize(dataset['train'][0]['sequence'])))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "9876c07a-6332-40cc-aa84-2d26e6a3a5ab",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "# tokens:  350\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('# tokens: ', len(tokenizer.tokenize(dataset['train'][0]['sequence'])))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "7c4b0442-5fdf-4845-8bf2-4c3b6cc86e84",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Map: 100%|██████████| 10656/10656 [00:01<00:00, 10066.72 examples/s]\n",
+      "Map: 100%|██████████| 1184/1184 [00:00<00:00, 10784.01 examples/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "def preprocess_labels(example):\n",
+    "  example['label'] = example['promoter_presence']\n",
+    "  return example\n",
+    "\n",
+    "dataset = dataset.map(preprocess_labels)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "46e912af-1aab-4ed2-8007-0f3f970b2255",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def preprocess_function(examples):\n",
+    "  # just truncate right, but for some tasks symmetric truncation from left and right is more reasonable\n",
+    "  # set max_length to 128 to make experiments faster\n",
+    "  return tokenizer(examples[\"sequence\"], truncation=True, max_length=256) #max_length 128"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "0ef16794-71fd-4404-a51e-b55b5cae9aab",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Map: 100%|██████████| 10656/10656 [00:01<00:00, 6044.89 examples/s]\n",
+      "Map: 100%|██████████| 1184/1184 [00:00<00:00, 6994.12 examples/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "tokenized_dataset = dataset.map(preprocess_function, batched=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "765c199b-9d13-4fd2-ac51-1e5d154766fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import DataCollatorWithPadding\n",
+    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "afd6f48f-5524-4740-b8ea-54a79fd79dad",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['sequence', 'promoter_presence', 'label', 'input_ids', 'attention_mask'],\n",
+       "        num_rows: 10656\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['sequence', 'promoter_presence', 'label', 'input_ids', 'attention_mask'],\n",
+       "        num_rows: 1184\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenized_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "e1b708bb-21e4-416d-b191-79f302ca4e93",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import TrainingArguments, Trainer\n",
+    "import numpy as np\n",
+    "\n",
+    "\n",
+    "def compute_metrics(eval_pred):\n",
+    "    predictions, labels = eval_pred\n",
+    "    predictions = np.argmax(predictions, axis=1)\n",
+    "    return {'accuracy': (predictions==labels).sum() / len(labels)}\n",
+    "\n",
+    "# change training hyperparameters to archive better quality\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"test_run\",\n",
+    "    learning_rate=1e-4,\n",
+    "    lr_scheduler_type=\"constant_with_warmup\",\n",
+    "    warmup_ratio=0.1,\n",
+    "    optim='adamw_torch',\n",
+    "    weight_decay=0.0,\n",
+    "    per_device_train_batch_size=32,\n",
+    "    per_device_eval_batch_size=32,\n",
+    "    num_train_epochs=5,\n",
+    "    evaluation_strategy=\"epoch\",\n",
+    "    save_strategy=\"epoch\",\n",
+    "    logging_strategy=\"epoch\",\n",
+    "    load_best_model_at_end=True\n",
+    ")\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_dataset[\"train\"],\n",
+    "    eval_dataset=tokenized_dataset[\"test\"],\n",
+    "    tokenizer=tokenizer,\n",
+    "    data_collator=data_collator,\n",
+    "    compute_metrics=compute_metrics,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "082754e6-53fc-4a95-9d1e-887c42975d71",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
+      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='835' max='835' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [835/835 08:19, Epoch 5/5]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Epoch</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "      <th>Accuracy</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>0.573000</td>\n",
+       "      <td>0.462363</td>\n",
+       "      <td>0.778716</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>0.360200</td>\n",
+       "      <td>0.504239</td>\n",
+       "      <td>0.760135</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>0.201600</td>\n",
+       "      <td>0.529274</td>\n",
+       "      <td>0.795608</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>0.103800</td>\n",
+       "      <td>0.946800</td>\n",
+       "      <td>0.784628</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>0.064900</td>\n",
+       "      <td>1.108120</td>\n",
+       "      <td>0.744932</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
+      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
+      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
+      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
+      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
+      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
+      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
+      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=835, training_loss=0.2607016295016169, metrics={'train_runtime': 499.203, 'train_samples_per_second': 106.73, 'train_steps_per_second': 1.673, 'total_flos': 6960945435770880.0, 'train_loss': 0.2607016295016169, 'epoch': 5.0})"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "55c9d1eb-6fc6-451f-b596-870ccaa81d8d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}