Initial Setup

.gitattributes

@@ -7,7 +7,7 @@
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
+.lfs. filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
@@ -23,8 +23,8 @@
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
+saved_model/*/ filter=lfs diff=lfs merge=lfs -text
+.tar. filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
@@ -32,4 +32,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+tfevents filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/data/GrammarGuru.parquet.gzip filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/dictionaries/GrammarGuru.dict filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/similarities_matrix/GrammarGuru filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/similarities_matrix/GrammarGuru.index.data.npy filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/similarities_matrix/GrammarGuru.index.indices.npy filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/similarities_matrix/GrammarGuru.index.indptr.npy filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/similarities_matrix/GrammarGuru.index.npy filter=lfs diff=lfs merge=lfs -text
+models/GrammarGuru/tdidf/GrammarGuru.model filter=lfs diff=lfs merge=lfs -text

models/GrammarGuru/data/GrammarGuru.parquet.gzip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d41d0d50492fc4f8f470aa5106359de11d5625d5dc1a502f04c26fc3cd472f5
+size 208673276

models/GrammarGuru/dictionaries/GrammarGuru.dict

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e1cf56046f5325c87d586761a6f9beea67599a5dba6f6bf7811b3d605e9838
+size 16358261

models/GrammarGuru/similarities_matrix/GrammarGuru

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82cb64d81faf4009561e0c0dd1861f60707aec87fd48be2147268154372d825a
+size 615

models/GrammarGuru/similarities_matrix/GrammarGuru.index.data.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17ee886caeb86b6afe13a87a3d86992fcdb76780a01a96264eba1d02cf7fbb0b
+size 62474552

models/GrammarGuru/similarities_matrix/GrammarGuru.index.indices.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a76c3a2ff5382688106ca638dbc5b5b7752a48069c8d199f8f260fd6b13e29fc
+size 62474552

models/GrammarGuru/similarities_matrix/GrammarGuru.index.indptr.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2189acf62052757ccfc57da9ff6a4910ea7ce9033f8be59cdb3623c67c2ef16
+size 1135612

models/GrammarGuru/similarities_matrix/GrammarGuru.index.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6fbdc8884bdc666b66b7c3aab8e6d2866ebbbeba041310a57b9cf9a979a3108
+size 118

models/GrammarGuru/tdidf/GrammarGuru.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78e4051d387687adc37cf59c2498829ffde41aef34f56d65e21c434deae6bf1c
+size 15251885

notebooks/feature-engineering.ipynb

@@ -0,0 +1,990 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from matplotlib import pyplot as plt \n",
+    "import seaborn as sns\n",
+    "\n",
+    "pd.set_option('display.float_format', '{:.2f}'.format)\n",
+    "sns.set_style(\"darkgrid\")\n",
+    "sns.set_palette(\"mako\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_cleaned = pd.read_parquet(\"../data/processed/arxiv_papers.parquet.gzip\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>submitter</th>\n",
+       "      <th>authors</th>\n",
+       "      <th>title</th>\n",
+       "      <th>comments</th>\n",
+       "      <th>journal-ref</th>\n",
+       "      <th>doi</th>\n",
+       "      <th>report-no</th>\n",
+       "      <th>categories</th>\n",
+       "      <th>license</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>versions</th>\n",
+       "      <th>update_date</th>\n",
+       "      <th>authors_parsed</th>\n",
+       "      <th>cleaned_abstracts</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0001</td>\n",
+       "      <td>Pavel Nadolsky</td>\n",
+       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
+       "      <td>Calculation of prompt diphoton production cros...</td>\n",
+       "      <td>37 pages, 15 figures; published version</td>\n",
+       "      <td>Phys.Rev.D76:013009,2007</td>\n",
+       "      <td>10.1103/PhysRevD.76.013009</td>\n",
+       "      <td>ANL-HEP-PR-07-12</td>\n",
+       "      <td>hep-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>A fully differential calculation in perturba...</td>\n",
+       "      <td>[{'created': 'Mon, 2 Apr 2007 19:18:42 GMT', '...</td>\n",
+       "      <td>2008-11-26</td>\n",
+       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
+       "      <td>fully differential calculation perturbative...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Louis Theran</td>\n",
+       "      <td>Ileana Streinu and Louis Theran</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>To appear in Graphs and Combinatorics</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO cs.CG</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 02:26:18 GMT', ...</td>\n",
+       "      <td>2008-12-13</td>\n",
+       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
+       "      <td>describe new algorithm $ k,\\ell)$-pebble ga...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>23 pages, 3 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>physics.gen-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td>[{'created': 'Sun, 1 Apr 2007 20:46:54 GMT', '...</td>\n",
+       "      <td>2008-01-13</td>\n",
+       "      <td>[[Pan, Hongjun, ]]</td>\n",
+       "      <td>evolution earth moon system describe dark m...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>11 pages</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that a determinant of Stirling cycle...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 03:16:14 GMT', ...</td>\n",
+       "      <td>2007-05-23</td>\n",
+       "      <td>[[Callan, David, ]]</td>\n",
+       "      <td>determinant stirling cycle number count unl...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>Alberto Torchinsky</td>\n",
+       "      <td>Wael Abu-Shammala and Alberto Torchinsky</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Illinois J. Math. 52 (2008) no.2, 681-689</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CA math.FA</td>\n",
+       "      <td>None</td>\n",
+       "      <td>In this paper we show how to compute the $\\L...</td>\n",
+       "      <td>[{'created': 'Mon, 2 Apr 2007 18:09:58 GMT', '...</td>\n",
+       "      <td>2013-10-15</td>\n",
+       "      <td>[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]</td>\n",
+       "      <td>paper compute $ \\lambda_{\\alpha}$ norm $ \\a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268247</th>\n",
+       "      <td>supr-con/9608008</td>\n",
+       "      <td>Ruslan Prozorov</td>\n",
+       "      <td>R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...</td>\n",
+       "      <td>On the origin of the irreversibility line in t...</td>\n",
+       "      <td>19 pages, LaTex, 6 PostScript figures; Author'...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.54.15530</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We report on measurements of the angular dep...</td>\n",
+       "      <td>[{'created': 'Mon, 26 Aug 1996 15:08:35 GMT', ...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Prozorov, R., ], [Konczykowski, M., ], [Schm...</td>\n",
+       "      <td>report measurement angular dependence irrev...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268248</th>\n",
+       "      <td>supr-con/9609001</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Durga P. Choudhury, Balam A. Willemsen, John S...</td>\n",
+       "      <td>Nonlinear Response of HTSC Thin Film Microwave...</td>\n",
+       "      <td>4 pages, LaTeX type, Uses IEEE style files, 60...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1109/77.620744</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The non-linear microwave surface impedance o...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Aug 1996 17:34:38 GMT', ...</td>\n",
+       "      <td>2016-11-18</td>\n",
+       "      <td>[[Choudhury, Durga P., , Physics Department, N...</td>\n",
+       "      <td>non linear microwave surface impedance patt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268249</th>\n",
+       "      <td>supr-con/9609002</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Balam A. Willemsen, J. S. Derov and S.Sridhar ...</td>\n",
+       "      <td>Critical State Flux Penetration and Linear Mic...</td>\n",
+       "      <td>20 pages, LaTeX type, Uses REVTeX style files,...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.56.11989</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The vortex contribution to the dc field (H) ...</td>\n",
+       "      <td>[{'created': 'Tue, 3 Sep 1996 14:08:26 GMT', '...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Willemsen, Balam A., , Physics Department,\\n...</td>\n",
+       "      <td>vortex contribution dc field h dependent mi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268250</th>\n",
+       "      <td>supr-con/9609003</td>\n",
+       "      <td>Hasegawa Yasumasa</td>\n",
+       "      <td>Yasumasa Hasegawa (Himeji Institute of Technol...</td>\n",
+       "      <td>Density of States and NMR Relaxation Rate in A...</td>\n",
+       "      <td>7 pages, 4 PostScript Figures, LaTeX, to appea...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1143/JPSJ.65.3131</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that the density of states in an ani...</td>\n",
+       "      <td>[{'created': 'Wed, 18 Sep 1996 07:57:29 GMT', ...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Hasegawa, Yasumasa, , Himeji Institute of Te...</td>\n",
+       "      <td>density state anisotropic superconductor \\n...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268251</th>\n",
+       "      <td>supr-con/9609004</td>\n",
+       "      <td>Masanori Ichioka</td>\n",
+       "      <td>Naoki Enomoto, Masanori Ichioka and Kazushige ...</td>\n",
+       "      <td>Ginzburg Landau theory for d-wave pairing and ...</td>\n",
+       "      <td>12 pages including 8 eps figs, LaTeX with jpsj...</td>\n",
+       "      <td>J. Phys. Soc. Jpn. 66, 204 (1997).</td>\n",
+       "      <td>10.1143/JPSJ.66.204</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The Ginzburg Landau theory for d_{x^2-y^2}-w...</td>\n",
+       "      <td>[{'created': 'Wed, 25 Sep 1996 14:17:09 GMT', ...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...</td>\n",
+       "      <td>ginzburg landau theory d_{x^2 y^2}-wave sup...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2268252 rows × 15 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       id           submitter   \n",
+       "0               0704.0001      Pavel Nadolsky  \\\n",
+       "1               0704.0002        Louis Theran   \n",
+       "2               0704.0003         Hongjun Pan   \n",
+       "3               0704.0004        David Callan   \n",
+       "4               0704.0005  Alberto Torchinsky   \n",
+       "...                   ...                 ...   \n",
+       "2268247  supr-con/9608008     Ruslan Prozorov   \n",
+       "2268248  supr-con/9609001  Durga P. Choudhury   \n",
+       "2268249  supr-con/9609002  Durga P. Choudhury   \n",
+       "2268250  supr-con/9609003   Hasegawa Yasumasa   \n",
+       "2268251  supr-con/9609004    Masanori Ichioka   \n",
+       "\n",
+       "                                                   authors   \n",
+       "0        C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
+       "1                          Ileana Streinu and Louis Theran   \n",
+       "2                                              Hongjun Pan   \n",
+       "3                                             David Callan   \n",
+       "4                 Wael Abu-Shammala and Alberto Torchinsky   \n",
+       "...                                                    ...   \n",
+       "2268247  R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...   \n",
+       "2268248  Durga P. Choudhury, Balam A. Willemsen, John S...   \n",
+       "2268249  Balam A. Willemsen, J. S. Derov and S.Sridhar ...   \n",
+       "2268250  Yasumasa Hasegawa (Himeji Institute of Technol...   \n",
+       "2268251  Naoki Enomoto, Masanori Ichioka and Kazushige ...   \n",
+       "\n",
+       "                                                     title   \n",
+       "0        Calculation of prompt diphoton production cros...  \\\n",
+       "1                 Sparsity-certifying Graph Decompositions   \n",
+       "2        The evolution of the Earth-Moon system based o...   \n",
+       "3        A determinant of Stirling cycle numbers counts...   \n",
+       "4        From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "...                                                    ...   \n",
+       "2268247  On the origin of the irreversibility line in t...   \n",
+       "2268248  Nonlinear Response of HTSC Thin Film Microwave...   \n",
+       "2268249  Critical State Flux Penetration and Linear Mic...   \n",
+       "2268250  Density of States and NMR Relaxation Rate in A...   \n",
+       "2268251  Ginzburg Landau theory for d-wave pairing and ...   \n",
+       "\n",
+       "                                                  comments   \n",
+       "0                  37 pages, 15 figures; published version  \\\n",
+       "1                    To appear in Graphs and Combinatorics   \n",
+       "2                                      23 pages, 3 figures   \n",
+       "3                                                 11 pages   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "2268247  19 pages, LaTex, 6 PostScript figures; Author'...   \n",
+       "2268248  4 pages, LaTeX type, Uses IEEE style files, 60...   \n",
+       "2268249  20 pages, LaTeX type, Uses REVTeX style files,...   \n",
+       "2268250  7 pages, 4 PostScript Figures, LaTeX, to appea...   \n",
+       "2268251  12 pages including 8 eps figs, LaTeX with jpsj...   \n",
+       "\n",
+       "                                       journal-ref   \n",
+       "0                         Phys.Rev.D76:013009,2007  \\\n",
+       "1                                             None   \n",
+       "2                                             None   \n",
+       "3                                             None   \n",
+       "4        Illinois J. Math. 52 (2008) no.2, 681-689   \n",
+       "...                                            ...   \n",
+       "2268247                                       None   \n",
+       "2268248                                       None   \n",
+       "2268249                                       None   \n",
+       "2268250                                       None   \n",
+       "2268251         J. Phys. Soc. Jpn. 66, 204 (1997).   \n",
+       "\n",
+       "                                doi         report-no   \n",
+       "0        10.1103/PhysRevD.76.013009  ANL-HEP-PR-07-12  \\\n",
+       "1                              None              None   \n",
+       "2                              None              None   \n",
+       "3                              None              None   \n",
+       "4                              None              None   \n",
+       "...                             ...               ...   \n",
+       "2268247   10.1103/PhysRevB.54.15530              None   \n",
+       "2268248           10.1109/77.620744              None   \n",
+       "2268249   10.1103/PhysRevB.56.11989              None   \n",
+       "2268250        10.1143/JPSJ.65.3131              None   \n",
+       "2268251         10.1143/JPSJ.66.204              None   \n",
+       "\n",
+       "                         categories   \n",
+       "0                            hep-ph  \\\n",
+       "1                     math.CO cs.CG   \n",
+       "2                    physics.gen-ph   \n",
+       "3                           math.CO   \n",
+       "4                   math.CA math.FA   \n",
+       "...                             ...   \n",
+       "2268247  supr-con cond-mat.supr-con   \n",
+       "2268248  supr-con cond-mat.supr-con   \n",
+       "2268249  supr-con cond-mat.supr-con   \n",
+       "2268250  supr-con cond-mat.supr-con   \n",
+       "2268251  supr-con cond-mat.supr-con   \n",
+       "\n",
+       "                                                   license   \n",
+       "0                                                     None  \\\n",
+       "1        http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "2                                                     None   \n",
+       "3                                                     None   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "2268247                                               None   \n",
+       "2268248                                               None   \n",
+       "2268249                                               None   \n",
+       "2268250                                               None   \n",
+       "2268251                                               None   \n",
+       "\n",
+       "                                                  abstract   \n",
+       "0          A fully differential calculation in perturba...  \\\n",
+       "1          We describe a new algorithm, the $(k,\\ell)$-...   \n",
+       "2          The evolution of Earth-Moon system is descri...   \n",
+       "3          We show that a determinant of Stirling cycle...   \n",
+       "4          In this paper we show how to compute the $\\L...   \n",
+       "...                                                    ...   \n",
+       "2268247    We report on measurements of the angular dep...   \n",
+       "2268248    The non-linear microwave surface impedance o...   \n",
+       "2268249    The vortex contribution to the dc field (H) ...   \n",
+       "2268250    We show that the density of states in an ani...   \n",
+       "2268251    The Ginzburg Landau theory for d_{x^2-y^2}-w...   \n",
+       "\n",
+       "                                                  versions update_date   \n",
+       "0        [{'created': 'Mon, 2 Apr 2007 19:18:42 GMT', '...  2008-11-26  \\\n",
+       "1        [{'created': 'Sat, 31 Mar 2007 02:26:18 GMT', ...  2008-12-13   \n",
+       "2        [{'created': 'Sun, 1 Apr 2007 20:46:54 GMT', '...  2008-01-13   \n",
+       "3        [{'created': 'Sat, 31 Mar 2007 03:16:14 GMT', ...  2007-05-23   \n",
+       "4        [{'created': 'Mon, 2 Apr 2007 18:09:58 GMT', '...  2013-10-15   \n",
+       "...                                                    ...         ...   \n",
+       "2268247  [{'created': 'Mon, 26 Aug 1996 15:08:35 GMT', ...  2009-10-30   \n",
+       "2268248  [{'created': 'Sat, 31 Aug 1996 17:34:38 GMT', ...  2016-11-18   \n",
+       "2268249  [{'created': 'Tue, 3 Sep 1996 14:08:26 GMT', '...  2009-10-30   \n",
+       "2268250  [{'created': 'Wed, 18 Sep 1996 07:57:29 GMT', ...  2009-10-30   \n",
+       "2268251  [{'created': 'Wed, 25 Sep 1996 14:17:09 GMT', ...  2009-10-30   \n",
+       "\n",
+       "                                            authors_parsed   \n",
+       "0        [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \\\n",
+       "1                 [[Streinu, Ileana, ], [Theran, Louis, ]]   \n",
+       "2                                       [[Pan, Hongjun, ]]   \n",
+       "3                                      [[Callan, David, ]]   \n",
+       "4        [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]   \n",
+       "...                                                    ...   \n",
+       "2268247  [[Prozorov, R., ], [Konczykowski, M., ], [Schm...   \n",
+       "2268248  [[Choudhury, Durga P., , Physics Department, N...   \n",
+       "2268249  [[Willemsen, Balam A., , Physics Department,\\n...   \n",
+       "2268250  [[Hasegawa, Yasumasa, , Himeji Institute of Te...   \n",
+       "2268251  [[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...   \n",
+       "\n",
+       "                                         cleaned_abstracts  \n",
+       "0           fully differential calculation perturbative...  \n",
+       "1           describe new algorithm $ k,\\ell)$-pebble ga...  \n",
+       "2           evolution earth moon system describe dark m...  \n",
+       "3           determinant stirling cycle number count unl...  \n",
+       "4           paper compute $ \\lambda_{\\alpha}$ norm $ \\a...  \n",
+       "...                                                    ...  \n",
+       "2268247     report measurement angular dependence irrev...  \n",
+       "2268248     non linear microwave surface impedance patt...  \n",
+       "2268249     vortex contribution dc field h dependent mi...  \n",
+       "2268250     density state anisotropic superconductor \\n...  \n",
+       "2268251     ginzburg landau theory d_{x^2 y^2}-wave sup...  \n",
+       "\n",
+       "[2268252 rows x 15 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_cleaned"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_cleaned['len_abstract'] =  df_cleaned['cleaned_abstracts'].str.len()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "count   2268252.00\n",
+       "mean        688.92\n",
+       "std         315.74\n",
+       "min           4.00\n",
+       "25%         446.00\n",
+       "50%         658.00\n",
+       "75%         908.00\n",
+       "max        4372.00\n",
+       "Name: len_abstract, dtype: float64"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_cleaned['len_abstract'].describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGxCAYAAACZa0njAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABInklEQVR4nO3de3hU1b3/8c+QSUIQYjQJkYhHQeUiQhISgdZQgQIiYItIPZUqIHi0lUt75KIBBSRcJFxUBBTkXlABsXhEqxWPtrWVwIkmCJTKTQwBQoJcJclkkvX7A7N/zCRI7rOTeb+eJ09m9ppZs9Z8J/Jx77X3OIwxRgAAALA08PUAAAAA7IaABAAA4IWABAAA4IWABAAA4IWABAAA4IWABAAA4IWABAAA4IWABAAA4IWABAAA4IWABAAA4MXp6wHUZSdPnlN1flGLwyGFhzep9n5RddTG3qiPfVEb+/LH2pTMuTwISFVgjGrkQ1VT/aLqqI29UR/7ojb2RW3KxiE2AAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQkAAAALwQk2Mr0pJm+HgIAAAQk2IvLVejrIQAAQEACAADwRkACAADwQkACAADwQkACAADw4tOAlJ2drTFjxqhTp07q2rWrZs2apYKCAklSZmamhg0bptjYWPXt21efffaZx3P/+c9/qn///oqJidGQIUOUmZnp0b5q1Sp17dpVcXFxmjhxovLy8qy2goICTZw4UQkJCUpMTNSKFStqfrIAAKDO8FlAMsZozJgxysvL07p16/TCCy/ok08+0YsvvihjjEaOHKmIiAht2rRJv/zlLzVq1CgdPXpUknT06FGNHDlSAwcO1FtvvaVrr71WTzzxhIwxkqQPP/xQCxcu1LRp07R69WplZGRozpw51munpKRo165dWr16taZMmaKFCxfqgw8+8Mn7AAAA7Mfpqxc+ePCg0tPT9Y9//EMRERGSpDFjxmj27Nn62c9+pszMTL355ptq1KiRbr75Zn3++efatGmTRo8erY0bN+r222/X8OHDJUmzZs3SnXfeqe3bt6tz585as2aNhg4dqu7du0uSnnvuOY0YMULjx4+XMUYbN27Ua6+9pnbt2qldu3bat2+f1q1bpz59+vjq7QAAADbisz1IkZGRWrZsmRWOSpw/f14ZGRm67bbb1KhRI2t7fHy80tPTJUkZGRlKSEiw2kJCQtSuXTulp6erqKhIX331lUd7bGysCgsLtXfvXu3du1dut1txcXEefWdkZKi4uLiGZgsAAOoSn+1BCg0NVdeuXa37xcXFWrt2rbp06aKcnBw1bdrU4/Hh4eE6fvy4JP1o+9mzZ1VQUODR7nQ6FRYWpuPHj6tBgwa65pprFBQUZLVHRESooKBAp0+f1rXXXlsT0wUAAHWIzwKStzlz5mjPnj166623tGrVKo8AI0lBQUFyuVySpLy8vMu25+fnW/fLajfGlNkmyeq/vByOCj283P1Vd791jR3nT23sjfrYF7WxL3+sTUXmaouANGfOHK1evVovvPCCWrVqpeDgYJ0+fdrjMS6XSw0bNpQkBQcHlwozLpdLoaGhCg4Otu57t4eEhKioqKjMNklW/+UVHt6kQo/3db91QaAzQBER9p2/P9emLqA+9kVt7IvalM3nASk5OVlvvPGG5syZo7vvvluSFBUVpf3793s8Ljc31zpsFhUVpdzc3FLtbdu2VVhYmIKDg5Wbm6ubb75ZkuR2u3X69GlFRkbKGKNTp07J7XbL6bw4/ZycHDVs2FChoaEVGvvJk+f0w4lz1cLhuPhBre5+65JCd5Fyc8/5ehilUBt7oz72RW3syx9rUzLn8vBpQFq4cKHefPNNzZ8/3+MMspiYGC1dulT5+fnWXp20tDTFx8db7Wlpadbj8/LytGfPHo0aNUoNGjRQ+/btlZaWps6dO0uS0tPT5XQ61aZNG0kX1ySlp6dbC7nT0tLUvn17NWhQsTXrxqhGPlQ11W9dYee5+3tt7I762Be1sS9qUzafncV24MABLV68WP/1X/+l+Ph45eTkWD+dOnVSs2bNlJSUpH379mnp0qXauXOnBg0aJEm6//779cUXX2jp0qXat2+fkpKS1Lx5cysQDR48WMuXL9fWrVu1c+dOTZ06VQ888IBCQkIUEhKiAQMGaOrUqdq5c6e2bt2qFStWaMiQIb56KwAAgM34bA/Sxx9/rKKiIr3yyit65ZVXPNr+/e9/a/HixZo0aZIGDhyoG2+8UYsWLVJ0dLQkqXnz5nr55Zc1c+ZMLVq0SHFxcVq0aJEcP6y+6tevn7KysjR58mS5XC717t1b48ePt/pPSkrS1KlTNXToUDVu3FijR49W7969a2/yAADA1hzGsGOtsnJzq38NUkREk2rvty6ZPPY5TZs3xdfDKIXa2Bv1sS9qY1/+WJuSOZcHX1YLAADghYAEAADghYAEAADghYAEAADghYAEAADghYAEAADghYAEAADghYAEAADghYCEGjN7coqmJ8309TAAAKgwAhKq1aWhyOUqlMtV6NFOYAIA1AUEJFSrskKRJI/QBACA3RGQUKMCAy9+HzLBCABQlxCQUCsCA52aPTnF18MAAKBcCEioNZfuRWItEgDAzghIqBaXBp7AQKfmJc+3bpfsOSo53CZJxpjaHSAAABVAQEK18F5jVHjJ/bLWHwUGOtmLBACwLQISatWle5RYuA0AsCsCEqrFpYfVrqQkGJWEJfYkAQDsxnnlhwDlU1iJPUIuV6EKC901MBoAACqPPUgAAABeCEjwqUvPbAMAwC4ISPA5zmgDANgNAQm2wBltAAA7ISABAAB4ISDBFkouE8Cp/wAAO2CFLKpk9uQUOatpoXXJZQI49R8A4GvsQUKVuFyFlbr+UXmxJwkA4AsEJNhSo0YNNXtyCou3AQA+QUCCbRGOAAC+QkBCpZV86SwAAPUNAQmVxh4eAEB9ZYuA5HK51L9/f6WmpkqSnn76abVu3brUz5AhQ6znJCQklGr//vvvJUkFBQWaOHGiEhISlJiYqBUrVni8XmZmpoYNG6bY2Fj17dtXn332We1NFgAA2J7PT/MvKCjQ2LFjtW/fPmvbpEmTNHbsWOt+VlaWHn74YSsgZWdn69y5c9q6dasaNmxoPa5Ro0aSpJSUFO3atUurV6/W0aNH9dRTTyk6Olp9+vSRMUYjR45Uq1attGnTJm3dulWjRo3S+++/r+jo6FqaNcqrUaOGmp40U8/MmujroQAA/IhPA9L+/fs1duxYGWM8tjdp0kRNmjSx7j/99NPq06ePevbsKUk6cOCAIiMjdcMNN5Tq88KFC9q4caNee+01tWvXTu3atdO+ffu0bt069enTR9u2bVNmZqbefPNNNWrUSDfffLM+//xzbdq0SaNHj67ZCaNSjDGEJABArfLpIbbt27erc+fOWr9+/WUf8/nnn2vHjh168sknrW379+9XixYtynz83r175Xa7FRcXZ22Lj49XRkaGiouLlZGRodtuu83a21TSnp6eXvUJocaUhCQAAGqDT/cgDR48+IqPWbp0qe677z41a9bM2nbgwAHl5eXp4Ycf1qFDh9S2bVtNnDhRLVq0UE5Ojq655hoFBQVZj4+IiFBBQYFOnz6tnJwcNW3a1OM1wsPDdfz48QqP3+Go8FPK1V9191tfGGN89t5QG3ujPvZFbezLH2tTkbn6fA3Sj8nMzNS2bds0adIkj+0HDx7UmTNn9OSTT6px48Z67bXXNGzYML333nvKy8vzCEeSrPsul+uy7S6Xq8LjCw9vcuUHVUJN9VvdAp0BkiSnM0Ayxvp9pW2VbZekiAjfvjd1pTb+ivrYF7WxL2pTNlsHpA8//FBt27bVLbfc4rF9+fLlKiws1FVXXSVJmjt3ru666y598sknCg4OLhV2Su43bNhQwcHBOn36dKn2Sxd7l9fJk+fktXyqShyOix/U6u63phS6iy7ecDgu3i75faVtlW2XlJt7rhZmVlpdq42/oT72RW3syx9rUzLn8rB1QPr73/+un//856W2BwUFeewFCg4OVvPmzZWdna2OHTvq1KlTcrvdcjovTi8nJ0cNGzZUaGiooqKitH//fo/+cnNzSx12Kw9jVCMfqprqtz7w9ftCbeyN+tgXtbEvalM2W1wHqSzGGH311Vfq2LFjqe09e/bU22+/bW27cOGCDh8+rJYtW6pt27ZyOp0ei67T0tLUvn17NWjQQDExMdq9e7fy8/M92mNiYmp8TgAAoG6wbUDKysrS999/X+rwmsPhULdu3fTyyy8rNTVV+/bt04QJE3TdddfprrvuUkhIiAYMGKCpU6dq586d2rp1q1asWGFdQ6lTp05q1qyZkpKStG/fPi1dulQ7d+7UoEGDfDFNAABgQ7Y9xHby5ElJ0tVXX12qbfz48XI6nRo7dqzOnz+vLl26aOnSpQoIuLhoOCkpSVOnTtXQoUPVuHFjjR49Wr1795YkBQQEaPHixZo0aZIGDhyoG2+8UYsWLeIikXVAYKBTsyenqLDQzTWRAAA1ymG8r9KIcsvNrf5F2hERTaq935qS/PQMSReDS2Gh2/p9pW1VbS8sdGvavCm1Ote6Vht/Q33si9rYlz/WpmTO5WHbQ2ywt3nJ8309BAAAagwBCZVS6Cr02WsHBjo1L3k+V9YGANQYAhLqpEJXoVw+DGkAgPqNgIQ6KzDQyV4kAECNICChTmMvEgCgJhCQUKcFBtr2ShUAgDqMgIR6gUNtAIDqREBChc2enOLrIZRijCEkAQCqDQEJFWandT8lV9eW7DUuAEDdRkBChdhx7xHBCABQ3QhIqBDCCADAHxCQAAAAvBCQAAAAvBCQUG9wZW0AQHUhIKFeYY0UAKA6EJBQr7AXCQBQHQhIKLd5yfN9PYQfVfK1I+xFAgBUFQEJ5VZI8AAA+AkCEgAAgBcCEgAAgBcCEsrFjl8xcjks1AYAVBUBCeVS1xY+G2MISQCASiMgod7hbDYAQFURkAAAALwQkAAAALwQkAAAALwQkOCBhc0AABCQ4IWFzQAAEJAAAABKISDhiuz+JbUAAFQ3WwQkl8ul/v37KzU11do2ffp0tW7d2uNn7dq1VvuWLVvUs2dPxcTEaOTIkfruu++sNmOM5s6dqy5duqhTp05KSUlRcXGx1X7q1CmNHj1acXFx6tGjh955553amWgdxZfUAgD8jdPXAygoKNDYsWO1b98+j+0HDhzQ2LFjdd9991nbGjduLEnauXOnJk2apOeee05t2rTRjBkzlJSUpCVLlkiSVq5cqS1btmjhwoVyu90aP368wsPDNWLECElSUlKS8vPztX79emVkZOiZZ55RixYt1KFDh1qaNQAAsDOf7kHav3+/HnjgAX377bel2g4cOKDbbrtNkZGR1k9ISIgkae3atbrnnns0YMAAtWnTRikpKfrrX/+qzMxMSdKaNWs0ZswYJSQkqEuXLho3bpzWrVsnSfr222/1ySefaPr06WrVqpV+9atf6Re/+IVef/312ps4agXfyQYAqCyfBqTt27erc+fOWr9+vcf28+fPKzs7WzfddFOZz8vIyFBCQoJ1v1mzZoqOjlZGRoays7N17Ngx3XHHHVZ7fHy8srKydOLECWVkZKhZs2Zq3ry5R/uXX35ZvZODLRhjfD0EAEAd5NNDbIMHDy5z+4EDB+RwOPTqq6/qb3/7m8LCwvTII49Yh9tOnDihpk2bejwnPDxcx48fV05OjiR5tEdEREiS1V7Wc7Ozs6ttXgAAoG7z+Rqkshw8eFAOh0MtW7bUQw89pB07dujZZ59V48aN1atXL+Xn5ysoKMjjOUFBQXK5XMrPz7fuX9omXVwMnpeXd9nnVpTDUeGnlKu/6u63suOQpOefTfHdQKpJdbyfdqkNykZ97Iva2Jc/1qYic7VlQBowYIC6d++usLAwSVKbNm30zTff6I033lCvXr0UHBxcKtC4XC6FhIR4hKHg4GDrtiSFhIRc9rkNGzas8DjDw5tU+Dm+7Lc8Ap0Bioj4/69viovlDHRKxsjpDJB+OGRV1u3ybqvpdo9tgU7Nema25r06vVreH1/WBldGfeyL2tgXtSmbLQOSw+GwwlGJli1batu2bZKkqKgo5ebmerTn5uYqMjJSUVFRkqScnBxrnVHJYbeS9ss9t6JOnjyn6lzi4nBc/KBWd78VUeguUm7uOY/7cjg8f/8w2FK3y7utptu9tl24kO8xp8qwQ21wedTHvqiNffljbUrmXB62DEgvvfSSvvzyS61atcratnfvXrVs2VKSFBMTo7S0NA0cOFCSdOzYMR07dkwxMTGKiopSdHS00tLSrICUlpam6OhoNW3aVLGxscrKytLx48d13XXXWe2xsbEVHqcxqpEPVU31W5HXl6TZk+v+4bUS1fV++ro2+HHUx76ojX1Rm7LZ4kKR3rp3764dO3Zo+fLl+vbbb/X6669r8+bNGj58uCTpwQcf1DvvvKONGzdq7969mjBhgrp166YbbrjBap87d65SU1OVmpqqefPmaciQIZKkG264QYmJiRo/frz27t2rjRs3asuWLfrNb37js/naVX35XjZO9wcAVJQt9yB16NBBL730khYsWKCXXnpJ119/vebNm6e4uDhJUlxcnKZNm6YFCxbozJkzuvPOO5WcnGw9f8SIETp58qRGjRqlgIAADRo0SMOGDbPaU1JSNGnSJD3wwAOKjIzUzJkzuUhkPVdfwh4AoHY4DBeKqbTc3OpfgxQR0aTa+62IyWOf07R5UyRJyU/PkHRxD0xhodv6fem2K7VX5jk19ZoOh0PPzJpYqffFDrXB5VEf+6I29uWPtSmZc3nY8hAbUBPYiwQAKC8CEgAAgBcCEgAAgBcCEgAAgBcCEgAAgBcCEvxGo0YNuR4SAKBcCEjwK5zJBgAoDwISAACAFwIS/Ep5vnaEw3AAAAIS/I4x5kdDEIfhAAAEJJRpXvJ8Xw+hRhGCAAA/hoCEMhUSIAAAfoyABL8TGOj09RAAADZHQAIAAPBCQAIAAPBCQIKHwECnZk9O8fUwAADwKQISSuEMLwCAvyMgwS+V54KRAAD/RUCC32JPGQDgcghIAAAAXghIAAAAXghI8FusQwIAXA4BCX7NGOPrIQAAbIiABAAA4IWABAAA4IWABL/GOiQAQFkISPB7XA8JAOCNgAS/x14kAIA3AhIs85Ln+3oIPsNeJADApQhIsBQSEgAAkERAAgAAKMUWAcnlcql///5KTU21tqWnp+vXv/614uLidPfdd2vjxo0ez/nFL36h1q1be/x8/fXXki5e/G/u3Lnq0qWLOnXqpJSUFBUXF1vPPXXqlEaPHq24uDj16NFD77zzTu1MFLbFOiQAwKWcvh5AQUGBxo4dq3379lnbcnJy9F//9V968MEH9fzzz2v37t1KSkpSZGSkunXrpqKiIn3zzTdau3atbrrpJut511xzjSRp5cqV2rJlixYuXCi3263x48crPDxcI0aMkCQlJSUpPz9f69evV0ZGhp555hm1aNFCHTp0qNW5w15YhwQAKOHTgLR//36NHTu21Nc9bN26VREREXryySclSTfddJNSU1P17rvvqlu3bjpy5IgKCwvVoUMHBQcHl+p3zZo1GjNmjBISEiRJ48aN00svvaQRI0bo22+/1SeffKKPP/5YzZs3V6tWrZSenq7XX3+dgAQAACT5+BDb9u3b1blzZ61fv95je9euXTVr1qxSjz9//ryki8GqWbNmZYaj7OxsHTt2THfccYe1LT4+XllZWTpx4oQyMjLUrFkzNW/e3KP9yy+/rK5pAQCAOs6ne5AGDx5c5vbmzZt7BJiTJ0/qvffe0+jRoyVJBw4cUGBgoB5//HHt2rVLLVq00IQJE9ShQwfl5ORIkpo2bWo9PyIiQpJ0/Phx5eTkeLRJUnh4uLKzsys8foejwk8pV3/V3S/KJzDQedkaUBt7oz72RW3syx9rU5G5+nwN0pXk5+dr9OjRioiI0H/+539Kkg4dOqQzZ87oV7/6lcaMGaMNGzZo6NChev/995Wfny9JCgoKsvooue1yuZSXl+fRVtLucrkqPLbw8CaVnZZP+r0SpzNA+uFwZ1m3K9NeE33W1GtKUkREEwU6AxQRUXYNfFUblA/1sS9qY1/Upmy2Dkjff/+9nnjiCX3zzTd6/fXXFRISIklKTk5Wfn6+GjduLEmaOnWqvvjiC73zzjv66U9/KuliGCo5BFcSfkJCQhQcHFwqDLlcLjVs2LDC4zt58py8lk9VicNx8YNa3f2Wl9tdpEJ3kTWYUrfL2nal9so8x1evKSk395wK3UXKzT3n8d74ujb4cdTHvqiNffljbUrmXB62DUjnz5/Xo48+qm+//VarV6/2OFvN6XRa4UiSHA6HWrZsqezsbEVFRUm6eCZcyWG6ksNukZGRioqKUm5ursdr5ebmKjIyssJjNEY18qGqqX5xZSXv++Xef2pjb9THvqiNfVGbstniOkjeiouLNWrUKB05ckR//OMfdeutt3q0P/zww1q4cKHH4//973+rZcuWioqKUnR0tNLS0qz2tLQ0RUdHq2nTpoqNjVVWVpaOHz/u0R4bG1vj87Kz2ZNTfD0EnwsMdPI+AAAk2XQP0ltvvaXU1FS98sorCg0NtfYABQYGKiwsTD169NCiRYvUtm1btWjRQmvWrNG5c+d03333SZIefPBBzZ07V9ddd50kad68eRo+fLgk6YYbblBiYqLGjx+vSZMm6auvvtKWLVu0du1a30zWJlyuQgUG2vLjUKu4FhIAQLJpQPrwww9VXFysxx9/3GN7p06d9Mc//lHDhg1TQUGBpk+frtzcXMXExGjlypXWYbcRI0bo5MmTGjVqlAICAjRo0CANGzbM6iclJUWTJk3SAw88oMjISM2cOZNrIAEAAIvDeF+lEeWWm1v9i7QjIppUe7/lkfz0DAUGOlVY6JakMm9Xpr0m+qzp1ywsdGvavCke748va4Mroz72RW3syx9rUzLn8rDlGiTAlzjUCAAgIAEAAHghIAEAAHip9oD03XffVXeXAAAAtapSAalt27ZlBqGsrCz9/Oc/r/KgAF8KDHRqetJMXw8DAOBD5V6NunnzZr399tuSJGOMRo4cqcDAQI/HnDhxolJXpAbshushAYB/K3dA6tWrl44cOSJJ2r59u2JjY3XVVVd5PKZRo0bq1atX9Y4QAACglpU7IF111VUaNWqUJOn6669X3759rS+DRd02L3m+r4cAAICtVOqCL/fdd58OHz6sXbt2qbCw9KGIAQMGVHVcqCWzJ6eIa4UCAOCpUgFp2bJlmjt3rq6++upSh9kcDgcBqQ7hO9h+3PSkmXpm1kRfDwMAUMsq9S/jihUrNH78eI0YMaK6xwPYCou1AcA/Veo0/4KCAvXu3bu6xwLYRmCgU7Mnp/h6GAAAH6lUQLr33nv1+uuvs3aljiMA/Dj2HgGA/6rUIbbz58/rrbfe0pYtW9S8efNS10Nas2ZNtQwONYsAcGWszwIA/1Sp//rfdNNN+u1vf1vdYwEAALCFSgWkkushAfVdydeOPPs8Z7IBgD+pVEBKSkr60fZZs2ZVajCAHXEoEgD8T6UWaXtzu906dOiQ3n//fV177bXV0SUAAIDPVGoP0uX2EC1btkxff/11lQYEAADga9WyB6lEnz599NFHH1VnlwAAALWu2gLShQsXtGHDBl1zzTXV1SVgC4GBTiU/PdPXwwAA1KJKHWJr06aNHA5Hqe3BwcGaPn16lQcF2I0xRk+NnqrxU8b6eigAgFpQqYDkfSFIh8OhwMBA3XLLLWrcuHG1DAywGzdnswGA36hUQOrUqZMk6ZtvvtGBAwdUXFysFi1aEI4AAEC9UKmAdPbsWSUlJenjjz/W1VdfraKiIn3//fe64447tGjRIjVp0qS6xwkAAFBrKrVIe/r06Tp+/Ljef/99paam6v/+7//07rvv6sKFC1wkEgAA1HmVCkj/+7//q6lTp6ply5bWtltuuUWTJ0/Wxx9/XG2DAwAA8IVKBaTg4GA1aFD6qQ6HQ0VFRVUeFGrevOT5vh5CnTU9iVP+AaC+q1RA6tGjh5577jl9++231rZvvvlG06dP11133VVtg0PNKeSMrApzBjr1/LMpfDcbAPiBSi3SHj9+vEaOHKm7775boaGhkqQzZ87oZz/7mZ599tlqHSBgJ4QjAPAPFd6DdPjwYYWEhOiPf/yjNm/erClTpmjmzJl67733tHTpUoWFhVV4EC6XS/3791dqaqq1LTMzU8OGDVNsbKz69u2rzz77zOM5//znP9W/f3/FxMRoyJAhyszM9GhftWqVunbtqri4OE2cOFF5eXlWW0FBgSZOnKiEhAQlJiZqxYoVFR4zAACov8odkIwxmj59uu655x59+eWXkqTWrVurb9++2rRpk/r376/nn39expgKDaCgoEBPPvmk9u3b5/FaI0eOVEREhDZt2qRf/vKXGjVqlI4ePSpJOnr0qEaOHKmBAwfqrbfe0rXXXqsnnnjCeu0PP/xQCxcu1LRp07R69WplZGRozpw5Vv8pKSnatWuXVq9erSlTpmjhwoX64IMPKjRuAABQf5U7IK1Zs0bvv/++Fi1aZF0ossTixYu1aNEi/elPf9Ibb7xR7hffv3+/HnjgAY+1TJK0bds2ZWZmatq0abr55pv1+OOPKzY2Vps2bZIkbdy4UbfffruGDx+uW2+9VbNmzVJWVpa2b99ujXXo0KHq3r27OnTooOeee06bNm1SXl6eLly4oI0bN2rSpElq166devXqpUcffVTr1q0r97jh3wIDnSzUBoB6rtwBacOGDXr22WfVvXv3Mtt79OihcePGVSggbd++XZ07d9b69es9tmdkZOi2225To0aNrG3x8fFKT0+32hMSEqy2kJAQtWvXTunp6SoqKtJXX33l0R4bG6vCwkLt3btXe/fuldvtVlxcnEffGRkZKi4uLvfY4b8CA52sRQKAeq7ci7SzsrLUoUOHH31Mly5dNGPGjHK/+ODBg8vcnpOTo6ZNm3psCw8P1/Hjx6/YfvbsWRUUFHi0O51OhYWF6fjx42rQoIGuueYaBQUFWe0REREqKCjQ6dOnde2115Z7/HXV7Mkpvh4CAAC2Vu6AFB4erqysLF1//fWXfczx48crtUjbW15enkeAkaSgoCC5XK4rtufn51v3y2o3xpTZJsnqv7wcjgo9vNz9VXe/3lyuQgUGVuoERr9VVm1quk4ov9r620HFURv78sfaVGSu5f5XslevXnr55Ze1YsUKBQYGlmp3u91auHChEhMTy//qlxEcHKzTp097bHO5XGrYsKHV7h1mXC6XQkNDFRwcbN33bg8JCVFRUVGZbZKs/ssrPLxmvnOupvotEegMkNMZIP2wqL3kdlnbqtpeE3366jUlyRlw8b0LdAYoIoLvHLSbmv7bQeVRG/uiNmUrd0B64oknNGjQIA0cOFAPP/ywbr/9djVp0kRnzpzR7t27tXbtWn3//fdKSan64ZuoqCjt37/fY1tubq512CwqKkq5ubml2tu2bauwsDAFBwcrNzdXN998s6SL4e306dOKjIyUMUanTp2S2+2W03lx+jk5OWrYsKF1TafyOnny3KX/dlaZw3Hxg1rd/XordBdJDsfF3z+88GW3VbW9Jvr0wWs6HBcvFOku+v/tubnnKl8EVKva+ttBxVEb+/LH2pTMuTzKHZBCQ0O1YcMGzZ07V88//7x1XSFjjJo0aaK+fftq9OjRioiIqNyoLxETE6OlS5cqPz/f2quTlpam+Ph4qz0tLc16fF5envbs2aNRo0apQYMGat++vdLS0tS5c2dJUnp6upxOp9q0aXNx0k6n0tPTrYXcaWlpat++fZlfn/JjjFGNfKhqql9UXkk9Lq0LNbIf/nbsi9rYF7UpW4UWooSFhWn69OmaPHmyMjMzdfbsWYWFhek//uM/FBAQUG2D6tSpk5o1a6akpCQ98cQT+uSTT7Rz507NmjVLknT//fdr+fLlWrp0qbp3765FixapefPmViAaPHiwJk+erFatWqlp06aaOnWqHnjgAYWEhEiSBgwYoKlTp2rmzJk6ceKEVqxYYfUNlEfJqf7PzJro66EAAGpApVbqBgUFWYevakJAQIAWL16sSZMmaeDAgbrxxhu1aNEiRUdHS5KaN2+ul19+WTNnztSiRYsUFxenRYsWyfHD6qt+/fopKytLkydPlsvlUu/evTV+/Hir/6SkJE2dOlVDhw5V48aNNXr0aPXu3bvG5mMnnMFWfTjVHwDqL4ep6KWvYcnNrf41SBERTaq930slP33xMgyBgU4VFro9bpe1rartNdGnr14zJCRYeXkF1rbCQremzZtSPYVBldTG3w4qh9rYlz/WpmTO5VHh72IDcBFX1AaA+ouABFQBh9kAoH4iIAEAAHghIAFVwBXJAaB+IiABVcA6JAConwhIQBWxDgkA6h8CEgAAgBcCEgAAgBcCEgAAgBcCElBFLNQGgPqHgORH5iXP9/UQ6i1jDCEJAOoRApIfKeRsqxrF2WwAUH8QkIBqwAUjAaB+ISABAAB4ISABAAB4ISABAAB4ISAB1YTT/QGg/iAgAdWIM9kAoH4gIAEAAHghIPmJ2ZNTfD0EAADqDAKSn+DQDwAA5UdAAgAA8EJAAqpRYKCTw5kAUA8QkIBqxhfXAkDdR0ACagBrvgCgbiMgAQAAeCEg+YF5yfN9PQS/w1W1AaBuIyD5gUIO9/gEh9kAoO4iIAEAAHghIAEAAHghIAE1pOSaSKxFAoC6x7YB6e2331br1q1L/bRp00aS9Lvf/a5U2yeffGI9f9WqVeratavi4uI0ceJE5eXlWW0FBQWaOHGiEhISlJiYqBUrVtT6/OAfjDGsRQKAOsjp6wFcTt++fdW1a1frvtvt1tChQ9WtWzdJ0oEDBzRnzhz95Cc/sR5z9dVXS5I+/PBDLVy4UHPmzFF4eLiSkpI0Z84cTZ48WZKUkpKiXbt2afXq1Tp69KieeuopRUdHq0+fPrU3wVrCVZ0BAKg42wakhg0bqmHDhtb9JUuWyBijcePGyeVy6ciRI2rfvr0iIyNLPXfNmjUaOnSounfvLkl67rnnNGLECI0fP17GGG3cuFGvvfaa2rVrp3bt2mnfvn1at25dvQxILlehAgNtW2YAAGzJtofYLnX69Gm99tprGjt2rIKCgnTw4EE5HA7dcMMNpR5bVFSkr776SgkJCda22NhYFRYWau/evdq7d6/cbrfi4uKs9vj4eGVkZKi4uLhW5gMAAOytTuxaeOONN9S0aVNrD8/BgwfVuHFjTZgwQdu3b9d1112n0aNH66677tLZs2dVUFCgpk2bWs93Op0KCwvT8ePH1aBBA11zzTUKCgqy2iMiIlRQUKDTp0/r2muvLfe4HI7qm+Ol/VV3v6i6qtaGmtYs/nbsi9rYlz/WpiJztX1AKjkk9uijj1rbDh48qPz8fCUmJuqxxx7TRx99pN/97ndav369IiIiJMkjAJXcd7lcMsaU2SZJLperQmMLD29SmSnVar+BzgA5nQGSMdZvSWXeron2+vSakuQMCKhwn4HOAEVE1MxnBZ5q6m8SVUdt7IvalM32Aemrr75Sdna2+vXrZ2174okn9PDDD1uLstu0aaPdu3drw4YN+u///m9JpcOOy+VSSEiIioqKymyT5LHmqTxOnjx36b+dVeZwXPygVme/he4iyeHw/P3Di5W6XRPt9eQ1HQ7JGeiUu6jirymHQ79/NEnPPj+x/IVDhdTE3w6qB7WxL3+sTcmcy8P2Aenvf/+7EhISrDAkSQ0aNPC4L0ktW7bU/v37FRYWpuDgYOXm5urmm2+WdPEMuNOnTysyMlLGGJ06dUput1tO58Xp5+TkqGHDhgoNDa3Q2IxRjXyoaqpfVF5JPSpbF5erkJrWAv527Iva2Be1KZvtF2nv3LlTHTt29Nj29NNPKykpyWPb3r171bJlSzVo0EDt27dXWlqa1Zaeni6n06k2bdqobdu2cjqdSk9Pt9rT0tLUvn17NWhg+7cDAADUAtsngn379umWW27x2NajRw+9++672rx5sw4fPqyFCxcqLS1NDz30kCRp8ODBWr58ubZu3aqdO3dq6tSpeuCBBxQSEqKQkBANGDBAU6dO1c6dO7V161atWLFCQ4YM8cX0AACADdn+EFtubm6pQ1+9e/fWlClT9Morr+jo0aO69dZbtWzZMjVv3lyS1K9fP2VlZWny5MlyuVzq3bu3xo8fbz0/KSlJU6dO1dChQ9W4cWONHj1avXv3rtV51QYuEmkfXIsKAOoWhzEceays3NzqX6QdEdGk2vpNfnqGpIv/OBcWuq3fl26r6fb69JohIcHKyyuodJ9ud5GemcVC7ZpQ3X87qD7Uxr78sTYlcy4P2x9iA+oLvpMNAOoOAhIAAIAXAhJQSwIDnZqeNNPXwwAAlAMBCahFHGYDgLqBgAQAAOCFgATUosBAJ5dfAIA6gIAE1DIOswGA/RGQAAAAvBCQAAAAvBCQgFrG6f4AYH8EJMAHStYhEZQAwJ4ISIAPlJzNVtaCbUITAPgeAQnwkcudzcZZbgDgewSkempe8nxfDwEAgDqLgFRPFbIXAgCASiMgAT4UGOj09RAAAGUgIAE+xCn/AGBPBCTAx1iUDQD2Q0ACfIwvsAUA+yEg1UP8Y1v3sBcJAOyFgFQP8Y8tAABVQ0ACbIDF2gBgLwQkwCaMMRweBQCbICABNsLhUQCwBwISAACAFwJSPcN3sAEAUHUEpHqG72ADAKDqCEiAzTRq1JAz2gDAxwhIgA2xWBsAfIuABAAA4MXWAemjjz5S69atPX7GjBkjSdqzZ49+9atfKSYmRvfff7927drl8dwtW7aoZ8+eiomJ0ciRI/Xdd99ZbcYYzZ07V126dFGnTp2UkpKi4uLiWp0b8GNKvp+NQ20A4Bu2Dkj79+9X9+7d9dlnn1k/06dP14ULF/TYY48pISFBb7/9tuLi4vT444/rwoULkqSdO3dq0qRJGjVqlNavX6+zZ88qKSnJ6nflypXasmWLFi5cqAULFujdd9/VypUrfTVNoEwuVyGH2gDAR2wdkA4cOKBWrVopMjLS+gkNDdX777+v4OBgTZgwQTfffLMmTZqkq666Sh988IEkae3atbrnnns0YMAAtWnTRikpKfrrX/+qzMxMSdKaNWs0ZswYJSQkqEuXLho3bpzWrVvny6kCAAAbsX1Auummm0ptz8jIUHx8vBwOhyTJ4XCoY8eOSk9Pt9oTEhKsxzdr1kzR0dHKyMhQdna2jh07pjvuuMNqj4+PV1ZWlk6cOFGj86lpfE0FAADVw7YByRijQ4cO6bPPPtPdd9+tnj17au7cuXK5XMrJyVHTpk09Hh8eHq7jx49Lkk6cOHHZ9pycHEnyaI+IiJAk6/l1FYdj6p/AQKevhwAAfsm2//U9evSo8vLyFBQUpBdffFFHjhzR9OnTlZ+fb22/VFBQkFwulyQpPz//su35+fnW/UvbJFnPL68fdmBVm5L+qrtfVJ0va8Pn4cr427EvamNf/libiszVtgHp+uuvV2pqqq6++mo5HA61bdtWxcXFGj9+vDp16lQqzLhcLjVs2FCSFBwcXGZ7SEiIRxgKDg62bktSSEhIhcYYHt6kUnOrqX4DnQFyOgMkY6zfkn50W02316fXlCRnQEDtzjPQqVnPzNa8V6eX/4Pgx2rqbxJVR23si9qUzbYBSZLCwsI87t98880qKChQZGSkcnNzPdpyc3Otw2ZRUVFltkdGRioqKkqSlJOTo+bNm1u3JSkyMrJC4zt58tyl/3ZWmcNx8YNa2X4L3UWSw+H5+4eOL7utptvryWs6HBfDirvIB/MsdOv3jybp2ecnll14VPlvBzWH2tiXP9amZM7lYds1SH//+9/VuXNn5eXlWdv+9a9/KSwsTPHx8fryyy9lfqioMUZffPGFYmJiJEkxMTFKS0uznnfs2DEdO3ZMMTExioqKUnR0tEd7WlqaoqOjS61buhJjqv+nsv0+/ywLtGvSpbXxzesbJT89s0Y+c/Xlp6Q+/Njvh9rY98cfa1Netg1IcXFxCg4O1jPPPKODBw/qr3/9q1JSUvToo4+qT58+Onv2rGbMmKH9+/drxowZysvL0z333CNJevDBB/XOO+9o48aN2rt3ryZMmKBu3brphhtusNrnzp2r1NRUpaamat68eRoyZIgvp1tlLNCu/6gxANQe2x5ia9y4sZYvX66ZM2fq/vvv11VXXaVf//rXevTRR+VwOLRkyRJNmTJFGzZsUOvWrbV06VI1atRI0sVwNW3aNC1YsEBnzpzRnXfeqeTkZKvvESNG6OTJkxo1apQCAgI0aNAgDRs2zEczBSpmetJMPTOLw20AUJNsG5Ak6dZbb73sFa47dOigP/3pT5d97sCBAzVw4MAy2wICApSUlORxdW3AzgIDnSosdEtiTxIA1AbbHmID4Knk+9kAADWPgATUIS5XIRePBIBaQEACAADwQkACAADwQkAC6pjAQKemJ8309TAAoF4jIAF1EGeyAUDNIiABdRB7kQCgZhGQgDqKvUgAUHMISAAAAF4ISAAAAF4ISPXAvOT5vh4CfIALRgJAzSEg1QOFrEXxSyzUBoCaQ0AC6jAWagNAzSAgAQAAeCEg1XF8u7t/4zAbANQMAlIdxyEWGGMISQBQzQhIQD1AUAaA6kVAAgAA8EJAAgAA8EJAqsNYoA0AQM0gINVhrDtBCa6qDQDVi4AEAADghYAE1ANcDwkAqhcBCagnOOQKANWHgAQAAOCFgATUExxmA4DqQ0AC6hEOswFA9SAg1VHzkuf7eggAANRbBKQ6qpA9BSgDh9kAoHoQkIB6hsNsAFB1BCQAAAAvtg5I2dnZGjNmjDp16qSuXbtq1qxZKigokCRNnz5drVu39vhZu3at9dwtW7aoZ8+eiomJ0ciRI/Xdd99ZbcYYzZ07V126dFGnTp2UkpKi4uLiWp8fAACwJ9t+gZMxRmPGjFFoaKjWrVunM2fOaOLEiWrQoIGeeuopHThwQGPHjtV9991nPadx48aSpJ07d2rSpEl67rnn1KZNG82YMUNJSUlasmSJJGnlypXasmWLFi5cKLfbrfHjxys8PFwjRozwyVyB6sT3sgFA1dl2D9LBgweVnp6uWbNm6dZbb1VCQoLGjBmjLVu2SJIOHDig2267TZGRkdZPSEiIJGnt2rW65557NGDAALVp00YpKSn661//qszMTEnSmjVrNGbMGCUkJKhLly4aN26c1q1b57O5AgAAe7FtQIqMjNSyZcsUERHhsf38+fM6f/68srOzddNNN5X53IyMDCUkJFj3mzVrpujoaGVkZCg7O1vHjh3THXfcYbXHx8crKytLJ06cqJG5VLfZk1N8PQQAAOo12+6LDw0NVdeuXa37xcXFWrt2rbp06aIDBw7I4XDo1Vdf1d/+9jeFhYXpkUcesQ63nThxQk2bNvXoLzw8XMePH1dOTo4kebSXhLDjx4+Xet6PcTgqPb0f7e9K/bpchRxGqWXlrY1d1JVxVpe6Vh9/Qm3syx9rU5G51pl/ZefMmaM9e/borbfe0u7du+VwONSyZUs99NBD2rFjh5599lk1btxYvXr1Un5+voKCgjyeHxQUJJfLpfz8fOv+pW2S5HK5KjSm8PAmVZxV5foNdAbI6QyQjJGkMm+Xd1tNt9en15QkZ0DZ772d5ilJERE189m0u5r6m0TVURv7ojZlqxMBac6cOVq9erVeeOEFtWrVSrfeequ6d++usLAwSVKbNm30zTff6I033lCvXr0UHBxcKuy4XC6FhIR4hKHg4GDrtiRrDVN5nTx57tJ/k6rM4bj4Qb1Sv4XuIsnhuPj7hyeWul3ebTXdXk9e0+GQnIFOuYvsP8/AQKd+/2iSnn1+ovxFef92UPuojX35Y21K5lwetg9IycnJeuONNzRnzhzdfffdkiSHw2GFoxItW7bUtm3bJElRUVHKzc31aM/NzVVkZKSioqIkSTk5OWrevLl1W7q47qkijFGNfKhqql9UXkk96kpdXK7COjPW6sTfjn1RG/uiNmWz7SJtSVq4cKHefPNNzZ8/X/369bO2v/TSSxo2bJjHY/fu3auWLVtKkmJiYpSWlma1HTt2TMeOHVNMTIyioqIUHR3t0Z6Wlqbo6OgKrT8C7IyvHAGAqrHtHqQDBw5o8eLFeuyxxxQfH2/t5ZGk7t27a+nSpVq+fLl69eqlzz77TJs3b9aaNWskSQ8++KAefvhhxcbGqn379poxY4a6deumG264wWqfO3eurrvuOknSvHnzNHz48NqfJFCD+MoRAKg82wakjz/+WEVFRXrllVf0yiuveLT9+9//1ksvvaQFCxbopZde0vXXX6958+YpLi5OkhQXF6dp06ZpwYIFOnPmjO68804lJydbzx8xYoROnjypUaNGKSAgQIMGDSq1RwoAAPgv2wakxx57TI899thl23v27KmePXtetn3gwIEaOHBgmW0BAQFKSkpSUlJSlccJAADqH1uvQUJp85Ln+3oIqCO4VhYAVB4BqY4pZF0JAAA1joAEAADghYAEAADghYAE1FNcCwkAKo+ABNRjXAsJACqHgAQAAOCFgATUYxxmA4DKISAB9RyH2QCg4ghIdcjsySm+HgLqIPYiAUDFEZDqEPYEoLL47ABAxRCQAAAAvBCQAAAAvBCQAD/AF9cCQMUQkAAAALwQkAAAALwQkOoITvFHVXCqPwBUDAGpjuA0bVQVnyEAKD8CEuAn2IsEAOVHQAL8CHuRAKB8CEgAAABeCEiAH+EwGwCUDwEJ8DMcZgOAKyMgAX6Gq2oDwJURkOqAecnzfT0EAAD8CgGpDijkkAiqEeuQAODKCEiAHzLGEJIA4EcQkGyOrxhBTWGxNgBcHgHJ5vhHDDWFxdoAcHkEJMBPsRYJAC7PbwNSQUGBJk6cqISEBCUmJmrFihW+HpKHecnzObyGGsdaJAAom98GpJSUFO3atUurV6/WlClTtHDhQn3wwQe+Hpal0FXI4TXUCkISAJTml4sQLly4oI0bN+q1115Tu3bt1K5dO+3bt0/r1q1Tnz59fD08oNYZY3w9BACwFb/cg7R371653W7FxcVZ2+Lj45WRkaHi4mIfjgzwjcBAp2ZPTmFPEgD8wC8DUk5Ojq655hoFBQVZ2yIiIlRQUKDTp0/7bmCAD7lchTLGaF7yfIISAL/nl4fY8vLyPMKRJOu+y+Uqdz8NGkjVeWTC4bj4e2HKIjUOvUruwiJJkjMwoNTtsrZdqb0yz+E1L952OKSg4CAFBgUqwFl/51lyOyAgQK++sERut/viNqdTbrfb+n3pNu/2/LwCPfnMGC2YvUglStrdhUV68pkx1vb50xdY9y+9fSXejy3526nuv0lUHbWxL3+sTcmcy/VY44eLD/785z9r+vTp+sc//mFtO3DggPr27avU1FSFhYX5bnAAAMDn/PIQW1RUlE6dOmX936908bBbw4YNFRoa6sORAQAAO/DLgNS2bVs5nU6lp6db29LS0tS+fXs1aOCXbwkAALiEX6aBkJAQDRgwQFOnTtXOnTu1detWrVixQkOGDPH10AAAgA345Rok6eJC7alTp+ovf/mLGjdurBEjRmjYsGG+HhYAALABvw1IAAAAl+OXh9gAAAB+DAEJAADACwEJAADACwHJJgoKCjRx4kQlJCQoMTFRK1as8PWQ6j2Xy6X+/fsrNTXV2paZmalhw4YpNjZWffv21WeffebxnH/+85/q37+/YmJiNGTIEGVmZnq0r1q1Sl27dlVcXJwmTpyovLy8WplLfZKdna0xY8aoU6dO6tq1q2bNmqWCggJJ1MfXDh8+rBEjRiguLk7dunXTsmXLrDZqYx+PPfaYnn76aev+nj179Ktf/UoxMTG6//77tWvXLo/Hb9myRT179lRMTIxGjhyp7777zmozxmju3Lnq0qWLOnXqpJSUFP/5zlIDW5g2bZq59957za5du8xf/vIXExcXZ/785z/7elj1Vn5+vhk5cqRp1aqV2bZtmzHGmOLiYnPvvfeasWPHmv3795tXX33VxMTEmKysLGOMMVlZWSY2NtYsX77cfP311+b3v/+96d+/vykuLjbGGPPBBx+Y+Ph487//+78mIyPD9O3b1zz33HM+m2NdVFxcbB544AHz6KOPmq+//trs2LHD9OrVyzz//PPUx8eKiopM7969zdixY82hQ4fMp59+ajp27Gj+53/+h9rYyJYtW0yrVq3MU089ZYwx5vvvvzd33nmnef75583+/ftNcnKy+elPf2q+//57Y4wxGRkZpkOHDuZPf/qT+de//mUeeugh89hjj1n9LV++3Nx1111mx44d5vPPPzeJiYlm2bJlPplbbSMg2cD3339v2rdvb/1DbYwxixYtMg899JAPR1V/7du3z/ziF78w9957r0dA+uc//2liY2Ot/3AYY8zQoUPNggULjDHGvPjiix41uXDhgomLi7OeP3jwYOuxxhizY8cO06FDB3PhwoXamFa9sH//ftOqVSuTk5NjbXv33XdNYmIi9fGx7Oxs8/vf/96cO3fO2jZy5EgzZcoUamMTp06dMj/72c/M/fffbwWkjRs3mh49elhhtLi42PTq1cts2rTJGGPM+PHjrccaY8zRo0dN69atzbfffmuMMeauu+6yHmuMMZs3bzbdu3evrSn5FIfYbGDv3r1yu92Ki4uztsXHxysjI8N/dmXWou3bt6tz585av369x/aMjAzddtttatSokbUtPj7euuJ6RkaGEhISrLaQkBC1a9dO6enpKioq0ldffeXRHhsbq8LCQu3du7dmJ1SPREZGatmyZYqIiPDYfv78eerjY02bNtWLL76oxo0byxijtLQ07dixQ506daI2NjF79mz98pe/1C233GJty8jIUHx8vBw/fEurw+FQx44dL1ubZs2aKTo6WhkZGcrOztaxY8d0xx13WO3x8fHKysrSiRMnamdSPkRAsoGcnBxdc801CgoKsrZFRESooKBAp0+f9t3A6qnBgwdr4sSJCgkJ8diek5Ojpk2bemwLDw/X8ePHr9h+9uxZFRQUeLQ7nU6FhYVZz8eVhYaGqmvXrtb94uJirV27Vl26dKE+NtKjRw8NHjxYcXFxuvvuu6mNDXz++ef6v//7Pz3xxBMe269UmxMnTly2PScnR5I82kv+58UfakNAsoG8vDyPcCTJuu9yuXwxJL90uTqU1ODH2vPz8637l3s+Km7OnDnas2eP/vu//5v62MiCBQv06quv6l//+pdmzZpFbXysoKBAU6ZM0eTJk9WwYUOPtivVJj8/v0K18ad/m5y+HgCk4ODgUh+2kvveH3bUnODg4FJ77Fwul1WDy9UpNDRUwcHB1n3vdu89VSifOXPmaPXq1XrhhRfUqlUr6mMj7du3l3TxH+Zx48bp/vvvL3XWGbWpPQsXLtTtt9/usfe1xOXe+yvVJiQkxCMMedfJH2rDHiQbiIqK0qlTp+R2u61tOTk5atiwoUJDQ304Mv8SFRWl3Nxcj225ubnW7uXLtUdGRiosLEzBwcEe7W63W6dPn1ZkZGTND76eSU5O1sqVKzVnzhzdfffdkqiPr+Xm5mrr1q0e22655RYVFhYqMjKS2vjQe++9p61btyouLk5xcXF699139e677youLq5KfzdRUVGSZB1qu/S2P9SGgGQDbdu2ldPptBbNSVJaWprat2+vBg0oUW2JiYnR7t27rd3K0sU6xMTEWO1paWlWW15envbs2aOYmBg1aNBA7du392hPT0+X0+lUmzZtam8S9cDChQv15ptvav78+erXr5+1nfr41pEjRzRq1ChlZ2db23bt2qVrr71W8fHx1MaH/vjHP+rdd9/V5s2btXnzZvXo0UM9evTQ5s2bFRMToy+//FLmh69dNcboiy++uGxtjh07pmPHjikmJkZRUVGKjo72aE9LS1N0dHSpdUv1ko/PosMPnn32WdOvXz+TkZFhPvroI9OxY0fz4Ycf+npY9d6lp/m73W7Tt29f84c//MF8/fXXZsmSJSY2Nta6lktmZqZp3769WbJkiXUtl3vvvdc6fXbLli2mY8eO5qOPPjIZGRmmX79+Jjk52Wdzq4v2799v2rZta1544QVz4sQJjx/q41tut9sMHDjQDB8+3Ozbt898+umn5qc//alZtWoVtbGZp556yjp1/9y5c6ZLly4mOTnZ7Nu3zyQnJ5s777zTuiTDF198Ydq1a2c2bNhgXQfp8ccft/pasmSJSUxMNNu2bTPbtm0ziYmJZsWKFT6ZV20jINnEhQsXzIQJE0xsbKxJTEw0K1eu9PWQ/MKlAckYY7755hvzm9/8xtx+++2mX79+5h//+IfH4z/99FPTu3dv06FDBzN06FDrWiEllixZYn7yk5+Y+Ph4k5SUZPLz82tlHvXFkiVLTKtWrcr8MYb6+Nrx48fNyJEjTceOHc2dd95pXnnlFSvkUBv7uDQgGXPxYpADBgww7du3N4MGDTK7d+/2ePymTZvMXXfdZWJjY83IkSPNd999Z7W53W4zc+ZMk5CQYDp37mzmzJlj1by+cxjzw343AAAASGINEgAAQCkEJAAAAC8EJAAAAC8EJAAAAC8EJAAAAC8EJAAAAC8EJAAAAC8EJAAAAC8EJAA+deTIEbVu3VpHjhzx6That26t1NTUSj33z3/+s06ePFnNI7ro888/14EDB2qkbwCXR0ACgCrIysrSH/7wB+Xl5dVI/8OGDSv1besAah4BCQCqgG9rAuonAhIA2zh79qzGjx+vjh07KjExUcnJycrPz5ckpaamqkePHnr99dfVtWtXxcbGavz48XK5XOXq+/z580pKStJPfvIT3X777erTp4+2bt3q8ZgdO3aod+/eiomJ0e9//3udOXPGaps/f74SExPVoUMHPfzww9q3b58k6ec//7n1++2339bLL7+sJ554Qr/5zW/UqVMnbd++XdnZ2RozZozuuOMO3X777brvvvuUlpZm9X348GGNGDFCcXFx6tatm9asWSNJ6tGjhyRpyJAhevnllyv5rgKoDAISANuYNGmSzp07pzfeeEOLFy/WV199pWnTplntJ06c0Icffqhly5bp5Zdf1l/+8hdt3ry5XH3PmDFDhw4d0ooVK7RlyxYlJCRo0qRJHgFr3bp1mjRpktatW6dDhw5p1qxZkqSPPvpI69ev14svvqgtW7YoIiJCSUlJkqSNGzdav/v27StJ+vjjj9W/f3+tXr1aHTp00Lhx41RUVKQ333xTmzdvVlRUlKZOnSpJKigo0PDhw3XVVVdpw4YNmjx5sl544QV98skneuuttyRJL7/8soYPH16l9xZABRkA8KHMzEzTqlUrc/jwYdOmTRtz9uxZq23v3r3Wtm3btplWrVqZr7/+2mofOXKkeeaZZ8r1Ops2bTL//ve/rfsHDhwwrVq1MkePHjXGGNOqVSuzdu1aqz01NdXcdttt5ty5c2blypXmzjvvNFlZWcYYY06ePGl27NjhMf7MzExjjDELFiwwP/3pT61+iouLzapVq8yxY8esbX/7299MmzZtjDHGbN261cTGxppz585Z7W+99Zb59NNPrXFt27atXHMEUH2cvg5oACBJf/vb31RcXKyf/exnHtuLi4t1+PBh6/6NN95o3W7cuLHcbne5+h8wYIC2bt2qDRs26ODBg9q9e7ckqaioyHpM+/btrdu33Xab3G63vv32W/Xr109r167Vz3/+c8XGxqpnz54aNGjQZV/r+uuvt247HA49+OCDev/99/XFF1/o0KFD2rVrl4qLiyVJhw4dUosWLdS4cWPrOffff3+55gSg5hCQANiC2+1WkyZNtGnTplJtUVFRysjIkCQFBQV5tJlyLpKeMGGCvvzyS/3yl7/Ugw8+qMjISP3nf/6nx2MCAgJK9RsYGKjIyEj9+c9/1j/+8Q998sknWr58uTZs2HDZw3vBwcHW7eLiYg0fPlxnz55V37591aNHDxUWFmrUqFGSJKeT/wwDdsQaJAC20LVrV507d04Oh0M33nijbrzxRuXn5yslJaXcC7Ev5/z589qyZYteeOEFjRkzRr169bIWYF8asL7++mvr9s6dOxUYGKjmzZvr008/1caNG9WtWzc999xzeuedd/TNN9/o66+/lsPh+NHX3r9/v3bs2KFVq1bpt7/9rbp166YTJ05Yr33TTTfp8OHDHpcJmD17tqZPn16lOQOoGgISAFsIDg5W165dNW7cOO3cuVO7d+9WUlKSLly4oNDQ0Cr1HRQUpJCQEP3lL3/RkSNH9Pe//91a/H1p+HrhhRf0+eefKz09XdOnT9evf/1rhYSEqLi4WCkpKfroo4905MgRvf322woJCdFNN92kkJAQSdLevXv1/fffl3rt0NBQNWjQQO+9956ysrL0wQcfWGekuVwuJSYmKiIiQpMnT9aBAwf08ccf680331RiYqIkqVGjRtq3b5/OnTtXpfcAQMUQkADYRkpKipo3b65hw4bpkUceUYsWLTR//vwq9xsUFKQ5c+boww8/VL9+/fT888/rd7/7nSIjI/Wvf/3LetwjjzyiSZMm6ZFHHlFcXJzGjRsn6eLp9mPGjNGsWbN0zz336P3339fixYt19dVX69prr9UvfvEL/eEPf7DOaLvUddddp6lTp+q1115T//79tXTpUj3zzDNyOp3as2ePnE6nFi9erBMnTui+++7TjBkzNGHCBHXr1k2S9PDDDyslJYXT/IFa5jDlPYAPAADgJ9iDBAAA4IXTJwDUeTNmzLAuqliWxx9/XL/97W9rcUQA6joOsQGo87777rsfXcR89dVXKywsrPYGBKDOIyABAAB4YQ0SAACAFwISAACAFwISAACAFwISAACAFwISAACAFwISAACAFwISAACAFwISAACAl/8HLdLj2gnQFo4AAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sns.histplot(x='len_abstract', data=df_cleaned)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Filter data to only have records with absract leght between 446.00 and 908.00"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df_cleaned[df_cleaned['len_abstract'].between(446.00, 908.00)].reset_index(drop=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>submitter</th>\n",
+       "      <th>authors</th>\n",
+       "      <th>title</th>\n",
+       "      <th>comments</th>\n",
+       "      <th>journal-ref</th>\n",
+       "      <th>doi</th>\n",
+       "      <th>report-no</th>\n",
+       "      <th>categories</th>\n",
+       "      <th>license</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>versions</th>\n",
+       "      <th>update_date</th>\n",
+       "      <th>authors_parsed</th>\n",
+       "      <th>cleaned_abstracts</th>\n",
+       "      <th>len_abstract</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0001</td>\n",
+       "      <td>Pavel Nadolsky</td>\n",
+       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
+       "      <td>Calculation of prompt diphoton production cros...</td>\n",
+       "      <td>37 pages, 15 figures; published version</td>\n",
+       "      <td>Phys.Rev.D76:013009,2007</td>\n",
+       "      <td>10.1103/PhysRevD.76.013009</td>\n",
+       "      <td>ANL-HEP-PR-07-12</td>\n",
+       "      <td>hep-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>A fully differential calculation in perturba...</td>\n",
+       "      <td>[{'created': 'Mon, 2 Apr 2007 19:18:42 GMT', '...</td>\n",
+       "      <td>2008-11-26</td>\n",
+       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
+       "      <td>fully differential calculation perturbative...</td>\n",
+       "      <td>695</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Louis Theran</td>\n",
+       "      <td>Ileana Streinu and Louis Theran</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>To appear in Graphs and Combinatorics</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO cs.CG</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 02:26:18 GMT', ...</td>\n",
+       "      <td>2008-12-13</td>\n",
+       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
+       "      <td>describe new algorithm $ k,\\ell)$-pebble ga...</td>\n",
+       "      <td>619</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>23 pages, 3 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>physics.gen-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td>[{'created': 'Sun, 1 Apr 2007 20:46:54 GMT', '...</td>\n",
+       "      <td>2008-01-13</td>\n",
+       "      <td>[[Pan, Hongjun, ]]</td>\n",
+       "      <td>evolution earth moon system describe dark m...</td>\n",
+       "      <td>631</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0006</td>\n",
+       "      <td>Yue Hin Pong</td>\n",
+       "      <td>Y. H. Pong and C. K. Law</td>\n",
+       "      <td>Bosonic characters of atomic Cooper pairs acro...</td>\n",
+       "      <td>6 pages, 4 figures, accepted by PRA</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevA.75.043613</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cond-mat.mes-hall</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We study the two-particle wave function of p...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 04:24:59 GMT', ...</td>\n",
+       "      <td>2015-05-13</td>\n",
+       "      <td>[[Pong, Y. H., ], [Law, C. K., ]]</td>\n",
+       "      <td>study particle wave function pair atom ferm...</td>\n",
+       "      <td>638</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0007</td>\n",
+       "      <td>Alejandro Corichi</td>\n",
+       "      <td>Alejandro Corichi, Tatjana Vukasinac and Jose ...</td>\n",
+       "      <td>Polymer Quantum Mechanics and its Continuum Limit</td>\n",
+       "      <td>16 pages, no figures. Typos corrected to match...</td>\n",
+       "      <td>Phys.Rev.D76:044016,2007</td>\n",
+       "      <td>10.1103/PhysRevD.76.044016</td>\n",
+       "      <td>IGPG-07/03-2</td>\n",
+       "      <td>gr-qc</td>\n",
+       "      <td>None</td>\n",
+       "      <td>A rather non-standard quantum representation...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 04:27:22 GMT', ...</td>\n",
+       "      <td>2008-11-26</td>\n",
+       "      <td>[[Corichi, Alejandro, ], [Vukasinac, Tatjana, ...</td>\n",
+       "      <td>non standard quantum representation canonic...</td>\n",
+       "      <td>734</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1135476</th>\n",
+       "      <td>supr-con/9608007</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Francesca Federici, Andrei A. Varlamov</td>\n",
+       "      <td>The Fluctuation Induced Pseudogap in the Infra...</td>\n",
+       "      <td>8 pages, 4 eps figures, Submitted to Phys. Rev. B</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.55.6070</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We study the effect of fluctuations on the {...</td>\n",
+       "      <td>[{'created': 'Fri, 23 Aug 1996 09:39:49 GMT', ...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Federici, Francesca, ], [Varlamov, Andrei A....</td>\n",
+       "      <td>study effect fluctuation \\bf ac conductivit...</td>\n",
+       "      <td>703</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1135477</th>\n",
+       "      <td>supr-con/9609001</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Durga P. Choudhury, Balam A. Willemsen, John S...</td>\n",
+       "      <td>Nonlinear Response of HTSC Thin Film Microwave...</td>\n",
+       "      <td>4 pages, LaTeX type, Uses IEEE style files, 60...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1109/77.620744</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The non-linear microwave surface impedance o...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Aug 1996 17:34:38 GMT', ...</td>\n",
+       "      <td>2016-11-18</td>\n",
+       "      <td>[[Choudhury, Durga P., , Physics Department, N...</td>\n",
+       "      <td>non linear microwave surface impedance patt...</td>\n",
+       "      <td>468</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1135478</th>\n",
+       "      <td>supr-con/9609002</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Balam A. Willemsen, J. S. Derov and S.Sridhar ...</td>\n",
+       "      <td>Critical State Flux Penetration and Linear Mic...</td>\n",
+       "      <td>20 pages, LaTeX type, Uses REVTeX style files,...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.56.11989</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The vortex contribution to the dc field (H) ...</td>\n",
+       "      <td>[{'created': 'Tue, 3 Sep 1996 14:08:26 GMT', '...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Willemsen, Balam A., , Physics Department,\\n...</td>\n",
+       "      <td>vortex contribution dc field h dependent mi...</td>\n",
+       "      <td>841</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1135479</th>\n",
+       "      <td>supr-con/9609003</td>\n",
+       "      <td>Hasegawa Yasumasa</td>\n",
+       "      <td>Yasumasa Hasegawa (Himeji Institute of Technol...</td>\n",
+       "      <td>Density of States and NMR Relaxation Rate in A...</td>\n",
+       "      <td>7 pages, 4 PostScript Figures, LaTeX, to appea...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1143/JPSJ.65.3131</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that the density of states in an ani...</td>\n",
+       "      <td>[{'created': 'Wed, 18 Sep 1996 07:57:29 GMT', ...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Hasegawa, Yasumasa, , Himeji Institute of Te...</td>\n",
+       "      <td>density state anisotropic superconductor \\n...</td>\n",
+       "      <td>449</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1135480</th>\n",
+       "      <td>supr-con/9609004</td>\n",
+       "      <td>Masanori Ichioka</td>\n",
+       "      <td>Naoki Enomoto, Masanori Ichioka and Kazushige ...</td>\n",
+       "      <td>Ginzburg Landau theory for d-wave pairing and ...</td>\n",
+       "      <td>12 pages including 8 eps figs, LaTeX with jpsj...</td>\n",
+       "      <td>J. Phys. Soc. Jpn. 66, 204 (1997).</td>\n",
+       "      <td>10.1143/JPSJ.66.204</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The Ginzburg Landau theory for d_{x^2-y^2}-w...</td>\n",
+       "      <td>[{'created': 'Wed, 25 Sep 1996 14:17:09 GMT', ...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...</td>\n",
+       "      <td>ginzburg landau theory d_{x^2 y^2}-wave sup...</td>\n",
+       "      <td>528</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1135481 rows × 16 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       id           submitter   \n",
+       "0               0704.0001      Pavel Nadolsky  \\\n",
+       "1               0704.0002        Louis Theran   \n",
+       "2               0704.0003         Hongjun Pan   \n",
+       "3               0704.0006        Yue Hin Pong   \n",
+       "4               0704.0007   Alejandro Corichi   \n",
+       "...                   ...                 ...   \n",
+       "1135476  supr-con/9608007                None   \n",
+       "1135477  supr-con/9609001  Durga P. Choudhury   \n",
+       "1135478  supr-con/9609002  Durga P. Choudhury   \n",
+       "1135479  supr-con/9609003   Hasegawa Yasumasa   \n",
+       "1135480  supr-con/9609004    Masanori Ichioka   \n",
+       "\n",
+       "                                                   authors   \n",
+       "0        C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
+       "1                          Ileana Streinu and Louis Theran   \n",
+       "2                                              Hongjun Pan   \n",
+       "3                                 Y. H. Pong and C. K. Law   \n",
+       "4        Alejandro Corichi, Tatjana Vukasinac and Jose ...   \n",
+       "...                                                    ...   \n",
+       "1135476             Francesca Federici, Andrei A. Varlamov   \n",
+       "1135477  Durga P. Choudhury, Balam A. Willemsen, John S...   \n",
+       "1135478  Balam A. Willemsen, J. S. Derov and S.Sridhar ...   \n",
+       "1135479  Yasumasa Hasegawa (Himeji Institute of Technol...   \n",
+       "1135480  Naoki Enomoto, Masanori Ichioka and Kazushige ...   \n",
+       "\n",
+       "                                                     title   \n",
+       "0        Calculation of prompt diphoton production cros...  \\\n",
+       "1                 Sparsity-certifying Graph Decompositions   \n",
+       "2        The evolution of the Earth-Moon system based o...   \n",
+       "3        Bosonic characters of atomic Cooper pairs acro...   \n",
+       "4        Polymer Quantum Mechanics and its Continuum Limit   \n",
+       "...                                                    ...   \n",
+       "1135476  The Fluctuation Induced Pseudogap in the Infra...   \n",
+       "1135477  Nonlinear Response of HTSC Thin Film Microwave...   \n",
+       "1135478  Critical State Flux Penetration and Linear Mic...   \n",
+       "1135479  Density of States and NMR Relaxation Rate in A...   \n",
+       "1135480  Ginzburg Landau theory for d-wave pairing and ...   \n",
+       "\n",
+       "                                                  comments   \n",
+       "0                  37 pages, 15 figures; published version  \\\n",
+       "1                    To appear in Graphs and Combinatorics   \n",
+       "2                                      23 pages, 3 figures   \n",
+       "3                      6 pages, 4 figures, accepted by PRA   \n",
+       "4        16 pages, no figures. Typos corrected to match...   \n",
+       "...                                                    ...   \n",
+       "1135476  8 pages, 4 eps figures, Submitted to Phys. Rev. B   \n",
+       "1135477  4 pages, LaTeX type, Uses IEEE style files, 60...   \n",
+       "1135478  20 pages, LaTeX type, Uses REVTeX style files,...   \n",
+       "1135479  7 pages, 4 PostScript Figures, LaTeX, to appea...   \n",
+       "1135480  12 pages including 8 eps figs, LaTeX with jpsj...   \n",
+       "\n",
+       "                                journal-ref                         doi   \n",
+       "0                  Phys.Rev.D76:013009,2007  10.1103/PhysRevD.76.013009  \\\n",
+       "1                                      None                        None   \n",
+       "2                                      None                        None   \n",
+       "3                                      None  10.1103/PhysRevA.75.043613   \n",
+       "4                  Phys.Rev.D76:044016,2007  10.1103/PhysRevD.76.044016   \n",
+       "...                                     ...                         ...   \n",
+       "1135476                                None    10.1103/PhysRevB.55.6070   \n",
+       "1135477                                None           10.1109/77.620744   \n",
+       "1135478                                None   10.1103/PhysRevB.56.11989   \n",
+       "1135479                                None        10.1143/JPSJ.65.3131   \n",
+       "1135480  J. Phys. Soc. Jpn. 66, 204 (1997).         10.1143/JPSJ.66.204   \n",
+       "\n",
+       "                report-no                  categories   \n",
+       "0        ANL-HEP-PR-07-12                      hep-ph  \\\n",
+       "1                    None               math.CO cs.CG   \n",
+       "2                    None              physics.gen-ph   \n",
+       "3                    None           cond-mat.mes-hall   \n",
+       "4            IGPG-07/03-2                       gr-qc   \n",
+       "...                   ...                         ...   \n",
+       "1135476              None  supr-con cond-mat.supr-con   \n",
+       "1135477              None  supr-con cond-mat.supr-con   \n",
+       "1135478              None  supr-con cond-mat.supr-con   \n",
+       "1135479              None  supr-con cond-mat.supr-con   \n",
+       "1135480              None  supr-con cond-mat.supr-con   \n",
+       "\n",
+       "                                                   license   \n",
+       "0                                                     None  \\\n",
+       "1        http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "2                                                     None   \n",
+       "3                                                     None   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "1135476                                               None   \n",
+       "1135477                                               None   \n",
+       "1135478                                               None   \n",
+       "1135479                                               None   \n",
+       "1135480                                               None   \n",
+       "\n",
+       "                                                  abstract   \n",
+       "0          A fully differential calculation in perturba...  \\\n",
+       "1          We describe a new algorithm, the $(k,\\ell)$-...   \n",
+       "2          The evolution of Earth-Moon system is descri...   \n",
+       "3          We study the two-particle wave function of p...   \n",
+       "4          A rather non-standard quantum representation...   \n",
+       "...                                                    ...   \n",
+       "1135476    We study the effect of fluctuations on the {...   \n",
+       "1135477    The non-linear microwave surface impedance o...   \n",
+       "1135478    The vortex contribution to the dc field (H) ...   \n",
+       "1135479    We show that the density of states in an ani...   \n",
+       "1135480    The Ginzburg Landau theory for d_{x^2-y^2}-w...   \n",
+       "\n",
+       "                                                  versions update_date   \n",
+       "0        [{'created': 'Mon, 2 Apr 2007 19:18:42 GMT', '...  2008-11-26  \\\n",
+       "1        [{'created': 'Sat, 31 Mar 2007 02:26:18 GMT', ...  2008-12-13   \n",
+       "2        [{'created': 'Sun, 1 Apr 2007 20:46:54 GMT', '...  2008-01-13   \n",
+       "3        [{'created': 'Sat, 31 Mar 2007 04:24:59 GMT', ...  2015-05-13   \n",
+       "4        [{'created': 'Sat, 31 Mar 2007 04:27:22 GMT', ...  2008-11-26   \n",
+       "...                                                    ...         ...   \n",
+       "1135476  [{'created': 'Fri, 23 Aug 1996 09:39:49 GMT', ...  2009-10-30   \n",
+       "1135477  [{'created': 'Sat, 31 Aug 1996 17:34:38 GMT', ...  2016-11-18   \n",
+       "1135478  [{'created': 'Tue, 3 Sep 1996 14:08:26 GMT', '...  2009-10-30   \n",
+       "1135479  [{'created': 'Wed, 18 Sep 1996 07:57:29 GMT', ...  2009-10-30   \n",
+       "1135480  [{'created': 'Wed, 25 Sep 1996 14:17:09 GMT', ...  2009-10-30   \n",
+       "\n",
+       "                                            authors_parsed   \n",
+       "0        [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \\\n",
+       "1                 [[Streinu, Ileana, ], [Theran, Louis, ]]   \n",
+       "2                                       [[Pan, Hongjun, ]]   \n",
+       "3                        [[Pong, Y. H., ], [Law, C. K., ]]   \n",
+       "4        [[Corichi, Alejandro, ], [Vukasinac, Tatjana, ...   \n",
+       "...                                                    ...   \n",
+       "1135476  [[Federici, Francesca, ], [Varlamov, Andrei A....   \n",
+       "1135477  [[Choudhury, Durga P., , Physics Department, N...   \n",
+       "1135478  [[Willemsen, Balam A., , Physics Department,\\n...   \n",
+       "1135479  [[Hasegawa, Yasumasa, , Himeji Institute of Te...   \n",
+       "1135480  [[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...   \n",
+       "\n",
+       "                                         cleaned_abstracts  len_abstract  \n",
+       "0           fully differential calculation perturbative...           695  \n",
+       "1           describe new algorithm $ k,\\ell)$-pebble ga...           619  \n",
+       "2           evolution earth moon system describe dark m...           631  \n",
+       "3           study particle wave function pair atom ferm...           638  \n",
+       "4           non standard quantum representation canonic...           734  \n",
+       "...                                                    ...           ...  \n",
+       "1135476     study effect fluctuation \\bf ac conductivit...           703  \n",
+       "1135477     non linear microwave surface impedance patt...           468  \n",
+       "1135478     vortex contribution dc field h dependent mi...           841  \n",
+       "1135479     density state anisotropic superconductor \\n...           449  \n",
+       "1135480     ginzburg landau theory d_{x^2 y^2}-wave sup...           528  \n",
+       "\n",
+       "[1135481 rows x 16 columns]"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGxCAYAAACZa0njAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFFklEQVR4nO3de3gU1eHG8XeTJZsgIBJCJISCteUeNiEh0gIVKFJubSkgFlsUoYVWkNoK9hexCAKiAdRyq6JyEyt3baXWC1ZrWxXbQIJIUW5iCAESNICSC5ud3x+YKTsksAmb3dnk+3mePGbnzOWcOYl5mTlzxmEYhiEAAACYIkJdAQAAALshIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWDhDXYFwdvLkGfGilktzOKTY2MacqxCjH0KPPrAH+sEeQtUPFcf1BwHpChiG+AXzE+fKHuiH0KMP7IF+sAc794MtbrGVlZVp6NCh2r59+0VlZ86cUe/evbVlyxaf5Vu3blX//v3ldrs1adIkffbZZ2aZYRhasGCBevToofT0dGVmZsrr9Zrln3/+ue666y6lpKSoX79++tOf/lR7jQMAAGEn5AGptLRUv/nNb7Rv375Ky+fPn68TJ074LNu1a5emT5+uyZMna/369Tp9+rQyMjLM8pUrV2rr1q1asmSJFi1apJdeekkrV640yzMyMnTmzBmtX79ev/zlL3X//fdr165dtdNAAAAQdkJ6i23//v265557ZFRxfe0///mP3nvvPcXFxfksX7t2rQYNGqRhw4ZJkjIzM9W3b1/l5uaqdevWWrNmjaZMmaK0tDRJ0tSpU/X73/9e48eP16effqo333xTb7zxhhITE9WuXTtlZ2frj3/8o7p27Vqr7QUAAOEhpFeQ3n//fd1www1av379RWVlZWX63e9+pxkzZigqKsqnLCcnxww/ktSyZUslJCQoJydHx48fV35+vrp3726Wp6amKi8vTydOnFBOTo5atmypxMREn/KdO3fWQgsBAEA4CukVpFtvvbXKsieeeEKdOnVSr169Lio7ceKEWrRo4bMsNjZWx44dU0FBgST5lDdv3lySzPLKtj1+/HiN2wEAAOoWWz7Ftn//fq1bt05//vOfKy0vKSm56KpSVFSUysrKVFJSYn6+sEw6f1WquLi4ym2ry+Go9ib1TsU54lyFFv0QevSBPdAP9hCqfqjO8WwXkAzD0P33368pU6aYV36sXC7XRYGmrKxMMTExPmHI5XKZ30tSTExMldtGR0dXu67+zqUAzpVd0A+hRx/YA/1gD3buB9sFpKNHj2rnzp366KOP9Mgjj0iSiouL9cADD+jll1/W008/rfj4eBUWFvpsV1hYqLi4OMXHx0uSCgoKzHFGFbfdKsqr2ra6mGjs8piUzR7oh9CjD+yBfrAHJoqsgfj4eL322ms+y8aMGaMxY8boBz/4gSTJ7XYrKytLw4cPlyTl5+crPz9fbrdb8fHxSkhIUFZWlhmQsrKylJCQoBYtWig5OVl5eXk6duyYrr32WrM8OTm52nW18wRXdsO5sgf6IfToA3ugH+zBzv1gu4DkdDrVpk2bi5bFxsaaV4dGjx6tMWPGKDk5WUlJSZo7d6769Omj1q1bm+ULFiwwA9DChQs1btw4SVLr1q3Vq1cvTZs2TdOnT9cHH3ygrVu3au3atUFsJQAAsDPbBSR/pKSk6MEHH9SiRYt06tQp9ezZU7NnzzbLx48fr5MnT2ry5MmKjIzUyJEjNXbsWLM8MzNT06dP16hRoxQXF6eHHnqIOZAAAIDJYVQ1SyMuq7CQe9iX43BIzZs35lyFGP0QevSBPdAP9hCqfqg4rj9C/qoRAAAAuyEgAQAAWITlGCSEntPpm609Hm+IagIAQOARkCCpeoHH6YzQxrWblZebL0lq1bqlbv7piGqHJEIWAMCuCEioUeDJy83Xof2Hg3pMAACChYAESVceeEJ1TK5CAQBqAwEJYYurUACA2kJAQlgLxZUvAEDdR0BCvRKIW3LV3Yd1/ZoeFwAQPAQk1BuBuCVX3X1Y16/pcQEAwUVAqgMYqOy/QNySq+4+uA0IAOGHgGRDoZiTCAAA/A8ByWZCMScRAADwRUCyIQIPrAJ1G9XpjDDfnM1VRgCoGgEJsLlADS5fsWytDn+SV+N9AEB9QkACwkBABpcf4cokAPjr4glaAAAA6jmuIAGoNTUZO8W0FQDsgIAEoFbUZOwU01YAsAsCEoBKBeIVKTUZO8VTnADsgIAE4CK8IgVAfUdAAlApruQAqM94ig0AAMCCK0gA6pRAjJ0CAAISgDqDsVMAAoWABKBOYewUgEBgDBIAAIAFAQkAAMCCgAQAAGDBGCQAsOB9cAAISABwAd4HB0AiIAHARWryJBxXnYC6hYAEAFeIq05A3UNAAoAAYP4loG7hKTYAAAALAhIAAIAFt9gAwAYY5A3YCwEJAEKMQd6A/RCQAMAGGOQN2AtjkAAAACxsEZDKyso0dOhQbd++3VyWnZ2tH//4x0pJSdH3vvc9bdy40Webd955R0OHDpXb7dZtt92m3Nxcn/JVq1apd+/eSklJ0X333afi4mKzrLS0VPfdd5/S0tLUq1cvrVixonYbCAC1wOmM8PkCEDgh/40qLS3Vb37zG+3bt89cVlBQoJ///OdKT0/XCy+8oClTpmj27Nl66623JElHjx7VpEmTNHz4cG3atEnNmjXTnXfeKcMwJEmvvvqqlixZogcffFCrV69WTk6O5s+fb+4/MzNTu3fv1urVq/XAAw9oyZIleuWVV4LabgC4EhXjlh6ft0SPz1uijWs3E5KAAArpGKT9+/frnnvuMYNNhW3btql58+b6zW9+I0lq27attm/frpdeekl9+vTRxo0b1aVLF40bN06SNG/ePPXs2VPvv/++brjhBq1Zs0a33367+vbtK0maNWuWxo8fr2nTpskwDG3cuFFPPfWUOnfurM6dO2vfvn167rnnNHDgwOCeAAC4AoxbAmpPSP+5URFo1q9f77O8d+/emjdv3kXrf/HFF5KknJwcpaWlmctjYmLUuXNnZWdnq7y8XB988IFPeXJyss6dO6e9e/dq79698ng8SklJMctTU1OVk5Mjr5cnRgDUH9ZbdDW5AsVtPtRVIb2CdOutt1a6PDExUYmJiebnkydP6i9/+YvuuusuSedvwbVo0cJnm9jYWB07dkynT59WaWmpT7nT6VTTpk117NgxRURE6JprrlFUVJRZ3rx5c5WWlqqoqEjNmjXzu/4Oh9+rXtE+HY6qj1Xd9YN9zIpl1jK717s2j1nd4wai3hduV3HB1o71vtK22rneVf0u1OYxL8XpjNCGZ/83tYB0fnqBUWP8n17Auo/qbh8Kl+oHBE+o+qE6x7P9Y/4lJSW666671Lx5c91yyy2SpOLiYp+AI0lRUVEqKytTSUmJ+bmycsMwKi2Tzg8Wr47Y2MbVWt9fTmeEGjSINL9v2vSqgK4fimNWdq7Cod6BPmZNjxuIep/f1v71vtK22r3eVf1/IxTn6nj+cR05fMRnf1eyjyv52Qy22vr/N6rHzv1g64D05Zdf6s4779Qnn3yiP/7xj4qJiZEkuVyui8JMWVmZmjRpIpfLZX62lsfExKi8vLzSMkmKjo6uVv1Onjwjy/CpK+Z0Rsjj8ercuXJJ52fTLSr6ssp/kVV3/WAf0+E4/wtgPVd2r3dtHbMmxw1Uvc9vWy7DsG+9r7Stdq53Vb8LtXnM6rQzEPuoyc9msF2qHxA8oeqHiuP6w7YB6YsvvtDPfvYzffrpp1q9erXatm1rlsXHx6uwsNBn/cLCQnXs2FFNmzaVy+VSYWGhrr/+ekmSx+NRUVGR4uLiZBiGPv/8c3k8Hjmd55tfUFCg6OhoNWnSpFp1NAwFvGMr29+ljlPd9UN1TGt5uNS7No5Z3eMGot6V7cuO9b7StoZDvStbXp/OlV2ESz3rOjv3gy1H1Hm9Xk2ePFlHjhzRs88+q29+85s+5W63W1lZWebn4uJi7dmzR263WxEREUpKSvIpz87OltPpVIcOHdSxY0c5nU5lZ2eb5VlZWUpKSlJEhC1PBwAACDJbJoJNmzZp+/btmjNnjpo0aaKCggIVFBSoqKhIkjRixAjt2LFDy5cv1759+5SRkaHExETdcMMNks4P/n7mmWe0bds27dq1SzNnztSoUaMUExOjmJgYDRs2TDNnztSuXbu0bds2rVixQrfddlsIWwwAAOzElrfYXn31VXm9Xk2cONFneXp6up599lklJiZq8eLFeuihh7R06VKlpKRo6dKlcnw1PH3IkCHKy8vTjBkzVFZWpgEDBmjatGnmfjIyMjRz5kzdfvvtatSoke666y4NGDAgqG0EAAD2ZZuA9NFHH5nfP/PMM5dd/8Ybb9SNN95YZfmECRM0YcKESstiYmL0yCOP6JFHHql+RQEAQJ1ny1tsAAAAoURAAgAAsLDNLTYAQP1gfSVJdedNquyVJnaeewnhiYAEAAgapzNCG9f6vp7k5p9W7/UmF25fk30A/iAgAQCCKi83X4f2Hw7Z9oA/GIMEAABgQUACAACwICABAABYEJAAAAAsGKQNAKjzKqYG+OqNVMBlEZAAAHWadWqANm1bafitP9K5c0wLgKoRkAAAdd6FUwNUNtEkYMVPCQAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWPAUG65YZGSEIiMvztq8WRsAEK4ISLhIZYHnUmEnvmULrV+zUUc+zTeXtWrdUjf/dITKy4MXkqpbbwAAqkJAqgcqm/OjOoGnIuxcapu8T/83x4g/9ajsitOVqkm9K2M9X4QsAKh/CEh1nHUGWSlwgae69djw7P/qkZzaRRfO+B+oqz9XWm/r+appyAIAhDcCUj1w4Qyy0sVhpDau5lyuHgmJ1/qUBerqT6DrCQConwhI9ZA1jFiv5gRCRQireDGkPyEs0FetAACoKQKSzdXWwOMLw4j1ak4gXBjCnM4IdXF3CngIAwCgthCQbK66t55CdfusMhUhrEGDSLW4tkXQj3+5c1FZ+LzcOv7s43IBlqftgPDEAxz1CwEpDFTn1lMwbp8FQ1VzK1nXuZTLnYvKpie43DqXK2/dppVGjfnf9AaV1fFy21S3nQBqHw9w1D8EpDqotm+fBUNV4aWw4GS1wt/lzoU1fF5uHX/K/Qmol9umuu0EUPt4gKN+ISCFGTvdQqttlYWX/CPHbB/+ahJQrdvUpJ1c/gfsrbpz0iG0CEhhJhS30OpTKAtX1sv//tzqAxA8NZ2TDqFDQApDwb6FVlfGNdV11nmmrH0GILS4RRdeCEjwS10Y11Tf0GcAUHMEJCDM+DM9QU32wfQEAPA/BCQgzPgzPUF19+HPWAg7vQ4GqC94+CJ0CEhAGPJneoLq7MPfgfhX8joYrkAB1cPcS6FFQAIQsIH4F/5r1xqGAnUFqrJHpas6JhDuGNgdOgQkAJKufFC39V+7lYWsS121qlhW3WMwqSaA2kBAAlAjld2Ws041cCk1HUtlPUY4TB4KIPwQkADUSCBuywViLBUA1AYCEoAaY64lAHWVLUY0lpWVaejQodq+fbu5LDc3V2PHjlVycrIGDx6sf/7znz7bvPPOOxo6dKjcbrduu+025ebm+pSvWrVKvXv3VkpKiu677z4VFxebZaWlpbrvvvuUlpamXr16acWKFbXbQAA1UnEbz+k8/1WTQdjWfVxqkDcAVAj5FaTS0lLdc8892rdvn7nMMAxNmjRJ7dq10+bNm7Vt2zZNnjxZL7/8shISEnT06FFNmjRJd911l3r37q2lS5fqzjvv1J///Gc5HA69+uqrWrJkiebPn6/Y2FhlZGRo/vz5mjFjhiQpMzNTu3fv1urVq3X06FH99re/VUJCggYOHBiq0wCgEoG4jefP03OXevqO6QmA+imkAWn//v265557ZBiGz/L33ntPubm5WrdunRo2bKjrr79e7777rjZv3qy77rpLGzduVJcuXTRu3DhJ0rx589SzZ0+9//77uuGGG7RmzRrdfvvt6tu3ryRp1qxZGj9+vKZNmybDMLRx40Y99dRT6ty5szp37qx9+/bpueeeIyABNhSI23iXm/Np3eqNVT59V5PpCSoLVRUvDgYQHkIakCoCza9//WslJyeby3NyctSpUyc1bNjQXJaamqrs7GyzPC0tzSyLiYlR586dlZ2drbS0NH3wwQeaPHmyWZ6cnKxz585p7969MgxDHo9HKSkpPvt+4okn5PV6FRHB5XegLqvsqtTRyzx9V90JMisLVaPGjAhA7YHax+zd54U0IN16662VLi8oKFCLFi18lsXGxurYsWOXLT99+rRKS0t9yp1Op5o2bapjx44pIiJC11xzjaKioszy5s2bq7S0VEVFRWrWrFmgmgfApq7kqpS/8zddyazjQCBVJ/Awe/f/hHwMUmWKi4t9AowkRUVFqays7LLlJSUl5ufKyg3DqLRMkrl/fzlqYUa62thnKFW0p441K+xUnH+HQ7Lc0UY1Xem78Cr7Ha9q2aX+f1CTbS63fSD2Ycd6X7i8qu9rctxg1rsmx5TOB54Nz/oGnlFjqg48DsfFs3dX95j+MP82BPmPQ7XOXe1Vo+ZcLpeKiop8lpWVlSk6Otost4aZsrIyNWnSRC6Xy/xsLY+JiVF5eXmlZZLM/fsrNrZxtdb3l9MZoQYNIiV99a/VAH6ujX1e6hhO5/llEWFW73A6hj/rRHx1haOiP+xQb7ucm5p8zs87piOHj5jHaN0m4ZLbRLmcatr0KklV/3/jwt97pzPCXP9SarJNVdsHYh92rrd1m+oe07qPUNS7JseUpOP5x82f12Ad01+19Xc0EGwZkOLj47V//36fZYWFheZts/j4eBUWFl5U3rFjRzVt2lQul0uFhYW6/vrrJUkej0dFRUWKi4uTYRj6/PPP5fF45HSeb35BQYGio6PVpEmTatXz5MkzAf/XuNMZIY/Hq3PnyiWdH9hZHsDPtbHPSx3D4ymX0xkpb5jVO5yO4c863q8GCHs85TIMe9TbLucmGMdoHtdcTy1Zo2NHj6vc41WC5V/x1t97j8eroqIvL3srpLrbXGr7QOzDrvW+cJuKKwjVOaYd6l2TY9ZkH4E4pj8cjvPhqDb+jvpzXH/YckSy2+3Whx9+aN4uk6SsrCy53W6zPCsryywrLi7Wnj175Ha7FRERoaSkJJ/y7OxsOZ1OdejQQR07dpTT6TQHfFfsOykpqdoDtA2jdr7qkor21LFmhZ2K81/Xfr7CSd6n+frkwKc6uP+w8nLzL/l7HxkZoYiICHO8U8XX5f5fEYj/11zpPmprmyut94XrBbOtgap3TY9Zk30E4pj+1qu29l3dfqiMLa8gpaenq2XLlsrIyNCdd96pN998U7t27dK8efMkSSNGjNAzzzyj5cuXq2/fvlq6dKkSExN1ww03SDo/+HvGjBlq166dWrRooZkzZ2rUqFGKiYmRJA0bNkwzZ87UQw89pBMnTmjFihXmvgEg1Cob53S5wbLM1wQEli0DUmRkpJYtW6bp06dr+PDhatOmjZYuXaqEhARJUmJiohYvXqyHHnpIS5cuVUpKipYuXSrHV9dOhwwZory8PM2YMUNlZWUaMGCApk2bZu4/IyNDM2fO1O23365GjRrprrvu0oABA0LSVgD1T2VzMVld6dQCrdu00qgxI3zmXyIwAf6zTUD66KOPfD63adNGa9eurXL9G2+8UTfeeGOV5RMmTNCECRMqLYuJidEjjzyiRx55pGaVBYArEIgZwitjnb6guhNcAvgf2wQkAKhPgvGiX+ZiAmqOgAQAYcCf23K1uT1Q3xCQACAMXOltudq6reePS70MGLArAhIAhIkrvS0XjNt6VtZXVwQzmAFXgoAEAJDk31QBNbkalHeZlwEDdkRAAgBIuvxUAZGREVq3eiNXg3BZ1XlBrl0RkAAApktNFZCc2kVHL3E1qLIrUIw5qn+st1X9mWLCjoGKgAQAqFJ1xi1VNgM4V5nqpwtvq15OTQJVMBCQAAABY517qSZjjux4NQG1qzqBKlgISAAA27Dr1QTUPwQkAICt2PFqAuofRs8BAABYEJAAAAAsuMUGAAiZy70jzp93yPkzwSVQXQQkAEDIXO4dcf68Q866DgO7EQgEJABASF1uriV/5mKyTi8AXCkCEgCg3mPuJVgRkAAA9RpzL6EyBCQAQL134dxLvFMOEgEJAFDHXO7Jt4ivyg2j8nLeKQeJgAQAqGMu+2TctS20bvWln4wLxDvlEN4ISACAOicQT8ahfuOmKgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWPAUGwAA1VTZZJLMvF23EJAAAKgm61xLlb2e5ML3uzETd/ghIAEAUAMXzqVU2ezd61ZvNN/vxkzc4YeABADAFaps9u6juUxGGc4ISAAABACzc9ct3BQFAACwICABAABYEJAAAAAsGIMEAEAIMJeSvRGQAAAIAeuTb63btNKoMSNUXn4+JDF3UmgRkAAACBHrk2/WqQKYOyl0CEgAANhEdacKuHC2bolbdIFEQAIAIAw5nRHauHazOVt3Za87Qc3Z+gZnfn6+Jk6cqG7duqlfv35atWqVWbZnzx7dfPPNcrvdGjFihHbv3u2z7datW9W/f3+53W5NmjRJn332mVlmGIYWLFigHj16KD09XZmZmfJ6+YECANhXxaBup/P8V2RkhPK+mq370P7DOnb0uE95xTqoGVtfQbr77ruVkJCgLVu2aP/+/Zo6dapatWqlnj17asKECfr+97+vhx9+WM8//7wmTpyo119/XQ0bNtSuXbs0ffp0zZo1Sx06dNDcuXOVkZGhJ598UpK0cuVKbd26VUuWLJHH49G0adMUGxur8ePHh7jFAABUrrLXmTguUV7ZOvCfbQPSqVOnlJ2drdmzZ6tt27Zq27atevfurXfffVenTp2Sy+XSvffeK4fDoenTp+vtt9/WK6+8ouHDh2vt2rUaNGiQhg0bJknKzMxU3759lZubq9atW2vNmjWaMmWK0tLSJElTp07V73//ewISAMDWLjdG6cLyqtaBf2x77S06OloxMTHasmWLzp07p4MHD2rHjh3q2LGjcnJylJqaKofjfC52OBzq1q2bsrOzJUk5OTlm+JGkli1bKiEhQTk5OTp+/Ljy8/PVvXt3szw1NVV5eXk6ceJEUNsIAADsybYByeVyacaMGVq/fr3cbrcGDRqk73znO7r55ptVUFCgFi1a+KwfGxurY8eOSZJOnDhRZXlBQYEk+ZQ3b95ckszt/eVw1M5XXVLRnjrWrLBTcf7r2s9XOKIPQqu+n/+KMUoNGlT+ZX0qTgrM37Gq1qnO+sH+G2vbW2ySdODAAfXt21d33HGH9u3bp9mzZ+tb3/qWiouLFRUV5bNuVFSUysrKJEklJSVVlpeUlJifLyyTZG7vr9jYxtVukz/O//BGSvpqUF4AP9fGPi91DKfz/LKIMKt3OB3Dn3UivhqoWdEfdqi3Xc5NMI9R0Qfh3o5wr7ekkLcjVOcmIfFabXx2s/KOnB+n5O7WRYUnTvp8vvBvkNMZoaZNr1J1+buPir+jgThmoNk2IL377rvatGmT/v73vys6OlpJSUk6fvy4/vCHP6h169YXhZmysjJFR0dLOn/1qbLymJgYnzDkcrnM7yUpJiamWnU8efKMDKNGzauS0xkhj8erc+fKJUnl5V6VB/BzbezzUsfweMrldEbKG2b1Dqdj+LOO96uZeT2echmGPeptl3MTzGNU9EG4tyOc611xBSHU7QjluTly5JgOfjVOqcW1LZRv+Xzh+oYhnTlTbM7uLV1+riXr3zGPx6uioi99tnM4zoejkyfPKDLy8usHSsVx/WHbgLR79261adPGDD2S1KlTJz3xxBNKS0tTYWGhz/qFhYXmbbP4+PhKy+Pi4hQfHy9JKigoUGJiovm9JMXFxVWrjoahgAekQO8v1CraU8eaFXYqzn9d+/kKR/RBaHH+qye+ZQutW/2/J+P8mWupsnNc1d/L6i4PJtuOQWrRooUOHz7scyXo4MGDSkxMlNvt1s6dO2V8dfYMw9COHTvkdrslSW63W1lZWeZ2+fn5ys/Pl9vtVnx8vBISEnzKs7KylJCQcNG4JQAA6ruKJ+OqmmupsnFLdUHAryB99tlnatas2RXvp1+/fpo/f77uv/9+/fKXv9ShQ4f0xBNP6Ne//rUGDhyohQsXau7cufrxj3+sdevWqbi4WIMGDZIkjR49WmPGjFFycrKSkpI0d+5c9enTR61btzbLFyxYoGuvPf/448KFCzVu3LgrrjMAAHVZZXMt1dUZvGsUkDp27Kh//etfFwWhvLw8DR06VDt37rziijVu3FirVq3S3LlzNXLkSDVr1ky//OUvdcstt8jhcOjJJ5/UAw88oA0bNqh9+/Zavny5GjZsKElKSUnRgw8+qEWLFunUqVPq2bOnZs+ebe57/PjxOnnypCZPnqzIyEiNHDlSY8eOveI6AwBQ11nnWqqr/A5IL774orZs2SLp/C2tSZMmqUGDBj7rnDhxotrjeC7lG9/4hlauXFlpWdeuXfXCCy9Uue3w4cM1fPjwSssiIyOVkZGhjIyMgNQTAADULX4HpJtuuklHjhyRJL3//vtKTk7WVVf5PobXsGFD3XTTTYGtIQAAQJD5HZCuuuoqTZ48WZLUqlUrDR482HxMHgAA1E8VL9G98HNdUKMxSD/60Y90+PBh7d69W+fOnbuovOIdaAAAoG673Et0rQFKks+8SnZVo4D09NNPa8GCBbr66qsvus3mcDgISAAA1COXeomuNUC1at1So8aMCHodq6tGAWnFihWaNm2axo8fH+j6AACAOiYcn3yrUUAqLS3VgAEDAl0XAABQx114y83pjFBEhD3HLNUoIH3/+9/XH//4R917771y1PdXIwMAAL9VvL7k2NHj8ni8F41ZsosaBaQvvvhCmzZt0tatW5WYmHjRfEhr1qwJSOUAAEDdk/dpvnIPH9G5c+UXjVmyixoFpLZt2+oXv/hFoOsCAABgCzUKSBXzIQEAANRFNQpIl3tFx7x582pUGQAAADsIyNBxj8ejQ4cO6eWXX77oBbYAAADhpkZXkKq6QvT000/r448/vqIKAQAAhFpAJx8YOHCgXn/99UDuEgAAIOgCFpDOnj2rDRs26JprrgnULgEAAEKiRrfYOnToUOkEkS6XS3PmzLniSgEAAIRSjQKSdSJIh8OhBg0a6Bvf+IYaNWoUkIoBAACESo0CUnp6uiTpk08+0YEDB+T1enXdddcRjgAAQJ1Qo4B0+vRpZWRk6I033tDVV1+t8vJyffnll+revbuWLl2qxo0bB7qeAAAAQVOjQdpz5szRsWPH9PLLL2v79u36z3/+o5deeklnz55lkkgAABD2ahSQ/va3v2nmzJn6+te/bi77xje+oRkzZuiNN94IWOUAAABCoUYByeVyKSLi4k0dDofKy8uvuFIAAAChVKOA1K9fP82aNUuffvqpueyTTz7RnDlzdOONNwascgAAAKFQo0Ha06ZN06RJk/S9731PTZo0kSSdOnVK3/nOd/S73/0uoBUEAAAItmoHpMOHDyshIUHPPvusPvroIx04cEAul0tt27bV9ddfXxt1BAAACCq/b7EZhqE5c+Zo0KBB2rlzpySpffv2Gjx4sDZv3qyhQ4fq4YcflmEYtVZZAACAYPA7IK1Zs0Yvv/yyli5dak4UWWHZsmVaunSpXnjhBT3//PMBryQAAEAw+R2QNmzYoN/97nfq27dvpeX9+vXT1KlTCUgAACDs+R2Q8vLy1LVr10uu06NHD+Xm5l5xpQAAAELJ74AUGxurvLy8S65z7NgxNW3a9ErrBAAAEFJ+B6SbbrpJixcv1rlz5yot93g8WrJkiXr16hWwygEAAISC34/533nnnRo5cqSGDx+uMWPGqEuXLmrcuLFOnTqlDz/8UGvXrtWXX36pzMzM2qwvAABArfM7IDVp0kQbNmzQggUL9PDDD6u4uFjS+cf/GzdurMGDB+uuu+5S8+bNa62yAAAAwVCtiSKbNm2qOXPmaMaMGcrNzdXp06fVtGlTfe1rX1NkZGRt1REAACCoavSqkaioKGbNBgAAdVaNXlYLAABQlxGQAAAALAhIAAAAFgQkAAAAC1sHpLKyMs2aNUvdu3fXt7/9bT366KMyDEOStGfPHt18881yu90aMWKEdu/e7bPt1q1b1b9/f7ndbk2aNEmfffaZWWYYhhYsWKAePXooPT1dmZmZ8nq9QW0bAACwL1sHpDlz5uidd97RM888o4ULF2rDhg1av369zp49qwkTJigtLU1btmxRSkqKJk6cqLNnz0qSdu3apenTp2vy5Mlav369Tp8+rYyMDHO/K1eu1NatW7VkyRItWrRIL730klauXBmqZgIAAJup0WP+wVBUVKTNmzdr5cqV5ktyx40bp5ycHDmdTrlcLt17771yOByaPn263n77bb3yyisaPny41q5dq0GDBmnYsGGSpMzMTPXt21e5ublq3bq11qxZoylTpigtLU2SNHXqVP3+97/X+PHjQ9VcAABgI7a9gpSVlaVGjRopPT3dXDZhwgTNmzdPOTk5Sk1NlcPhkCQ5HA5169ZN2dnZkqScnBwz/EhSy5YtlZCQoJycHB0/flz5+fnq3r27WZ6amqq8vDydOHEiOI0DAAC2ZtuAlJubq1atWunFF1/UwIED9d3vfldLly6V1+tVQUGBWrRo4bN+bGysjh07Jkk6ceJEleUFBQWS5FNe8XqUiu0BAED9ZttbbGfPntXhw4e1bt06zZs3TwUFBZoxY4ZiYmJUXFysqKgon/WjoqJUVlYmSSopKamyvKSkxPx8YZkkc3t/fXUBK6BqY5+hVNGeOtassFNx/h0O6avnHBAide13PNxw/u2lqv5wOEL/N9a2AcnpdOqLL77QwoUL1apVK0nS0aNH9fzzz6tNmzYXhZmysjJFR0dLklwuV6XlMTExPmHI5XKZ30tSTExMteoYG9u4+g3zg9MZoQYNzr/bLjIyQpEB/Fwb+7zUMZzO88siwqze4XQMf9aJiDx/sbiiP+xQb7ucm2Aeo6IPwr0d4V5vSSFvh13PTTCPIVX+++B0Rqhp06sUarYNSHFxcXK5XGY4kqTrrrtO+fn5Sk9PV2Fhoc/6hYWF5m2z+Pj4Ssvj4uIUHx8vSSooKFBiYqL5fcUxq+PkyTMB/9e40xkhj8erc+fKJUnl5V6VB/BzbezzUsfweMrldEbKG2b1Dqdj+LOOt/z8NBYeT7kMwx71tsu5CeYxKvog3NsRzvWuuIIQ6nbY8dwE+xhS5b8PHo9XRUVfyuMJ/PQ7Dof/FzdsOwbJ7XartLRUhw4dMpcdPHhQrVq1ktvt1s6dO805kQzD0I4dO+R2u81ts7KyzO3y8/OVn58vt9ut+Ph4JSQk+JRnZWUpISHhonFLl2MYtfNVl1S0p441K+xUnP+69vMVjuiD0OL820tV/VFbf1+r0/+2DUhf//rX1adPH2VkZGjv3r36xz/+oeXLl2v06NEaOHCgTp8+rblz52r//v2aO3euiouLNWjQIEnS6NGj9ac//UkbN27U3r17de+996pPnz5q3bq1Wb5gwQJt375d27dv18KFC3XbbbeFsrkAAMBGbHuLTZIWLFig2bNna/To0YqJidFPfvITjRkzRg6HQ08++aQeeOABbdiwQe3bt9fy5cvVsGFDSVJKSooefPBBLVq0SKdOnVLPnj01e/Zsc7/jx4/XyZMnNXnyZEVGRmrkyJEaO3ZsiFoJAADsxtYBqXHjxsrMzKy0rGvXrnrhhReq3Hb48OEaPnx4pWWRkZHKyMjwmV0bAACggm1vsQEAAIQKAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAAi7AJSBMmTND//d//mZ/37Nmjm2++WW63WyNGjNDu3bt91t+6dav69+8vt9utSZMm6bPPPjPLDMPQggUL1KNHD6WnpyszM1NerzdobQEAAPYWFgHpL3/5i/7+97+bn8+ePasJEyYoLS1NW7ZsUUpKiiZOnKizZ89Kknbt2qXp06dr8uTJWr9+vU6fPq2MjAxz+5UrV2rr1q1asmSJFi1apJdeekkrV64MersAAIA92T4gFRUVKTMzU0lJSeayl19+WS6XS/fee6+uv/56TZ8+XVdddZVeeeUVSdLatWs1aNAgDRs2TB06dFBmZqb+/ve/Kzc3V5K0Zs0aTZkyRWlpaerRo4emTp2q5557LiTtAwAA9mP7gPTII4/ohz/8ob7xjW+Yy3JycpSamiqHwyFJcjgc6tatm7Kzs83ytLQ0c/2WLVsqISFBOTk5On78uPLz89W9e3ezPDU1VXl5eTpx4kRwGgUAAGzN1gHp3Xff1X/+8x/deeedPssLCgrUokULn2WxsbE6duyYJOnEiRNVlhcUFEiST3nz5s0lydzeXw5H7XzVJRXtqWPNCjsV57+u/XyFI/ogtDj/9lJVf9TW39fq9L8zME0MvNLSUj3wwAOaMWOGoqOjfcqKi4sVFRXlsywqKkplZWWSpJKSkirLS0pKzM8Xlkkyt/dXbGzjaq3vL6czQg0aREqSIiMjFBnAz7Wxz0sdw+k8vywizOodTsfwZ52IyPP/FqroDzvU2y7nJpjHqOiDcG9HuNdbUsjbYddzE8xjSJX/PjidEWra9CqFmm0D0pIlS9SlSxf17t37ojKXy3VRmCkrKzODVFXlMTExPmHI5XKZ30tSTExMtep48uQZGUa1NrkspzNCHo9X586VS5LKy70qD+Dn2tjnpY7h8ZTL6YyUN8zqHU7H8Gcdb/n5pzQ9nnIZhj3qbZdzE8xjVPRBuLcjnOtdcQUh1O2w47kJ9jGkyn8fPB6vioq+lMcT+KfLHQ7/L27YNiD95S9/UWFhoVJSUiT9L8S8+uqrGjp0qAoLC33WLywsNG+bxcfHV1oeFxen+Ph4Sedv0yUmJprfS1JcXFy16mgYCnhACvT+Qq2iPXWsWWGn4vzXtZ+vcEQfhBbn316q6o/a+PtaXbYdg/Tss8/qpZde0osvvqgXX3xR/fr1U79+/fTiiy/K7XZr586dMr46e4ZhaMeOHXK73ZIkt9utrKwsc1/5+fnKz8+X2+1WfHy8EhISfMqzsrKUkJBw0bglAABQP9n2ClKrVq18Pl911fn7kW3atFFsbKwWLlyouXPn6sc//rHWrVun4uJiDRo0SJI0evRojRkzRsnJyUpKStLcuXPVp08ftW7d2ixfsGCBrr32WknSwoULNW7cuCC2DgAA2JltA9KlNGrUSE8++aQeeOABbdiwQe3bt9fy5cvVsGFDSVJKSooefPBBLVq0SKdOnVLPnj01e/Zsc/vx48fr5MmTmjx5siIjIzVy5EiNHTs2RK0BAAB2EzYB6eGHH/b53LVrV73wwgtVrj98+HANHz680rLIyEhlZGT4zK4NAABQwbZjkAAAAEKFgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACAha0D0vHjxzVlyhSlp6erd+/emjdvnkpLSyVJubm5Gjt2rJKTkzV48GD985//9Nn2nXfe0dChQ+V2u3XbbbcpNzfXp3zVqlXq3bu3UlJSdN9996m4uDho7QIAAPZm24BkGIamTJmi4uJiPffcc3rsscf05ptv6vHHH5dhGJo0aZKaN2+uzZs364c//KEmT56so0ePSpKOHj2qSZMmafjw4dq0aZOaNWumO++8U4ZhSJJeffVVLVmyRA8++KBWr16tnJwczZ8/P5TNBQAANmLbgHTw4EFlZ2dr3rx5+uY3v6m0tDRNmTJFW7du1Xvvvafc3Fw9+OCDuv766zVx4kQlJydr8+bNkqSNGzeqS5cuGjdunL75zW9q3rx5ysvL0/vvvy9JWrNmjW6//Xb17dtXXbt21axZs7R582auIgEAAEk2DkhxcXF6+umn1bx5c5/lX3zxhXJyctSpUyc1bNjQXJ6amqrs7GxJUk5OjtLS0syymJgYde7cWdnZ2SovL9cHH3zgU56cnKxz585p7969tdsoAAAQFpyhrkBVmjRpot69e5ufvV6v1q5dqx49eqigoEAtWrTwWT82NlbHjh2TpEuWnz59WqWlpT7lTqdTTZs2Nbf3l8NR3VaFZp+hVNGeOtassFNx/h0O6as7zQiRuvY7Hm44//ZSVX84HKH/G2vbgGQ1f/587dmzR5s2bdKqVasUFRXlUx4VFaWysjJJUnFxcZXlJSUl5ueqtvdXbGzj6jbDL05nhBo0iJQkRUZGKDKAn2tjn5c6htN5fllEmNU7nI7hzzoRkecvFlf0hx3qbZdzE8xjVPRBuLcj3OstKeTtsOu5CeYxpMp/H5zOCDVtepVCLSwC0vz587V69Wo99thjateunVwul4qKinzWKSsrU3R0tCTJ5XJdFHbKysrUpEkTuVwu87O1PCYmplr1OnnyTMD/Ne50Rsjj8ercuXJJUnm5V+UB/Fwb+7zUMTyecjmdkfKGWb3D6Rj+rOMt90qSPJ5yGYY96m2XcxPMY1T0Qbi3I5zrXXEFIdTtsOO5CfYxpMp/Hzwer4qKvpTnq3UCyeHw/+KGbccgVZg9e7ZWrlyp+fPn63vf+54kKT4+XoWFhT7rFRYWmrfNqiqPi4tT06ZN5XK5fMo9Ho+KiooUFxdXrboZRu181SUV7aljzQo7Fee/rv18hSP6ILQ4//ZSVX/U1t/X6vS/rQPSkiVLtG7dOj366KMaMmSIudztduvDDz80b5dJUlZWltxut1melZVllhUXF2vPnj1yu92KiIhQUlKST3l2dracTqc6dOgQhFYBAAC7s21AOnDggJYtW6af//znSk1NVUFBgfmVnp6uli1bKiMjQ/v27dPy5cu1a9cujRw5UpI0YsQI7dixQ8uXL9e+ffuUkZGhxMRE3XDDDZKkW2+9Vc8884y2bdumXbt2aebMmRo1alS1b7EBAIC6ybZjkN544w2Vl5frD3/4g/7whz/4lH300UdatmyZpk+fruHDh6tNmzZaunSpEhISJEmJiYlavHixHnroIS1dulQpKSlaunSpHF/dfB4yZIjy8vI0Y8YMlZWVacCAAZo2bVrQ2wgAAOzJtgFpwoQJmjBhQpXlbdq00dq1a6ssv/HGG3XjjTfWeP8AAKD+su0tNgAAgFAhIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABYEJAAAAAsCEgAAAAWBCQAAAALAhIAAIAFAQkAAMCCgAQAAGBBQAIAALAgIAEAAFgQkAAAACwISAAAABYEJAAAAAsCEgAAgAUBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAot4GpNLSUt13331KS0tTr169tGLFilBXCQAA2IQz1BUIlczMTO3evVurV6/W0aNH9dvf/lYJCQkaOHBgqKsGAABCrF4GpLNnz2rjxo166qmn1LlzZ3Xu3Fn79u3Tc889R0ACAAD18xbb3r175fF4lJKSYi5LTU1VTk6OvF5vCGsGAADsoF4GpIKCAl1zzTWKiooylzVv3lylpaUqKioKXcUAAIAt1MtbbMXFxT7hSJL5uayszO/9RERIhhHQqsnhkNp+/Wtyuc7XJ6HVtXK5ohQVoM+1sc9LHcPlilKEM0IJieFV73A6hl/bJF6rhjHRckY6Zdik3rY5N0E8RsxV0fJ6vGHfjnCut+Orz06nk3MT4mNU9fvQstW1cjjO/40NNIfD/3XrZUByuVwXBaGKz9HR0X7vp1mzxgGtV4U7fjG6VvYLAAD8Uy9vscXHx+vzzz+Xx+MxlxUUFCg6OlpNmjQJYc0AAIAd1MuA1LFjRzmdTmVnZ5vLsrKylJSUpIjauKYHAADCSr1MAzExMRo2bJhmzpypXbt2adu2bVqxYoVuu+22UFcNAADYgMMwAj3MODwUFxdr5syZeu2119SoUSONHz9eY8eODXW1AACADdTbgAQAAFCVenmLDQAA4FIISAAAABYEJAAAAAsCEmrs9ddfV/v27X2+pkyZIknas2ePbr75Zrndbo0YMUK7d+/22Xbr1q3q37+/3G63Jk2apM8++ywUTagTysrKNGvWLHXv3l3f/va39eijj6piaCH9EBxbtmy56Hehffv26tChgyT6IVjy8/M1ceJEdevWTf369dOqVavMMvogeE6ePKkpU6YoLS1NN910k7Zs2WKW5ebmauzYsUpOTtbgwYP1z3/+02fbd955R0OHDpXb7dZtt92m3NzcYFf/fwyghpYtW2ZMnDjROHHihPl16tQp48svvzR69uxpPPzww8b+/fuN2bNnG9/+9reNL7/80jAMw8jJyTG6du1qvPDCC8Z///tf46c//akxYcKEELcmfP3ud78zBgwYYOTk5BjvvPOOccMNNxjPP/88/RBExcXFPr8HR48eNW666SZj7ty59EMQjRo1yrj77ruNQ4cOGa+//rrhdruN1157jT4IIq/Xa9xyyy3GzTffbHz44YfG3/72N6N79+7Gq6++ani9XuP73/++cc899xj79+83nnjiCcPtdht5eXmGYRhGXl6ekZycbDzzzDPGxx9/bPzqV78yhg4dani93pC0hYCEGrvnnnuMhQsXXrR848aNRr9+/cwfaq/Xa9x0003G5s2bDcMwjGnTphm//e1vzfWPHj1qtG/f3vj000+DU/E65PPPPzc6depkbN++3Vz25JNPGv/3f/9HP4TQE088YfTv398oLS2lH4KkqKjIaNeunfHRRx+ZyyZPnmzMmjWLPgiiXbt2Ge3atfM5d08++aQxatQo45133jGSk5PNYGoYhnH77bcbixYtMgzDMB5//HHjpz/9qVl29uxZIyUlxXjvvfeC14ALcIsNNXbgwAG1bdv2ouU5OTlKTU2V46u3AjocDnXr1s2cuTwnJ0dpaWnm+i1btlRCQoJycnKCUe06JSsrS40aNVJ6erq5bMKECZo3bx79ECJFRUV66qmndM899ygqKop+CJLo6GjFxMRoy5YtOnfunA4ePKgdO3aoY8eO9EEQ5ebmqlmzZmrdurW5rH379tq9e7eysrLUqVMnNWzY0CxLTU2tsh9iYmLUuXNnn7deBBMBCTViGIYOHTqkf/7zn/re976n/v37a8GCBSorK1NBQYFatGjhs35sbKyOHTsmSTpx4sQly+G/3NxctWrVSi+++KIGDhyo7373u1q6dKm8Xi/9ECLPP/+8WrRooYEDB0oS/RAkLpdLM2bM0Pr16+V2uzVo0CB95zvf0c0330wfBFHz5s115swZFRcXm8uOHTsmj8dz2X64XHmwOUNyVIS9o0ePqri4WFFRUXr88cd15MgRzZkzRyUlJebyC0VFRamsrEySVFJScsly+O/s2bM6fPiw1q1bp3nz5qmgoEAzZsxQTEwM/RAChmFo48aN+tnPfmYuox+C58CBA+rbt6/uuOMO7du3T7Nnz9a3vvUt+iCI3G63WrRoodmzZ+v+++9XQUGBVq5cKen8AyWXOs+X66dgIyChRlq1aqXt27fr6quvlsPhUMeOHeX1ejVt2jSlp6df9ANdVlam6OhoSef/pVdZeUxMTNDqX1c4nU598cUXWrhwoVq1aiXpfHh9/vnn1aZNG/ohyD744AMdP35cQ4YMMZdVdZ7ph8B69913tWnTJv39739XdHS0kpKSdPz4cf3hD39Q69at6YMgcblcevzxx3X33XcrNTVVsbGx+tnPfqZ58+bJ4XDUqB+aNGkStPpfiFtsqLGmTZua9/Ql6frrr1dpaani4uJUWFjos25hYaF56TQ+Pr7S8ri4uNqvdB0TFxcnl8tlhiNJuu6665Sfn1/leaYfas8//vEPpaWl6eqrrzaX0Q/BsXv3brVp08b8YytJnTp10tGjR+mDIOvatav+9re/6e2339Zbb72l6667Ttdcc42+9rWvhVU/EJBQI//4xz90ww03+Nxn/u9//6umTZsqNTVVO3fuNOfiMQxDO3bskNvtlnT+EmxWVpa5XX5+vvLz881y+M/tdqu0tFSHDh0ylx08eFCtWrWS2+2mH4Js165d6tatm88y+iE4WrRoocOHD/tcgTh48KASExPpgyAqKirS6NGj9fnnnysuLk5Op1NvvfWW0tPT5Xa79eGHH6qkpMRcPysrq8p+KC4u1p49e0LXDyF5dg5h78yZM0bv3r2N3/zmN8aBAweMt956y+jVq5exfPly48yZM0aPHj2M2bNnG/v27TNmz55t9OzZ03y0c8eOHUbnzp2NDRs2mHOOTJw4McQtCl8TJkwwbrnlFuO///2v8fbbbxs9evQwVq9eTT+EQN++fY2tW7f6LKMfguP06dNGz549jWnTphkHDx403njjDSM9Pd14/vnn6YMg+8EPfmBkZGQYn376qbFhwwYjKSnJyMnJMTwejzF48GDj7rvvNj7++GPjySefNJKTk815kHJzc42kpCTjySefNOdB+v73v888SAg/H3/8sTF27FgjOTnZ6Nmzp7F48WLzBzknJ8cYNmyYkZSUZIwcOdL48MMPfbbdvHmzceONNxrJycnGpEmTjM8++ywUTagTTp8+bUybNs1ITk42vvWtb9EPIZSUlGS8/fbbFy2nH4Jj3759xtixY41u3boZ/fv3N1auXMnvQggcOHDA+OlPf2q43W5jyJAhxt/+9jez7JNPPjF+8pOfGF26dDGGDBli/Otf//LZ9q233jIGDBhgdO3a1bj99ttDOheVwzC+uuYIAAAASYxBAgAAuAgBCQAAwIKABAAAYEFAAgAAsCAgAQAAWBCQAAAALAhIAAAAFgQkAAAACwISgJA6cuSI2rdvryNHjoS0Hu3bt9f27dtrtO1f//pXnTx5MsA1Ou/dd9/VgQMHamXfAKpGQAKAK5CXl6e7777b58XNgTR27NiL3nAOoPYRkADgCvC2JqBuIiABsI3Tp09r2rRp6tatm3r16qXZs2erpKREkrR9+3b169dPf/zjH9W7d28lJydr2rRpKisr82vfX3zxhTIyMvStb31LXbp00cCBA7Vt2zafdf79739rwIABcrvd+tWvfqVTp06ZZY8++qh69eqlrl27asyYMdq3b58k6bvf/a753y1btmjx4sW688479ZOf/ETp6el6//33dfz4cU2ZMkXdu3dXly5d9KMf/UhZWVnmvg8fPqzx48crJSVFffr00Zo1ayRJ/fr1kyTddtttWrx4cQ3PKoCaICABsI3p06frzJkzev7557Vs2TJ98MEHevDBB83yEydO6NVXX9XTTz+txYsX67XXXtOLL77o177nzp2rQ4cOacWKFdq6davS0tI0ffp0n4D13HPPafr06Xruued06NAhzZs3T5L0+uuva/369Xr88ce1detWNW/eXBkZGZKkjRs3mv8dPHiwJOmNN97Q0KFDtXr1anXt2lVTp05VeXm51q1bpxdffFHx8fGaOXOmJKm0tFTjxo3TVVddpQ0bNmjGjBl67LHH9Oabb2rTpk2SpMWLF2vcuHFXdG4BVJMBACGUm5trtGvXzjh8+LDRoUMH4/Tp02bZ3r17zWXvvfee0a5dO+Pjjz82yydNmmTcf//9fh1n8+bNxkcffWR+PnDggNGuXTvj6NGjhmEYRrt27Yy1a9ea5du3bzc6depknDlzxli5cqXRs2dPIy8vzzAMwzh58qTx73//26f+ubm5hmEYxqJFi4xvf/vb5n68Xq+xatUqIz8/31z29ttvGx06dDAMwzC2bdtmJCcnG2fOnDHLN23aZLz11ltmvd577z2/2gggcJyhDmgAIElvv/22vF6vvvOd7/gs93q9Onz4sPm5TZs25veNGjWSx+Pxa//Dhg3Ttm3btGHDBh08eFAffvihJKm8vNxcJykpyfy+U6dO8ng8+vTTTzVkyBCtXbtW3/3ud5WcnKz+/ftr5MiRVR6rVatW5vcOh0OjR4/Wyy+/rB07dujQoUPavXu3vF6vJOnQoUO67rrr1KhRI3ObESNG+NUmALWHgATAFjwejxo3bqzNmzdfVBYfH6+cnBxJUlRUlE+Z4ecg6XvvvVc7d+7UD3/4Q40ePVpxcXG65ZZbfNaJjIy8aL8NGjRQXFyc/vrXv+pf//qX3nzzTT3zzDPasGFDlbf3XC6X+b3X69W4ceN0+vRpDR48WP369dO5c+c0efJkSZLTyf+GATtiDBIAW+jdu7fOnDkjh8OhNm3aqE2bNiopKVFmZqbfA7Gr8sUXX2jr1q167LHHNGXKFN10003mAOwLA9bHH39sfr9r1y41aNBAiYmJeuutt7Rx40b16dNHs2bN0p/+9Cd98skn+vjjj+VwOC557P379+vf//63Vq1apV/84hfq06ePTpw4YR67bdu2Onz4sM80AY888ojmzJlzRW0GcGUISABsweVyqXfv3po6dap27dqlDz/8UBkZGTp79qyaNGlyRfuOiopSTEyMXnvtNR05ckT/+Mc/zMHfF4avxx57TO+++66ys7M1Z84c/fjHP1ZMTIy8Xq8yMzP1+uuv68iRI9qyZYtiYmLUtm1bxcTESJL27t2rL7/88qJjN2nSRBEREfrLX/6ivLw8vfLKK+YTaWVlZerVq5eaN2+uGTNm6MCBA3rjjTe0bt069erVS5LUsGFD7du3T2fOnLmicwCgeghIAGwjMzNTiYmJGjt2rO644w5dd911evTRR694v1FRUZo/f75effVVDRkyRA8//LB++ctfKi4uTv/973/N9e644w5Nnz5dd9xxh1JSUjR16lRJ5x+3nzJliubNm6dBgwbp5Zdf1rJly3T11VerWbNm+sEPfqC7777bfKLtQtdee61mzpypp556SkOHDtXy5ct1//33y+l0as+ePXI6nVq2bJlOnDihH/3oR5o7d67uvfde9enTR5I0ZswYZWZm8pg/EGQOw98b+AAAAPUEV5AAAAAseHwCQNibO3euOaliZSZOnKhf/OIXQawRgHDHLTYAYe+zzz675CDmq6++Wk2bNg1ehQCEPQISAACABWOQAAAALAhIAAAAFgQkAAAACwISAACABQEJAADAgoAEAABgQUACAACwICABAABY/D8tOJIRUK1lBwAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sns.histplot(x='len_abstract', data=df)\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.to_parquet(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/reduced_arxiv_papers.parquet.gzip\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.11.4 ('arxiv-env': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "aae17c2ae2f38cc6f211be9b71a2aa280701d8462782cbc1f67caa83a1603363"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/nlp_cleansing.ipynb

@@ -0,0 +1,388 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-06-24 16:49:13.031488: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy\n",
+    "from matplotlib import pyplot as plt\n",
+    "from typing import List, Dict\n",
+    "from collections import Counter\n",
+    "from pprint import pprint\n",
+    "\n",
+    "import seaborn as sns\n",
+    "sns.set_style(\"darkgrid\")\n",
+    "sns.set_palette(\"mako\")\n",
+    "\n",
+    "import spacy\n",
+    "from spacy.lang.en import English\n",
+    "from nltk.corpus import stopwords\n",
+    "\n",
+    "nlp = spacy.load('en_core_web_sm')\n",
+    "\n",
+    "pd.set_option('display.float_format', '{:.2f}'.format)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "spacy.lang.en.English"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(nlp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_raw = pd.read_parquet(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/arxiv_papers_raw.parquet.gzip\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>submitter</th>\n",
+       "      <th>authors</th>\n",
+       "      <th>title</th>\n",
+       "      <th>comments</th>\n",
+       "      <th>journal-ref</th>\n",
+       "      <th>doi</th>\n",
+       "      <th>report-no</th>\n",
+       "      <th>categories</th>\n",
+       "      <th>license</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>versions</th>\n",
+       "      <th>update_date</th>\n",
+       "      <th>authors_parsed</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0001</td>\n",
+       "      <td>Pavel Nadolsky</td>\n",
+       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
+       "      <td>Calculation of prompt diphoton production cros...</td>\n",
+       "      <td>37 pages, 15 figures; published version</td>\n",
+       "      <td>Phys.Rev.D76:013009,2007</td>\n",
+       "      <td>10.1103/PhysRevD.76.013009</td>\n",
+       "      <td>ANL-HEP-PR-07-12</td>\n",
+       "      <td>hep-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>A fully differential calculation in perturba...</td>\n",
+       "      <td>[{'created': 'Mon, 2 Apr 2007 19:18:42 GMT', '...</td>\n",
+       "      <td>2008-11-26</td>\n",
+       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Louis Theran</td>\n",
+       "      <td>Ileana Streinu and Louis Theran</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>To appear in Graphs and Combinatorics</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO cs.CG</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 02:26:18 GMT', ...</td>\n",
+       "      <td>2008-12-13</td>\n",
+       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>23 pages, 3 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>physics.gen-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td>[{'created': 'Sun, 1 Apr 2007 20:46:54 GMT', '...</td>\n",
+       "      <td>2008-01-13</td>\n",
+       "      <td>[[Pan, Hongjun, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>11 pages</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that a determinant of Stirling cycle...</td>\n",
+       "      <td>[{'created': 'Sat, 31 Mar 2007 03:16:14 GMT', ...</td>\n",
+       "      <td>2007-05-23</td>\n",
+       "      <td>[[Callan, David, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>Alberto Torchinsky</td>\n",
+       "      <td>Wael Abu-Shammala and Alberto Torchinsky</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Illinois J. Math. 52 (2008) no.2, 681-689</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CA math.FA</td>\n",
+       "      <td>None</td>\n",
+       "      <td>In this paper we show how to compute the $\\L...</td>\n",
+       "      <td>[{'created': 'Mon, 2 Apr 2007 18:09:58 GMT', '...</td>\n",
+       "      <td>2013-10-15</td>\n",
+       "      <td>[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          id           submitter   \n",
+       "0  0704.0001      Pavel Nadolsky  \\\n",
+       "1  0704.0002        Louis Theran   \n",
+       "2  0704.0003         Hongjun Pan   \n",
+       "3  0704.0004        David Callan   \n",
+       "4  0704.0005  Alberto Torchinsky   \n",
+       "\n",
+       "                                             authors   \n",
+       "0  C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
+       "1                    Ileana Streinu and Louis Theran   \n",
+       "2                                        Hongjun Pan   \n",
+       "3                                       David Callan   \n",
+       "4           Wael Abu-Shammala and Alberto Torchinsky   \n",
+       "\n",
+       "                                               title   \n",
+       "0  Calculation of prompt diphoton production cros...  \\\n",
+       "1           Sparsity-certifying Graph Decompositions   \n",
+       "2  The evolution of the Earth-Moon system based o...   \n",
+       "3  A determinant of Stirling cycle numbers counts...   \n",
+       "4  From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "\n",
+       "                                  comments   \n",
+       "0  37 pages, 15 figures; published version  \\\n",
+       "1    To appear in Graphs and Combinatorics   \n",
+       "2                      23 pages, 3 figures   \n",
+       "3                                 11 pages   \n",
+       "4                                     None   \n",
+       "\n",
+       "                                 journal-ref                         doi   \n",
+       "0                   Phys.Rev.D76:013009,2007  10.1103/PhysRevD.76.013009  \\\n",
+       "1                                       None                        None   \n",
+       "2                                       None                        None   \n",
+       "3                                       None                        None   \n",
+       "4  Illinois J. Math. 52 (2008) no.2, 681-689                        None   \n",
+       "\n",
+       "          report-no       categories   \n",
+       "0  ANL-HEP-PR-07-12           hep-ph  \\\n",
+       "1              None    math.CO cs.CG   \n",
+       "2              None   physics.gen-ph   \n",
+       "3              None          math.CO   \n",
+       "4              None  math.CA math.FA   \n",
+       "\n",
+       "                                             license   \n",
+       "0                                               None  \\\n",
+       "1  http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "2                                               None   \n",
+       "3                                               None   \n",
+       "4                                               None   \n",
+       "\n",
+       "                                            abstract   \n",
+       "0    A fully differential calculation in perturba...  \\\n",
+       "1    We describe a new algorithm, the $(k,\\ell)$-...   \n",
+       "2    The evolution of Earth-Moon system is descri...   \n",
+       "3    We show that a determinant of Stirling cycle...   \n",
+       "4    In this paper we show how to compute the $\\L...   \n",
+       "\n",
+       "                                            versions update_date   \n",
+       "0  [{'created': 'Mon, 2 Apr 2007 19:18:42 GMT', '...  2008-11-26  \\\n",
+       "1  [{'created': 'Sat, 31 Mar 2007 02:26:18 GMT', ...  2008-12-13   \n",
+       "2  [{'created': 'Sun, 1 Apr 2007 20:46:54 GMT', '...  2008-01-13   \n",
+       "3  [{'created': 'Sat, 31 Mar 2007 03:16:14 GMT', ...  2007-05-23   \n",
+       "4  [{'created': 'Mon, 2 Apr 2007 18:09:58 GMT', '...  2013-10-15   \n",
+       "\n",
+       "                                      authors_parsed  \n",
+       "0  [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \n",
+       "1           [[Streinu, Ileana, ], [Theran, Louis, ]]  \n",
+       "2                                 [[Pan, Hongjun, ]]  \n",
+       "3                                [[Callan, David, ]]  \n",
+       "4  [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]  "
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_raw.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def cleanData(doc: pd.Series, stemming=False, nlp = spacy.load('en_core_web_sm')):\n",
+    "    \"\"\"\n",
+    "    TODO: Optimize NLP Object to only obtain stopwords, lemmas, and tokenize docs.\n",
+    "    \n",
+    "    Cleans and processes the input documents by performing various text cleaning operations.\n",
+    "\n",
+    "    Args:\n",
+    "        doc (pd.Series): The documents to be cleaned, passed in a Series object.\n",
+    "        stemming (bool, optional): Specifies whether stemming should be applied. Defaults to False.\n",
+    "\n",
+    "    Returns:\n",
+    "        str: The cleaned and processed document as a single string.\n",
+    "    \"\"\"\n",
+    "    doc = doc.lower()\n",
+    "    doc = nlp(doc)\n",
+    "    tokens = [tokens.lower_ for tokens in doc]\n",
+    "    tokens = [tokens for tokens in doc if (tokens.is_stop == False)]\n",
+    "    tokens = [tokens for tokens in tokens if (tokens.is_punct == False)]\n",
+    "    final_token = [token.lemma_ for token in tokens]\n",
+    "    \n",
+    "    return \" \".join(final_token)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "  In this paper we show how to compute the $\\Lambda_{\\alpha}$ norm, $\\alpha\\ge\n",
+      "0$, using the dyadic grid. This result is a consequence of the description of\n",
+      "the Hardy spaces $H^p(R^N)$ in terms of dyadic and special atoms.\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'   paper compute $ \\\\lambda_{\\\\alpha}$ norm $ \\\\alpha\\\\ge \\n 0 $ dyadic grid result consequence description \\n hardy space $ h^p(r^n)$ term dyadic special atom \\n'"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(df_raw['abstract'][4])\n",
+    "test= cleanData(df_raw['abstract'][4])\n",
+    "test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_raw[\"cleaned_abstracts\"] = df_raw[\"abstract\"].map(lambda x: cleanData(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "df_raw.to_parquet(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/arxiv_papers.parquet.gzip\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.11.4 ('arxiv-env': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "aae17c2ae2f38cc6f211be9b71a2aa280701d8462782cbc1f67caa83a1603363"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/read_raw_data.ipynb

@@ -0,0 +1,875 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 13 µs, sys: 0 ns, total: 13 µs\n",
+      "Wall time: 20.3 µs\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "import pandas as pd\n",
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "import pyarrow as pa\n",
+    "import pyarrow.parquet as pq"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2268252it [04:21, 8672.99it/s] \n"
+     ]
+    }
+   ],
+   "source": [
+    "# df = pd.read_json(\"../data/raw/arxiv-metadata-oai-snapshot.json\")\n",
+    "\n",
+    "# Initialize an empty list to store the data from the JSON file\n",
+    "arxiv_data = []\n",
+    "\n",
+    "# Open the JSON file using the 'with' statement, ensuring it's closed automatically\n",
+    "with open(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/raw/arxiv-metadata-oai-snapshot.json\", 'r') as file:\n",
+    "    # Iterate over each line in the file\n",
+    "    for line in tqdm(file):\n",
+    "        # Load the JSON data from each line and append it to the arxiv_data list\n",
+    "        arxiv_data.append(json.loads(line))\n",
+    "\n",
+    "# Create a DataFrame from the arxiv_data list using the pd.DataFrame.from_records() method\n",
+    "df = pd.DataFrame.from_records(arxiv_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>submitter</th>\n",
+       "      <th>authors</th>\n",
+       "      <th>title</th>\n",
+       "      <th>comments</th>\n",
+       "      <th>journal-ref</th>\n",
+       "      <th>doi</th>\n",
+       "      <th>report-no</th>\n",
+       "      <th>categories</th>\n",
+       "      <th>license</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>versions</th>\n",
+       "      <th>update_date</th>\n",
+       "      <th>authors_parsed</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0001</td>\n",
+       "      <td>Pavel Nadolsky</td>\n",
+       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
+       "      <td>Calculation of prompt diphoton production cros...</td>\n",
+       "      <td>37 pages, 15 figures; published version</td>\n",
+       "      <td>Phys.Rev.D76:013009,2007</td>\n",
+       "      <td>10.1103/PhysRevD.76.013009</td>\n",
+       "      <td>ANL-HEP-PR-07-12</td>\n",
+       "      <td>hep-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>A fully differential calculation in perturba...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
+       "      <td>2008-11-26</td>\n",
+       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Louis Theran</td>\n",
+       "      <td>Ileana Streinu and Louis Theran</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>To appear in Graphs and Combinatorics</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO cs.CG</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
+       "      <td>2008-12-13</td>\n",
+       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>23 pages, 3 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>physics.gen-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sun, 1 Apr 2007...</td>\n",
+       "      <td>2008-01-13</td>\n",
+       "      <td>[[Pan, Hongjun, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>11 pages</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that a determinant of Stirling cycle...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
+       "      <td>2007-05-23</td>\n",
+       "      <td>[[Callan, David, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>Alberto Torchinsky</td>\n",
+       "      <td>Wael Abu-Shammala and Alberto Torchinsky</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Illinois J. Math. 52 (2008) no.2, 681-689</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CA math.FA</td>\n",
+       "      <td>None</td>\n",
+       "      <td>In this paper we show how to compute the $\\L...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
+       "      <td>2013-10-15</td>\n",
+       "      <td>[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268247</th>\n",
+       "      <td>supr-con/9608008</td>\n",
+       "      <td>Ruslan Prozorov</td>\n",
+       "      <td>R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...</td>\n",
+       "      <td>On the origin of the irreversibility line in t...</td>\n",
+       "      <td>19 pages, LaTex, 6 PostScript figures; Author'...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.54.15530</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We report on measurements of the angular dep...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Mon, 26 Aug 199...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Prozorov, R., ], [Konczykowski, M., ], [Schm...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268248</th>\n",
+       "      <td>supr-con/9609001</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Durga P. Choudhury, Balam A. Willemsen, John S...</td>\n",
+       "      <td>Nonlinear Response of HTSC Thin Film Microwave...</td>\n",
+       "      <td>4 pages, LaTeX type, Uses IEEE style files, 60...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1109/77.620744</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The non-linear microwave surface impedance o...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Aug 199...</td>\n",
+       "      <td>2016-11-18</td>\n",
+       "      <td>[[Choudhury, Durga P., , Physics Department, N...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268249</th>\n",
+       "      <td>supr-con/9609002</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Balam A. Willemsen, J. S. Derov and S.Sridhar ...</td>\n",
+       "      <td>Critical State Flux Penetration and Linear Mic...</td>\n",
+       "      <td>20 pages, LaTeX type, Uses REVTeX style files,...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.56.11989</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The vortex contribution to the dc field (H) ...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Tue, 3 Sep 1996...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Willemsen, Balam A., , Physics Department,\\n...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268250</th>\n",
+       "      <td>supr-con/9609003</td>\n",
+       "      <td>Hasegawa Yasumasa</td>\n",
+       "      <td>Yasumasa Hasegawa (Himeji Institute of Technol...</td>\n",
+       "      <td>Density of States and NMR Relaxation Rate in A...</td>\n",
+       "      <td>7 pages, 4 PostScript Figures, LaTeX, to appea...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1143/JPSJ.65.3131</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that the density of states in an ani...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Wed, 18 Sep 199...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Hasegawa, Yasumasa, , Himeji Institute of Te...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268251</th>\n",
+       "      <td>supr-con/9609004</td>\n",
+       "      <td>Masanori Ichioka</td>\n",
+       "      <td>Naoki Enomoto, Masanori Ichioka and Kazushige ...</td>\n",
+       "      <td>Ginzburg Landau theory for d-wave pairing and ...</td>\n",
+       "      <td>12 pages including 8 eps figs, LaTeX with jpsj...</td>\n",
+       "      <td>J. Phys. Soc. Jpn. 66, 204 (1997).</td>\n",
+       "      <td>10.1143/JPSJ.66.204</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The Ginzburg Landau theory for d_{x^2-y^2}-w...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Wed, 25 Sep 199...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2268252 rows × 14 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       id           submitter   \n",
+       "0               0704.0001      Pavel Nadolsky  \\\n",
+       "1               0704.0002        Louis Theran   \n",
+       "2               0704.0003         Hongjun Pan   \n",
+       "3               0704.0004        David Callan   \n",
+       "4               0704.0005  Alberto Torchinsky   \n",
+       "...                   ...                 ...   \n",
+       "2268247  supr-con/9608008     Ruslan Prozorov   \n",
+       "2268248  supr-con/9609001  Durga P. Choudhury   \n",
+       "2268249  supr-con/9609002  Durga P. Choudhury   \n",
+       "2268250  supr-con/9609003   Hasegawa Yasumasa   \n",
+       "2268251  supr-con/9609004    Masanori Ichioka   \n",
+       "\n",
+       "                                                   authors   \n",
+       "0        C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
+       "1                          Ileana Streinu and Louis Theran   \n",
+       "2                                              Hongjun Pan   \n",
+       "3                                             David Callan   \n",
+       "4                 Wael Abu-Shammala and Alberto Torchinsky   \n",
+       "...                                                    ...   \n",
+       "2268247  R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...   \n",
+       "2268248  Durga P. Choudhury, Balam A. Willemsen, John S...   \n",
+       "2268249  Balam A. Willemsen, J. S. Derov and S.Sridhar ...   \n",
+       "2268250  Yasumasa Hasegawa (Himeji Institute of Technol...   \n",
+       "2268251  Naoki Enomoto, Masanori Ichioka and Kazushige ...   \n",
+       "\n",
+       "                                                     title   \n",
+       "0        Calculation of prompt diphoton production cros...  \\\n",
+       "1                 Sparsity-certifying Graph Decompositions   \n",
+       "2        The evolution of the Earth-Moon system based o...   \n",
+       "3        A determinant of Stirling cycle numbers counts...   \n",
+       "4        From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "...                                                    ...   \n",
+       "2268247  On the origin of the irreversibility line in t...   \n",
+       "2268248  Nonlinear Response of HTSC Thin Film Microwave...   \n",
+       "2268249  Critical State Flux Penetration and Linear Mic...   \n",
+       "2268250  Density of States and NMR Relaxation Rate in A...   \n",
+       "2268251  Ginzburg Landau theory for d-wave pairing and ...   \n",
+       "\n",
+       "                                                  comments   \n",
+       "0                  37 pages, 15 figures; published version  \\\n",
+       "1                    To appear in Graphs and Combinatorics   \n",
+       "2                                      23 pages, 3 figures   \n",
+       "3                                                 11 pages   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "2268247  19 pages, LaTex, 6 PostScript figures; Author'...   \n",
+       "2268248  4 pages, LaTeX type, Uses IEEE style files, 60...   \n",
+       "2268249  20 pages, LaTeX type, Uses REVTeX style files,...   \n",
+       "2268250  7 pages, 4 PostScript Figures, LaTeX, to appea...   \n",
+       "2268251  12 pages including 8 eps figs, LaTeX with jpsj...   \n",
+       "\n",
+       "                                       journal-ref   \n",
+       "0                         Phys.Rev.D76:013009,2007  \\\n",
+       "1                                             None   \n",
+       "2                                             None   \n",
+       "3                                             None   \n",
+       "4        Illinois J. Math. 52 (2008) no.2, 681-689   \n",
+       "...                                            ...   \n",
+       "2268247                                       None   \n",
+       "2268248                                       None   \n",
+       "2268249                                       None   \n",
+       "2268250                                       None   \n",
+       "2268251         J. Phys. Soc. Jpn. 66, 204 (1997).   \n",
+       "\n",
+       "                                doi         report-no   \n",
+       "0        10.1103/PhysRevD.76.013009  ANL-HEP-PR-07-12  \\\n",
+       "1                              None              None   \n",
+       "2                              None              None   \n",
+       "3                              None              None   \n",
+       "4                              None              None   \n",
+       "...                             ...               ...   \n",
+       "2268247   10.1103/PhysRevB.54.15530              None   \n",
+       "2268248           10.1109/77.620744              None   \n",
+       "2268249   10.1103/PhysRevB.56.11989              None   \n",
+       "2268250        10.1143/JPSJ.65.3131              None   \n",
+       "2268251         10.1143/JPSJ.66.204              None   \n",
+       "\n",
+       "                         categories   \n",
+       "0                            hep-ph  \\\n",
+       "1                     math.CO cs.CG   \n",
+       "2                    physics.gen-ph   \n",
+       "3                           math.CO   \n",
+       "4                   math.CA math.FA   \n",
+       "...                             ...   \n",
+       "2268247  supr-con cond-mat.supr-con   \n",
+       "2268248  supr-con cond-mat.supr-con   \n",
+       "2268249  supr-con cond-mat.supr-con   \n",
+       "2268250  supr-con cond-mat.supr-con   \n",
+       "2268251  supr-con cond-mat.supr-con   \n",
+       "\n",
+       "                                                   license   \n",
+       "0                                                     None  \\\n",
+       "1        http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "2                                                     None   \n",
+       "3                                                     None   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "2268247                                               None   \n",
+       "2268248                                               None   \n",
+       "2268249                                               None   \n",
+       "2268250                                               None   \n",
+       "2268251                                               None   \n",
+       "\n",
+       "                                                  abstract   \n",
+       "0          A fully differential calculation in perturba...  \\\n",
+       "1          We describe a new algorithm, the $(k,\\ell)$-...   \n",
+       "2          The evolution of Earth-Moon system is descri...   \n",
+       "3          We show that a determinant of Stirling cycle...   \n",
+       "4          In this paper we show how to compute the $\\L...   \n",
+       "...                                                    ...   \n",
+       "2268247    We report on measurements of the angular dep...   \n",
+       "2268248    The non-linear microwave surface impedance o...   \n",
+       "2268249    The vortex contribution to the dc field (H) ...   \n",
+       "2268250    We show that the density of states in an ani...   \n",
+       "2268251    The Ginzburg Landau theory for d_{x^2-y^2}-w...   \n",
+       "\n",
+       "                                                  versions update_date   \n",
+       "0        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2008-11-26  \\\n",
+       "1        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2008-12-13   \n",
+       "2        [{'version': 'v1', 'created': 'Sun, 1 Apr 2007...  2008-01-13   \n",
+       "3        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2007-05-23   \n",
+       "4        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2013-10-15   \n",
+       "...                                                    ...         ...   \n",
+       "2268247  [{'version': 'v1', 'created': 'Mon, 26 Aug 199...  2009-10-30   \n",
+       "2268248  [{'version': 'v1', 'created': 'Sat, 31 Aug 199...  2016-11-18   \n",
+       "2268249  [{'version': 'v1', 'created': 'Tue, 3 Sep 1996...  2009-10-30   \n",
+       "2268250  [{'version': 'v1', 'created': 'Wed, 18 Sep 199...  2009-10-30   \n",
+       "2268251  [{'version': 'v1', 'created': 'Wed, 25 Sep 199...  2009-10-30   \n",
+       "\n",
+       "                                            authors_parsed  \n",
+       "0        [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \n",
+       "1                 [[Streinu, Ileana, ], [Theran, Louis, ]]  \n",
+       "2                                       [[Pan, Hongjun, ]]  \n",
+       "3                                      [[Callan, David, ]]  \n",
+       "4        [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]  \n",
+       "...                                                    ...  \n",
+       "2268247  [[Prozorov, R., ], [Konczykowski, M., ], [Schm...  \n",
+       "2268248  [[Choudhury, Durga P., , Physics Department, N...  \n",
+       "2268249  [[Willemsen, Balam A., , Physics Department,\\n...  \n",
+       "2268250  [[Hasegawa, Yasumasa, , Himeji Institute of Te...  \n",
+       "2268251  [[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...  \n",
+       "\n",
+       "[2268252 rows x 14 columns]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.to_parquet('/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/arxiv_papers_raw.parquet.gzip', compression='gzip')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>submitter</th>\n",
+       "      <th>authors</th>\n",
+       "      <th>title</th>\n",
+       "      <th>comments</th>\n",
+       "      <th>journal-ref</th>\n",
+       "      <th>doi</th>\n",
+       "      <th>report-no</th>\n",
+       "      <th>categories</th>\n",
+       "      <th>license</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>versions</th>\n",
+       "      <th>update_date</th>\n",
+       "      <th>authors_parsed</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0001</td>\n",
+       "      <td>Pavel Nadolsky</td>\n",
+       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
+       "      <td>Calculation of prompt diphoton production cros...</td>\n",
+       "      <td>37 pages, 15 figures; published version</td>\n",
+       "      <td>Phys.Rev.D76:013009,2007</td>\n",
+       "      <td>10.1103/PhysRevD.76.013009</td>\n",
+       "      <td>ANL-HEP-PR-07-12</td>\n",
+       "      <td>hep-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>A fully differential calculation in perturba...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
+       "      <td>2008-11-26</td>\n",
+       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Louis Theran</td>\n",
+       "      <td>Ileana Streinu and Louis Theran</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>To appear in Graphs and Combinatorics</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO cs.CG</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
+       "      <td>2008-12-13</td>\n",
+       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>Hongjun Pan</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>23 pages, 3 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>physics.gen-ph</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sun, 1 Apr 2007...</td>\n",
+       "      <td>2008-01-13</td>\n",
+       "      <td>[[Pan, Hongjun, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>David Callan</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>11 pages</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CO</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that a determinant of Stirling cycle...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
+       "      <td>2007-05-23</td>\n",
+       "      <td>[[Callan, David, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>Alberto Torchinsky</td>\n",
+       "      <td>Wael Abu-Shammala and Alberto Torchinsky</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Illinois J. Math. 52 (2008) no.2, 681-689</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.CA math.FA</td>\n",
+       "      <td>None</td>\n",
+       "      <td>In this paper we show how to compute the $\\L...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
+       "      <td>2013-10-15</td>\n",
+       "      <td>[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268247</th>\n",
+       "      <td>supr-con/9608008</td>\n",
+       "      <td>Ruslan Prozorov</td>\n",
+       "      <td>R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...</td>\n",
+       "      <td>On the origin of the irreversibility line in t...</td>\n",
+       "      <td>19 pages, LaTex, 6 PostScript figures; Author'...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.54.15530</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We report on measurements of the angular dep...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Mon, 26 Aug 199...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Prozorov, R., ], [Konczykowski, M., ], [Schm...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268248</th>\n",
+       "      <td>supr-con/9609001</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Durga P. Choudhury, Balam A. Willemsen, John S...</td>\n",
+       "      <td>Nonlinear Response of HTSC Thin Film Microwave...</td>\n",
+       "      <td>4 pages, LaTeX type, Uses IEEE style files, 60...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1109/77.620744</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The non-linear microwave surface impedance o...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Aug 199...</td>\n",
+       "      <td>2016-11-18</td>\n",
+       "      <td>[[Choudhury, Durga P., , Physics Department, N...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268249</th>\n",
+       "      <td>supr-con/9609002</td>\n",
+       "      <td>Durga P. Choudhury</td>\n",
+       "      <td>Balam A. Willemsen, J. S. Derov and S.Sridhar ...</td>\n",
+       "      <td>Critical State Flux Penetration and Linear Mic...</td>\n",
+       "      <td>20 pages, LaTeX type, Uses REVTeX style files,...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevB.56.11989</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The vortex contribution to the dc field (H) ...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Tue, 3 Sep 1996...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Willemsen, Balam A., , Physics Department,\\n...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268250</th>\n",
+       "      <td>supr-con/9609003</td>\n",
+       "      <td>Hasegawa Yasumasa</td>\n",
+       "      <td>Yasumasa Hasegawa (Himeji Institute of Technol...</td>\n",
+       "      <td>Density of States and NMR Relaxation Rate in A...</td>\n",
+       "      <td>7 pages, 4 PostScript Figures, LaTeX, to appea...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1143/JPSJ.65.3131</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We show that the density of states in an ani...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Wed, 18 Sep 199...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Hasegawa, Yasumasa, , Himeji Institute of Te...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2268251</th>\n",
+       "      <td>supr-con/9609004</td>\n",
+       "      <td>Masanori Ichioka</td>\n",
+       "      <td>Naoki Enomoto, Masanori Ichioka and Kazushige ...</td>\n",
+       "      <td>Ginzburg Landau theory for d-wave pairing and ...</td>\n",
+       "      <td>12 pages including 8 eps figs, LaTeX with jpsj...</td>\n",
+       "      <td>J. Phys. Soc. Jpn. 66, 204 (1997).</td>\n",
+       "      <td>10.1143/JPSJ.66.204</td>\n",
+       "      <td>None</td>\n",
+       "      <td>supr-con cond-mat.supr-con</td>\n",
+       "      <td>None</td>\n",
+       "      <td>The Ginzburg Landau theory for d_{x^2-y^2}-w...</td>\n",
+       "      <td>[{'version': 'v1', 'created': 'Wed, 25 Sep 199...</td>\n",
+       "      <td>2009-10-30</td>\n",
+       "      <td>[[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2268252 rows × 14 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       id           submitter   \n",
+       "0               0704.0001      Pavel Nadolsky  \\\n",
+       "1               0704.0002        Louis Theran   \n",
+       "2               0704.0003         Hongjun Pan   \n",
+       "3               0704.0004        David Callan   \n",
+       "4               0704.0005  Alberto Torchinsky   \n",
+       "...                   ...                 ...   \n",
+       "2268247  supr-con/9608008     Ruslan Prozorov   \n",
+       "2268248  supr-con/9609001  Durga P. Choudhury   \n",
+       "2268249  supr-con/9609002  Durga P. Choudhury   \n",
+       "2268250  supr-con/9609003   Hasegawa Yasumasa   \n",
+       "2268251  supr-con/9609004    Masanori Ichioka   \n",
+       "\n",
+       "                                                   authors   \n",
+       "0        C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
+       "1                          Ileana Streinu and Louis Theran   \n",
+       "2                                              Hongjun Pan   \n",
+       "3                                             David Callan   \n",
+       "4                 Wael Abu-Shammala and Alberto Torchinsky   \n",
+       "...                                                    ...   \n",
+       "2268247  R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...   \n",
+       "2268248  Durga P. Choudhury, Balam A. Willemsen, John S...   \n",
+       "2268249  Balam A. Willemsen, J. S. Derov and S.Sridhar ...   \n",
+       "2268250  Yasumasa Hasegawa (Himeji Institute of Technol...   \n",
+       "2268251  Naoki Enomoto, Masanori Ichioka and Kazushige ...   \n",
+       "\n",
+       "                                                     title   \n",
+       "0        Calculation of prompt diphoton production cros...  \\\n",
+       "1                 Sparsity-certifying Graph Decompositions   \n",
+       "2        The evolution of the Earth-Moon system based o...   \n",
+       "3        A determinant of Stirling cycle numbers counts...   \n",
+       "4        From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "...                                                    ...   \n",
+       "2268247  On the origin of the irreversibility line in t...   \n",
+       "2268248  Nonlinear Response of HTSC Thin Film Microwave...   \n",
+       "2268249  Critical State Flux Penetration and Linear Mic...   \n",
+       "2268250  Density of States and NMR Relaxation Rate in A...   \n",
+       "2268251  Ginzburg Landau theory for d-wave pairing and ...   \n",
+       "\n",
+       "                                                  comments   \n",
+       "0                  37 pages, 15 figures; published version  \\\n",
+       "1                    To appear in Graphs and Combinatorics   \n",
+       "2                                      23 pages, 3 figures   \n",
+       "3                                                 11 pages   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "2268247  19 pages, LaTex, 6 PostScript figures; Author'...   \n",
+       "2268248  4 pages, LaTeX type, Uses IEEE style files, 60...   \n",
+       "2268249  20 pages, LaTeX type, Uses REVTeX style files,...   \n",
+       "2268250  7 pages, 4 PostScript Figures, LaTeX, to appea...   \n",
+       "2268251  12 pages including 8 eps figs, LaTeX with jpsj...   \n",
+       "\n",
+       "                                       journal-ref   \n",
+       "0                         Phys.Rev.D76:013009,2007  \\\n",
+       "1                                             None   \n",
+       "2                                             None   \n",
+       "3                                             None   \n",
+       "4        Illinois J. Math. 52 (2008) no.2, 681-689   \n",
+       "...                                            ...   \n",
+       "2268247                                       None   \n",
+       "2268248                                       None   \n",
+       "2268249                                       None   \n",
+       "2268250                                       None   \n",
+       "2268251         J. Phys. Soc. Jpn. 66, 204 (1997).   \n",
+       "\n",
+       "                                doi         report-no   \n",
+       "0        10.1103/PhysRevD.76.013009  ANL-HEP-PR-07-12  \\\n",
+       "1                              None              None   \n",
+       "2                              None              None   \n",
+       "3                              None              None   \n",
+       "4                              None              None   \n",
+       "...                             ...               ...   \n",
+       "2268247   10.1103/PhysRevB.54.15530              None   \n",
+       "2268248           10.1109/77.620744              None   \n",
+       "2268249   10.1103/PhysRevB.56.11989              None   \n",
+       "2268250        10.1143/JPSJ.65.3131              None   \n",
+       "2268251         10.1143/JPSJ.66.204              None   \n",
+       "\n",
+       "                         categories   \n",
+       "0                            hep-ph  \\\n",
+       "1                     math.CO cs.CG   \n",
+       "2                    physics.gen-ph   \n",
+       "3                           math.CO   \n",
+       "4                   math.CA math.FA   \n",
+       "...                             ...   \n",
+       "2268247  supr-con cond-mat.supr-con   \n",
+       "2268248  supr-con cond-mat.supr-con   \n",
+       "2268249  supr-con cond-mat.supr-con   \n",
+       "2268250  supr-con cond-mat.supr-con   \n",
+       "2268251  supr-con cond-mat.supr-con   \n",
+       "\n",
+       "                                                   license   \n",
+       "0                                                     None  \\\n",
+       "1        http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "2                                                     None   \n",
+       "3                                                     None   \n",
+       "4                                                     None   \n",
+       "...                                                    ...   \n",
+       "2268247                                               None   \n",
+       "2268248                                               None   \n",
+       "2268249                                               None   \n",
+       "2268250                                               None   \n",
+       "2268251                                               None   \n",
+       "\n",
+       "                                                  abstract   \n",
+       "0          A fully differential calculation in perturba...  \\\n",
+       "1          We describe a new algorithm, the $(k,\\ell)$-...   \n",
+       "2          The evolution of Earth-Moon system is descri...   \n",
+       "3          We show that a determinant of Stirling cycle...   \n",
+       "4          In this paper we show how to compute the $\\L...   \n",
+       "...                                                    ...   \n",
+       "2268247    We report on measurements of the angular dep...   \n",
+       "2268248    The non-linear microwave surface impedance o...   \n",
+       "2268249    The vortex contribution to the dc field (H) ...   \n",
+       "2268250    We show that the density of states in an ani...   \n",
+       "2268251    The Ginzburg Landau theory for d_{x^2-y^2}-w...   \n",
+       "\n",
+       "                                                  versions update_date   \n",
+       "0        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2008-11-26  \\\n",
+       "1        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2008-12-13   \n",
+       "2        [{'version': 'v1', 'created': 'Sun, 1 Apr 2007...  2008-01-13   \n",
+       "3        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2007-05-23   \n",
+       "4        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2013-10-15   \n",
+       "...                                                    ...         ...   \n",
+       "2268247  [{'version': 'v1', 'created': 'Mon, 26 Aug 199...  2009-10-30   \n",
+       "2268248  [{'version': 'v1', 'created': 'Sat, 31 Aug 199...  2016-11-18   \n",
+       "2268249  [{'version': 'v1', 'created': 'Tue, 3 Sep 1996...  2009-10-30   \n",
+       "2268250  [{'version': 'v1', 'created': 'Wed, 18 Sep 199...  2009-10-30   \n",
+       "2268251  [{'version': 'v1', 'created': 'Wed, 25 Sep 199...  2009-10-30   \n",
+       "\n",
+       "                                            authors_parsed  \n",
+       "0        [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \n",
+       "1                 [[Streinu, Ileana, ], [Theran, Louis, ]]  \n",
+       "2                                       [[Pan, Hongjun, ]]  \n",
+       "3                                      [[Callan, David, ]]  \n",
+       "4        [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]  \n",
+       "...                                                    ...  \n",
+       "2268247  [[Prozorov, R., ], [Konczykowski, M., ], [Schm...  \n",
+       "2268248  [[Choudhury, Durga P., , Physics Department, N...  \n",
+       "2268249  [[Willemsen, Balam A., , Physics Department,\\n...  \n",
+       "2268250  [[Hasegawa, Yasumasa, , Himeji Institute of Te...  \n",
+       "2268251  [[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...  \n",
+       "\n",
+       "[2268252 rows x 14 columns]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.11.3 ('arxiv-env': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "aae17c2ae2f38cc6f211be9b71a2aa280701d8462782cbc1f67caa83a1603363"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,81 @@
+altair==5.0.1
+attrs==23.1.0
+beautifulsoup4==4.12.2
+blinker==1.6.2
+cachetools==5.3.1
+certifi==2023.5.7
+charset-normalizer==3.2.0
+click==8.1.4
+contourpy==1.1.0
+cramjam==2.6.2
+cycler==0.11.0
+decorator==5.1.1
+Faker==19.1.0
+fastparquet==2023.7.0
+favicon==0.7.0
+fonttools==4.41.0
+fsspec==2023.6.0
+gensim==4.3.1
+gitdb==4.0.10
+GitPython==3.1.32
+htbuilder==0.6.1
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.0.0
+Jinja2==3.1.2
+jsonschema==4.18.2
+jsonschema-specifications==2023.6.1
+kiwisolver==1.4.4
+lxml==4.9.3
+Markdown==3.4.3
+markdown-it-py==3.0.0
+markdownlit==0.0.7
+MarkupSafe==2.1.3
+matplotlib==3.7.2
+mdurl==0.1.2
+more-itertools==9.1.0
+numpy==1.25.1
+packaging==23.1
+pandas==2.0.3
+Pillow==9.5.0
+protobuf==4.23.4
+pyarrow==12.0.1
+pydeck==0.8.1b0
+Pygments==2.15.1
+pymdown-extensions==10.0.1
+Pympler==1.0.1
+pyparsing==3.0.9
+python-dateutil==2.8.2
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+referencing==0.29.1
+requests==2.31.0
+rich==13.4.2
+rpds-py==0.8.10
+scipy==1.11.1
+six==1.16.0
+smart-open==6.3.0
+smmap==5.0.0
+soupsieve==2.4.1
+st-annotated-text==4.0.0
+streamlit==1.24.1
+streamlit-camera-input-live==0.2.0
+streamlit-card==0.0.61
+streamlit-embedcode==0.1.2
+streamlit-extras==0.2.7
+streamlit-faker==0.0.2
+streamlit-image-coordinates==0.1.5
+streamlit-keyup==0.2.0
+streamlit-toggle-switch==1.0.2
+streamlit-vertical-slider==1.0.2
+tenacity==8.2.2
+toml==0.10.2
+toolz==0.12.0
+tornado==6.3.2
+typing_extensions==4.7.1
+tzdata==2023.3
+tzlocal==4.3.1
+urllib3==2.0.3
+validators==0.20.0
+zipp==3.16.0

src/app.py

@@ -0,0 +1,226 @@
+import streamlit as st
+from streamlit_extras.no_default_selectbox import selectbox
+import pandas as pd
+from PIL import Image
+from random import choices
+import zipfile
+import os
+
+from gensim.corpora import Dictionary
+from gensim.models import TfidfModel
+from gensim.similarities import SparseMatrixSimilarity
+
+from models.utils.constants import user_requests_tests, TEST_INPUTS
+from models.utils.mlutilities import gensim_tokenizer, get_recomendations_metadata
+
+
+st.set_page_config(page_title="Papers Recomendation App")
+
+model_name = "GrammarGuru"
+
+def folder_exists(folder_path):
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        return True
+    else:
+        return False
+
+
+
+def unzip_file(zip_file_path: str, modelname: str = model_name):
+    if not folder_exists(f"models/{modelname}"):
+        try:
+            with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
+                zip_ref.extractall(f"models/")
+            st.write("Model Zip file Extraction completed!.")
+        except FileNotFoundError:
+            raise("Error: The specified zip file was not found.")
+        except zipfile.BadZipFile:
+            raise("Error: The specified file is not a valid zip file.")
+
+
+hide_default_format = """
+       <style>
+       #MainMenu {visibility: hidden; }
+       footer {visibility: hidden;}
+       </style>
+       """
+st.markdown(hide_default_format, unsafe_allow_html=True)
+
+image = Image.open('reports/figures/arxiv-logo.jpg')
+
+st.sidebar.image(image , caption="Arxiv Papers Recomendation System",width = 256)
+app_mode = st.sidebar.selectbox("Choose app mode", ["Generate Recomendations", "About this Project", "About Me"])
+
+st.title("ResearchRadar")
+
+
+@st.cache_data
+def load_papers_corpus(path: str):
+    return pd.read_parquet(path)
+
+@st.cache_resource
+def load_dict(path: str):
+    dict_corpus = Dictionary.load(path)
+    return dict_corpus
+
+@st.cache_resource
+def load_model(path: str ):
+    tfidf_model = TfidfModel.load(path)
+    return tfidf_model
+
+@st.cache_resource
+def load_sparse_matrix(path: str):
+    similarities = SparseMatrixSimilarity.load(path)
+    return similarities
+
+
+if app_mode == "Generate Recomendations":    
+    welcome_text = """    
+    <div style="text-align: justify">Welcome to my paper recommendation project! This App is here to simplify your search for relevant scientific and academic papers. Our intelligent recommendation system, powered by  <strong>Machine Learning and natural language processing</strong>, analyzes keywords, abstracts, titles, authors, and more to provide personalized suggestions based on your interests. Say goodbye to information overload and let us guide you towards new horizons in your quest for knowledge. 
+    """
+    subjects = """
+    Our model is trained to recommend papers in various domains, including:
+    - Mathematics
+    - Statistics
+    - Electrical Engineering
+    - Quantitative Biology
+    - Economics
+
+    Say goodbye to information overload and let us guide you towards **new horizons** in your quest for knowledge. Join us and discover a streamlined way to **explore, learn, and stay ahead** in your field. Welcome aboard!
+    """
+    st.markdown(welcome_text, unsafe_allow_html=True)
+    st.markdown(subjects)
+    st.divider()
+
+    
+    with st.container():
+        examples = """
+        ### Examples of prompts
+        - "Can you recommend papers that explore the application of deep learning in computer vision for object detection, image segmentation, and video analysis?"
+        - "Can you recommend papers that explore the use of deep reinforcement learning for autonomous driving, including perception, planning, and control?"
+        - "Could you provide papers on image and video compression algorithms based on the latest video coding standards, such as HEVC and AV1?"
+        - "Can you suggest recent papers on behavioral economics that investigate the role of emotions and biases in decision-making under uncertainty, particularly in the context of financial markets?"
+        
+        """
+        st.markdown(examples)
+        st.divider()
+    
+    
+    with st.spinner('The model binaries are unziping ...'):
+        zip_file_path = "models/GrammarGuru.zip"
+        unzip_file(zip_file_path)            
+
+    with st.spinner('The model binaries are loading, please wait...'):
+
+        df = load_papers_corpus("models/GrammarGuru/data/GrammarGuru.parquet.gzip") 
+        dictionary = load_dict("models/GrammarGuru/dictionaries/GrammarGuru.dict") 
+        model = load_model("models/GrammarGuru/tdidf/GrammarGuru.model") 
+        matrix = load_sparse_matrix("models/GrammarGuru/similarities_matrix/GrammarGuru") 
+        st.success('Models Loaded, yei!', icon="🚀")
+    
+    st.markdown("#### Generate Recommendations")
+    # recs_number = st.slider("Enter the number of papers you need", min_value=1, max_value=10, value=3)
+    query = st.text_input("Enter the description of the Paper you need (the more descriptive, the better)", value="")
+    
+    if query != "":
+        cleaned_prompt = gensim_tokenizer(query)
+
+        with st.spinner('Generating Recommendations ... '):
+            results_df = get_recomendations_metadata(query=query, df=df, n=3, dictionary=dictionary, index=matrix, tfidf_model=model)
+        
+            ids = results_df['id'].to_list()
+            titles = results_df['title'].to_list()
+            authors = results_df['authors'].to_list()
+            categories = results_df['categories'].to_list()
+            abstracts = results_df['abstract'].to_list()
+            release_date = results_df['update_date'].to_list()
+            
+            results = list(zip(ids, titles, authors, categories, abstracts, release_date))
+            
+            st.write("Your top 3 papers:")
+            for result in results:
+                with st.container():
+                    col1, col2 = st.columns([1,3])
+
+                    with col1:
+                        st.markdown(f"**Title:**")
+                        st.markdown(f"**Author:**")
+                        st.markdown(f"**Categories:**")
+                        st.markdown(f"**release_date:**")
+                        st.markdown(f"**Abstract:**")
+
+                        
+                    with col2:
+                        st.write(f"Title: {result[1]}")
+                        st.write(f"Author: {result[2]}")
+                        st.write(f"Categories: {result[3]}")
+                        st.write(f"release_date: {result[5]}")
+                        st.write(f"Abstract: {result[4]}")
+                        st.markdown(f"""[Paper Link](https://arxiv.org/abs/{result[0]})""")
+                    st.divider()
+            st.balloons()
+
+    else:
+        st.write("Please enter your prompt :)")
+
+        
+    
+
+    
+elif app_mode == "About this Project":
+    intro_text = """
+    Welcome to my paper recommendation project! This application aims to simplify and speed up the process of finding relevant scientific and academic papers. It utilizes Machine Learning techniques and natural language processing to provide an effective solution for students, researchers, and general users.
+
+    ### Key Features
+
+    - **Intelligent Recommendation System:** The application uses advanced algorithms to analyze keywords, abstracts, titles, authors, and other metadata associated with each paper.
+    - **Efficient Discovery Process:** By leveraging machine learning, the system identifies and suggests the most relevant papers based on the user's interests and areas of study.
+    - **Comprehensive Analysis:** The recommendation system performs an exhaustive analysis of various aspects of each paper to ensure accurate and targeted recommendations.
+    - **Time-saving Solution:** Instead of manually searching through vast amounts of information, users can rely on this application to streamline the paper discovery process.
+
+    ### Available Models
+
+    - SemanticSherlock: trained on 100% of the data
+    - LanguageLiberator: trained on 75% of the data
+    - TextualTango: trained on 50% of the data
+    - GrammarGuru: trained on 25% of the data **(Deployed Version)**
+
+    **Note:** Due to resource limitations on the free tier of Streamlit, only the GrammarGuru version of the model is available for deployment.
+
+
+    ### Benefits
+
+    - **Saves Time and Effort:** With the application's intelligent algorithms, users can avoid the challenges and time-consuming nature of searching for papers on their own.
+    - **Increased Relevance:** By considering keywords, abstracts, titles, authors, and other metadata, the recommendation system provides users with highly relevant paper suggestions.
+    - **Tailored to User Interests:** The system takes into account each user's interests and areas of study, ensuring that the recommended papers align with their specific needs.
+    - **Accessible to All Users:** Whether you are a student, researcher, or general user, this application is designed to cater to a wide range of users' needs.
+
+    ### Get Started
+
+    Explore, discover, and reach new horizons in your search for knowledge with our paper recommendation application. Simplify your journey to finding relevant papers and stay ahead in your field.
+    
+    Take a look to this proyect in my [GitHub Repo](https://github.com/LewisPons/arxiv-paper-recommender-system)
+    """
+
+
+
+    st.markdown(intro_text)
+
+
+
+elif app_mode == "About Me":
+    st.title('About Me')
+    mkdn = """
+    <p style="text-align: justify;">Hey there! I'm  <strong>Luis Morales</strong>, a passionate data professional with a background in Actuarial Sciences and expertise in Data Engineering and Machine Learning. I love diving into complex data projects and helping organizations unlock the power of their data. From designing robust data pipelines to building powerful ML models, I enjoy the thrill of turning raw data into actionable insights. With my coding skills in Python and R, I'm always up for tackling challenging projects and learning new technologies along the way.
+    Thank you for taking the time to learn a little bit about me!</p>
+    """
+    st.markdown(mkdn, unsafe_allow_html=True)
+    st.success("Feel free to contact me here 👇 ")
+
+    col1,col2,col3,col4 = st.columns((2,1,2,1))
+    col1.markdown('* [LinkedIn](https://www.linkedin.com/in/luis-morales-ponce/)')
+    col1.markdown('* [GitHub](https://github.com/LewisPons)')
+    image2 = Image.open('reports/figures/profile.jpeg')
+    st.image(image2, width=400)
+    
+

src/data/make_dataset.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+import click
+import logging
+from pathlib import Path
+from dotenv import find_dotenv, load_dotenv
+
+
+@click.command()
+@click.argument('input_filepath', type=click.Path(exists=True))
+@click.argument('output_filepath', type=click.Path())
+def main(input_filepath, output_filepath):
+    """ Runs data processing scripts to turn raw data from (../raw) into
+        cleaned data ready to be analyzed (saved in ../processed).
+    """
+    logger = logging.getLogger(__name__)
+    logger.info('making final data set from raw data')
+
+
+if __name__ == '__main__':
+    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    logging.basicConfig(level=logging.INFO, format=log_fmt)
+
+    # not used in this stub but often useful for finding various files
+    project_dir = Path(__file__).resolve().parents[2]
+
+    # find .env automagically by walking up directories until it's found, then
+    # load up the .env entries as environment variables
+    load_dotenv(find_dotenv())
+
+    main()

src/data/transform_raw_data.py

@@ -0,0 +1,53 @@
+"""
+    This is a utility script for use in sagemaker
+"""
+
+import json
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+import os
+from tqdm import tqdm
+
+# File paths
+json_file_path = "/home/studio-lab-user/arxiv-paper-recommender-system/arxiv-metadata-oai-snapshot.json"
+parquet_file_path = "/home/studio-lab-user/arxiv-paper-recommender-system/data/processed/arxiv_papers_raw.parquet.gzip"
+
+# Batch size
+batch_size = 10000
+
+# Create the parent directory if it doesn't exist
+parent_dir = os.path.dirname(parquet_file_path)
+os.makedirs(parent_dir, exist_ok=True)
+
+# Open the JSON file
+with open(json_file_path, 'r') as file:
+    # Initialize an empty list to store the data
+    arxiv_data = []
+    processed_count = 0
+
+    # Iterate over each line in the file
+    for line in tqdm(file):
+        # Load the JSON data from each line and append it to the arxiv_data list
+        arxiv_data.append(json.loads(line))
+
+        processed_count += 1
+
+        # Process a batch of data
+        if processed_count % batch_size == 0:
+            df = pd.DataFrame.from_records(arxiv_data)
+            # Convert the batch to parquet and append it to the file
+            # df.to_parquet(parquet_file_path, compression='gzip', engine='pyarrow', index=False, append=True)
+            # Create a parquet table from your dataframe
+            table = pa.Table.from_pandas(df)
+
+            # Write direct to your parquet file
+            pq.write_to_dataset(table , root_path=parquet_file_path)
+            arxiv_data = []
+
+    # Process the remaining data (if any)
+    if arxiv_data:
+        df = pd.DataFrame.from_records(arxiv_data)
+        # Convert the remaining batch to parquet and append it to the file
+        # df.to_parquet(parquet_file_path, compression='gzip', engine='pyarrow', index=False, append=True)
+        pq.write_to_dataset(parquet_file_path , root_path=parquet_file_path)

src/models/configs.yaml

@@ -0,0 +1,18 @@
+
+GensimConfig:
+  Large:
+    ModelName: SemanticSherlock
+    DataSetFracSplit: null
+    RandomSeedSplit: null
+  SubLarge:
+    ModelName: LanguageLiberator
+    DataSetFracSplit: 0.75
+    RandomSeedSplit: 98765
+  Medium:
+    ModelName: TextualTango
+    DataSetFracSplit: 0.5
+    RandomSeedSplit: 42355
+  Small:
+    ModelName: GrammarGuru
+    DataSetFracSplit: 0.25
+    RandomSeedSplit: 76438

src/models/gensim_tfidf.ipynb

@@ -0,0 +1,862 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-07-07 17:13:01.457105: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import gensim\n",
+    "import pprint\n",
+    "from gensim import corpora\n",
+    "from gensim.utils import simple_preprocess\n",
+    "from gensim.models import TfidfModel\n",
+    "from gensim.parsing import strip_tags, strip_numeric, \\\n",
+    "    strip_multiple_whitespaces, stem_text, strip_punctuation, \\\n",
+    "    remove_stopwords, preprocess_string\n",
+    "import re\n",
+    "import os\n",
+    "\n",
+    "from typing import List\n",
+    "import spacy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "transform_to_lower = lambda s: s.lower()\n",
+    "remove_single_char = lambda s: re.sub(r'\\s+\\w{1}\\s+', '', s)\n",
+    "\n",
+    "cleaning_filters = [\n",
+    "    strip_tags,\n",
+    "    strip_numeric,\n",
+    "    strip_punctuation, \n",
+    "    strip_multiple_whitespaces, \n",
+    "    transform_to_lower,\n",
+    "    remove_stopwords,\n",
+    "    remove_single_char\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/reduced_arxiv_papers.parquet.gzip\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "638707"
+      ]
+     },
+     "execution_count": 94,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "int(df.shape[0] * 0.75) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 73,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/reduced_arxiv_papers.parquet.gzip\").sample().reset_index(drop=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 74,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>submitter</th>\n",
+       "      <th>authors</th>\n",
+       "      <th>title</th>\n",
+       "      <th>comments</th>\n",
+       "      <th>journal-ref</th>\n",
+       "      <th>doi</th>\n",
+       "      <th>report-no</th>\n",
+       "      <th>categories</th>\n",
+       "      <th>license</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>versions</th>\n",
+       "      <th>update_date</th>\n",
+       "      <th>authors_parsed</th>\n",
+       "      <th>cleaned_abstracts</th>\n",
+       "      <th>len_abstract</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2007.00905</td>\n",
+       "      <td>Song Qingheng</td>\n",
+       "      <td>Qingheng Song, Yong Zeng, Jie Xu, and Shi Jin</td>\n",
+       "      <td>A Survey of Prototype and Experiment for UAV C...</td>\n",
+       "      <td>24 pages, 6 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cs.IT eess.SP math.IT</td>\n",
+       "      <td>http://creativecommons.org/licenses/by-nc-sa/4.0/</td>\n",
+       "      <td>Unmanned aerial vehicle (UAV) communications...</td>\n",
+       "      <td>[{'created': 'Thu, 2 Jul 2020 06:26:20 GMT', '...</td>\n",
+       "      <td>2020-07-03</td>\n",
+       "      <td>[[Song, Qingheng, ], [Zeng, Yong, ], [Xu, Jie,...</td>\n",
+       "      <td>unmanned aerial vehicle uav communication a...</td>\n",
+       "      <td>865</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2102.04209</td>\n",
+       "      <td>Michael Stuart</td>\n",
+       "      <td>Michael T. Stuart and Markus Kneer</td>\n",
+       "      <td>Guilty Artificial Minds</td>\n",
+       "      <td>20 pages, 4 figures, 1 table</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cs.CY cs.AI cs.HC</td>\n",
+       "      <td>http://creativecommons.org/licenses/by/4.0/</td>\n",
+       "      <td>The concepts of blameworthiness and wrongnes...</td>\n",
+       "      <td>[{'created': 'Sun, 24 Jan 2021 21:37:35 GMT', ...</td>\n",
+       "      <td>2021-02-09</td>\n",
+       "      <td>[[Stuart, Michael T., ], [Kneer, Markus, ]]</td>\n",
+       "      <td>concept blameworthiness wrongness fundament...</td>\n",
+       "      <td>739</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>1201.5796</td>\n",
+       "      <td>Denis Jerome</td>\n",
+       "      <td>Denis Jerome</td>\n",
+       "      <td>Organic Superconductors: when correlations and...</td>\n",
+       "      <td>41 pages, 21 figures to be published in Journa...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1007/s10948-012-1475-7</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cond-mat.supr-con</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>This survey provides a brief account for the...</td>\n",
+       "      <td>[{'created': 'Fri, 27 Jan 2012 15:24:46 GMT', ...</td>\n",
+       "      <td>2012-02-21</td>\n",
+       "      <td>[[Jerome, Denis, ]]</td>\n",
+       "      <td>survey provide brief account start organic ...</td>\n",
+       "      <td>649</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1511.03076</td>\n",
+       "      <td>Emma Platts Miss</td>\n",
+       "      <td>George F.R. Ellis, Emma Platts, David Sloan an...</td>\n",
+       "      <td>Current observations with a decaying cosmologi...</td>\n",
+       "      <td>23 pages, 11 figures</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1088/1475-7516/2016/04/026</td>\n",
+       "      <td>None</td>\n",
+       "      <td>astro-ph.CO gr-qc hep-th</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We use the phase plane analysis technique of...</td>\n",
+       "      <td>[{'created': 'Tue, 10 Nov 2015 12:08:23 GMT', ...</td>\n",
+       "      <td>2016-04-27</td>\n",
+       "      <td>[[Ellis, George F. R., ], [Platts, Emma, ], [S...</td>\n",
+       "      <td>use phase plane analysis technique madsen e...</td>\n",
+       "      <td>554</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>1710.02954</td>\n",
+       "      <td>Kirk Bansak</td>\n",
+       "      <td>Kirk Bansak</td>\n",
+       "      <td>Estimating Causal Moderation Effects with Rand...</td>\n",
+       "      <td>Forthcoming, Journal of the Royal Statistical ...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>stat.ME</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>Researchers are often interested in analyzin...</td>\n",
+       "      <td>[{'created': 'Mon, 9 Oct 2017 06:34:01 GMT', '...</td>\n",
+       "      <td>2020-08-25</td>\n",
+       "      <td>[[Bansak, Kirk, ]]</td>\n",
+       "      <td>researcher interested analyze conditional t...</td>\n",
+       "      <td>799</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>851605</th>\n",
+       "      <td>1301.0707</td>\n",
+       "      <td>Sebastian Klein</td>\n",
+       "      <td>Sebastian Klein</td>\n",
+       "      <td>Chow groups of tensor triangulated categories</td>\n",
+       "      <td>40 pages. The presentation of the article has ...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.AG math.CT math.RT</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We recall P. Balmer's definition of tensor t...</td>\n",
+       "      <td>[{'created': 'Fri, 4 Jan 2013 11:06:40 GMT', '...</td>\n",
+       "      <td>2015-10-02</td>\n",
+       "      <td>[[Klein, Sebastian, ]]</td>\n",
+       "      <td>recall p. balmer definition tensor triangul...</td>\n",
+       "      <td>787</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>851606</th>\n",
+       "      <td>1707.00341</td>\n",
+       "      <td>Giorgos Anastasiou</td>\n",
+       "      <td>Giorgos Anastasiou, Rodrigo Olea, David Rivera...</td>\n",
+       "      <td>Noether-Wald energy in Critical Gravity</td>\n",
+       "      <td>7 pages, no figures, Final version for PLB</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1016/j.physletb.2018.11.021</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hep-th gr-qc</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>Criticality represents a specific point in t...</td>\n",
+       "      <td>[{'created': 'Sun, 2 Jul 2017 19:52:32 GMT', '...</td>\n",
+       "      <td>2018-11-21</td>\n",
+       "      <td>[[Anastasiou, Giorgos, ], [Olea, Rodrigo, ], [...</td>\n",
+       "      <td>criticality represent specific point parame...</td>\n",
+       "      <td>631</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>851607</th>\n",
+       "      <td>1610.08526</td>\n",
+       "      <td>Blagoje Oblak</td>\n",
+       "      <td>Blagoje Oblak</td>\n",
+       "      <td>BMS Particles in Three Dimensions</td>\n",
+       "      <td>437 pages (including index), 33 figures. Appen...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1007/978-3-319-61878-4</td>\n",
+       "      <td>None</td>\n",
+       "      <td>hep-th gr-qc math-ph math.GR math.MP math.RT</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>This thesis is devoted to the group-theoreti...</td>\n",
+       "      <td>[{'created': 'Wed, 26 Oct 2016 20:00:16 GMT', ...</td>\n",
+       "      <td>2018-01-29</td>\n",
+       "      <td>[[Oblak, Blagoje, ]]</td>\n",
+       "      <td>thesis devoted group theoretic aspect dimen...</td>\n",
+       "      <td>542</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>851608</th>\n",
+       "      <td>1211.6629</td>\n",
+       "      <td>Philippe Joyez</td>\n",
+       "      <td>Philippe Joyez</td>\n",
+       "      <td>Self-consistent dynamics of a Josephson juncti...</td>\n",
+       "      <td>7 pages, 1 figure</td>\n",
+       "      <td>None</td>\n",
+       "      <td>10.1103/PhysRevLett.110.217003</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cond-mat.supr-con cond-mat.mes-hall</td>\n",
+       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
+       "      <td>We derive microscopically the dynamics assoc...</td>\n",
+       "      <td>[{'created': 'Tue, 27 Nov 2012 17:29:04 GMT', ...</td>\n",
+       "      <td>2013-05-29</td>\n",
+       "      <td>[[Joyez, Philippe, ]]</td>\n",
+       "      <td>derive microscopically dynamic associate d....</td>\n",
+       "      <td>558</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>851609</th>\n",
+       "      <td>0705.2878</td>\n",
+       "      <td>Benoit Perthame</td>\n",
+       "      <td>Benoit Perthame (DMA), Panagiotis E. Souganidis</td>\n",
+       "      <td>Asymmetric potentials and motor effect: a larg...</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>math.AP</td>\n",
+       "      <td>None</td>\n",
+       "      <td>We provide a mathematical analysis of appear...</td>\n",
+       "      <td>[{'created': 'Sun, 20 May 2007 17:43:39 GMT', ...</td>\n",
+       "      <td>2007-05-23</td>\n",
+       "      <td>[[Perthame, Benoit, , DMA], [Souganidis, Panag...</td>\n",
+       "      <td>provide mathematical analysis appearance co...</td>\n",
+       "      <td>518</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>851610 rows × 16 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                id           submitter   \n",
+       "0       2007.00905       Song Qingheng  \\\n",
+       "1       2102.04209      Michael Stuart   \n",
+       "2        1201.5796        Denis Jerome   \n",
+       "3       1511.03076    Emma Platts Miss   \n",
+       "4       1710.02954         Kirk Bansak   \n",
+       "...            ...                 ...   \n",
+       "851605   1301.0707     Sebastian Klein   \n",
+       "851606  1707.00341  Giorgos Anastasiou   \n",
+       "851607  1610.08526       Blagoje Oblak   \n",
+       "851608   1211.6629      Philippe Joyez   \n",
+       "851609   0705.2878     Benoit Perthame   \n",
+       "\n",
+       "                                                  authors   \n",
+       "0           Qingheng Song, Yong Zeng, Jie Xu, and Shi Jin  \\\n",
+       "1                      Michael T. Stuart and Markus Kneer   \n",
+       "2                                            Denis Jerome   \n",
+       "3       George F.R. Ellis, Emma Platts, David Sloan an...   \n",
+       "4                                             Kirk Bansak   \n",
+       "...                                                   ...   \n",
+       "851605                                    Sebastian Klein   \n",
+       "851606  Giorgos Anastasiou, Rodrigo Olea, David Rivera...   \n",
+       "851607                                      Blagoje Oblak   \n",
+       "851608                                     Philippe Joyez   \n",
+       "851609    Benoit Perthame (DMA), Panagiotis E. Souganidis   \n",
+       "\n",
+       "                                                    title   \n",
+       "0       A Survey of Prototype and Experiment for UAV C...  \\\n",
+       "1                                 Guilty Artificial Minds   \n",
+       "2       Organic Superconductors: when correlations and...   \n",
+       "3       Current observations with a decaying cosmologi...   \n",
+       "4       Estimating Causal Moderation Effects with Rand...   \n",
+       "...                                                   ...   \n",
+       "851605      Chow groups of tensor triangulated categories   \n",
+       "851606            Noether-Wald energy in Critical Gravity   \n",
+       "851607                  BMS Particles in Three Dimensions   \n",
+       "851608  Self-consistent dynamics of a Josephson juncti...   \n",
+       "851609  Asymmetric potentials and motor effect: a larg...   \n",
+       "\n",
+       "                                                 comments journal-ref   \n",
+       "0                                     24 pages, 6 figures        None  \\\n",
+       "1                            20 pages, 4 figures, 1 table        None   \n",
+       "2       41 pages, 21 figures to be published in Journa...        None   \n",
+       "3                                    23 pages, 11 figures        None   \n",
+       "4       Forthcoming, Journal of the Royal Statistical ...        None   \n",
+       "...                                                   ...         ...   \n",
+       "851605  40 pages. The presentation of the article has ...        None   \n",
+       "851606         7 pages, no figures, Final version for PLB        None   \n",
+       "851607  437 pages (including index), 33 figures. Appen...        None   \n",
+       "851608                                  7 pages, 1 figure        None   \n",
+       "851609                                               None        None   \n",
+       "\n",
+       "                                   doi report-no   \n",
+       "0                                 None      None  \\\n",
+       "1                                 None      None   \n",
+       "2            10.1007/s10948-012-1475-7      None   \n",
+       "3        10.1088/1475-7516/2016/04/026      None   \n",
+       "4                                 None      None   \n",
+       "...                                ...       ...   \n",
+       "851605                            None      None   \n",
+       "851606  10.1016/j.physletb.2018.11.021      None   \n",
+       "851607       10.1007/978-3-319-61878-4      None   \n",
+       "851608  10.1103/PhysRevLett.110.217003      None   \n",
+       "851609                            None      None   \n",
+       "\n",
+       "                                          categories   \n",
+       "0                              cs.IT eess.SP math.IT  \\\n",
+       "1                                  cs.CY cs.AI cs.HC   \n",
+       "2                                  cond-mat.supr-con   \n",
+       "3                           astro-ph.CO gr-qc hep-th   \n",
+       "4                                            stat.ME   \n",
+       "...                                              ...   \n",
+       "851605                       math.AG math.CT math.RT   \n",
+       "851606                                  hep-th gr-qc   \n",
+       "851607  hep-th gr-qc math-ph math.GR math.MP math.RT   \n",
+       "851608           cond-mat.supr-con cond-mat.mes-hall   \n",
+       "851609                                       math.AP   \n",
+       "\n",
+       "                                                  license   \n",
+       "0       http://creativecommons.org/licenses/by-nc-sa/4.0/  \\\n",
+       "1             http://creativecommons.org/licenses/by/4.0/   \n",
+       "2       http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "3       http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "4       http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "...                                                   ...   \n",
+       "851605  http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "851606  http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "851607  http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "851608  http://arxiv.org/licenses/nonexclusive-distrib...   \n",
+       "851609                                               None   \n",
+       "\n",
+       "                                                 abstract   \n",
+       "0         Unmanned aerial vehicle (UAV) communications...  \\\n",
+       "1         The concepts of blameworthiness and wrongnes...   \n",
+       "2         This survey provides a brief account for the...   \n",
+       "3         We use the phase plane analysis technique of...   \n",
+       "4         Researchers are often interested in analyzin...   \n",
+       "...                                                   ...   \n",
+       "851605    We recall P. Balmer's definition of tensor t...   \n",
+       "851606    Criticality represents a specific point in t...   \n",
+       "851607    This thesis is devoted to the group-theoreti...   \n",
+       "851608    We derive microscopically the dynamics assoc...   \n",
+       "851609    We provide a mathematical analysis of appear...   \n",
+       "\n",
+       "                                                 versions update_date   \n",
+       "0       [{'created': 'Thu, 2 Jul 2020 06:26:20 GMT', '...  2020-07-03  \\\n",
+       "1       [{'created': 'Sun, 24 Jan 2021 21:37:35 GMT', ...  2021-02-09   \n",
+       "2       [{'created': 'Fri, 27 Jan 2012 15:24:46 GMT', ...  2012-02-21   \n",
+       "3       [{'created': 'Tue, 10 Nov 2015 12:08:23 GMT', ...  2016-04-27   \n",
+       "4       [{'created': 'Mon, 9 Oct 2017 06:34:01 GMT', '...  2020-08-25   \n",
+       "...                                                   ...         ...   \n",
+       "851605  [{'created': 'Fri, 4 Jan 2013 11:06:40 GMT', '...  2015-10-02   \n",
+       "851606  [{'created': 'Sun, 2 Jul 2017 19:52:32 GMT', '...  2018-11-21   \n",
+       "851607  [{'created': 'Wed, 26 Oct 2016 20:00:16 GMT', ...  2018-01-29   \n",
+       "851608  [{'created': 'Tue, 27 Nov 2012 17:29:04 GMT', ...  2013-05-29   \n",
+       "851609  [{'created': 'Sun, 20 May 2007 17:43:39 GMT', ...  2007-05-23   \n",
+       "\n",
+       "                                           authors_parsed   \n",
+       "0       [[Song, Qingheng, ], [Zeng, Yong, ], [Xu, Jie,...  \\\n",
+       "1             [[Stuart, Michael T., ], [Kneer, Markus, ]]   \n",
+       "2                                     [[Jerome, Denis, ]]   \n",
+       "3       [[Ellis, George F. R., ], [Platts, Emma, ], [S...   \n",
+       "4                                      [[Bansak, Kirk, ]]   \n",
+       "...                                                   ...   \n",
+       "851605                             [[Klein, Sebastian, ]]   \n",
+       "851606  [[Anastasiou, Giorgos, ], [Olea, Rodrigo, ], [...   \n",
+       "851607                               [[Oblak, Blagoje, ]]   \n",
+       "851608                              [[Joyez, Philippe, ]]   \n",
+       "851609  [[Perthame, Benoit, , DMA], [Souganidis, Panag...   \n",
+       "\n",
+       "                                        cleaned_abstracts  len_abstract  \n",
+       "0          unmanned aerial vehicle uav communication a...           865  \n",
+       "1          concept blameworthiness wrongness fundament...           739  \n",
+       "2          survey provide brief account start organic ...           649  \n",
+       "3          use phase plane analysis technique madsen e...           554  \n",
+       "4          researcher interested analyze conditional t...           799  \n",
+       "...                                                   ...           ...  \n",
+       "851605     recall p. balmer definition tensor triangul...           787  \n",
+       "851606     criticality represent specific point parame...           631  \n",
+       "851607     thesis devoted group theoretic aspect dimen...           542  \n",
+       "851608     derive microscopically dynamic associate d....           558  \n",
+       "851609     provide mathematical analysis appearance co...           518  \n",
+       "\n",
+       "[851610 rows x 16 columns]"
+      ]
+     },
+     "execution_count": 74,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 75,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus = df['cleaned_abstracts'].to_list()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def gensim_tokenizer(docs: List[str]):\n",
+    "    tokenized_docs = list()\n",
+    "    for doc in docs:\n",
+    "        processed_words = preprocess_string(doc, cleaning_filters)\n",
+    "        tokenized_docs.append(processed_words)\n",
+    "    \n",
+    "    return tokenized_docs\n",
+    "\n",
+    "tokenized_corpus = gensim_tokenizer(corpus)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "851610"
+      ]
+     },
+     "execution_count": 77,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(tokenized_corpus)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def cleaning_pipe(document):\n",
+    "    # Invoking gensim.parsing.preprocess_string method with set of filters\n",
+    "    processed_words = preprocess_string(document, cleaning_filters)\n",
+    "    return processed_words"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/Users/luis.morales/Desktop/arxiv-paper-recommender/models\n",
+      "/Users/luis.morales/Desktop/arxiv-paper-recommender\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "False"
+      ]
+     },
+     "execution_count": 78,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def validate_if_dictionary_exists(dictionaty_name: str) -> bool:\n",
+    "    dicts_folder = \"models/nlp_dictionaries\"\n",
+    "    current_dir = os.getcwd()\n",
+    "    parent_dir = os.path.dirname(current_dir)\n",
+    "    dict_path = f\"{parent_dir}/{dicts_folder}/{dictionaty_name}\"\n",
+    "    print(current_dir)\n",
+    "    print(parent_dir)\n",
+    "    return os.path.isfile(dict_path)\n",
+    "    \n",
+    "\n",
+    "validate_if_dictionary_exists('30ktokens.dict')    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_gensim_dictionary(tokenized_docs: List[str], dict_name: str = \"corpus\", save_dict: bool = False):\n",
+    "    \"\"\"\n",
+    "        Create dictionary of words in preprocessed corpus and saves the dict object\n",
+    "    \"\"\"\n",
+    "    dictionary = corpora.Dictionary(tokenized_docs)\n",
+    "    if save_dict:    \n",
+    "        dict_lenght = len(tokenized_corpus)\n",
+    "        parent_folder = \"/Users/luis.morales/Desktop/arxiv-paper-recommender/models/dictionaries\"\n",
+    "        #if validate_if_dictionary_exists('30ktokens.dict'):\n",
+    "        dictionary.save(f'{parent_folder}/{dict_name}.dict')\n",
+    "        \n",
+    "    return dictionary\n",
+    "\n",
+    "dictionary = get_gensim_dictionary(tokenized_docs=tokenized_corpus, dict_name=\"TextualTango\", save_dict=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# def get_gensim_dictionary(tokenized_docs: List[str], dict_name: str = \"corpus\", save_dict: bool = False):\n",
+    "#     \"\"\"\n",
+    "#         Create dictionary of words in preprocessed corpus and saves the dict object\n",
+    "#     \"\"\"\n",
+    "#     dictionary = corpora.Dictionary(tokenized_docs)\n",
+    "#     if save_dict:    \n",
+    "#         dict_lenght = len(tokenized_corpus)\n",
+    "#         parent_folder = \"/Users/luis.morales/Desktop/arxiv-paper-recommender/models/nlp_dictionaries\"\n",
+    "#         if validate_if_dictionary_exists('30ktokens.dict'):\n",
+    "#             dictionary.save(f'{parent_folder}/{dict_name}.dict')\n",
+    "        \n",
+    "#     return dictionary\n",
+    "\n",
+    "# dictionary = get_gensim_dictionary(tokenized_docs=tokenized_corpus, dict_name=\"300Ktokens\", save_dict=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 80,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BoW_corpus = [dictionary.doc2bow(doc, allow_update=True) for doc in tokenized_corpus]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 81,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tfidf_model = TfidfModel(BoW_corpus)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 82,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tfidf_model.save(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/models/tfidf/TextualTango.model\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# index the tfidf vector of corpus as sparse matrix\n",
+    "from gensim import similarities\n",
+    "index = similarities.SparseMatrixSimilarity(tfidf_model[BoW_corpus], num_features=len(dictionary))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 84,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.save(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/models/similarities_matrix/TextualTangoSimilarities/TextualTango\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_closest_n(query, n):\n",
+    "    '''get the top matching docs as per cosine similarity\n",
+    "    between tfidf vector of query and all docs'''\n",
+    "    query_document = cleaning_pipe(query)\n",
+    "    query_bow = dictionary.doc2bow(query_document)\n",
+    "    sims = index[tfidf_model[query_bow]]\n",
+    "    top_idx = sims.argsort()[-1*n:][::-1]\n",
+    "    return top_idx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 85,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_recomendations_metadata(query: str, n: int, df: pd.DataFrame):\n",
+    "    recommendations_idxs = get_closest_n(query, n)\n",
+    "    return df.iloc[recommendations_idxs].reset_index(drop=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "User Request ---- : \n",
+      " Which papers discuss the use of statistical models and Bayesian inference for uncertainty quantification and risk assessment in engineering systems?\n",
+      "User Request ---- : \n",
+      " \n",
+      "Title: A framework for benchmarking uncertainty in deep regression\n",
+      "Abstract:   We propose a framework for the assessment of uncertainty quantification in\n",
+      "deep regression. The framework is based on regression problems where the\n",
+      "regression function is a linear combination of nonlinear functions. Basically,\n",
+      "any level of complexity can be realized through the choice of the nonlinear\n",
+      "functions and the dimensionality of their domain. Results of an uncertainty\n",
+      "quantification for deep regression are compared against those obtained by a\n",
+      "statistical reference method. The reference method utilizes knowledge of the\n",
+      "underlying nonlinear functions and is based on a Bayesian linear regression\n",
+      "using a reference prior. Reliability of uncertainty quantification is assessed\n",
+      "in terms of coverage probabilities, and accuracy through the size of calculated\n",
+      "uncertainties. We illustrate the proposed framework by applying it to current\n",
+      "approaches for uncertainty quantification in deep regression. The flexibility,\n",
+      "together with the availability of a reference solution, makes the framework\n",
+      "suitable for defining benchmark sets for uncertainty quantification.\n",
+      "\n",
+      "\n",
+      "--------------------------\n",
+      "User Request ---- : \n",
+      " Which papers discuss the use of statistical models and Bayesian inference for uncertainty quantification and risk assessment in engineering systems?\n",
+      "User Request ---- : \n",
+      " \n",
+      "Title: Generative Parameter Sampler For Scalable Uncertainty Quantification\n",
+      "Abstract:   Uncertainty quantification has been a core of the statistical machine\n",
+      "learning, but its computational bottleneck has been a serious challenge for\n",
+      "both Bayesians and frequentists. We propose a model-based framework in\n",
+      "quantifying uncertainty, called predictive-matching Generative Parameter\n",
+      "Sampler (GPS). This procedure considers an Uncertainty Quantification (UQ)\n",
+      "distribution on the targeted parameter, which matches the corresponding\n",
+      "predictive distribution to the observed data. This framework adopts a\n",
+      "hierarchical modeling perspective such that each observation is modeled by an\n",
+      "individual parameter. This individual parameterization permits the resulting\n",
+      "inference to be computationally scalable and robust to outliers. Our approach\n",
+      "is illustrated for linear models, Poisson processes, and deep neural networks\n",
+      "for classification. The results show that the GPS is successful in providing\n",
+      "uncertainty quantification as well as additional flexibility beyond what is\n",
+      "allowed by classical statistical procedures under the postulated statistical\n",
+      "models.\n",
+      "\n",
+      "\n",
+      "--------------------------\n",
+      "User Request ---- : \n",
+      " Which papers discuss the use of statistical models and Bayesian inference for uncertainty quantification and risk assessment in engineering systems?\n",
+      "User Request ---- : \n",
+      " \n",
+      "Title: Recent Advances in Uncertainty Quantification Methods for Engineering\n",
+      "  Problems\n",
+      "Abstract:   In the last few decades, uncertainty quantification (UQ) methods have been\n",
+      "used widely to ensure the robustness of engineering designs. This chapter aims\n",
+      "to detail recent advances in popular uncertainty quantification methods used in\n",
+      "engineering applications. This chapter describes the two most popular\n",
+      "meta-modeling methods for uncertainty quantification suitable for engineering\n",
+      "applications (Polynomial Chaos Method and Gaussian Process). Further, the UQ\n",
+      "methods are applied to an engineering test problem under multiple\n",
+      "uncertainties. The test problem considered here is a supersonic nozzle under\n",
+      "operational uncertainties. For the deterministic solution, an open-source\n",
+      "computational fluid dynamics (CFD) solver SU2 is used. The UQ methods are\n",
+      "developed in Matlab and are further combined with SU2 for the uncertainty and\n",
+      "sensitivity estimates. The results are presented in terms of the mean and\n",
+      "standard deviation of the output quantities.\n",
+      "\n",
+      "\n",
+      "--------------------------\n"
+     ]
+    }
+   ],
+   "source": [
+    "_input = \"Which papers discuss the use of statistical models and Bayesian inference for uncertainty quantification and risk assessment in engineering systems?\"\n",
+    "results_df = get_recomendations_metadata(query=_input, df=df, n=3)\n",
+    "\n",
+    "\n",
+    "for abstract in list(zip(results_df['abstract'].to_list(), results_df['title'].to_list())):\n",
+    "    print(f\"User Request ---- : \\n {_input}\")\n",
+    "    print(f\"User Request ---- : \\n \")\n",
+    "    print(f\"Title: {abstract[1]}\")\n",
+    "    print(f\"Abstract: {abstract[0]}\\n\")\n",
+    "    print(f\"--------------------------\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.11.4 ('arxiv-env': venv)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "aae17c2ae2f38cc6f211be9b71a2aa280701d8462782cbc1f67caa83a1603363"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}