{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13 µs, sys: 0 ns, total: 13 µs\n",
      "Wall time: 20.3 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import pandas as pd\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2268252it [04:21, 8672.99it/s] \n"
     ]
    }
   ],
   "source": [
    "# df = pd.read_json(\"../data/raw/arxiv-metadata-oai-snapshot.json\")\n",
    "\n",
    "# Initialize an empty list to store the data from the JSON file\n",
    "arxiv_data = []\n",
    "\n",
    "# Open the JSON file using the 'with' statement, ensuring it's closed automatically\n",
    "with open(\"/Users/luis.morales/Desktop/arxiv-paper-recommender/data/raw/arxiv-metadata-oai-snapshot.json\", 'r') as file:\n",
    "    # Iterate over each line in the file\n",
    "    for line in tqdm(file):\n",
    "        # Load the JSON data from each line and append it to the arxiv_data list\n",
    "        arxiv_data.append(json.loads(line))\n",
    "\n",
    "# Create a DataFrame from the arxiv_data list using the pd.DataFrame.from_records() method\n",
    "df = pd.DataFrame.from_records(arxiv_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submitter</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "      <th>comments</th>\n",
       "      <th>journal-ref</th>\n",
       "      <th>doi</th>\n",
       "      <th>report-no</th>\n",
       "      <th>categories</th>\n",
       "      <th>license</th>\n",
       "      <th>abstract</th>\n",
       "      <th>versions</th>\n",
       "      <th>update_date</th>\n",
       "      <th>authors_parsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0704.0001</td>\n",
       "      <td>Pavel Nadolsky</td>\n",
       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
       "      <td>Calculation of prompt diphoton production cros...</td>\n",
       "      <td>37 pages, 15 figures; published version</td>\n",
       "      <td>Phys.Rev.D76:013009,2007</td>\n",
       "      <td>10.1103/PhysRevD.76.013009</td>\n",
       "      <td>ANL-HEP-PR-07-12</td>\n",
       "      <td>hep-ph</td>\n",
       "      <td>None</td>\n",
       "      <td>A fully differential calculation in perturba...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
       "      <td>2008-11-26</td>\n",
       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0704.0002</td>\n",
       "      <td>Louis Theran</td>\n",
       "      <td>Ileana Streinu and Louis Theran</td>\n",
       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
       "      <td>To appear in Graphs and Combinatorics</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>math.CO cs.CG</td>\n",
       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
       "      <td>2008-12-13</td>\n",
       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0704.0003</td>\n",
       "      <td>Hongjun Pan</td>\n",
       "      <td>Hongjun Pan</td>\n",
       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
       "      <td>23 pages, 3 figures</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>physics.gen-ph</td>\n",
       "      <td>None</td>\n",
       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sun, 1 Apr 2007...</td>\n",
       "      <td>2008-01-13</td>\n",
       "      <td>[[Pan, Hongjun, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0704.0004</td>\n",
       "      <td>David Callan</td>\n",
       "      <td>David Callan</td>\n",
       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
       "      <td>11 pages</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>math.CO</td>\n",
       "      <td>None</td>\n",
       "      <td>We show that a determinant of Stirling cycle...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
       "      <td>2007-05-23</td>\n",
       "      <td>[[Callan, David, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0704.0005</td>\n",
       "      <td>Alberto Torchinsky</td>\n",
       "      <td>Wael Abu-Shammala and Alberto Torchinsky</td>\n",
       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
       "      <td>None</td>\n",
       "      <td>Illinois J. Math. 52 (2008) no.2, 681-689</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>math.CA math.FA</td>\n",
       "      <td>None</td>\n",
       "      <td>In this paper we show how to compute the $\\L...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268247</th>\n",
       "      <td>supr-con/9608008</td>\n",
       "      <td>Ruslan Prozorov</td>\n",
       "      <td>R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...</td>\n",
       "      <td>On the origin of the irreversibility line in t...</td>\n",
       "      <td>19 pages, LaTex, 6 PostScript figures; Author'...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1103/PhysRevB.54.15530</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>We report on measurements of the angular dep...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Mon, 26 Aug 199...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Prozorov, R., ], [Konczykowski, M., ], [Schm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268248</th>\n",
       "      <td>supr-con/9609001</td>\n",
       "      <td>Durga P. Choudhury</td>\n",
       "      <td>Durga P. Choudhury, Balam A. Willemsen, John S...</td>\n",
       "      <td>Nonlinear Response of HTSC Thin Film Microwave...</td>\n",
       "      <td>4 pages, LaTeX type, Uses IEEE style files, 60...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1109/77.620744</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>The non-linear microwave surface impedance o...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Aug 199...</td>\n",
       "      <td>2016-11-18</td>\n",
       "      <td>[[Choudhury, Durga P., , Physics Department, N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268249</th>\n",
       "      <td>supr-con/9609002</td>\n",
       "      <td>Durga P. Choudhury</td>\n",
       "      <td>Balam A. Willemsen, J. S. Derov and S.Sridhar ...</td>\n",
       "      <td>Critical State Flux Penetration and Linear Mic...</td>\n",
       "      <td>20 pages, LaTeX type, Uses REVTeX style files,...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1103/PhysRevB.56.11989</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>The vortex contribution to the dc field (H) ...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Tue, 3 Sep 1996...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Willemsen, Balam A., , Physics Department,\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268250</th>\n",
       "      <td>supr-con/9609003</td>\n",
       "      <td>Hasegawa Yasumasa</td>\n",
       "      <td>Yasumasa Hasegawa (Himeji Institute of Technol...</td>\n",
       "      <td>Density of States and NMR Relaxation Rate in A...</td>\n",
       "      <td>7 pages, 4 PostScript Figures, LaTeX, to appea...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1143/JPSJ.65.3131</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>We show that the density of states in an ani...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Wed, 18 Sep 199...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Hasegawa, Yasumasa, , Himeji Institute of Te...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268251</th>\n",
       "      <td>supr-con/9609004</td>\n",
       "      <td>Masanori Ichioka</td>\n",
       "      <td>Naoki Enomoto, Masanori Ichioka and Kazushige ...</td>\n",
       "      <td>Ginzburg Landau theory for d-wave pairing and ...</td>\n",
       "      <td>12 pages including 8 eps figs, LaTeX with jpsj...</td>\n",
       "      <td>J. Phys. Soc. Jpn. 66, 204 (1997).</td>\n",
       "      <td>10.1143/JPSJ.66.204</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>The Ginzburg Landau theory for d_{x^2-y^2}-w...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Wed, 25 Sep 199...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2268252 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       id           submitter   \n",
       "0               0704.0001      Pavel Nadolsky  \\\n",
       "1               0704.0002        Louis Theran   \n",
       "2               0704.0003         Hongjun Pan   \n",
       "3               0704.0004        David Callan   \n",
       "4               0704.0005  Alberto Torchinsky   \n",
       "...                   ...                 ...   \n",
       "2268247  supr-con/9608008     Ruslan Prozorov   \n",
       "2268248  supr-con/9609001  Durga P. Choudhury   \n",
       "2268249  supr-con/9609002  Durga P. Choudhury   \n",
       "2268250  supr-con/9609003   Hasegawa Yasumasa   \n",
       "2268251  supr-con/9609004    Masanori Ichioka   \n",
       "\n",
       "                                                   authors   \n",
       "0        C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
       "1                          Ileana Streinu and Louis Theran   \n",
       "2                                              Hongjun Pan   \n",
       "3                                             David Callan   \n",
       "4                 Wael Abu-Shammala and Alberto Torchinsky   \n",
       "...                                                    ...   \n",
       "2268247  R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...   \n",
       "2268248  Durga P. Choudhury, Balam A. Willemsen, John S...   \n",
       "2268249  Balam A. Willemsen, J. S. Derov and S.Sridhar ...   \n",
       "2268250  Yasumasa Hasegawa (Himeji Institute of Technol...   \n",
       "2268251  Naoki Enomoto, Masanori Ichioka and Kazushige ...   \n",
       "\n",
       "                                                     title   \n",
       "0        Calculation of prompt diphoton production cros...  \\\n",
       "1                 Sparsity-certifying Graph Decompositions   \n",
       "2        The evolution of the Earth-Moon system based o...   \n",
       "3        A determinant of Stirling cycle numbers counts...   \n",
       "4        From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
       "...                                                    ...   \n",
       "2268247  On the origin of the irreversibility line in t...   \n",
       "2268248  Nonlinear Response of HTSC Thin Film Microwave...   \n",
       "2268249  Critical State Flux Penetration and Linear Mic...   \n",
       "2268250  Density of States and NMR Relaxation Rate in A...   \n",
       "2268251  Ginzburg Landau theory for d-wave pairing and ...   \n",
       "\n",
       "                                                  comments   \n",
       "0                  37 pages, 15 figures; published version  \\\n",
       "1                    To appear in Graphs and Combinatorics   \n",
       "2                                      23 pages, 3 figures   \n",
       "3                                                 11 pages   \n",
       "4                                                     None   \n",
       "...                                                    ...   \n",
       "2268247  19 pages, LaTex, 6 PostScript figures; Author'...   \n",
       "2268248  4 pages, LaTeX type, Uses IEEE style files, 60...   \n",
       "2268249  20 pages, LaTeX type, Uses REVTeX style files,...   \n",
       "2268250  7 pages, 4 PostScript Figures, LaTeX, to appea...   \n",
       "2268251  12 pages including 8 eps figs, LaTeX with jpsj...   \n",
       "\n",
       "                                       journal-ref   \n",
       "0                         Phys.Rev.D76:013009,2007  \\\n",
       "1                                             None   \n",
       "2                                             None   \n",
       "3                                             None   \n",
       "4        Illinois J. Math. 52 (2008) no.2, 681-689   \n",
       "...                                            ...   \n",
       "2268247                                       None   \n",
       "2268248                                       None   \n",
       "2268249                                       None   \n",
       "2268250                                       None   \n",
       "2268251         J. Phys. Soc. Jpn. 66, 204 (1997).   \n",
       "\n",
       "                                doi         report-no   \n",
       "0        10.1103/PhysRevD.76.013009  ANL-HEP-PR-07-12  \\\n",
       "1                              None              None   \n",
       "2                              None              None   \n",
       "3                              None              None   \n",
       "4                              None              None   \n",
       "...                             ...               ...   \n",
       "2268247   10.1103/PhysRevB.54.15530              None   \n",
       "2268248           10.1109/77.620744              None   \n",
       "2268249   10.1103/PhysRevB.56.11989              None   \n",
       "2268250        10.1143/JPSJ.65.3131              None   \n",
       "2268251         10.1143/JPSJ.66.204              None   \n",
       "\n",
       "                         categories   \n",
       "0                            hep-ph  \\\n",
       "1                     math.CO cs.CG   \n",
       "2                    physics.gen-ph   \n",
       "3                           math.CO   \n",
       "4                   math.CA math.FA   \n",
       "...                             ...   \n",
       "2268247  supr-con cond-mat.supr-con   \n",
       "2268248  supr-con cond-mat.supr-con   \n",
       "2268249  supr-con cond-mat.supr-con   \n",
       "2268250  supr-con cond-mat.supr-con   \n",
       "2268251  supr-con cond-mat.supr-con   \n",
       "\n",
       "                                                   license   \n",
       "0                                                     None  \\\n",
       "1        http://arxiv.org/licenses/nonexclusive-distrib...   \n",
       "2                                                     None   \n",
       "3                                                     None   \n",
       "4                                                     None   \n",
       "...                                                    ...   \n",
       "2268247                                               None   \n",
       "2268248                                               None   \n",
       "2268249                                               None   \n",
       "2268250                                               None   \n",
       "2268251                                               None   \n",
       "\n",
       "                                                  abstract   \n",
       "0          A fully differential calculation in perturba...  \\\n",
       "1          We describe a new algorithm, the $(k,\\ell)$-...   \n",
       "2          The evolution of Earth-Moon system is descri...   \n",
       "3          We show that a determinant of Stirling cycle...   \n",
       "4          In this paper we show how to compute the $\\L...   \n",
       "...                                                    ...   \n",
       "2268247    We report on measurements of the angular dep...   \n",
       "2268248    The non-linear microwave surface impedance o...   \n",
       "2268249    The vortex contribution to the dc field (H) ...   \n",
       "2268250    We show that the density of states in an ani...   \n",
       "2268251    The Ginzburg Landau theory for d_{x^2-y^2}-w...   \n",
       "\n",
       "                                                  versions update_date   \n",
       "0        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2008-11-26  \\\n",
       "1        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2008-12-13   \n",
       "2        [{'version': 'v1', 'created': 'Sun, 1 Apr 2007...  2008-01-13   \n",
       "3        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2007-05-23   \n",
       "4        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2013-10-15   \n",
       "...                                                    ...         ...   \n",
       "2268247  [{'version': 'v1', 'created': 'Mon, 26 Aug 199...  2009-10-30   \n",
       "2268248  [{'version': 'v1', 'created': 'Sat, 31 Aug 199...  2016-11-18   \n",
       "2268249  [{'version': 'v1', 'created': 'Tue, 3 Sep 1996...  2009-10-30   \n",
       "2268250  [{'version': 'v1', 'created': 'Wed, 18 Sep 199...  2009-10-30   \n",
       "2268251  [{'version': 'v1', 'created': 'Wed, 25 Sep 199...  2009-10-30   \n",
       "\n",
       "                                            authors_parsed  \n",
       "0        [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \n",
       "1                 [[Streinu, Ileana, ], [Theran, Louis, ]]  \n",
       "2                                       [[Pan, Hongjun, ]]  \n",
       "3                                      [[Callan, David, ]]  \n",
       "4        [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]  \n",
       "...                                                    ...  \n",
       "2268247  [[Prozorov, R., ], [Konczykowski, M., ], [Schm...  \n",
       "2268248  [[Choudhury, Durga P., , Physics Department, N...  \n",
       "2268249  [[Willemsen, Balam A., , Physics Department,\\n...  \n",
       "2268250  [[Hasegawa, Yasumasa, , Himeji Institute of Te...  \n",
       "2268251  [[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...  \n",
       "\n",
       "[2268252 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_parquet('/Users/luis.morales/Desktop/arxiv-paper-recommender/data/processed/arxiv_papers_raw.parquet.gzip', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submitter</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "      <th>comments</th>\n",
       "      <th>journal-ref</th>\n",
       "      <th>doi</th>\n",
       "      <th>report-no</th>\n",
       "      <th>categories</th>\n",
       "      <th>license</th>\n",
       "      <th>abstract</th>\n",
       "      <th>versions</th>\n",
       "      <th>update_date</th>\n",
       "      <th>authors_parsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0704.0001</td>\n",
       "      <td>Pavel Nadolsky</td>\n",
       "      <td>C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...</td>\n",
       "      <td>Calculation of prompt diphoton production cros...</td>\n",
       "      <td>37 pages, 15 figures; published version</td>\n",
       "      <td>Phys.Rev.D76:013009,2007</td>\n",
       "      <td>10.1103/PhysRevD.76.013009</td>\n",
       "      <td>ANL-HEP-PR-07-12</td>\n",
       "      <td>hep-ph</td>\n",
       "      <td>None</td>\n",
       "      <td>A fully differential calculation in perturba...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
       "      <td>2008-11-26</td>\n",
       "      <td>[[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0704.0002</td>\n",
       "      <td>Louis Theran</td>\n",
       "      <td>Ileana Streinu and Louis Theran</td>\n",
       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
       "      <td>To appear in Graphs and Combinatorics</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>math.CO cs.CG</td>\n",
       "      <td>http://arxiv.org/licenses/nonexclusive-distrib...</td>\n",
       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
       "      <td>2008-12-13</td>\n",
       "      <td>[[Streinu, Ileana, ], [Theran, Louis, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0704.0003</td>\n",
       "      <td>Hongjun Pan</td>\n",
       "      <td>Hongjun Pan</td>\n",
       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
       "      <td>23 pages, 3 figures</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>physics.gen-ph</td>\n",
       "      <td>None</td>\n",
       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sun, 1 Apr 2007...</td>\n",
       "      <td>2008-01-13</td>\n",
       "      <td>[[Pan, Hongjun, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0704.0004</td>\n",
       "      <td>David Callan</td>\n",
       "      <td>David Callan</td>\n",
       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
       "      <td>11 pages</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>math.CO</td>\n",
       "      <td>None</td>\n",
       "      <td>We show that a determinant of Stirling cycle...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Mar 200...</td>\n",
       "      <td>2007-05-23</td>\n",
       "      <td>[[Callan, David, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0704.0005</td>\n",
       "      <td>Alberto Torchinsky</td>\n",
       "      <td>Wael Abu-Shammala and Alberto Torchinsky</td>\n",
       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
       "      <td>None</td>\n",
       "      <td>Illinois J. Math. 52 (2008) no.2, 681-689</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>math.CA math.FA</td>\n",
       "      <td>None</td>\n",
       "      <td>In this paper we show how to compute the $\\L...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Mon, 2 Apr 2007...</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>[[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268247</th>\n",
       "      <td>supr-con/9608008</td>\n",
       "      <td>Ruslan Prozorov</td>\n",
       "      <td>R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...</td>\n",
       "      <td>On the origin of the irreversibility line in t...</td>\n",
       "      <td>19 pages, LaTex, 6 PostScript figures; Author'...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1103/PhysRevB.54.15530</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>We report on measurements of the angular dep...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Mon, 26 Aug 199...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Prozorov, R., ], [Konczykowski, M., ], [Schm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268248</th>\n",
       "      <td>supr-con/9609001</td>\n",
       "      <td>Durga P. Choudhury</td>\n",
       "      <td>Durga P. Choudhury, Balam A. Willemsen, John S...</td>\n",
       "      <td>Nonlinear Response of HTSC Thin Film Microwave...</td>\n",
       "      <td>4 pages, LaTeX type, Uses IEEE style files, 60...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1109/77.620744</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>The non-linear microwave surface impedance o...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Sat, 31 Aug 199...</td>\n",
       "      <td>2016-11-18</td>\n",
       "      <td>[[Choudhury, Durga P., , Physics Department, N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268249</th>\n",
       "      <td>supr-con/9609002</td>\n",
       "      <td>Durga P. Choudhury</td>\n",
       "      <td>Balam A. Willemsen, J. S. Derov and S.Sridhar ...</td>\n",
       "      <td>Critical State Flux Penetration and Linear Mic...</td>\n",
       "      <td>20 pages, LaTeX type, Uses REVTeX style files,...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1103/PhysRevB.56.11989</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>The vortex contribution to the dc field (H) ...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Tue, 3 Sep 1996...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Willemsen, Balam A., , Physics Department,\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268250</th>\n",
       "      <td>supr-con/9609003</td>\n",
       "      <td>Hasegawa Yasumasa</td>\n",
       "      <td>Yasumasa Hasegawa (Himeji Institute of Technol...</td>\n",
       "      <td>Density of States and NMR Relaxation Rate in A...</td>\n",
       "      <td>7 pages, 4 PostScript Figures, LaTeX, to appea...</td>\n",
       "      <td>None</td>\n",
       "      <td>10.1143/JPSJ.65.3131</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>We show that the density of states in an ani...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Wed, 18 Sep 199...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Hasegawa, Yasumasa, , Himeji Institute of Te...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2268251</th>\n",
       "      <td>supr-con/9609004</td>\n",
       "      <td>Masanori Ichioka</td>\n",
       "      <td>Naoki Enomoto, Masanori Ichioka and Kazushige ...</td>\n",
       "      <td>Ginzburg Landau theory for d-wave pairing and ...</td>\n",
       "      <td>12 pages including 8 eps figs, LaTeX with jpsj...</td>\n",
       "      <td>J. Phys. Soc. Jpn. 66, 204 (1997).</td>\n",
       "      <td>10.1143/JPSJ.66.204</td>\n",
       "      <td>None</td>\n",
       "      <td>supr-con cond-mat.supr-con</td>\n",
       "      <td>None</td>\n",
       "      <td>The Ginzburg Landau theory for d_{x^2-y^2}-w...</td>\n",
       "      <td>[{'version': 'v1', 'created': 'Wed, 25 Sep 199...</td>\n",
       "      <td>2009-10-30</td>\n",
       "      <td>[[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2268252 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       id           submitter   \n",
       "0               0704.0001      Pavel Nadolsky  \\\n",
       "1               0704.0002        Louis Theran   \n",
       "2               0704.0003         Hongjun Pan   \n",
       "3               0704.0004        David Callan   \n",
       "4               0704.0005  Alberto Torchinsky   \n",
       "...                   ...                 ...   \n",
       "2268247  supr-con/9608008     Ruslan Prozorov   \n",
       "2268248  supr-con/9609001  Durga P. Choudhury   \n",
       "2268249  supr-con/9609002  Durga P. Choudhury   \n",
       "2268250  supr-con/9609003   Hasegawa Yasumasa   \n",
       "2268251  supr-con/9609004    Masanori Ichioka   \n",
       "\n",
       "                                                   authors   \n",
       "0        C. Bal\\'azs, E. L. Berger, P. M. Nadolsky, C.-...  \\\n",
       "1                          Ileana Streinu and Louis Theran   \n",
       "2                                              Hongjun Pan   \n",
       "3                                             David Callan   \n",
       "4                 Wael Abu-Shammala and Alberto Torchinsky   \n",
       "...                                                    ...   \n",
       "2268247  R. Prozorov, M. Konczykowski, B. Schmidt, Y. Y...   \n",
       "2268248  Durga P. Choudhury, Balam A. Willemsen, John S...   \n",
       "2268249  Balam A. Willemsen, J. S. Derov and S.Sridhar ...   \n",
       "2268250  Yasumasa Hasegawa (Himeji Institute of Technol...   \n",
       "2268251  Naoki Enomoto, Masanori Ichioka and Kazushige ...   \n",
       "\n",
       "                                                     title   \n",
       "0        Calculation of prompt diphoton production cros...  \\\n",
       "1                 Sparsity-certifying Graph Decompositions   \n",
       "2        The evolution of the Earth-Moon system based o...   \n",
       "3        A determinant of Stirling cycle numbers counts...   \n",
       "4        From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
       "...                                                    ...   \n",
       "2268247  On the origin of the irreversibility line in t...   \n",
       "2268248  Nonlinear Response of HTSC Thin Film Microwave...   \n",
       "2268249  Critical State Flux Penetration and Linear Mic...   \n",
       "2268250  Density of States and NMR Relaxation Rate in A...   \n",
       "2268251  Ginzburg Landau theory for d-wave pairing and ...   \n",
       "\n",
       "                                                  comments   \n",
       "0                  37 pages, 15 figures; published version  \\\n",
       "1                    To appear in Graphs and Combinatorics   \n",
       "2                                      23 pages, 3 figures   \n",
       "3                                                 11 pages   \n",
       "4                                                     None   \n",
       "...                                                    ...   \n",
       "2268247  19 pages, LaTex, 6 PostScript figures; Author'...   \n",
       "2268248  4 pages, LaTeX type, Uses IEEE style files, 60...   \n",
       "2268249  20 pages, LaTeX type, Uses REVTeX style files,...   \n",
       "2268250  7 pages, 4 PostScript Figures, LaTeX, to appea...   \n",
       "2268251  12 pages including 8 eps figs, LaTeX with jpsj...   \n",
       "\n",
       "                                       journal-ref   \n",
       "0                         Phys.Rev.D76:013009,2007  \\\n",
       "1                                             None   \n",
       "2                                             None   \n",
       "3                                             None   \n",
       "4        Illinois J. Math. 52 (2008) no.2, 681-689   \n",
       "...                                            ...   \n",
       "2268247                                       None   \n",
       "2268248                                       None   \n",
       "2268249                                       None   \n",
       "2268250                                       None   \n",
       "2268251         J. Phys. Soc. Jpn. 66, 204 (1997).   \n",
       "\n",
       "                                doi         report-no   \n",
       "0        10.1103/PhysRevD.76.013009  ANL-HEP-PR-07-12  \\\n",
       "1                              None              None   \n",
       "2                              None              None   \n",
       "3                              None              None   \n",
       "4                              None              None   \n",
       "...                             ...               ...   \n",
       "2268247   10.1103/PhysRevB.54.15530              None   \n",
       "2268248           10.1109/77.620744              None   \n",
       "2268249   10.1103/PhysRevB.56.11989              None   \n",
       "2268250        10.1143/JPSJ.65.3131              None   \n",
       "2268251         10.1143/JPSJ.66.204              None   \n",
       "\n",
       "                         categories   \n",
       "0                            hep-ph  \\\n",
       "1                     math.CO cs.CG   \n",
       "2                    physics.gen-ph   \n",
       "3                           math.CO   \n",
       "4                   math.CA math.FA   \n",
       "...                             ...   \n",
       "2268247  supr-con cond-mat.supr-con   \n",
       "2268248  supr-con cond-mat.supr-con   \n",
       "2268249  supr-con cond-mat.supr-con   \n",
       "2268250  supr-con cond-mat.supr-con   \n",
       "2268251  supr-con cond-mat.supr-con   \n",
       "\n",
       "                                                   license   \n",
       "0                                                     None  \\\n",
       "1        http://arxiv.org/licenses/nonexclusive-distrib...   \n",
       "2                                                     None   \n",
       "3                                                     None   \n",
       "4                                                     None   \n",
       "...                                                    ...   \n",
       "2268247                                               None   \n",
       "2268248                                               None   \n",
       "2268249                                               None   \n",
       "2268250                                               None   \n",
       "2268251                                               None   \n",
       "\n",
       "                                                  abstract   \n",
       "0          A fully differential calculation in perturba...  \\\n",
       "1          We describe a new algorithm, the $(k,\\ell)$-...   \n",
       "2          The evolution of Earth-Moon system is descri...   \n",
       "3          We show that a determinant of Stirling cycle...   \n",
       "4          In this paper we show how to compute the $\\L...   \n",
       "...                                                    ...   \n",
       "2268247    We report on measurements of the angular dep...   \n",
       "2268248    The non-linear microwave surface impedance o...   \n",
       "2268249    The vortex contribution to the dc field (H) ...   \n",
       "2268250    We show that the density of states in an ani...   \n",
       "2268251    The Ginzburg Landau theory for d_{x^2-y^2}-w...   \n",
       "\n",
       "                                                  versions update_date   \n",
       "0        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2008-11-26  \\\n",
       "1        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2008-12-13   \n",
       "2        [{'version': 'v1', 'created': 'Sun, 1 Apr 2007...  2008-01-13   \n",
       "3        [{'version': 'v1', 'created': 'Sat, 31 Mar 200...  2007-05-23   \n",
       "4        [{'version': 'v1', 'created': 'Mon, 2 Apr 2007...  2013-10-15   \n",
       "...                                                    ...         ...   \n",
       "2268247  [{'version': 'v1', 'created': 'Mon, 26 Aug 199...  2009-10-30   \n",
       "2268248  [{'version': 'v1', 'created': 'Sat, 31 Aug 199...  2016-11-18   \n",
       "2268249  [{'version': 'v1', 'created': 'Tue, 3 Sep 1996...  2009-10-30   \n",
       "2268250  [{'version': 'v1', 'created': 'Wed, 18 Sep 199...  2009-10-30   \n",
       "2268251  [{'version': 'v1', 'created': 'Wed, 25 Sep 199...  2009-10-30   \n",
       "\n",
       "                                            authors_parsed  \n",
       "0        [[Balázs, C., ], [Berger, E. L., ], [Nadolsky,...  \n",
       "1                 [[Streinu, Ileana, ], [Theran, Louis, ]]  \n",
       "2                                       [[Pan, Hongjun, ]]  \n",
       "3                                      [[Callan, David, ]]  \n",
       "4        [[Abu-Shammala, Wael, ], [Torchinsky, Alberto, ]]  \n",
       "...                                                    ...  \n",
       "2268247  [[Prozorov, R., ], [Konczykowski, M., ], [Schm...  \n",
       "2268248  [[Choudhury, Durga P., , Physics Department, N...  \n",
       "2268249  [[Willemsen, Balam A., , Physics Department,\\n...  \n",
       "2268250  [[Hasegawa, Yasumasa, , Himeji Institute of Te...  \n",
       "2268251  [[Enomoto, Naoki, , Okayama Univ.], [Ichioka, ...  \n",
       "\n",
       "[2268252 rows x 14 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.11.3 ('arxiv-env': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "aae17c2ae2f38cc6f211be9b71a2aa280701d8462782cbc1f67caa83a1603363"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}