wrote class for holding arxiv data

arxiv_query_retrieval.py

@@ -1,5 +1,59 @@
 import arxiv
 import pandas as pd
+import data_cleaning as clean
+from sklearn.preprocessing import MultiLabelBinarizer
+
+class ArXivData():
+    """A light class for storing the metadata of a collection of arXiv papers.
+    """   
+
+    def __init__(self):
+        """
+        data: dataframe holding the metadata. Each row represents a paper and each column is
+        a separate piece of metadata.
+        
+        query: A tuple of the form (query_string,max_results) where query_string is the formatted 
+        string that produced the raw data and max_results is the value of that parameter passed to the
+        arXiv API.
+
+        raw: The original, raw dataset as returned by the arXiv API, if current data is clean.
+
+        cats: A DataFrame containing one-hot-encoded categories of the self.data DataFrame.
+        """
+
+        self.data = None
+        self.query = None
+        self.raw = None
+        self.categories = None
+
+    def get_from_query(self,query_string,max_results):
+        self.data = query_to_df(query=query_string,max_results=max_results)        
+        self.query = (query_string,max_results)
+        self.raw = self.data
+        self.categories = self.get_OHE_cats()
+        
+        
+    def clean(self,dataset):
+        """Constructs this dataset by cleaning another one.
+
+        Args:
+            dataset: An ArXivData object containing data to be cleaned.
+        """
+        self.data = clean.clean(dataset)
+        self.query = dataset.query
+        self.raw = dataset.raw
+        self.categories = dataset.categories
+
+    def get_OHE_cats(self):
+        mlb = MultiLabelBinarizer()
+        OHE_category_array = mlb.fit_transform(self.data.categories)
+        return pd.DataFrame(
+            OHE_category_array, columns = mlb.classes_).rename(
+            mapper=clean.category_map())
+
+
+
+
 
 def format_query(author='',title='',cat='',abstract=''):
     """Returns a formatted arxiv query string to handle simple queries of at most one instance each of these fields. To leave a field unspecified,

cleaning-abstracts.ipynb

@@ -6486,6 +6486,227 @@
    "source": [
     " Include the cleaning utilities applied to the data in the util file."
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import arxiv_query_retrieval\n",
+    "from arxiv_query_retrieval import ArXivData\n",
+    "import importlib\n",
+    "importlib.reload(arxiv_query_retrieval)\n",
+    "\n",
+    "data = ArXivData()\n",
+    "\n",
+    "data.get_from_query(query_string='cat:math.AP',max_results=10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>35K55, 35K65, 35Q74, 74A30, 35A01, 76S05, 35K51, 74B20</th>\n",
+       "      <th>35K58, 49K20, 90C46, 92D25</th>\n",
+       "      <th>35L05, 35L71, 35B44</th>\n",
+       "      <th>35P30, 35A15, 53C17</th>\n",
+       "      <th>35Q53, 93D15, 93D30, 93C20</th>\n",
+       "      <th>35R25 (Primary) 42B35 35B30 (Secondary)</th>\n",
+       "      <th>39A12, 39A14, 39A22</th>\n",
+       "      <th>math.AP</th>\n",
+       "      <th>math.DS</th>\n",
+       "      <th>math.OC</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   35K55, 35K65, 35Q74, 74A30, 35A01, 76S05, 35K51, 74B20  ...  math.OC\n",
+       "0                                                  0       ...        0\n",
+       "1                                                  0       ...        0\n",
+       "2                                                  0       ...        1\n",
+       "3                                                  0       ...        0\n",
+       "4                                                  0       ...        0\n",
+       "5                                                  0       ...        0\n",
+       "6                                                  1       ...        0\n",
+       "7                                                  0       ...        1\n",
+       "8                                                  0       ...        0\n",
+       "9                                                  0       ...        1\n",
+       "\n",
+       "[10 rows x 10 columns]"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data.categories"
+   ]
   }
  ],
  "metadata": {