Added data cleaner.

loader-cleaner.ipynb

@@ -64,6 +64,27 @@
     "    except Exception as error:\n",
     "        print(f\"Error: Unable to read file '{file_candidate}' ({str(type(error))}: {error})\".ljust(120))"
    ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Clean data to remove duplicates and rows with missing values."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DROP_MISSING = True\n",
+    "REMOVE_DUPLICATES = True\n",
+    "\n",
+    "df = df.dropna(how=\"any\" if DROP_MISSING else \"all\")\n",
+    "if REMOVE_DUPLICATES: df = df.drop_duplicates()"
+   ]
   }
  ],
  "metadata": {