Initial commit

.gitattributes

@@ -27,3 +27,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 *.tar.gz filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
+*.csv filter=lfs diff=lfs merge=lfs -text

code/reuters_dataset_prep.ipynb

@@ -0,0 +1,420 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "1383f909",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: transformers in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (4.11.3)\n",
+      "Collecting datasets\n",
+      "  Downloading datasets-1.15.1-py3-none-any.whl (290 kB)\n",
+      "\u001b[K     |████████████████████████████████| 290 kB 2.4 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting fsspec[http]>=2021.05.0\n",
+      "  Downloading fsspec-2021.11.0-py3-none-any.whl (132 kB)\n",
+      "\u001b[K     |████████████████████████████████| 132 kB 3.8 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting multiprocess\n",
+      "  Downloading multiprocess-0.70.12.2-py38-none-any.whl (128 kB)\n",
+      "\u001b[K     |████████████████████████████████| 128 kB 5.5 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting dill\n",
+      "  Downloading dill-0.3.4-py2.py3-none-any.whl (86 kB)\n",
+      "\u001b[K     |████████████████████████████████| 86 kB 4.4 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: pandas in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from datasets) (1.2.4)\n",
+      "Collecting huggingface-hub<1.0.0,>=0.1.0\n",
+      "  Downloading huggingface_hub-0.1.2-py3-none-any.whl (59 kB)\n",
+      "\u001b[K     |████████████████████████████████| 59 kB 4.4 MB/s eta 0:00:01\n",
+      "\u001b[?25hCollecting xxhash\n",
+      "  Downloading xxhash-2.0.2-cp38-cp38-macosx_10_9_x86_64.whl (31 kB)\n",
+      "Requirement already satisfied: requests>=2.19.0 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from datasets) (2.25.1)\n",
+      "Collecting tqdm>=4.62.1\n",
+      "  Using cached tqdm-4.62.3-py2.py3-none-any.whl (76 kB)\n",
+      "Collecting pyarrow!=4.0.0,>=1.0.0\n",
+      "  Downloading pyarrow-6.0.1-cp38-cp38-macosx_10_13_x86_64.whl (19.1 MB)\n",
+      "\u001b[K     |████████████████████████████████| 19.1 MB 5.9 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from datasets) (1.20.1)\n",
+      "Collecting aiohttp\n",
+      "  Downloading aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl (574 kB)\n",
+      "\u001b[K     |████████████████████████████████| 574 kB 5.0 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: packaging in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from datasets) (20.9)\n",
+      "Requirement already satisfied: pyyaml in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (5.4.1)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.4.3)\n",
+      "Requirement already satisfied: filelock in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.0.12)\n",
+      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from packaging->datasets) (2.4.7)\n",
+      "Requirement already satisfied: idna<3,>=2.5 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2.10)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.4)\n",
+      "Requirement already satisfied: chardet<5,>=3.0.2 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (4.0.0)\n",
+      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from transformers) (0.10.3)\n",
+      "Requirement already satisfied: sacremoses in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from transformers) (0.0.46)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from transformers) (2021.4.4)\n",
+      "Collecting charset-normalizer<3.0,>=2.0\n",
+      "  Using cached charset_normalizer-2.0.7-py3-none-any.whl (38 kB)\n",
+      "Collecting frozenlist>=1.1.1\n",
+      "  Downloading frozenlist-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl (81 kB)\n",
+      "\u001b[K     |████████████████████████████████| 81 kB 7.0 MB/s eta 0:00:011\n",
+      "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
+      "  Downloading multidict-5.2.0-cp38-cp38-macosx_10_9_x86_64.whl (45 kB)\n",
+      "\u001b[K     |████████████████████████████████| 45 kB 4.0 MB/s eta 0:00:011\n",
+      "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
+      "  Downloading yarl-1.7.2-cp38-cp38-macosx_10_9_x86_64.whl (121 kB)\n",
+      "\u001b[K     |████████████████████████████████| 121 kB 5.8 MB/s eta 0:00:01\n",
+      "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from aiohttp->datasets) (20.3.0)\n",
+      "Collecting async-timeout<5.0,>=4.0.0a3\n",
+      "  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n",
+      "Collecting aiosignal>=1.1.2\n",
+      "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
+      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from pandas->datasets) (2.8.1)\n",
+      "Requirement already satisfied: pytz>=2017.3 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from pandas->datasets) (2021.1)\n",
+      "Requirement already satisfied: six>=1.5 in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
+      "Requirement already satisfied: joblib in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from sacremoses->transformers) (1.0.1)\n",
+      "Requirement already satisfied: click in /Users/markmcquade/opt/anaconda3/lib/python3.8/site-packages (from sacremoses->transformers) (7.1.2)\n",
+      "Installing collected packages: multidict, frozenlist, yarl, charset-normalizer, async-timeout, aiosignal, tqdm, fsspec, dill, aiohttp, xxhash, pyarrow, multiprocess, huggingface-hub, datasets\n",
+      "  Attempting uninstall: tqdm\n",
+      "    Found existing installation: tqdm 4.59.0\n",
+      "    Uninstalling tqdm-4.59.0:\n",
+      "      Successfully uninstalled tqdm-4.59.0\n",
+      "  Attempting uninstall: fsspec\n",
+      "    Found existing installation: fsspec 0.9.0\n",
+      "    Uninstalling fsspec-0.9.0:\n",
+      "      Successfully uninstalled fsspec-0.9.0\n",
+      "  Attempting uninstall: huggingface-hub\n",
+      "    Found existing installation: huggingface-hub 0.0.19\n",
+      "    Uninstalling huggingface-hub-0.0.19:\n",
+      "      Successfully uninstalled huggingface-hub-0.0.19\n",
+      "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.1 charset-normalizer-2.0.7 datasets-1.15.1 dill-0.3.4 frozenlist-1.2.0 fsspec-2021.11.0 huggingface-hub-0.1.2 multidict-5.2.0 multiprocess-0.70.12.2 pyarrow-6.0.1 tqdm-4.62.3 xxhash-2.0.2 yarl-1.7.2\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install transformers datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "d0ec8542",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "248be0df",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Reusing dataset reuters21578 (/Users/markmcquade/.cache/huggingface/datasets/reuters21578/ModHayes/1.0.0/bd91fac5a25fc818873c02a7281cc276c9b326a9e6a89288fc6ba6967772240f)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6cbd2502b9d54505b2a60aa8f809a2f5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\"reuters21578\", \"ModHayes\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "906c5716",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    test: Dataset({\n",
+       "        features: ['text', 'text_type', 'topics', 'lewis_split', 'cgis_split', 'old_id', 'new_id', 'places', 'people', 'orgs', 'exchanges', 'date', 'title'],\n",
+       "        num_rows: 722\n",
+       "    })\n",
+       "    train: Dataset({\n",
+       "        features: ['text', 'text_type', 'topics', 'lewis_split', 'cgis_split', 'old_id', 'new_id', 'places', 'people', 'orgs', 'exchanges', 'date', 'title'],\n",
+       "        num_rows: 20856\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ee660fd8",
+   "metadata": {},
+   "source": [
+    "Remove unwanted columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "5a66ae17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = dataset.remove_columns(\n",
+    "['text_type', 'topics', 'lewis_split', 'cgis_split', 'old_id', \n",
+    " 'new_id', 'places', 'people', 'orgs', 'exchanges', 'date'\n",
+    "])\n",
+    "\n",
+    "dataset = dataset.rename_column('title', 'target')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "bae56dc5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    test: Dataset({\n",
+       "        features: ['text', 'target'],\n",
+       "        num_rows: 722\n",
+       "    })\n",
+       "    train: Dataset({\n",
+       "        features: ['text', 'target'],\n",
+       "        num_rows: 20856\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "2d89fd84",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'text': 'Standard Oil Co and BP North America\\nInc said they plan to form a venture to manage the money market\\nborrowing and investment activities of both companies.\\n    BP North America is a subsidiary of British Petroleum Co\\nPlc &lt;BP>, which also owns a 55 pct interest in Standard Oil.\\n    The venture will be called BP/Standard Financial Trading\\nand will be operated by Standard Oil under the oversight of a\\njoint management committee.\\n\\n Reuter\\n',\n",
+       " 'target': 'STANDARD OIL &lt;SRD> TO FORM FINANCIAL UNIT'}"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset['train'][1]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4151eddf",
+   "metadata": {},
+   "source": [
+    "Drop new lines and tabs, replace with white space. Remove commas, remove quotes. Replace \"Reuter\" from end of text. Drop extra white space agains. Remove html tags"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "76bc67d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean(row):\n",
+    "    row['text'] = row['text'].replace('\\n',' ').replace('\\t',' ')\\\n",
+    "                             .replace(',','').replace('\\'','').replace('\\\"','')\\\n",
+    "                             .replace(' Reuter','').replace(' REUTER','')\n",
+    "    row['text'] = \" \".join(row['text'].split())\n",
+    "    row['target'] = row['target'].replace('&lt;','<').replace('&gt;','>')\n",
+    "    return row"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "40bff5a5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7a4b761c23b84a138fa74ef9dfe9ed49",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/722 [00:00<?, ?ex/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "73c80676ca004de29295c8e5ef2512ea",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/20856 [00:00<?, ?ex/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset = dataset.map(clean)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "d882cc08",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'text': 'Standard Oil Co and BP North America Inc said they plan to form a venture to manage the money market borrowing and investment activities of both companies. BP North America is a subsidiary of British Petroleum Co Plc &lt;BP> which also owns a 55 pct interest in Standard Oil. The venture will be called BP/Standard Financial Trading and will be operated by Standard Oil under the oversight of a joint management committee.',\n",
+       " 'target': 'STANDARD OIL <SRD> TO FORM FINANCIAL UNIT'}"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset['train'][1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "22c4da2e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset.save_to_disk('reuters_processed')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "95ef34aa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "17b994c03fe84d0082f285fd972c4140",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating CSV from Arrow format:   0%|          | 0/3 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4a3f180722184a2f90a288c35d313f3c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "807451"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset['train'].to_csv('reuters_train.csv', index=False, header=True)\n",
+    "dataset['test'].to_csv('reuters_test.csv', index=False, header=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "489257d1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

code/reuters_processed/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["test", "train"]}

code/reuters_processed/test/dataset.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5674ef9fdb039939b76e2bb55d2883b076a91557de80d13e8cba40267b2daac9
+size 812240

code/reuters_processed/test/dataset_info.json

@@ -0,0 +1,54 @@
+{
+  "builder_name": "reuters21578",
+  "citation": "@article{APTE94,\nauthor = {Chidanand Apt{'{e}} and Fred Damerau and Sholom M. Weiss},\ntitle = {Automated Learning of Decision Rules for Text Categorization},\njournal = {ACM Transactions on Information Systems},\nyear = {1994},\nnote = {To appear.}\n}\n\n@inproceedings{APTE94b,\nauthor = {Chidanand Apt{'{e}} and Fred Damerau and Sholom M. Weiss},\ntitle = {Toward Language Independent Automated Learning of Text Categorization Models},\nbooktitle = {sigir94},\nyear = {1994},\nnote = {To appear.}\n}\n\n@inproceedings{HAYES8},\nauthor = {Philip J. Hayes and Peggy M. Anderson and Irene B. Nirenburg and\nLinda M. Schmandt},\ntitle = {{TCS}: A Shell for Content-Based Text Categorization},\nbooktitle = {IEEE Conference on Artificial Intelligence Applications},\nyear = {1990}\n}\n\n@inproceedings{HAYES90b,\nauthor = {Philip J. Hayes and Steven P. Weinstein},\ntitle = {{CONSTRUE/TIS:} A System for Content-Based Indexing of a\nDatabase of News Stories},\nbooktitle = {Second Annual Conference on Innovative Applications of\nArtificial Intelligence},\nyear = {1990}\n}\n\n@incollection{HAYES92 ,\nauthor = {Philip J. Hayes},\ntitle = {Intelligent High-Volume Text Processing using Shallow,\nDomain-Specific Techniques},\nbooktitle = {Text-Based Intelligent Systems},\npublisher = {Lawrence Erlbaum},\naddress =  {Hillsdale, NJ},\nyear = {1992},\neditor = {Paul S. Jacobs}\n}\n\n@inproceedings{LEWIS91c ,\nauthor = {David D. Lewis},\ntitle = {Evaluating Text Categorization},\nbooktitle = {Proceedings of Speech and Natural Language Workshop},\nyear = {1991},\nmonth = {feb},\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\npages = {312--318}\n\n}\n\n@phdthesis{LEWIS91d,\nauthor = {David Dolan Lewis},\ntitle = {Representation and Learning in Information Retrieval},\nschool = {Computer Science Dept.; Univ. of Massachusetts; Amherst, MA 01003},\nyear = 1992},\nnote = {Technical Report 91--93.}\n}\n\n@inproceedings{LEWIS91e,\nauthor = {David D. Lewis},\ntitle = {Data Extraction as Text Categorization: An Experiment with\nthe {MUC-3} Corpus},\nbooktitle = {Proceedings of the Third Message Understanding Evaluation\nand Conference},\nyear = {1991},\nmonth = {may},\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\naddress = {Los Altos, CA}\n\n}\n\n@inproceedings{LEWIS92b,\nauthor = {David D. Lewis},\ntitle = {An Evaluation of Phrasal and Clustered Representations on a Text\nCategorization Task},\nbooktitle = {Fifteenth Annual International ACM SIGIR Conference on\nResearch and Development in Information Retrieval},\nyear = {1992},\npages = {37--50}\n}\n\n@inproceedings{LEWIS92d ,\nauthor = {David D. Lewis and Richard M. Tong},\ntitle = {Text Filtering in {MUC-3} and {MUC-4}},\nbooktitle = {Proceedings of the Fourth Message Understanding Conference ({MUC-4})},\nyear = {1992},\nmonth = {jun},\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\naddress = {Los Altos, CA}\n}\n\n@inproceedings{LEWIS92e,\nauthor = {David D. Lewis},\ntitle = {Feature Selection and Feature Extraction for Text Categorization},\nbooktitle = {Proceedings of Speech and Natural Language Workshop},\nyear = {1992},\nmonth = {feb} ,\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\npages = {212--217}\n}\n\n@inproceedings{LEWIS94b,\nauthor = {David D. Lewis and Marc Ringuette},\ntitle = {A Comparison of Two Learning Algorithms for Text Categorization},\nbooktitle = {Symposium on Document Analysis and Information Retrieval},\nyear = {1994},\norganization = {ISRI; Univ. of Nevada, Las Vegas},\naddress = {Las Vegas, NV},\nmonth = {apr},\npages = {81--93}\n}\n\n@article{LEWIS94d,\nauthor = {David D. Lewis and Philip J. Hayes},\ntitle = {Guest Editorial},\njournal = {ACM Transactions on Information Systems},\nyear = {1994},\nvolume  = {12},\nnumber  = {3},\npages = {231},\nmonth = {jul}\n}\n\n@article{SPARCKJONES76,\nauthor = {K. {Sparck Jones} and  C. J. {van Rijsbergen}},\ntitle =  {Information Retrieval Test Collections},\njournal = {Journal of Documentation},\nyear = {1976},\nvolume = {32},\nnumber = {1},\npages = {59--75}\n}\n\n@book{WEISS91,\nauthor = {Sholom M. Weiss and Casimir A. Kulikowski},\ntitle = {Computer Systems That Learn},\npublisher = {Morgan Kaufmann},\nyear = {1991},\naddress = {San Mateo, CA}\n}\n",
+  "config_name": "ModHayes",
+  "dataset_size": 20019638,
+  "description": "The Reuters-21578 dataset  is one of the most widely used data collections for text\ncategorization research. It is collected from the Reuters financial newswire service in 1987.\n",
+  "download_checksums": {
+    "https://kdd.ics.uci.edu/databases/reuters21578/reuters21578.tar.gz": {
+      "num_bytes": 8150596,
+      "checksum": "3bae43c9b14e387f76a61b6d82bf98a4fb5d3ef99ef7e7075ff2ccbcf59f9d30"
+    }
+  },
+  "download_size": 8150596,
+  "features": {
+    "text": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    },
+    "target": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    }
+  },
+  "homepage": "https://kdd.ics.uci.edu/databases/reuters21578/reuters21578.html",
+  "license": "",
+  "post_processed": null,
+  "post_processing_size": null,
+  "size_in_bytes": 28170234,
+  "splits": {
+    "test": {
+      "name": "test",
+      "num_bytes": 948316,
+      "num_examples": 722,
+      "dataset_name": "reuters21578"
+    },
+    "train": {
+      "name": "train",
+      "num_bytes": 19071322,
+      "num_examples": 20856,
+      "dataset_name": "reuters21578"
+    }
+  },
+  "supervised_keys": null,
+  "task_templates": null,
+  "version": {
+    "version_str": "1.0.0",
+    "description": "",
+    "major": 1,
+    "minor": 0,
+    "patch": 0
+  }
+}

code/reuters_processed/test/state.json

@@ -0,0 +1,14 @@
+{
+  "_data_files": [
+    {
+      "filename": "dataset.arrow"
+    }
+  ],
+  "_fingerprint": "226b9d2adcbef331",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_indexes": {},
+  "_output_all_columns": false,
+  "_split": "test"
+}

code/reuters_processed/train/dataset.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972f47423ef2291c6c6e7d6ed8849434c964270f87ad1f91d51edea98a0f1df4
+size 15716672

code/reuters_processed/train/dataset_info.json

@@ -0,0 +1,54 @@
+{
+  "builder_name": "reuters21578",
+  "citation": "@article{APTE94,\nauthor = {Chidanand Apt{'{e}} and Fred Damerau and Sholom M. Weiss},\ntitle = {Automated Learning of Decision Rules for Text Categorization},\njournal = {ACM Transactions on Information Systems},\nyear = {1994},\nnote = {To appear.}\n}\n\n@inproceedings{APTE94b,\nauthor = {Chidanand Apt{'{e}} and Fred Damerau and Sholom M. Weiss},\ntitle = {Toward Language Independent Automated Learning of Text Categorization Models},\nbooktitle = {sigir94},\nyear = {1994},\nnote = {To appear.}\n}\n\n@inproceedings{HAYES8},\nauthor = {Philip J. Hayes and Peggy M. Anderson and Irene B. Nirenburg and\nLinda M. Schmandt},\ntitle = {{TCS}: A Shell for Content-Based Text Categorization},\nbooktitle = {IEEE Conference on Artificial Intelligence Applications},\nyear = {1990}\n}\n\n@inproceedings{HAYES90b,\nauthor = {Philip J. Hayes and Steven P. Weinstein},\ntitle = {{CONSTRUE/TIS:} A System for Content-Based Indexing of a\nDatabase of News Stories},\nbooktitle = {Second Annual Conference on Innovative Applications of\nArtificial Intelligence},\nyear = {1990}\n}\n\n@incollection{HAYES92 ,\nauthor = {Philip J. Hayes},\ntitle = {Intelligent High-Volume Text Processing using Shallow,\nDomain-Specific Techniques},\nbooktitle = {Text-Based Intelligent Systems},\npublisher = {Lawrence Erlbaum},\naddress =  {Hillsdale, NJ},\nyear = {1992},\neditor = {Paul S. Jacobs}\n}\n\n@inproceedings{LEWIS91c ,\nauthor = {David D. Lewis},\ntitle = {Evaluating Text Categorization},\nbooktitle = {Proceedings of Speech and Natural Language Workshop},\nyear = {1991},\nmonth = {feb},\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\npages = {312--318}\n\n}\n\n@phdthesis{LEWIS91d,\nauthor = {David Dolan Lewis},\ntitle = {Representation and Learning in Information Retrieval},\nschool = {Computer Science Dept.; Univ. of Massachusetts; Amherst, MA 01003},\nyear = 1992},\nnote = {Technical Report 91--93.}\n}\n\n@inproceedings{LEWIS91e,\nauthor = {David D. Lewis},\ntitle = {Data Extraction as Text Categorization: An Experiment with\nthe {MUC-3} Corpus},\nbooktitle = {Proceedings of the Third Message Understanding Evaluation\nand Conference},\nyear = {1991},\nmonth = {may},\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\naddress = {Los Altos, CA}\n\n}\n\n@inproceedings{LEWIS92b,\nauthor = {David D. Lewis},\ntitle = {An Evaluation of Phrasal and Clustered Representations on a Text\nCategorization Task},\nbooktitle = {Fifteenth Annual International ACM SIGIR Conference on\nResearch and Development in Information Retrieval},\nyear = {1992},\npages = {37--50}\n}\n\n@inproceedings{LEWIS92d ,\nauthor = {David D. Lewis and Richard M. Tong},\ntitle = {Text Filtering in {MUC-3} and {MUC-4}},\nbooktitle = {Proceedings of the Fourth Message Understanding Conference ({MUC-4})},\nyear = {1992},\nmonth = {jun},\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\naddress = {Los Altos, CA}\n}\n\n@inproceedings{LEWIS92e,\nauthor = {David D. Lewis},\ntitle = {Feature Selection and Feature Extraction for Text Categorization},\nbooktitle = {Proceedings of Speech and Natural Language Workshop},\nyear = {1992},\nmonth = {feb} ,\norganization = {Defense Advanced Research Projects Agency},\npublisher = {Morgan Kaufmann},\npages = {212--217}\n}\n\n@inproceedings{LEWIS94b,\nauthor = {David D. Lewis and Marc Ringuette},\ntitle = {A Comparison of Two Learning Algorithms for Text Categorization},\nbooktitle = {Symposium on Document Analysis and Information Retrieval},\nyear = {1994},\norganization = {ISRI; Univ. of Nevada, Las Vegas},\naddress = {Las Vegas, NV},\nmonth = {apr},\npages = {81--93}\n}\n\n@article{LEWIS94d,\nauthor = {David D. Lewis and Philip J. Hayes},\ntitle = {Guest Editorial},\njournal = {ACM Transactions on Information Systems},\nyear = {1994},\nvolume  = {12},\nnumber  = {3},\npages = {231},\nmonth = {jul}\n}\n\n@article{SPARCKJONES76,\nauthor = {K. {Sparck Jones} and  C. J. {van Rijsbergen}},\ntitle =  {Information Retrieval Test Collections},\njournal = {Journal of Documentation},\nyear = {1976},\nvolume = {32},\nnumber = {1},\npages = {59--75}\n}\n\n@book{WEISS91,\nauthor = {Sholom M. Weiss and Casimir A. Kulikowski},\ntitle = {Computer Systems That Learn},\npublisher = {Morgan Kaufmann},\nyear = {1991},\naddress = {San Mateo, CA}\n}\n",
+  "config_name": "ModHayes",
+  "dataset_size": 20019638,
+  "description": "The Reuters-21578 dataset  is one of the most widely used data collections for text\ncategorization research. It is collected from the Reuters financial newswire service in 1987.\n",
+  "download_checksums": {
+    "https://kdd.ics.uci.edu/databases/reuters21578/reuters21578.tar.gz": {
+      "num_bytes": 8150596,
+      "checksum": "3bae43c9b14e387f76a61b6d82bf98a4fb5d3ef99ef7e7075ff2ccbcf59f9d30"
+    }
+  },
+  "download_size": 8150596,
+  "features": {
+    "text": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    },
+    "target": {
+      "dtype": "string",
+      "id": null,
+      "_type": "Value"
+    }
+  },
+  "homepage": "https://kdd.ics.uci.edu/databases/reuters21578/reuters21578.html",
+  "license": "",
+  "post_processed": null,
+  "post_processing_size": null,
+  "size_in_bytes": 28170234,
+  "splits": {
+    "test": {
+      "name": "test",
+      "num_bytes": 948316,
+      "num_examples": 722,
+      "dataset_name": "reuters21578"
+    },
+    "train": {
+      "name": "train",
+      "num_bytes": 19071322,
+      "num_examples": 20856,
+      "dataset_name": "reuters21578"
+    }
+  },
+  "supervised_keys": null,
+  "task_templates": null,
+  "version": {
+    "version_str": "1.0.0",
+    "description": "",
+    "major": 1,
+    "minor": 0,
+    "patch": 0
+  }
+}

code/reuters_processed/train/state.json

@@ -0,0 +1,14 @@
+{
+  "_data_files": [
+    {
+      "filename": "dataset.arrow"
+    }
+  ],
+  "_fingerprint": "ed601f62656c294e",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_indexes": {},
+  "_output_all_columns": false,
+  "_split": "train"
+}

code/reuters_test.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:925c8919afd37bb4acf70b916ae7c0a24d4ae96aae40d687dfbcb83df267c22c
+size 807451

code/reuters_train.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c41f0ae30ba16e6b1ddb087371caa91cc44c165f86cf91d85b766397c3109415
+size 15589359