migrate to huggingface spaces

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.3.1
+    hooks:
+      - id: nbqa-black
+        args: [--line-length=99]
+      - id: nbqa-pyupgrade
+        args: [--py36-plus]
+      - id: nbqa-isort
+        args: [--profile=black]

pkmn-classifier/evaluate.ipynb

@@ -0,0 +1,541 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "name": "some_evaluation.ipynb",
+      "provenance": [],
+      "authorship_tag": "ABX9TyOnJhILbrhl8aZ1wDYochYn",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/mrcoombes/projects/blob/main/some_evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "zSH83k5bXBVV",
+        "outputId": "6bccc72d-1764-4ecd-ea55-3b05fa5f255a"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Collecting transformers\n",
+            "  Downloading transformers-4.19.2-py3-none-any.whl (4.2 MB)\n",
+            "\u001b[K     |████████████████████████████████| 4.2 MB 5.1 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
+            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
+            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
+            "\u001b[K     |████████████████████████████████| 6.6 MB 45.6 MB/s \n",
+            "\u001b[?25hCollecting pyyaml>=5.1\n",
+            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
+            "\u001b[K     |████████████████████████████████| 596 kB 45.0 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
+            "Collecting huggingface-hub<1.0,>=0.1.0\n",
+            "  Downloading huggingface_hub-0.7.0-py3-none-any.whl (86 kB)\n",
+            "\u001b[K     |████████████████████████████████| 86 kB 5.0 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
+            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.4)\n",
+            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
+            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
+            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
+            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.5.18.1)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
+            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
+            "  Attempting uninstall: pyyaml\n",
+            "    Found existing installation: PyYAML 3.13\n",
+            "    Uninstalling PyYAML-3.13:\n",
+            "      Successfully uninstalled PyYAML-3.13\n",
+            "Successfully installed huggingface-hub-0.7.0 pyyaml-6.0 tokenizers-0.12.1 transformers-4.19.2\n",
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Collecting datasets\n",
+            "  Downloading datasets-2.2.2-py3-none-any.whl (346 kB)\n",
+            "\u001b[K     |████████████████████████████████| 346 kB 5.1 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.13)\n",
+            "Collecting xxhash\n",
+            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
+            "\u001b[K     |████████████████████████████████| 212 kB 58.9 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.7.0)\n",
+            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.0)\n",
+            "Collecting dill<0.3.5\n",
+            "  Downloading dill-0.3.4-py2.py3-none-any.whl (86 kB)\n",
+            "\u001b[K     |████████████████████████████████| 86 kB 5.4 MB/s \n",
+            "\u001b[?25hCollecting aiohttp\n",
+            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
+            "\u001b[K     |████████████████████████████████| 1.1 MB 52.8 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
+            "Collecting responses<0.19\n",
+            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
+            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.11.4)\n",
+            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
+            "Collecting fsspec[http]>=2021.05.0\n",
+            "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
+            "\u001b[K     |████████████████████████████████| 140 kB 56.3 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
+            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
+            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.2.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.0)\n",
+            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.5.18.1)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
+            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
+            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
+            "\u001b[K     |████████████████████████████████| 127 kB 60.8 MB/s \n",
+            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
+            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
+            "Collecting yarl<2.0,>=1.0\n",
+            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
+            "\u001b[K     |████████████████████████████████| 271 kB 50.7 MB/s \n",
+            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
+            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
+            "\u001b[K     |████████████████████████████████| 94 kB 1.0 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.4.0)\n",
+            "Collecting asynctest==0.13.0\n",
+            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
+            "Collecting async-timeout<5.0,>=4.0.0a3\n",
+            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
+            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.12)\n",
+            "Collecting frozenlist>=1.1.1\n",
+            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
+            "\u001b[K     |████████████████████████████████| 144 kB 57.3 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.8.0)\n",
+            "Collecting multiprocess\n",
+            "  Downloading multiprocess-0.70.12.2-py37-none-any.whl (112 kB)\n",
+            "\u001b[K     |████████████████████████████████| 112 kB 64.6 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
+            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.1)\n",
+            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
+            "Installing collected packages: multidict, frozenlist, yarl, urllib3, asynctest, async-timeout, aiosignal, fsspec, dill, aiohttp, xxhash, responses, multiprocess, datasets\n",
+            "  Attempting uninstall: urllib3\n",
+            "    Found existing installation: urllib3 1.24.3\n",
+            "    Uninstalling urllib3-1.24.3:\n",
+            "      Successfully uninstalled urllib3-1.24.3\n",
+            "  Attempting uninstall: dill\n",
+            "    Found existing installation: dill 0.3.5.1\n",
+            "    Uninstalling dill-0.3.5.1:\n",
+            "      Successfully uninstalled dill-0.3.5.1\n",
+            "  Attempting uninstall: multiprocess\n",
+            "    Found existing installation: multiprocess 0.70.13\n",
+            "    Uninstalling multiprocess-0.70.13:\n",
+            "      Successfully uninstalled multiprocess-0.70.13\n",
+            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
+            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 datasets-2.2.2 dill-0.3.4 frozenlist-1.3.0 fsspec-2022.5.0 multidict-6.0.2 multiprocess-0.70.12.2 responses-0.18.0 urllib3-1.25.11 xxhash-3.0.0 yarl-1.7.2\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install transformers\n",
+        "!pip install datasets"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+        "from transformers import pipeline\n",
+        "\n",
+        "model = AutoModelForSequenceClassification.from_pretrained('mrcoombes/distilbert-wikipedia-pokemon')\n",
+        "tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')\n",
+        "\n",
+        "classifier = pipeline('text-classification', model = model, tokenizer=tokenizer, return_all_scores=True)\n",
+        "\n",
+        "clf = lambda x: sorted(classifier(x)[0], key=lambda y: y['score'], reverse=True)"
+      ],
+      "metadata": {
+        "id": "3PGrgNFoXje6"
+      },
+      "execution_count": 90,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "pooh = 'Pooh is a small yellow bear. He is nearly 22 inches (560 mm) tall. He wears an old red t-shirt. His favorite food is honey. The first thing he says when he gets up in the morning is \"What is for breakfast?\". He invented the game \"Poohsticks\".'\n",
+        "\n",
+        "clf(pooh)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "SG72dknXqD7K",
+        "outputId": "8a923810-4757-48f0-de13-0fe127428bf9"
+      },
+      "execution_count": 91,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "[{'label': 'Normal', 'score': 0.2600723206996918},\n",
+              " {'label': 'Psychic', 'score': 0.1543654203414917},\n",
+              " {'label': 'Fire', 'score': 0.06567206978797913},\n",
+              " {'label': 'Electric', 'score': 0.05511392652988434},\n",
+              " {'label': 'Bug', 'score': 0.054607123136520386},\n",
+              " {'label': 'Water', 'score': 0.05065950006246567},\n",
+              " {'label': 'Ghost', 'score': 0.04469247907400131},\n",
+              " {'label': 'Fairy', 'score': 0.03949982300400734},\n",
+              " {'label': 'Dark', 'score': 0.039207249879837036},\n",
+              " {'label': 'Grass', 'score': 0.03893379494547844},\n",
+              " {'label': 'Dragon', 'score': 0.034857217222452164},\n",
+              " {'label': 'Poison', 'score': 0.02820383384823799},\n",
+              " {'label': 'Ice', 'score': 0.028034506365656853},\n",
+              " {'label': 'Fighting', 'score': 0.02752748504281044},\n",
+              " {'label': 'Ground', 'score': 0.021409546956419945},\n",
+              " {'label': 'Steel', 'score': 0.021021585911512375},\n",
+              " {'label': 'Rock', 'score': 0.020723488181829453},\n",
+              " {'label': 'Flying', 'score': 0.015398567542433739}]"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 91
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "clf('Tigger is a fictional tiger. He has orange fur with black stripes. He is easily recognized by his beady eyes, long chin, springy tail, and bouncy personality. As he says himself, \"Bouncing is what Tiggers do best.\" Like other Pooh characters, Tigger is based on one of the stuffed animals of Christopher Robin Milne.')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "o3-gMTkxe44B",
+        "outputId": "632deac3-f378-4fda-c9a3-112edaf0e966"
+      },
+      "execution_count": 93,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "[{'label': 'Normal', 'score': 0.335999995470047},\n",
+              " {'label': 'Water', 'score': 0.09296873211860657},\n",
+              " {'label': 'Psychic', 'score': 0.07045886665582657},\n",
+              " {'label': 'Fire', 'score': 0.06954856216907501},\n",
+              " {'label': 'Bug', 'score': 0.044683054089546204},\n",
+              " {'label': 'Dragon', 'score': 0.0426957830786705},\n",
+              " {'label': 'Dark', 'score': 0.04223953187465668},\n",
+              " {'label': 'Electric', 'score': 0.040662020444869995},\n",
+              " {'label': 'Fairy', 'score': 0.03830638527870178},\n",
+              " {'label': 'Grass', 'score': 0.03206343948841095},\n",
+              " {'label': 'Poison', 'score': 0.027593791484832764},\n",
+              " {'label': 'Ground', 'score': 0.026110077276825905},\n",
+              " {'label': 'Ghost', 'score': 0.025732463225722313},\n",
+              " {'label': 'Ice', 'score': 0.02510858327150345},\n",
+              " {'label': 'Fighting', 'score': 0.023818649351596832},\n",
+              " {'label': 'Steel', 'score': 0.022579438984394073},\n",
+              " {'label': 'Rock', 'score': 0.021684883162379265},\n",
+              " {'label': 'Flying', 'score': 0.017745744436979294}]"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 93
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# sourced from https://avengers.marvelhq.com/characters\n",
+        "avengers = {\n",
+        "\"black_panther\":\"Driven by an extremely lifelike artificial intelligence and possessing a nearly indestructible robotic body, the synthetic android called the Vision has taken his place among Earth’s Mightiest Heroes, the Avengers!\",\n",
+        "\"iron_man\":\"When billionaire industrialist Tony Stark dons his sophisticated steel-mesh armor, he becomes a living high-tech weapon - the world's greatest fighting machine. Tony has primed his ultra modern creation for waging state of the art campaigns, attaining sonic flight, and defending the greater good! He is the Armored Avenger - driven by a heart that is part machine, but all hero! He is the INVINCIBLE IRON MAN!\",\n",
+        "\"thor\":\"Nordic legend tells the tale of the son of Odin, the heir to the throne of Asgard - he is THOR, renowned as the mightiest hero of mythology! Thor's strength, endurance, and quest for battle are far greater than his Asgardian brethren. The mighty Thor wields an enchanted Uru hammer, Mjolnir, and is master of thunder and lightning.\",\n",
+        "\"black_widow\":\"Natasha Romanoff is the super-spy known as the Black Widow! Trained extensively in the art of espionage and outfitted with state-of-the-art equipment, Black Widow's combat skills are virtually unmatched. One of S.H.I.E.L.D's most valuable agents, she has carried out numerous black-ops missions and has recently been assigned by Director Nick Fury to keep an eye on the Avengers.\",\n",
+        "\"hulk\":\"A massive dose of gamma radiation transformed the brilliant but meek scientist Bruce Banner's DNA, awakening the hidden, adrenaline-fed hero in his genes... HULK! A hero of few words and incredible strength, the Hulk has long been pursued by those who want to use his immense power for their own purposes, or by those who thought the Jade Giant's anger was too dangerous to be controlled. Now, as a member of the Avengers, Hulk helps smash the unimaginable threats that no Hero could face alone, hoping to at least prove to the world that he is the strongest HERO there is!\",\n",
+        "\"cap\":\"During WWII, the patriotic Steve Rogers was offered a place in the military's top operation: Rebirth. Injected with an experimental super-serum, Rogers emerged from the treatment with heightened endurance, strength, and reaction time. With extensive training and an indestructible Vibranium shield, Rogers soon became the country's ultimate weapon: CAPTAIN AMERICA! Though frozen in ice during a climactic battle toward the end of the war, Rogers was discovered and revived decades later. Now the living legend continues the war against evil in modern times as a member of The Avengers!\",\n",
+        "}"
+      ],
+      "metadata": {
+        "id": "s950R1RBgztY"
+      },
+      "execution_count": 94,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "tokenizer.model_max_length"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "pVv2MH8-iNKE",
+        "outputId": "adae6f5e-2788-47b7-c382-f46c1c6b17c1"
+      },
+      "execution_count": 23,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "512"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 23
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "for avenger, description in avengers.items():\n",
+        "  print(len(tokenizer(description)['input_ids']))\n",
+        "  assert len(tokenizer(description)['input_ids']) <= tokenizer.model_max_length"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "rUp5CaCIi-E4",
+        "outputId": "724e56f4-5d15-4a65-fe5f-4794e760b944"
+      },
+      "execution_count": 95,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "43\n",
+            "87\n",
+            "81\n",
+            "93\n",
+            "125\n",
+            "118\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "types = {}\n",
+        "\n",
+        "for avenger, description in avengers.items():\n",
+        "  types[avenger] = clf(description)"
+      ],
+      "metadata": {
+        "id": "piSuS63cjZDR"
+      },
+      "execution_count": 96,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "types"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "lohLMB1WkD3j",
+        "outputId": "92fdf9b4-2671-498a-99c6-001ad02c22f1"
+      },
+      "execution_count": 97,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "{'black_panther': [{'label': 'Psychic', 'score': 0.14039084315299988},\n",
+              "  {'label': 'Normal', 'score': 0.11493027955293655},\n",
+              "  {'label': 'Electric', 'score': 0.08732766658067703},\n",
+              "  {'label': 'Bug', 'score': 0.07257580012083054},\n",
+              "  {'label': 'Ghost', 'score': 0.05607102811336517},\n",
+              "  {'label': 'Fire', 'score': 0.05543342977762222},\n",
+              "  {'label': 'Dark', 'score': 0.05199592188000679},\n",
+              "  {'label': 'Steel', 'score': 0.05135666951537132},\n",
+              "  {'label': 'Dragon', 'score': 0.04997771978378296},\n",
+              "  {'label': 'Water', 'score': 0.04398226737976074},\n",
+              "  {'label': 'Fighting', 'score': 0.042541105300188065},\n",
+              "  {'label': 'Fairy', 'score': 0.04088388383388519},\n",
+              "  {'label': 'Rock', 'score': 0.0403033122420311},\n",
+              "  {'label': 'Grass', 'score': 0.0353127047419548},\n",
+              "  {'label': 'Ice', 'score': 0.03269997984170914},\n",
+              "  {'label': 'Ground', 'score': 0.030976610258221626},\n",
+              "  {'label': 'Flying', 'score': 0.026838229969143867},\n",
+              "  {'label': 'Poison', 'score': 0.02640254609286785}],\n",
+              " 'black_widow': [{'label': 'Psychic', 'score': 0.11771736294031143},\n",
+              "  {'label': 'Normal', 'score': 0.09108898788690567},\n",
+              "  {'label': 'Electric', 'score': 0.0777704268693924},\n",
+              "  {'label': 'Ghost', 'score': 0.0661557987332344},\n",
+              "  {'label': 'Bug', 'score': 0.06480831652879715},\n",
+              "  {'label': 'Fire', 'score': 0.06366308778524399},\n",
+              "  {'label': 'Dark', 'score': 0.06326092034578323},\n",
+              "  {'label': 'Fighting', 'score': 0.05556448549032211},\n",
+              "  {'label': 'Steel', 'score': 0.05402420461177826},\n",
+              "  {'label': 'Dragon', 'score': 0.050193771719932556},\n",
+              "  {'label': 'Grass', 'score': 0.04514629766345024},\n",
+              "  {'label': 'Water', 'score': 0.0421665795147419},\n",
+              "  {'label': 'Fairy', 'score': 0.04069530591368675},\n",
+              "  {'label': 'Ice', 'score': 0.03823541849851608},\n",
+              "  {'label': 'Rock', 'score': 0.03812859579920769},\n",
+              "  {'label': 'Ground', 'score': 0.03384226933121681},\n",
+              "  {'label': 'Flying', 'score': 0.02902897447347641},\n",
+              "  {'label': 'Poison', 'score': 0.028509100899100304}],\n",
+              " 'cap': [{'label': 'Psychic', 'score': 0.1309943050146103},\n",
+              "  {'label': 'Electric', 'score': 0.1047213152050972},\n",
+              "  {'label': 'Fire', 'score': 0.08712606877088547},\n",
+              "  {'label': 'Normal', 'score': 0.07033843547105789},\n",
+              "  {'label': 'Bug', 'score': 0.061330799013376236},\n",
+              "  {'label': 'Dragon', 'score': 0.058846309781074524},\n",
+              "  {'label': 'Ghost', 'score': 0.056743498891592026},\n",
+              "  {'label': 'Steel', 'score': 0.052078839391469955},\n",
+              "  {'label': 'Rock', 'score': 0.05096956342458725},\n",
+              "  {'label': 'Ice', 'score': 0.050031181424856186},\n",
+              "  {'label': 'Fighting', 'score': 0.042880505323410034},\n",
+              "  {'label': 'Dark', 'score': 0.042242322117090225},\n",
+              "  {'label': 'Fairy', 'score': 0.037094846367836},\n",
+              "  {'label': 'Water', 'score': 0.03594202175736427},\n",
+              "  {'label': 'Grass', 'score': 0.03098415769636631},\n",
+              "  {'label': 'Ground', 'score': 0.03098156489431858},\n",
+              "  {'label': 'Poison', 'score': 0.030156349763274193},\n",
+              "  {'label': 'Flying', 'score': 0.026537923142313957}],\n",
+              " 'hulk': [{'label': 'Psychic', 'score': 0.19098739326000214},\n",
+              "  {'label': 'Electric', 'score': 0.09343517571687698},\n",
+              "  {'label': 'Normal', 'score': 0.0896320790052414},\n",
+              "  {'label': 'Fire', 'score': 0.06670381128787994},\n",
+              "  {'label': 'Ghost', 'score': 0.062663234770298},\n",
+              "  {'label': 'Bug', 'score': 0.0534147247672081},\n",
+              "  {'label': 'Dragon', 'score': 0.05224861577153206},\n",
+              "  {'label': 'Dark', 'score': 0.048941321671009064},\n",
+              "  {'label': 'Steel', 'score': 0.04724843055009842},\n",
+              "  {'label': 'Fighting', 'score': 0.042670682072639465},\n",
+              "  {'label': 'Water', 'score': 0.038225673139095306},\n",
+              "  {'label': 'Fairy', 'score': 0.038099996745586395},\n",
+              "  {'label': 'Ice', 'score': 0.03596445918083191},\n",
+              "  {'label': 'Rock', 'score': 0.033795569092035294},\n",
+              "  {'label': 'Grass', 'score': 0.028919294476509094},\n",
+              "  {'label': 'Flying', 'score': 0.027448808774352074},\n",
+              "  {'label': 'Ground', 'score': 0.02604469284415245},\n",
+              "  {'label': 'Poison', 'score': 0.023556090891361237}],\n",
+              " 'iron_man': [{'label': 'Electric', 'score': 0.08547740429639816},\n",
+              "  {'label': 'Steel', 'score': 0.08327846229076385},\n",
+              "  {'label': 'Fire', 'score': 0.08254428952932358},\n",
+              "  {'label': 'Psychic', 'score': 0.08083238452672958},\n",
+              "  {'label': 'Normal', 'score': 0.0718364492058754},\n",
+              "  {'label': 'Bug', 'score': 0.07048758119344711},\n",
+              "  {'label': 'Fighting', 'score': 0.06302879750728607},\n",
+              "  {'label': 'Dragon', 'score': 0.06081298366189003},\n",
+              "  {'label': 'Rock', 'score': 0.05577201768755913},\n",
+              "  {'label': 'Dark', 'score': 0.05418119952082634},\n",
+              "  {'label': 'Ice', 'score': 0.04405869543552399},\n",
+              "  {'label': 'Grass', 'score': 0.043908316642045975},\n",
+              "  {'label': 'Ground', 'score': 0.043028734624385834},\n",
+              "  {'label': 'Ghost', 'score': 0.04071647301316261},\n",
+              "  {'label': 'Fairy', 'score': 0.03440254554152489},\n",
+              "  {'label': 'Water', 'score': 0.03119415044784546},\n",
+              "  {'label': 'Flying', 'score': 0.027475930750370026},\n",
+              "  {'label': 'Poison', 'score': 0.0269634909927845}],\n",
+              " 'thor': [{'label': 'Psychic', 'score': 0.1594998836517334},\n",
+              "  {'label': 'Normal', 'score': 0.08447221666574478},\n",
+              "  {'label': 'Fire', 'score': 0.08234388381242752},\n",
+              "  {'label': 'Electric', 'score': 0.08180674910545349},\n",
+              "  {'label': 'Ghost', 'score': 0.06200374290347099},\n",
+              "  {'label': 'Dark', 'score': 0.06072374805808067},\n",
+              "  {'label': 'Dragon', 'score': 0.05554042384028435},\n",
+              "  {'label': 'Bug', 'score': 0.0517977699637413},\n",
+              "  {'label': 'Steel', 'score': 0.04996638000011444},\n",
+              "  {'label': 'Fighting', 'score': 0.04767195135354996},\n",
+              "  {'label': 'Water', 'score': 0.04199497774243355},\n",
+              "  {'label': 'Fairy', 'score': 0.03844171389937401},\n",
+              "  {'label': 'Ice', 'score': 0.03652443364262581},\n",
+              "  {'label': 'Rock', 'score': 0.03268720209598541},\n",
+              "  {'label': 'Grass', 'score': 0.031752120703458786},\n",
+              "  {'label': 'Ground', 'score': 0.03069768100976944},\n",
+              "  {'label': 'Flying', 'score': 0.030208230018615723},\n",
+              "  {'label': 'Poison', 'score': 0.021866854280233383}]}"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 97
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Well, that's dissapointing... anyway...\n",
+        "print(clf('Thor thunder and lightning'))\n",
+        "print('\\n\\n')\n",
+        "print(clf('Thor is the god'))\n",
+        "print('\\n\\n')\n",
+        "print(clf('Thor is the god of thunder and lightning'))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "BsmZe_kSkFNs",
+        "outputId": "149879af-01f2-4df0-d32c-0f6f56db0868"
+      },
+      "execution_count": 102,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "[{'label': 'Electric', 'score': 0.11147855967283249}, {'label': 'Psychic', 'score': 0.10549111664295197}, {'label': 'Fire', 'score': 0.08532625436782837}, {'label': 'Normal', 'score': 0.07318642735481262}, {'label': 'Bug', 'score': 0.06542624533176422}, {'label': 'Ghost', 'score': 0.06079600378870964}, {'label': 'Steel', 'score': 0.05627329647541046}, {'label': 'Dragon', 'score': 0.051239024847745895}, {'label': 'Water', 'score': 0.04977051168680191}, {'label': 'Rock', 'score': 0.04720067232847214}, {'label': 'Fighting', 'score': 0.045852769166231155}, {'label': 'Ice', 'score': 0.041491683572530746}, {'label': 'Dark', 'score': 0.041257862001657486}, {'label': 'Grass', 'score': 0.041162408888339996}, {'label': 'Ground', 'score': 0.03666074201464653}, {'label': 'Fairy', 'score': 0.030792705714702606}, {'label': 'Poison', 'score': 0.029198039323091507}, {'label': 'Flying', 'score': 0.027395661920309067}]\n",
+            "\n",
+            "\n",
+            "\n",
+            "[{'label': 'Psychic', 'score': 0.10292913764715195}, {'label': 'Normal', 'score': 0.07830759137868881}, {'label': 'Electric', 'score': 0.07660140842199326}, {'label': 'Water', 'score': 0.06917692720890045}, {'label': 'Fire', 'score': 0.06355269998311996}, {'label': 'Bug', 'score': 0.06285296380519867}, {'label': 'Ghost', 'score': 0.06255136430263519}, {'label': 'Dark', 'score': 0.06148482486605644}, {'label': 'Dragon', 'score': 0.04900471866130829}, {'label': 'Grass', 'score': 0.047270145267248154}, {'label': 'Fighting', 'score': 0.04712704196572304}, {'label': 'Steel', 'score': 0.04605235531926155}, {'label': 'Rock', 'score': 0.04519566521048546}, {'label': 'Ground', 'score': 0.04106030985713005}, {'label': 'Ice', 'score': 0.039930786937475204}, {'label': 'Fairy', 'score': 0.03901135176420212}, {'label': 'Poison', 'score': 0.03707478940486908}, {'label': 'Flying', 'score': 0.030815837904810905}]\n",
+            "\n",
+            "\n",
+            "\n",
+            "[{'label': 'Psychic', 'score': 0.12898413836956024}, {'label': 'Electric', 'score': 0.10279341787099838}, {'label': 'Fire', 'score': 0.08279084414243698}, {'label': 'Normal', 'score': 0.07643641531467438}, {'label': 'Ghost', 'score': 0.06414799392223358}, {'label': 'Bug', 'score': 0.060264818370342255}, {'label': 'Water', 'score': 0.052734874188899994}, {'label': 'Dragon', 'score': 0.05206158012151718}, {'label': 'Steel', 'score': 0.047912657260894775}, {'label': 'Dark', 'score': 0.04722539335489273}, {'label': 'Fighting', 'score': 0.042949989438056946}, {'label': 'Rock', 'score': 0.042036380618810654}, {'label': 'Ice', 'score': 0.03920420631766319}, {'label': 'Grass', 'score': 0.036403022706508636}, {'label': 'Fairy', 'score': 0.03535248339176178}, {'label': 'Ground', 'score': 0.03424106165766716}, {'label': 'Flying', 'score': 0.027838196605443954}, {'label': 'Poison', 'score': 0.02662259340286255}]\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        ""
+      ],
+      "metadata": {
+        "id": "i-oI9LS1vq7e"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

pkmn-classifier/transformer.ipynb

@@ -0,0 +1,1503 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install transformers\n",
+        "!pip install datasets"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "v6Hl9dT9xI3h",
+        "outputId": "99c0c9dc-432d-43b4-ce25-828e9e467326"
+      },
+      "execution_count": 1,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.19.2)\n",
+            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
+            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
+            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.12.1)\n",
+            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
+            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.4)\n",
+            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.7.0)\n",
+            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
+            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
+            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
+            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.25.11)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.5.18.1)\n",
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.2.2)\n",
+            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
+            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.0)\n",
+            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
+            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.7.0)\n",
+            "Requirement already satisfied: dill<0.3.5 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
+            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.18.0)\n",
+            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
+            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
+            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
+            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.11.4)\n",
+            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.5.0)\n",
+            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
+            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.1)\n",
+            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.2.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.0)\n",
+            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.5.18.1)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.25.11)\n",
+            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
+            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.0)\n",
+            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.4.0)\n",
+            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
+            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
+            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
+            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.7.2)\n",
+            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.12)\n",
+            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.8.0)\n",
+            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.1)\n",
+            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
+            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "i-B5sPHELBBj"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import pandas as pd\n",
+        "import seaborn as sns"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "id": "uxhFjfeHLBBr"
+      },
+      "outputs": [],
+      "source": [
+        "# Read the pokedex we scraped in web_scrape.ipynb into a DataFrame\n",
+        "pkmn = pd.read_csv(\"pokemon.csv\")\n",
+        "pkmn.rename(columns={\"Unnamed: 0\": \"wiki_index\"}, inplace=True)\n",
+        "pkmn = pkmn[pkmn.primary_type != \"Bird\"] # MissingNo is special, but not special enough to break the rules."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Fixing Inference.\n",
+        "\n",
+        "lil = pkmn[['primary_type', 'Notes']].copy()"
+      ],
+      "metadata": {
+        "id": "y_n4NNSOxzXL"
+      },
+      "execution_count": 4,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from datasets.dataset_dict import DatasetDict\n",
+        "from datasets import Dataset\n",
+        "import datasets"
+      ],
+      "metadata": {
+        "id": "F_HSR6Kk2M0H"
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "lil['primary_type'] = lil['primary_type'].astype('category') \n",
+        "lil['label'] = lil['primary_type'].cat.codes\n",
+        "df = lil[['label', 'Notes']].copy()\n",
+        "df = df.rename(columns={'Notes': 'text'})"
+      ],
+      "metadata": {
+        "id": "Dt-gNH3D4P9Y"
+      },
+      "execution_count": 6,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "id2label = {k: v for k, v in enumerate(lil['primary_type'].cat.categories)}\n",
+        "label2id = {v: k for k, v in enumerate(lil['primary_type'].cat.categories)}"
+      ],
+      "metadata": {
+        "id": "3eJghd1m4TSC"
+      },
+      "execution_count": 7,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "VyS7Nc_Iv7ys",
+        "outputId": "4558a505-5e59-4c41-f6e5-33e637891a49"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_projector.weight', 'vocab_transform.bias']\n",
+            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'classifier.weight', 'pre_classifier.bias', 'classifier.bias']\n",
+            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+          ]
+        }
+      ],
+      "source": [
+        "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
+        "\n",
+        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
+        "model = AutoModelForSequenceClassification.from_pretrained(\n",
+        "    \"distilbert-base-uncased\", num_labels=18, id2label=id2label, label2id=label2id\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "train_df = df.sample(frac=0.7)\n",
+        "test_df = df.drop(train_df.index, inplace=False)\n",
+        "\n",
+        "train_dataset = Dataset.from_dict(train_df)\n",
+        "test_dataset = Dataset.from_dict(test_df)\n",
+        "my_dataset_dict = DatasetDict({\"train\":train_dataset,\"test\":test_dataset})\n",
+        "\n",
+        "my_dataset_dict"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "GhlYrzASzDUS",
+        "outputId": "5c41102d-0c54-498f-f24e-0dae59a0ed56"
+      },
+      "execution_count": 10,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "DatasetDict({\n",
+              "    train: Dataset({\n",
+              "        features: ['label', 'text'],\n",
+              "        num_rows: 654\n",
+              "    })\n",
+              "    test: Dataset({\n",
+              "        features: ['label', 'text'],\n",
+              "        num_rows: 281\n",
+              "    })\n",
+              "})"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 10
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def tokenize_function(examples):\n",
+        "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
+        "\n",
+        "dataset = my_dataset_dict\n",
+        "tokenized_datasets = dataset.map(tokenize_function, batched=True)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 81,
+          "referenced_widgets": [
+            "e397276a3494413a8537fa5ba48f7015",
+            "3681f628eb844524bd4a121dd313d2ef",
+            "4699b0136d4d449ca4accf9bc90fc45a",
+            "0435b641146f45488248e042ff0d4f31",
+            "8233bb294c7549b6a8ae5e140e8ca5b6",
+            "608d0d01fe4d4a14877920e51b10233a",
+            "246696956b774368ad7572db606a2414",
+            "cd1d0bd35b464f74aff7c7421de03fc9",
+            "eac0a76ba4cf4b61ae82e9aa2f770dce",
+            "68e198c9120f4451bd9b9f033a9d4bae",
+            "bd8b8f2e781e4de1a477c8af4c450d1f",
+            "b2eba59325544c0790fea0b5a08916b1",
+            "deeb1a85968d4619a7928712c09168d2",
+            "39d408f797bf46aa8a9617a68c8ba913",
+            "e643c45c934d414d9b46abdc64eccbbc",
+            "f7972177ace147fbb45918eebe106915",
+            "aa8e9ad5e1ce4b84b49c194c61f90820",
+            "93c35117cea74e5b9de2b871168b7095",
+            "09542f17885d4ce1b6fb5e8682beb6de",
+            "e1fce70e1a67446982a09c9d4948b48d",
+            "7712b1987ee143fe9ceb6ef13bded85d",
+            "51f19ba89e5f4d208564f03ff6f2b0da"
+          ]
+        },
+        "id": "HB1hGkNLy9rG",
+        "outputId": "365046b5-635d-402d-9c81-3fec9d05ac64"
+      },
+      "execution_count": 11,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "  0%|          | 0/1 [00:00<?, ?ba/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "e397276a3494413a8537fa5ba48f7015"
+            }
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "  0%|          | 0/1 [00:00<?, ?ba/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "b2eba59325544c0790fea0b5a08916b1"
+            }
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "tokenized_datasets"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "ZXN4Vn-l3dYd",
+        "outputId": "5fcee721-01eb-48c4-b111-af15c88b5078"
+      },
+      "execution_count": 12,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "DatasetDict({\n",
+              "    train: Dataset({\n",
+              "        features: ['label', 'text', 'input_ids', 'attention_mask'],\n",
+              "        num_rows: 654\n",
+              "    })\n",
+              "    test: Dataset({\n",
+              "        features: ['label', 'text', 'input_ids', 'attention_mask'],\n",
+              "        num_rows: 281\n",
+              "    })\n",
+              "})"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 12
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "small_train_dataset = tokenized_datasets[\"train\"]\n",
+        "small_eval_dataset = tokenized_datasets[\"test\"]"
+      ],
+      "metadata": {
+        "id": "PuYI3C5vyIdB"
+      },
+      "execution_count": 13,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from transformers import TrainingArguments\n",
+        "\n",
+        "training_args = TrainingArguments(output_dir=\"test_trainer\")"
+      ],
+      "metadata": {
+        "id": "d9jCOTqy3mJl"
+      },
+      "execution_count": 14,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import numpy as np\n",
+        "from datasets import load_metric\n",
+        "\n",
+        "metric = load_metric(\"accuracy\")\n"
+      ],
+      "metadata": {
+        "id": "SSmBwKCy3wO9"
+      },
+      "execution_count": 15,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def compute_metrics(eval_pred):\n",
+        "    logits, labels = eval_pred\n",
+        "    predictions = np.argmax(logits, axis=-1)\n",
+        "    return metric.compute(predictions=predictions, references=labels)"
+      ],
+      "metadata": {
+        "id": "8dOc3PsL30Q4"
+      },
+      "execution_count": 16,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from transformers import TrainingArguments, Trainer\n",
+        "\n",
+        "training_args = TrainingArguments(output_dir=\"test_trainer\", evaluation_strategy=\"epoch\")"
+      ],
+      "metadata": {
+        "id": "-P9jBD1U37PZ"
+      },
+      "execution_count": 17,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "trainer = Trainer(\n",
+        "    model=model,\n",
+        "    args=training_args,\n",
+        "    train_dataset=small_train_dataset,\n",
+        "    eval_dataset=small_eval_dataset,\n",
+        "    compute_metrics=compute_metrics,\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "wuKk1EvI4DSo"
+      },
+      "execution_count": 18,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "trainer.train()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 675
+        },
+        "id": "CCTgG2cl4GZb",
+        "outputId": "ebe231ae-ad4e-4a44-eb0a-070ff395d6e9"
+      },
+      "execution_count": 19,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "The following columns in the training set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
+            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+            "  FutureWarning,\n",
+            "***** Running training *****\n",
+            "  Num examples = 654\n",
+            "  Num Epochs = 3\n",
+            "  Instantaneous batch size per device = 8\n",
+            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
+            "  Gradient Accumulation steps = 1\n",
+            "  Total optimization steps = 246\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "\n",
+              "    <div>\n",
+              "      \n",
+              "      <progress value='246' max='246' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+              "      [246/246 01:46, Epoch 3/3]\n",
+              "    </div>\n",
+              "    <table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              " <tr style=\"text-align: left;\">\n",
+              "      <th>Epoch</th>\n",
+              "      <th>Training Loss</th>\n",
+              "      <th>Validation Loss</th>\n",
+              "      <th>Accuracy</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <td>1</td>\n",
+              "      <td>No log</td>\n",
+              "      <td>2.471577</td>\n",
+              "      <td>0.274021</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2</td>\n",
+              "      <td>No log</td>\n",
+              "      <td>2.191889</td>\n",
+              "      <td>0.437722</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3</td>\n",
+              "      <td>No log</td>\n",
+              "      <td>2.077948</td>\n",
+              "      <td>0.473310</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table><p>"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
+            "***** Running Evaluation *****\n",
+            "  Num examples = 281\n",
+            "  Batch size = 8\n",
+            "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
+            "***** Running Evaluation *****\n",
+            "  Num examples = 281\n",
+            "  Batch size = 8\n",
+            "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
+            "***** Running Evaluation *****\n",
+            "  Num examples = 281\n",
+            "  Batch size = 8\n",
+            "\n",
+            "\n",
+            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
+            "\n",
+            "\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "TrainOutput(global_step=246, training_loss=2.312268264894563, metrics={'train_runtime': 106.6333, 'train_samples_per_second': 18.4, 'train_steps_per_second': 2.307, 'total_flos': 259975195619328.0, 'train_loss': 2.312268264894563, 'epoch': 3.0})"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 19
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "model.save_pretrained(\"./model\")"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "svA3ZvuKW6LA",
+        "outputId": "ca4674d4-92c2-4c9a-a151-1d093ddbf954"
+      },
+      "execution_count": 20,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Configuration saved in ./config.json\n",
+            "Model weights saved in ./pytorch_model.bin\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "model2 = AutoModelForSequenceClassification.from_pretrained('./model')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "rHGVDBRc3eqe",
+        "outputId": "29d1d861-c58a-41a0-8179-19abc44efac1"
+      },
+      "execution_count": 21,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "loading configuration file ./config.json\n",
+            "Model config DistilBertConfig {\n",
+            "  \"_name_or_path\": \".\",\n",
+            "  \"activation\": \"gelu\",\n",
+            "  \"architectures\": [\n",
+            "    \"DistilBertForSequenceClassification\"\n",
+            "  ],\n",
+            "  \"attention_dropout\": 0.1,\n",
+            "  \"dim\": 768,\n",
+            "  \"dropout\": 0.1,\n",
+            "  \"hidden_dim\": 3072,\n",
+            "  \"id2label\": {\n",
+            "    \"0\": \"Bug\",\n",
+            "    \"1\": \"Dark\",\n",
+            "    \"2\": \"Dragon\",\n",
+            "    \"3\": \"Electric\",\n",
+            "    \"4\": \"Fairy\",\n",
+            "    \"5\": \"Fighting\",\n",
+            "    \"6\": \"Fire\",\n",
+            "    \"7\": \"Flying\",\n",
+            "    \"8\": \"Ghost\",\n",
+            "    \"9\": \"Grass\",\n",
+            "    \"10\": \"Ground\",\n",
+            "    \"11\": \"Ice\",\n",
+            "    \"12\": \"Normal\",\n",
+            "    \"13\": \"Poison\",\n",
+            "    \"14\": \"Psychic\",\n",
+            "    \"15\": \"Rock\",\n",
+            "    \"16\": \"Steel\",\n",
+            "    \"17\": \"Water\"\n",
+            "  },\n",
+            "  \"initializer_range\": 0.02,\n",
+            "  \"label2id\": {\n",
+            "    \"Bug\": 0,\n",
+            "    \"Dark\": 1,\n",
+            "    \"Dragon\": 2,\n",
+            "    \"Electric\": 3,\n",
+            "    \"Fairy\": 4,\n",
+            "    \"Fighting\": 5,\n",
+            "    \"Fire\": 6,\n",
+            "    \"Flying\": 7,\n",
+            "    \"Ghost\": 8,\n",
+            "    \"Grass\": 9,\n",
+            "    \"Ground\": 10,\n",
+            "    \"Ice\": 11,\n",
+            "    \"Normal\": 12,\n",
+            "    \"Poison\": 13,\n",
+            "    \"Psychic\": 14,\n",
+            "    \"Rock\": 15,\n",
+            "    \"Steel\": 16,\n",
+            "    \"Water\": 17\n",
+            "  },\n",
+            "  \"max_position_embeddings\": 512,\n",
+            "  \"model_type\": \"distilbert\",\n",
+            "  \"n_heads\": 12,\n",
+            "  \"n_layers\": 6,\n",
+            "  \"pad_token_id\": 0,\n",
+            "  \"problem_type\": \"single_label_classification\",\n",
+            "  \"qa_dropout\": 0.1,\n",
+            "  \"seq_classif_dropout\": 0.2,\n",
+            "  \"sinusoidal_pos_embds\": false,\n",
+            "  \"tie_weights_\": true,\n",
+            "  \"torch_dtype\": \"float32\",\n",
+            "  \"transformers_version\": \"4.19.2\",\n",
+            "  \"vocab_size\": 30522\n",
+            "}\n",
+            "\n",
+            "loading weights file ./pytorch_model.bin\n",
+            "All model checkpoint weights were used when initializing DistilBertForSequenceClassification.\n",
+            "\n",
+            "All the weights of DistilBertForSequenceClassification were initialized from the model checkpoint at ..\n",
+            "If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForSequenceClassification for predictions without further training.\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from transformers import pipeline\n",
+        "\n",
+        "classifier = pipeline(task=\"text-classification\", tokenizer=tokenizer, model=model2.to('cpu'))"
+      ],
+      "metadata": {
+        "id": "yP--Matd4WCe"
+      },
+      "execution_count": 22,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "classifier('This pokemon climbs buildings at night.')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "qVAlioYj6jBO",
+        "outputId": "c820c497-da3e-4e1e-c1d0-ef2494bffb03"
+      },
+      "execution_count": 41,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "[{'label': 'Bug', 'score': 0.17771221697330475}]"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 41
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "classifier('This pokemon climbs buildings at night. They frequent midnight pool parties')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "3AJcXRR54Yk4",
+        "outputId": "5bd7cdd1-fd34-483f-d30c-560d54a28493"
+      },
+      "execution_count": 36,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "[{'label': 'Water', 'score': 0.4050225019454956}]"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 36
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "classifier('This pokemon climbs buildings at night. They frequent midnight garden parties')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "kJ54PeHz4pbO",
+        "outputId": "3e1571bd-6a0c-4800-ab4d-60835cb4f3a5"
+      },
+      "execution_count": 37,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "[{'label': 'Grass', 'score': 0.38808730244636536}]"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 37
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "classifier('This pokemon climbs buildings at night. They frequent midnight flame-throwing parties')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "GXVKzk-N8_So",
+        "outputId": "61e50b07-f3a5-4ff9-c152-1ea2d846dcfe"
+      },
+      "execution_count": 38,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "[{'label': 'Fire', 'score': 0.22531799972057343}]"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 38
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        ""
+      ],
+      "metadata": {
+        "id": "idRtKyjM9KIE"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "nlp.ipynb",
+      "provenance": []
+    },
+    "interpreter": {
+      "hash": "45e1260056979d5382785f386f12ee00f44622d9a136ee7663e9a61a67ca2a68"
+    },
+    "kernelspec": {
+      "display_name": "Python 3.10.0 ('projects-vBrzsZbN-py3.10')",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.0"
+    },
+    "orig_nbformat": 4,
+    "accelerator": "GPU",
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "e397276a3494413a8537fa5ba48f7015": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_3681f628eb844524bd4a121dd313d2ef",
+              "IPY_MODEL_4699b0136d4d449ca4accf9bc90fc45a",
+              "IPY_MODEL_0435b641146f45488248e042ff0d4f31"
+            ],
+            "layout": "IPY_MODEL_8233bb294c7549b6a8ae5e140e8ca5b6"
+          }
+        },
+        "3681f628eb844524bd4a121dd313d2ef": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_608d0d01fe4d4a14877920e51b10233a",
+            "placeholder": "​",
+            "style": "IPY_MODEL_246696956b774368ad7572db606a2414",
+            "value": "100%"
+          }
+        },
+        "4699b0136d4d449ca4accf9bc90fc45a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_cd1d0bd35b464f74aff7c7421de03fc9",
+            "max": 1,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_eac0a76ba4cf4b61ae82e9aa2f770dce",
+            "value": 1
+          }
+        },
+        "0435b641146f45488248e042ff0d4f31": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_68e198c9120f4451bd9b9f033a9d4bae",
+            "placeholder": "​",
+            "style": "IPY_MODEL_bd8b8f2e781e4de1a477c8af4c450d1f",
+            "value": " 1/1 [00:00&lt;00:00,  2.22ba/s]"
+          }
+        },
+        "8233bb294c7549b6a8ae5e140e8ca5b6": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "608d0d01fe4d4a14877920e51b10233a": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "246696956b774368ad7572db606a2414": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "cd1d0bd35b464f74aff7c7421de03fc9": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "eac0a76ba4cf4b61ae82e9aa2f770dce": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "68e198c9120f4451bd9b9f033a9d4bae": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "bd8b8f2e781e4de1a477c8af4c450d1f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "b2eba59325544c0790fea0b5a08916b1": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_deeb1a85968d4619a7928712c09168d2",
+              "IPY_MODEL_39d408f797bf46aa8a9617a68c8ba913",
+              "IPY_MODEL_e643c45c934d414d9b46abdc64eccbbc"
+            ],
+            "layout": "IPY_MODEL_f7972177ace147fbb45918eebe106915"
+          }
+        },
+        "deeb1a85968d4619a7928712c09168d2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_aa8e9ad5e1ce4b84b49c194c61f90820",
+            "placeholder": "​",
+            "style": "IPY_MODEL_93c35117cea74e5b9de2b871168b7095",
+            "value": "100%"
+          }
+        },
+        "39d408f797bf46aa8a9617a68c8ba913": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_09542f17885d4ce1b6fb5e8682beb6de",
+            "max": 1,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_e1fce70e1a67446982a09c9d4948b48d",
+            "value": 1
+          }
+        },
+        "e643c45c934d414d9b46abdc64eccbbc": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_7712b1987ee143fe9ceb6ef13bded85d",
+            "placeholder": "​",
+            "style": "IPY_MODEL_51f19ba89e5f4d208564f03ff6f2b0da",
+            "value": " 1/1 [00:00&lt;00:00,  3.74ba/s]"
+          }
+        },
+        "f7972177ace147fbb45918eebe106915": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "aa8e9ad5e1ce4b84b49c194c61f90820": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "93c35117cea74e5b9de2b871168b7095": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "09542f17885d4ce1b6fb5e8682beb6de": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e1fce70e1a67446982a09c9d4948b48d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "7712b1987ee143fe9ceb6ef13bded85d": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "51f19ba89e5f4d208564f03ff6f2b0da": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        }
+      }
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

pkmn-classifier/web_scrape.ipynb

@@ -0,0 +1,476 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import bs4 as BeautifulSoup\n",
+    "import pandas as pd\n",
+    "import requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "list_of_urls = [\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_I_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_II_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_III_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_IV_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_V_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_VI_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_VII_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_VIII_Pok%C3%A9mon\",\n",
+    "    \"https://en.wikipedia.org/wiki/List_of_generation_IX_Pok%C3%A9mon\",\n",
+    "]\n",
+    "\n",
+    "generations = [\"I\", \"II\", \"III\", \"IV\", \"V\", \"VI\", \"VII\", \"VIII\", \"IX\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "I\n",
+      "II\n",
+      "III\n",
+      "IV\n",
+      "V\n",
+      "VI\n",
+      "VII\n",
+      "VIII\n",
+      "IX\n"
+     ]
+    }
+   ],
+   "source": [
+    "import time\n",
+    "\n",
+    "generation_list = []\n",
+    "for generation, url in zip(generations, list_of_urls):\n",
+    "    print(generation)\n",
+    "    generation_list.append(pd.read_html(url))\n",
+    "    time.sleep(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "biggest_table_from_each_url = []\n",
+    "\n",
+    "for tables in generation_list[:-1]:\n",
+    "    largest_in_generation = tables[0]\n",
+    "    for table in tables:\n",
+    "        if table.size > largest_in_generation.size:\n",
+    "            largest_in_generation = table\n",
+    "\n",
+    "    biggest_table_from_each_url.append(largest_in_generation)\n",
+    "\n",
+    "biggest_table_from_each_url.append(\n",
+    "    generation_list[-1][1]\n",
+    ")  # hacky, gen IX has fewer new pokemon than the table at the bottom of the wikipedia page."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead tr th {\n",
+       "        text-align: left;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr>\n",
+       "      <th></th>\n",
+       "      <th colspan=\"2\" halign=\"left\">Name</th>\n",
+       "      <th>National Pokédexnumber</th>\n",
+       "      <th colspan=\"2\" halign=\"left\">Type(s)</th>\n",
+       "      <th>Evolves from</th>\n",
+       "      <th>Evolves into</th>\n",
+       "      <th>Notes</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th></th>\n",
+       "      <th>English</th>\n",
+       "      <th>Japanese</th>\n",
+       "      <th>National Pokédexnumber</th>\n",
+       "      <th>Primary</th>\n",
+       "      <th>Secondary</th>\n",
+       "      <th>Evolves from</th>\n",
+       "      <th>Evolves into</th>\n",
+       "      <th>Notes</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Sprigatito</td>\n",
+       "      <td>Nyaoha (ニャオハ)</td>\n",
+       "      <td>TBA</td>\n",
+       "      <td>Grass</td>\n",
+       "      <td>Grass</td>\n",
+       "      <td>Beginning of evolution</td>\n",
+       "      <td>Unknown</td>\n",
+       "      <td>Sprigatito is a cat-like Pokémon and the Grass...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Fuecoco</td>\n",
+       "      <td>Hogēta (ホゲータ)</td>\n",
+       "      <td>TBA</td>\n",
+       "      <td>Fire</td>\n",
+       "      <td>Fire</td>\n",
+       "      <td>Beginning of evolution</td>\n",
+       "      <td>Unknown</td>\n",
+       "      <td>Fuecoco is a crocodile-like Pokémon and the Fi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Quaxly</td>\n",
+       "      <td>Kuwassu (クワッス)</td>\n",
+       "      <td>TBA</td>\n",
+       "      <td>Water</td>\n",
+       "      <td>Water</td>\n",
+       "      <td>Beginning of evolution</td>\n",
+       "      <td>Unknown</td>\n",
+       "      <td>Quaxly is a duck-like Pokémon and the Water-ty...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         Name                 National Pokédexnumber Type(s)            \\\n",
+       "      English        Japanese National Pokédexnumber Primary Secondary   \n",
+       "0  Sprigatito   Nyaoha (ニャオハ)                    TBA   Grass     Grass   \n",
+       "1     Fuecoco   Hogēta (ホゲータ)                    TBA    Fire      Fire   \n",
+       "2      Quaxly  Kuwassu (クワッス)                    TBA   Water     Water   \n",
+       "\n",
+       "             Evolves from Evolves into  \\\n",
+       "             Evolves from Evolves into   \n",
+       "0  Beginning of evolution      Unknown   \n",
+       "1  Beginning of evolution      Unknown   \n",
+       "2  Beginning of evolution      Unknown   \n",
+       "\n",
+       "                                               Notes  \n",
+       "                                               Notes  \n",
+       "0  Sprigatito is a cat-like Pokémon and the Grass...  \n",
+       "1  Fuecoco is a crocodile-like Pokémon and the Fi...  \n",
+       "2  Quaxly is a duck-like Pokémon and the Water-ty...  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "biggest_table_from_each_url[-1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "9"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(biggest_table_from_each_url)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[(152, 8),\n",
+       " (100, 8),\n",
+       " (138, 8),\n",
+       " (117, 8),\n",
+       " (158, 8),\n",
+       " (73, 8),\n",
+       " (94, 8),\n",
+       " (101, 8),\n",
+       " (3, 8)]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "[i.shape for i in biggest_table_from_each_url]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_pkmn = pd.concat(biggest_table_from_each_url)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "MultiIndex([(                  'Name',                'English'),\n",
+       "            (                  'Name',               'Japanese'),\n",
+       "            ('National Pokédexnumber', 'National Pokédexnumber'),\n",
+       "            (               'Type(s)',                'Primary'),\n",
+       "            (               'Type(s)',              'Secondary'),\n",
+       "            (          'Evolves from',           'Evolves from'),\n",
+       "            (          'Evolves into',           'Evolves into'),\n",
+       "            (                 'Notes',                  'Notes')],\n",
+       "           )"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_pkmn.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_pkmn = (\n",
+    "    all_pkmn.T.reset_index()\n",
+    "    .drop(columns=[\"level_0\"])\n",
+    "    .set_index(\"level_1\")\n",
+    "    .T.rename(\n",
+    "        columns={\n",
+    "            \"English\": \"english_name\",\n",
+    "            \"Japanese\": \"japanese_name\",\n",
+    "            \"Primary\": \"primary_type\",\n",
+    "            \"Secondary\": \"secondary_type\",\n",
+    "        }\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = all_pkmn[[\"english_name\", \"primary_type\", \"secondary_type\", \"Notes\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def select_text_before_first_bracket(string):\n",
+    "    return string.split(\"[\")[0]\n",
+    "\n",
+    "\n",
+    "all_pkmn[\"primary_type\"] = all_pkmn[\"primary_type\"].apply(select_text_before_first_bracket)\n",
+    "all_pkmn[\"secondary_type\"] = all_pkmn[\"secondary_type\"].apply(select_text_before_first_bracket)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Water       126\n",
+       "Normal      111\n",
+       "Grass        89\n",
+       "Bug          80\n",
+       "Psychic      64\n",
+       "Fire         61\n",
+       "Electric     55\n",
+       "Rock         50\n",
+       "Fighting     38\n",
+       "Dark         37\n",
+       "Ground       36\n",
+       "Poison       36\n",
+       "Ghost        32\n",
+       "Dragon       31\n",
+       "Steel        30\n",
+       "Ice          29\n",
+       "Fairy        23\n",
+       "Flying        7\n",
+       "Bird          1\n",
+       "Name: primary_type, dtype: int64"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Wikipedia adds a number of footnotes to the types, this output is much cleaner.\n",
+    "all_pkmn[\"primary_type\"].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th>level_1</th>\n",
+       "      <th>english_name</th>\n",
+       "      <th>japanese_name</th>\n",
+       "      <th>National Pokédexnumber</th>\n",
+       "      <th>primary_type</th>\n",
+       "      <th>secondary_type</th>\n",
+       "      <th>Evolves from</th>\n",
+       "      <th>Evolves into</th>\n",
+       "      <th>Notes</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>151</th>\n",
+       "      <td>MissingNo.</td>\n",
+       "      <td>Ketsuban (けつばん)</td>\n",
+       "      <td>None[nb 8]</td>\n",
+       "      <td>Bird</td>\n",
+       "      <td>Normal</td>\n",
+       "      <td>No evolution</td>\n",
+       "      <td>No evolution</td>\n",
+       "      <td>An error handler species, \"Missing Number\" was...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "level_1 english_name    japanese_name National Pokédexnumber primary_type  \\\n",
+       "151       MissingNo.  Ketsuban (けつばん)             None[nb 8]         Bird   \n",
+       "\n",
+       "level_1 secondary_type  Evolves from  Evolves into  \\\n",
+       "151             Normal  No evolution  No evolution   \n",
+       "\n",
+       "level_1                                              Notes  \n",
+       "151      An error handler species, \"Missing Number\" was...  "
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Bird Type? Is that like flying or something?\n",
+    "all_pkmn[all_pkmn.primary_type == \"Bird\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_pkmn.to_csv(\"pokemon.csv\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "45e1260056979d5382785f386f12ee00f44622d9a136ee7663e9a61a67ca2a68"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.10.0 ('projects-vBrzsZbN-py3.10')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

pyproject.toml

@@ -0,0 +1,27 @@
+[tool.poetry]
+name = "projects"
+version = "0.1.0"
+description = ""
+authors = ["Martin Coombes <mrcjdc@protonmail.com>"]
+
+[tool.poetry.dependencies]
+python = ">=3.10, <3.11"
+beautifulsoup4 = "^4.11.1"
+jupyterlab = "^3.4.2"
+lxml = "^4.8.0"
+numpy = "^1.22.4"
+scikit-learn = "^1.1.1"
+scipy = "^1.8.1"
+seaborn = "^0.11.2"
+streamlit = "^1.9.2"
+transformers = "^4.19.2"
+torch = "^1.11.0"
+
+[tool.poetry.dev-dependencies]
+black = "^22.3.0"
+nbqa = "^1.3.1"
+pre-commit = "^2.19.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"