adding states

get_party.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 16,
    "id": "d179ded1-6235-47ed-bbfb-6d72468188d5",
    "metadata": {},
    "outputs": [],
@@ -30,31 +30,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "ab644102-c725-4cf4-915c-8550a0a74c32",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "       year                    key current_party previous_party\n",
-      "0      2004      AUTAUGA COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
-      "1      2004    COVINGTON COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
-      "2      2004       ELMORE COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
-      "3      2004        HENRY COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
-      "4      2004      LOWNDES COUNTY-AL      DEMOCRAT       DEMOCRAT\n",
-      "...     ...                    ...           ...            ...\n",
-      "14904  2020          ELK COUNTY-PA    REPUBLICAN     REPUBLICAN\n",
-      "14905  2020       LOVING COUNTY-TX    REPUBLICAN     REPUBLICAN\n",
-      "14906  2020    RED RIVER COUNTY-TX    REPUBLICAN     REPUBLICAN\n",
-      "14907  2020      WHATCOM COUNTY-WA      DEMOCRAT       DEMOCRAT\n",
-      "14908  2020  DISTRICT 12 COUNTY-AK    REPUBLICAN     REPUBLICAN\n",
-      "\n",
-      "[14909 rows x 4 columns]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "filtered = county.filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))\n",
     "\n",
@@ -102,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "id": "ce0d80bf-3b78-4aa9-8048-5cc0dbf970d9",
    "metadata": {},
    "outputs": [],
@@ -118,7 +97,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 30,
    "id": "87bef5e2-a40a-4aff-aa27-e7d49ec68aac",
    "metadata": {},
    "outputs": [
@@ -177,7 +156,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "b086e76c-4285-4036-8033-e4e45cb6966b",
    "metadata": {},
    "outputs": [],
@@ -265,240 +244,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "b3429fea-7c0d-4838-bcbb-6552079dc3b6",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>key</th>\n",
-       "      <th>geometry</th>\n",
-       "      <th>Status</th>\n",
-       "      <th>yes</th>\n",
-       "      <th>year</th>\n",
-       "      <th>amount</th>\n",
-       "      <th>log_amount</th>\n",
-       "      <th>jurisdiction</th>\n",
-       "      <th>current_party</th>\n",
-       "      <th>previous_party</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>San Luis Obispo-CA</td>\n",
-       "      <td>POLYGON ((-120.58962 35.28275, -120.59096 35.2...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>58.23%</td>\n",
-       "      <td>2020</td>\n",
-       "      <td>$21,600,000</td>\n",
-       "      <td>16.8882</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>Fruita-CO</td>\n",
-       "      <td>POLYGON ((-108.65899 39.15887, -108.66033 39.1...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>60.73%</td>\n",
-       "      <td>2020</td>\n",
-       "      <td>$1,300,000</td>\n",
-       "      <td>14.0779</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>Gig Harbor-WA</td>\n",
-       "      <td>POLYGON ((-122.51787 47.32597, -122.51921 47.3...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>62.05%</td>\n",
-       "      <td>2004</td>\n",
-       "      <td>$3,500,000</td>\n",
-       "      <td>15.0683</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>Jupiter-FL</td>\n",
-       "      <td>POLYGON ((-80.02421 26.93422, -80.02555 26.920...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>70.84%</td>\n",
-       "      <td>2004</td>\n",
-       "      <td>$17,000,000</td>\n",
-       "      <td>16.6487</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>Bozeman-MT</td>\n",
-       "      <td>POLYGON ((-110.97405 45.67943, -110.97539 45.6...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>73.39%</td>\n",
-       "      <td>2012</td>\n",
-       "      <td>$7,500,000</td>\n",
-       "      <td>15.8304</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>268</th>\n",
-       "      <td>Chandler-AZ</td>\n",
-       "      <td>POLYGON ((-111.77118 33.3062, -111.77253 33.29...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>75.16%</td>\n",
-       "      <td>2004</td>\n",
-       "      <td>$17,350,000</td>\n",
-       "      <td>16.6691</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>269</th>\n",
-       "      <td>Medfield-MA</td>\n",
-       "      <td>POLYGON ((-71.23546 42.18656, -71.23681 42.172...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>84.17%</td>\n",
-       "      <td>2008</td>\n",
-       "      <td>$1,550,000</td>\n",
-       "      <td>14.2538</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>270</th>\n",
-       "      <td>Somerville-MA</td>\n",
-       "      <td>POLYGON ((-71.0295 42.3876, -71.03084 42.37394...</td>\n",
-       "      <td>Pass</td>\n",
-       "      <td>76.07%</td>\n",
-       "      <td>2012</td>\n",
-       "      <td>$9,659,333</td>\n",
-       "      <td>16.0834</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>271</th>\n",
-       "      <td>Dryden-NY</td>\n",
-       "      <td>POLYGON ((-76.22716 42.49091, -76.2285 42.4772...</td>\n",
-       "      <td>Fail</td>\n",
-       "      <td>44.07%</td>\n",
-       "      <td>2016</td>\n",
-       "      <td>$71,800</td>\n",
-       "      <td>11.1816</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "      <td>DEMOCRAT</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>272</th>\n",
-       "      <td>Grafton-WI</td>\n",
-       "      <td>POLYGON ((-87.88178 43.31952, -87.88313 43.305...</td>\n",
-       "      <td>Fail</td>\n",
-       "      <td>28.78%</td>\n",
-       "      <td>2004</td>\n",
-       "      <td>$1,120,000</td>\n",
-       "      <td>13.9288</td>\n",
-       "      <td>Municipal</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "      <td>REPUBLICAN</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>273 rows × 10 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                    key                                           geometry  \\\n",
-       "0    San Luis Obispo-CA  POLYGON ((-120.58962 35.28275, -120.59096 35.2...   \n",
-       "1             Fruita-CO  POLYGON ((-108.65899 39.15887, -108.66033 39.1...   \n",
-       "2         Gig Harbor-WA  POLYGON ((-122.51787 47.32597, -122.51921 47.3...   \n",
-       "3            Jupiter-FL  POLYGON ((-80.02421 26.93422, -80.02555 26.920...   \n",
-       "4            Bozeman-MT  POLYGON ((-110.97405 45.67943, -110.97539 45.6...   \n",
-       "..                  ...                                                ...   \n",
-       "268         Chandler-AZ  POLYGON ((-111.77118 33.3062, -111.77253 33.29...   \n",
-       "269         Medfield-MA  POLYGON ((-71.23546 42.18656, -71.23681 42.172...   \n",
-       "270       Somerville-MA  POLYGON ((-71.0295 42.3876, -71.03084 42.37394...   \n",
-       "271           Dryden-NY  POLYGON ((-76.22716 42.49091, -76.2285 42.4772...   \n",
-       "272          Grafton-WI  POLYGON ((-87.88178 43.31952, -87.88313 43.305...   \n",
-       "\n",
-       "    Status     yes  year       amount  log_amount jurisdiction current_party  \\\n",
-       "0     Pass  58.23%  2020  $21,600,000     16.8882    Municipal      DEMOCRAT   \n",
-       "1     Pass  60.73%  2020   $1,300,000     14.0779    Municipal    REPUBLICAN   \n",
-       "2     Pass  62.05%  2004   $3,500,000     15.0683    Municipal      DEMOCRAT   \n",
-       "3     Pass  70.84%  2004  $17,000,000     16.6487    Municipal      DEMOCRAT   \n",
-       "4     Pass  73.39%  2012   $7,500,000     15.8304    Municipal    REPUBLICAN   \n",
-       "..     ...     ...   ...          ...         ...          ...           ...   \n",
-       "268   Pass  75.16%  2004  $17,350,000     16.6691    Municipal    REPUBLICAN   \n",
-       "269   Pass  84.17%  2008   $1,550,000     14.2538    Municipal      DEMOCRAT   \n",
-       "270   Pass  76.07%  2012   $9,659,333     16.0834    Municipal      DEMOCRAT   \n",
-       "271   Fail  44.07%  2016      $71,800     11.1816    Municipal      DEMOCRAT   \n",
-       "272   Fail  28.78%  2004   $1,120,000     13.9288    Municipal    REPUBLICAN   \n",
-       "\n",
-       "    previous_party  \n",
-       "0         DEMOCRAT  \n",
-       "1       REPUBLICAN  \n",
-       "2         DEMOCRAT  \n",
-       "3         DEMOCRAT  \n",
-       "4         DEMOCRAT  \n",
-       "..             ...  \n",
-       "268     REPUBLICAN  \n",
-       "269       DEMOCRAT  \n",
-       "270       DEMOCRAT  \n",
-       "271       DEMOCRAT  \n",
-       "272     REPUBLICAN  \n",
-       "\n",
-       "[273 rows x 10 columns]"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "\n",
     "localities_boundaries = \"us_localities.parquet\"\n",
@@ -556,11 +305,172 @@
     "# hf_upload(\"municipal_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "06e24a7e-5f7f-42bc-b515-43082016d496",
+   "metadata": {},
+   "source": [
+    "# Get States"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 28,
+   "id": "217170ca-b732-4875-b4cb-f8a8cd2fc405",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "states = (conn\n",
+    "          .read_csv(\"1976-2020-president.csv\")\n",
+    "          .filter(_. year >=2000)\n",
+    "         )\n",
+    "# states.execute()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
    "id": "322b9a85-bdf9-45f9-9b19-695cc1b996e8",
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "     year key current_party previous_party\n",
+      "0    2004  HI      DEMOCRAT       DEMOCRAT\n",
+      "1    2004  ME      DEMOCRAT       DEMOCRAT\n",
+      "2    2004  NJ      DEMOCRAT       DEMOCRAT\n",
+      "3    2004  NM    REPUBLICAN       DEMOCRAT\n",
+      "4    2004  ND    REPUBLICAN     REPUBLICAN\n",
+      "..    ...  ..           ...            ...\n",
+      "250  2020  VT      DEMOCRAT       DEMOCRAT\n",
+      "251  2020  AL    REPUBLICAN     REPUBLICAN\n",
+      "252  2020  IA    REPUBLICAN     REPUBLICAN\n",
+      "253  2020  SD    REPUBLICAN     REPUBLICAN\n",
+      "254  2020  GA      DEMOCRAT     REPUBLICAN\n",
+      "\n",
+      "[255 rows x 4 columns]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# filtered = county.filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))\n",
+    "\n",
+    "# Find the winning party for each year, state, and county\n",
+    "most_votes= (\n",
+    "    states\n",
+    "    .group_by(['year', 'state_po', 'party_simplified'])\n",
+    "    .aggregate(winning_votes=_.candidatevotes.sum())\n",
+    ")\n",
+    "\n",
+    "# For each year, state, and county, select the party with the highest total votes\n",
+    "winning_party = (\n",
+    "    most_votes\n",
+    "    .group_by('year', 'state_po')\n",
+    "    .aggregate(\n",
+    "        max_votes=_.winning_votes.max(),  # Max votes in this group\n",
+    "    )\n",
+    "    .join(\n",
+    "        most_votes,\n",
+    "        [\"year\",\"state_po\",most_votes['winning_votes'] == _.max_votes],\n",
+    "        how='inner'\n",
+    "    )\n",
+    "    .select(\"year\",\"state_po\",most_votes['party_simplified'].name('current_party')\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "# Self-join to get the previous year's winning party\n",
+    "previous_year = winning_party.view()\n",
+    "\n",
+    "joined = (\n",
+    "    winning_party\n",
+    "    .join(\n",
+    "        previous_year, [\"state_po\",winning_party['year'] == previous_year['year'] + 4],\n",
+    "        how='left'\n",
+    "    )\n",
+    "    .rename(key = \"state_po\")\n",
+    "    # .mutate(key = _.county_name + ibis.literal(\" COUNTY-\") + _.state_id)\n",
+    "    .select(\"year\",\"key\",\"current_party\",previous_year['current_party'].name('previous_party'))\n",
+    ")\n",
+    "\n",
+    "state_parties = joined.filter(_.year >2000).order_by(\"year\")\n",
+    "\n",
+    "print(state_parties.execute())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "2c03920e-76da-4034-8eaf-1e80a56f5b0d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3574af6546ff4cd1949e27c63cd15cd7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "states_parties.parquet:   0%|          | 0.00/2.36M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "For layer 0, using name \"states_parties\"\n",
+      "12 features, 833591 bytes of geometry and attributes, 542 bytes of string pool, 0 bytes of vertices, 0 bytes of nodes\n",
+      "Choosing a maxzoom of -z0 for features typically 7514540 feet (2290432 meters) apart, and at least 2073685 feet (632060 meters) apart\n",
+      "Choosing a maxzoom of -z10 for resolution of about 376 feet (114 meters) within features\n",
+      "  99.9%  10/271/383  \n",
+      "  100.0%  10/187/380  \r"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully generated PMTiles file: states_parties.pmtiles\n"
+     ]
+    }
+   ],
+   "source": [
+    "# state_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-state-territory.parquet\"\n",
+    "states = conn.read_parquet(\"vote_states.parquet\")\n",
+    "\n",
+    "df_states = (states\n",
+    "        .mutate(key = _.key.upper())\n",
+    "        # .filter(_.jurisdiction == \"Municipal\")\n",
+    "        .join(state_parties, [\"key\",\"year\"],how='inner'\n",
+    "        )\n",
+    "        .cast({\"geometry\": \"geometry\"})\n",
+    ")\n",
+    "\n",
+    "gdf_states = df_states.execute()\n",
+    "gdf_states = gdf_states.set_crs(\"EPSG:4326\")\n",
+    "gdf_states\n",
+    "\n",
+    "\n",
+    "gdf_states.to_parquet(\"states_parties.parquet\")\n",
+    "hf_upload(\"states_parties.parquet\", \"boettiger-lab/landvote\")\n",
+    "\n",
+    "gdf_states.to_file(\"states_parties.geojson\")\n",
+    "hf_upload(\"states_parties.geojson\", \"boettiger-lab/landvote\")\n",
+    "\n",
+    "generate_pmtiles(\"states_parties.geojson\", \"states_parties.pmtiles\")\n",
+    "hf_upload(\"states_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dcf5d049-68ae-4d73-be77-72d985c9ed1c",
+   "metadata": {},
    "outputs": [],
    "source": []
   }