adding municipal data

preprocess.ipynb

@@ -0,0 +1,100 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e5756d2-382b-49e9-93b5-2ecf6d0eb812",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import duckdb\n",
+    "\n",
+    "con = duckdb.connect()\n",
+    "\n",
+    "con.execute(\"SET s3_region='us-west-2';\")\n",
+    "con.execute(\"LOAD spatial;\")\n",
+    "con.execute(\"LOAD httpfs;\")\n",
+    "\n",
+    "query = \"\"\"\n",
+    "    COPY (\n",
+    "        SELECT * \n",
+    "        FROM read_parquet('s3://overturemaps-us-west-2/release/2024-09-18.0/theme=divisions/*/*')\n",
+    "        WHERE country = 'US' AND subtype IN ('locality', 'neighborhood')\n",
+    "    ) TO 'us_localities_neighborhoods.parquet' (FORMAT 'parquet');\n",
+    "\"\"\"\n",
+    "con.execute(query)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25f62dd7-5539-438b-8f0a-1d85c9bc78ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ibis\n",
+    "from ibis import _\n",
+    "\n",
+    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
+    "\n",
+    "df = (conn\n",
+    "      .read_parquet(\"us_localities_neighborhoods.parquet\")\n",
+    "      .cast({\"geometry\": \"geometry\"})\n",
+    "      .filter(_[\"type\"] == \"division\")\n",
+    "      .filter(_[\"subtype\"] == \"locality\")\n",
+    "      .mutate(name = _.names[\"primary\"])\n",
+    "      .mutate(state_id = _.region.replace(\"US-\", \"\")) \n",
+    "      .mutate(county = _.hierarchies[0][2]['name'] )\n",
+    "      .mutate(key_long = _.name + ibis.literal('-') + _.county + ibis.literal('-') + _.state_id)\n",
+    "      .select(\"key_long\",\"name\", \"county\",\"state_id\" ,\"geometry\")\n",
+    "     )\n",
+    "\n",
+    "\n",
+    "## Dropping rows with same locality and state, with differing counties \n",
+    "county_count = (\n",
+    "    df.group_by([\"name\", \"state_id\"])\n",
+    "    .aggregate(county_count=_.county.nunique())  # Count unique counties for each group\n",
+    ") \n",
+    "valid_names = county_count.filter(county_count.county_count == 1).select(\"name\", \"state_id\")\n",
+    "df_filtered = df.join(valid_names, [\"name\", \"state_id\"], how=\"inner\")\n",
+    "\n",
+    "\n",
+    "# if two records have the same name but different geometries, only keep the first one.\n",
+    "df_first = (\n",
+    "    df_filtered.group_by(\"key_long\")\n",
+    "    .aggregate(\n",
+    "        name=df_filtered.name.first(),\n",
+    "        county=df_filtered.county.first(),\n",
+    "        state_id=df_filtered.state_id.first(),\n",
+    "        geometry=df_filtered.geometry.first()\n",
+    "    )\n",
+    "\n",
+    ")\n",
+    "\n",
+    "df_first.execute().to_parquet(\"us_localities.parquet\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -157,3 +157,5 @@ whiteboxgui==2.3.0
 widgetsnbextension==4.0.13
 xyzservices==2024.9.0
 yarl==1.9.7
+overturemaps
+tippecanoe

static-maps.ipynb

@@ -18,27 +18,79 @@
     "states = conn.read_parquet(state_boundaries).rename(state_id = \"STUSPS\", state = \"NAME\")\n",
     "county = conn.read_parquet(county_boundaries).rename(county = \"NAMELSAD\", state = \"STATE_NAME\")\n",
     "\n",
-    "votes = conn.read_csv(\"landvote.csv\")\n",
-    "votes.count().execute()\n",
-    "\n",
-    "vote = (votes\n",
-    "       .filter(_[\"Jurisdiction Type\"] == \"County\")\n",
-    "       .rename(county = \"Jurisdiction Name\", state_id = \"State\")\n",
-    "       .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
-    "       .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
-    "       .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
-    "       .mutate(log_amount=_.amount_n.log())\n",
-    "       .mutate(year=_['Date'].year().cast('int32'))\n",
-    "       .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "       )\n",
-    "df = (county\n",
-    "      .join(states.select(\"state\", \"state_id\"), \"state\")\n",
-    "      .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
-    "      .select('key', 'geometry')\n",
-    "      .right_join(vote, \"key\")\n",
-    "      .drop('key_right')\n",
-    "     )\n",
-    "\n"
+    "localities_boundaries = \"us_localities.parquet\"\n",
+    "locality = conn.read_parquet(localities_boundaries)\n",
+    "\n",
+    "\n",
+    "votes = conn.read_csv(\"landvote.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ba4d8915-cde3-4ef9-ad8c-7759ed2c8a13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vote_county = (votes\n",
+    "               .filter(_[\"Jurisdiction Type\"] == \"County\")\n",
+    "               .rename(county = \"Jurisdiction Name\", state_id = \"State\")\n",
+    "               .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
+    "               .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
+    "               .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
+    "               .mutate(log_amount=_.amount_n.log())\n",
+    "               .mutate(year=_['Date'].year().cast('int32'))\n",
+    "               .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "               )\n",
+    "df_county = (county\n",
+    "            .join(states.select(\"state\", \"state_id\"), \"state\")\n",
+    "            .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
+    "            .select('key', 'geometry')\n",
+    "            .right_join(vote_county, \"key\")\n",
+    "            .drop('key_right')\n",
+    "            .mutate(jurisdiction = ibis.literal(\"County\"))\n",
+    "            .cast({\"geometry\": \"geometry\"})\n",
+    "             )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0cce23c9-245c-4c28-9523-0231eb5acc17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vote_local = (votes\n",
+    "                .filter(_[\"Jurisdiction Type\"] == \"Municipal\")\n",
+    "                .rename(city = \"Jurisdiction Name\", state_id = \"State\")\n",
+    "                .mutate(key = _.city + ibis.literal('-') + _.state_id)\n",
+    "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
+    "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
+    "                .mutate(log_amount=_.amount_n.log())\n",
+    "                .mutate(year=_['Date'].year().cast('int32'))\n",
+    "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "                )\n",
+    "\n",
+    "df_local = (locality\n",
+    "            .mutate(key = _.name + ibis.literal('-') + _.state_id)\n",
+    "            .select('key', 'geometry')\n",
+    "            .right_join(vote_local, \"key\")\n",
+    "            .drop('key_right')\n",
+    "            .mutate(jurisdiction = ibis.literal(\"Municipal\"))\n",
+    "            .cast({\"geometry\": \"geometry\"})\n",
+    "            \n",
+    "            )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1e81807-8ce3-44bf-9a1c-8563fa33817c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df_county.union(df_local)\n",
+    "df.execute()"
    ]
   },
   {
@@ -54,7 +106,8 @@
     "import streamlit as st\n",
     "\n",
     "login(st.secrets[\"HF_TOKEN\"])\n",
-    "api = HfApi(add_to_git_credential=False)\n",
+    "# api = HfApi(add_to_git_credential=False)\n",
+    "api = HfApi()\n",
     "\n",
     "def hf_upload(file, repo_id):\n",
     "    info = api.upload_file(\n",
@@ -163,6 +216,14 @@
     "#m.add_gdf(gdf, \"fill-extrusion\", paint = paint)\n",
     "#m"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e521f00-1b04-4016-9a6a-71a12e846dd3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -181,7 +242,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.10"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,