and so it begins :seedling:

.github/workflows/deploy.yml

@@ -0,0 +1,20 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://cboettig:$HF_TOKEN@huggingface.co/spaces/boettiger-lab/redlining main

.gitignore

@@ -160,3 +160,10 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+
+*.parquet
+*.tiff
+*.tif
+
+

Dockerfile

@@ -0,0 +1,11 @@
+FROM  ghcr.io/boettiger-lab/k8s:latest
+WORKDIR /app
+
+COPY . .
+
+# huggingface uses port 7860 by default
+CMD streamlit run app.py \
+    --server.address 0.0.0.0 \
+    --server.port 7860 \
+    --server.headless true \
+    --server.fileWatcherType none

README.md

@@ -1,2 +1,19 @@
-# redlining-app
-Source code for streamlit app for exploring GBIF distribution & other biodiversity variables by redlined area
+---
+title: Streamlit Demo
+emoji: 🌍
+colorFrom: yellow
+colorTo: indigo
+sdk: docker
+pinned: false
+license: bsd-2-clause
+---
+
+For ESPM-157 students using <https://nature.datahub.berkeley.edu> servers: 
+To preview locally, use
+
+```
+make
+```
+
+and click the link provided
+

app.py

@@ -0,0 +1,176 @@
+import streamlit as st
+
+import ibis
+from ibis import _
+import pydeck as pdk
+from utilities import *
+import leafmap.maplibregl as leafmap
+import requests
+import geopandas as gpd
+
+st.set_page_config(page_title="Redlining & GBIF", layout="wide")
+st.title("Redlining & GBIF")
+
+con = ibis.duckdb.connect(extensions=['httpfs', 'spatial', 'h3'])
+set_secrets(con) # s3 credentials
+#set_aws_secrets(con)
+#set_source_secrets(con)
+
+distinct_taxa = "" # default
+
+col1, col2, col3, col4 = st.columns([1,3,3,3])
+
+# placed outside the form so that toggling this immediately updates the form options available
+with col1:
+    st.markdown("#### Start 👇")
+    area_source = st.radio("Area types", ["City", "All"])  
+    nunique = st.toggle("unique taxa only", False)
+
+
+# config with different default settings by area
+config = {
+    "City": {
+        "names": con.read_parquet("s3://public-gbif/app/city_names.parquet").select("name").execute(),
+        "index": 183,
+        "zoom": 11,
+        "vertical": 0.1,
+        "rank_index": 2,
+        "taxa": "Aves",
+    },
+     "All": {
+        "names": ["All"],
+        "index": 0,
+        "zoom": 9,
+        "vertical": 1.0,
+        "rank_index": 2,
+        "taxa": "Aves",
+    }
+}
+
+with st.form("my_form"):
+    
+    taxonomic_ranks = ["kingdom", "phylum", "class", "order", "family","genus", "species"]
+    default = config[area_source]
+
+    with col2:
+        ## Add additional layer toggles here, e.g. SVI?
+        st.markdown("####  🗺️ Select map layers")
+        gdf_name = st.selectbox("Area", default["names"], index=default["index"])
+        
+    with col3:
+        st.markdown("#### 🐦 Select taxonomic groups")
+        ## add support for multiple taxa!
+        rank = st.selectbox("Taxonomic Rank", options=taxonomic_ranks, index = default["rank_index"])
+        taxa = st.text_input("taxa", default["taxa"])
+        if nunique:
+            distinct_taxa = st.selectbox("Count only unique occurrences by:", options=taxonomic_ranks, index = default["rank_index"])
+
+    with col4: 
+        st.markdown('''
+        #### 🔎 Set spatial resolution
+        See [H3 cell size by zoom](https://h3geo.org/docs/core-library/restable/#cell-areas)
+        ''')
+        zoom = st.slider("H3 resolution", min_value=1, max_value=11, value = default["zoom"])
+        v_scale = st.number_input("vertical scale", min_value = 0.0, value = default["vertical"])
+
+    submitted = st.form_submit_button("Go")
+
+@st.cache_data
+def compute_hexes(_gdf, gdf_name, rank, taxa, zoom, distinct_taxa = ""):
+
+    # FIXME check if dest exists in cache
+    dest = unique_path(gdf_name, rank, taxa, zoom, distinct_taxa)
+    bucket = "public-gbif"
+    url = base_url + f"/{bucket}/" + dest
+
+    response = requests.head(url)
+    if response.status_code != 404:
+        return url
+
+    sel = con.read_parquet("s3://public-gbif/app/redlined_cities_gbif.parquet")
+
+    sel = (sel
+           .rename(hex = "h" + str(zoom)) # h3 == 41,150 hexes.  h5 == 2,016,830 hexes
+           .group_by(_.hex)
+           )
+
+    if distinct_taxa != "": # count n unique taxa
+        sel = sel.agg(n = _[distinct_taxa].nunique()) 
+    else: # count occurrences
+        sel = sel.agg(n = _.count())
+
+    sel = (sel
+           .filter(_.n > 0)
+           .mutate(logn = _.n.log())
+           .mutate(value = (255 * _.logn / _.logn.max()).cast("int")) # normalized color-scale
+    )
+
+    # .to_json() doesn't exist in ibis, use SQL
+    query = ibis.to_sql(sel)
+    con.raw_sql(f"COPY ({query}) TO 's3://{bucket}/{dest}' (FORMAT JSON, ARRAY true);")
+
+    return url
+
+
+
+import altair as alt
+
+@st.cache_data
+def bar_chart(gdf_name, rank, taxa, zoom, distinct_taxa = ""):
+    sel = con.read_parquet("s3://public-gbif/app/redlined_cities_gbif.parquet")
+    sel = sel.filter(_[rank] == taxa)
+
+    if gdf_name != "All":
+        sel = sel.filter(_.city == gdf_name)
+    
+    sel = (sel
+      .group_by(_.city, _.grade)
+      .agg(n = _.count(), area = _.area.sum())
+      .mutate(density = _.n /_.area)
+      .group_by(_.grade)
+      .agg(mean = _.density.mean(),sd = _.density.std())
+      .order_by(_.mean.desc())
+    )
+    
+    plt = alt.Chart(sel.execute()).mark_bar().encode(x = "grade", y = "mean")
+    return st.altair_chart(plt)
+
+mappinginequality = 'https://data.source.coop/cboettig/us-boundaries/mappinginequality.pmtiles'
+
+redlines = {'version': 8,
+ 'sources': {'source': {'type': 'vector',
+   'url': 'pmtiles://' + mappinginequality,
+   'attribution': 'PMTiles'}},
+ 'layers': [{'id': 'mappinginequality_fill',
+   'source': 'source',
+   'source-layer': 'mappinginequality',
+   'type': 'fill',
+   'paint': {'fill-color': ["get", "fill"], 'fill-opacity': 0.9},}
+    ]}
+
+
+count = "occurrences"
+if nunique:
+    count = "unique " + distinct_taxa
+
+mapcol, chartcol = st.columns([4,1])
+
+if submitted:
+    gdf = get_polygon(gdf_name, area_source, con)
+    url = compute_hexes(gdf, gdf_name, rank, taxa, zoom, distinct_taxa = distinct_taxa)
+    layer = HexagonLayer(url, v_scale)
+
+    
+    m = leafmap.Map(style= terrain_style, center=[-120, 37.6], zoom=2, pitch=35, bearing=10)
+    if gdf is not None:
+        m.add_gdf(gdf[[gdf.geometry.name]], "fill", paint = {"fill-opacity": 0.2}) # adds area of interest & zooms in
+    m.add_pmtiles(mappinginequality, style=redlines, visible=True, opacity = 0.9,  fit_bounds=False)
+    m.add_deck_layers([layer])
+    m.add_layer_control()
+
+    with mapcol:
+        m.to_streamlit()
+    with chartcol:
+        st.markdown("Mean number of " + count + " by redline grade")
+        bar_chart(gdf_name, rank, taxa, zoom, distinct_taxa = "")
+

utilities.py

@@ -0,0 +1,203 @@
+import os
+
+base_url = "https://minio.carlboettiger.info"
+
+# make sure h3 is installed.
+import duckdb
+db = duckdb.connect()
+db.install_extension("h3", repository = "community")
+db.close()
+
+# enable ibis to use built-in function from the h3 extension
+import ibis
+from ibis import _
+@ibis.udf.scalar.builtin
+def h3_cell_to_boundary_wkt	(array) -> str:
+    ...
+
+
+# Configure write-access to source.coop
+import streamlit as st
+def set_source_secrets(con):
+    source_key = st.secrets["SOURCE_KEY"]
+    source_secret = st.secrets["SOURCE_SECRET"]
+    
+    query=   f'''
+    CREATE OR REPLACE SECRET source (
+        TYPE S3,
+        KEY_ID '{source_key}',
+        SECRET '{source_secret}',
+        ENDPOINT 'data.source.coop',
+        URL_STYLE 'path',
+        SCOPE 's3://cboettig'
+    );
+    
+    set THREADS=100;
+    '''
+
+
+    con.raw_sql(query)
+
+def set_aws_secrets(con):    
+    query=   f'''
+    CREATE OR REPLACE SECRET aws (
+        TYPE S3,
+        ENDPOINT 's3.us-west-2.amazonaws.com',
+        SCOPE 's3://overturemaps-us-west-2/release/'
+    );
+    '''
+
+    # ENDPOINT 'data.source.coop',
+
+    con.raw_sql(query)
+
+# or write access to minio
+def set_secrets(con):   
+    minio_key = st.secrets["MINIO_KEY"]
+    minio_secret = st.secrets["MINIO_SECRET"]
+    query=   f'''
+    CREATE OR REPLACE SECRET secret2 (
+        TYPE S3,
+        KEY_ID '{minio_key}',
+        SECRET '{minio_secret}',
+        ENDPOINT 'minio.carlboettiger.info',
+        URL_STYLE 'path',
+        SCOPE 's3://public-gbif/'
+    );
+    '''
+    con.raw_sql(query)
+
+
+import minio
+def s3_client(type="minio"):
+    minio_key = st.secrets["MINIO_KEY"]
+    minio_secret = st.secrets["MINIO_SECRET"]
+    client = minio.Minio("minio.carlboettiger.info", minio_key, minio_secret)
+    if type == "minio":
+        return client
+
+    source_key = st.secrets["SOURCE_KEY"]
+    source_secret = st.secrets["SOURCE_SECRET"]
+    client = minio.Minio("data.source.coop", source_key, source_secret)
+    return client
+    
+
+import pydeck as pdk
+def HexagonLayer(data, v_scale = 1):
+    return pdk.Layer(
+            "H3HexagonLayer",
+            id="gbif",
+            data=data,
+            extruded=True,
+            get_elevation="value",
+            get_hexagon="hex",
+            elevation_scale = 50 * v_scale,
+            elevation_range = [0,1],
+            pickable=True,
+            auto_highlight=True,
+            get_fill_color="[255 - value, 255, value]",
+            )
+
+def DeckGlobe(layer):
+    view_state = pdk.ViewState(latitude=51.47, longitude=0.45, zoom=0)
+    view = pdk.View(type="_GlobeView", controller=True, width=1000, height=600)
+    COUNTRIES = "https://d2ad6b4ur7yvpq.cloudfront.net/naturalearth-3.3.0/ne_50m_admin_0_scale_rank.geojson"
+    
+    layers = [
+        pdk.Layer(
+            "GeoJsonLayer",
+            id="base-map",
+            data=COUNTRIES,
+            stroked=False,
+            filled=True,
+            get_fill_color=[200, 200, 200],
+        ),
+        layer,
+    ]
+    deck = pdk.Deck(
+        views=[view],
+        initial_view_state=view_state,
+        layers=layers,
+        map_provider=None,
+        # Note that this must be set for the globe to be opaque
+        parameters={"cull": True},
+    )
+    return deck
+
+key = st.secrets['MAPTILER_KEY']
+terrain_style = {
+    "version": 8,
+    "sources": {
+        "osm": {
+            "type": "raster",
+            "tiles": ["https://server.arcgisonline.com/ArcGIS/rest/services/NatGeo_World_Map/MapServer/tile/{z}/{y}/{x}.png"],
+            "tileSize": 256,
+            "attribution": "© National Geographic",
+            "maxzoom": 19,
+        },
+        "terrainSource": {
+            "type": "raster-dem",
+            "url": f"https://api.maptiler.com/tiles/terrain-rgb-v2/tiles.json?key={key}",
+            "tileSize": 256,
+        },
+        "hillshadeSource": {
+            "type": "raster-dem",
+            "url": f"https://api.maptiler.com/tiles/terrain-rgb-v2/tiles.json?key={key}",
+            "tileSize": 256,
+        },
+    },
+    "layers": [
+        {"id": "osm", "type": "raster", "source": "osm"},
+        {
+            "id": "hills",
+            "type": "hillshade",
+            "source": "hillshadeSource",
+            "layout": {"visibility": "visible"},
+            "paint": {"hillshade-shadow-color": "#473B24"},
+        },
+    ],
+    "terrain": {"source": "terrainSource", "exaggeration": .1},
+}
+####
+
+
+## grab polygon of a National park:
+import ibis
+from ibis import _
+
+import geopandas as gpd
+def get_city(name = "Oakland", con = ibis.duckdb.connect()):
+    gdf = (con
+        .read_geo("/vsicurl/https://data.source.coop/cboettig/us-boundaries/mappinginequality.json")
+        .filter(_.city == name)
+        .agg(geom = _.geom.unary_union())
+       ).execute()
+    return gdf 
+
+
+@st.cache_data
+def get_polygon(name = "New Haven", 
+                source = "City",
+                _con = ibis.duckdb.connect()):
+    match source:
+        case 'City':
+            gdf = get_city(name, _con)
+        case 'Custom':
+            gdf = gpd.read_file(name)
+        case "All":
+            gdf = None
+        case _:
+            gdf = None
+    return gdf
+
+import hashlib
+import pandas as pd
+def unique_path(gdf_name, rank, taxa, zoom, distinct_taxa):
+    #gdf_hash = str(pd.util.hash_pandas_object(gdf).sum())
+    text = gdf_name + rank + taxa + str(zoom) + distinct_taxa
+    hash_object = hashlib.sha1(text.encode())
+    sig = hash_object.hexdigest()
+    dest = "cache/gbif_" + sig + ".json"
+    return dest
+
+