Merge pull request #3 from openbiodiversity/develop

.gitignore

@@ -1,3 +1,7 @@
 .venv
 __pycache__/
 service_account.json
+ee_service_account.json
+md_service_token.txt
+.env
+.vscode

app.py

@@ -1,64 +1,360 @@
+import datetime
+import logging
+import os
+
+import duckdb
+import ee
 import gradio as gr
+import pandas as pd
 import plotly.graph_objects as go
-from datasets import load_dataset
-import ee
-# import geemap
-
-# GEE
-service_account = 'climatebase-july-2023@ee-geospatialml-aquarry.iam.gserviceaccount.com'
-credentials = ee.ServiceAccountCredentials(service_account, 'service_account.json')
-ee.Initialize(credentials)
-
-# Gradio dataset
-dataset = load_dataset("gradio/NYC-Airbnb-Open-Data", split="train")
-df = dataset.to_pandas()
-
-def filter_map(min_price, max_price, boroughs):
-
-    filtered_df = df[(df['neighbourhood_group'].isin(boroughs)) & 
-          (df['price'] > min_price) & (df['price'] < max_price)]
-    names = filtered_df["name"].tolist()
-    prices = filtered_df["price"].tolist()
-    text_list = [(names[i], prices[i]) for i in range(0, len(names))]
-    fig = go.Figure(go.Scattermapbox(
-            customdata=text_list,
-            lat=filtered_df['latitude'].tolist(),
-            lon=filtered_df['longitude'].tolist(),
-            mode='markers',
-            marker=go.scattermapbox.Marker(
-                size=6
-            ),
-            hoverinfo="text",
-            hovertemplate='<b>Name</b>: %{customdata[0]}<br><b>Price</b>: $%{customdata[1]}'
-        ))
-
-    fig.update_layout(
-        mapbox_style="open-street-map",
-        hovermode='closest',
-        mapbox=dict(
-            bearing=0,
-            center=go.layout.mapbox.Center(
-                lat=40.67,
-                lon=-73.90
-            ),
-            pitch=0,
-            zoom=9
-        ),
+import yaml
+
+# Logging
+logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG)
+
+# Define constants
+DATE = "2020-01-01"
+YEAR = 2020
+LOCATION = [-74.653370, 5.845328]
+ROI_RADIUS = 20000
+GEE_SERVICE_ACCOUNT = (
+    "climatebase-july-2023@ee-geospatialml-aquarry.iam.gserviceaccount.com"
+)
+GEE_SERVICE_ACCOUNT_CREDENTIALS_FILE = "ee_service_account.json"
+INDICES_FILE = "indices.yaml"
+START_YEAR = 2015
+END_YEAR = 2022
+
+
+class IndexGenerator:
+    """
+    A class to generate indices and compute zonal means.
+
+        Args:
+            centroid (tuple): The centroid coordinates (latitude, longitude) of the region of interest.
+            year (int): The year for which indices are generated.
+            roi_radius (int, optional): The radius (in meters) for creating a buffer around the centroid as the region of interest. Defaults to 20000.
+            project_name (str, optional): The name of the project. Defaults to "".
+            map (geemap.Map, optional): Map object for mapping. Defaults to None (i.e. no map created)
+    """
+
+    def __init__(
+        self,
+        centroid,
+        roi_radius,
+        year,
+        indices_file,
+        project_name="",
+        map=None,
+    ):
+        self.indices = self._load_indices(indices_file)
+        self.centroid = centroid
+        self.roi = ee.Geometry.Point(*centroid).buffer(roi_radius)
+        self.year = year
+        self.start_date = str(datetime.date(self.year, 1, 1))
+        self.end_date = str(datetime.date(self.year, 12, 31))
+        self.daterange = [self.start_date, self.end_date]
+        self.project_name = project_name
+        self.map = map
+        if self.map is not None:
+            self.show = True
+        else:
+            self.show = False
+
+    def _cloudfree(self, gee_path):
+        """
+        Internal method to generate a cloud-free composite.
+
+        Args:
+            gee_path (str): The path to the Google Earth Engine (GEE) image or image collection.
+
+        Returns:
+            ee.Image: The cloud-free composite clipped to the region of interest.
+        """
+        # Load a raw Landsat ImageCollection for a single year.
+        collection = (
+            ee.ImageCollection(gee_path)
+            .filterDate(*self.daterange)
+            .filterBounds(self.roi)
+        )
+
+        # Create a cloud-free composite with custom parameters for cloud score threshold and percentile.
+        composite_cloudfree = ee.Algorithms.Landsat.simpleComposite(
+            **{"collection": collection, "percentile": 75, "cloudScoreRange": 5}
+        )
+        return composite_cloudfree.clip(self.roi)
+
+    def _load_indices(self, indices_file):
+        # Read index configurations
+        with open(indices_file, "r") as stream:
+            try:
+                return yaml.safe_load(stream)
+            except yaml.YAMLError as e:
+                logging.error(e)
+                return None
+
+    def show_map(self, map=None):
+        if map is not None:
+            self.map = map
+            self.show = True
+
+    def disable_map(self):
+        self.show = False
+
+    def generate_index(self, index_config):
+        """
+        Generates an index based on the provided index configuration.
+
+        Args:
+            index_config (dict): Configuration for generating the index.
+
+        Returns:
+            ee.Image: The generated index clipped to the region of interest.
+        """
+        match index_config["gee_type"]:
+            case "image":
+                dataset = ee.Image(index_config["gee_path"]).clip(self.roi)
+                if index_config.get("select"):
+                    dataset = dataset.select(index_config["select"])
+            case "image_collection":
+                dataset = (
+                    ee.ImageCollection(index_config["gee_path"])
+                    .filterBounds(self.roi)
+                    .map(lambda image: image.clip(self.roi))
+                    .mean()
+                )
+                if index_config.get("select"):
+                    dataset = dataset.select(index_config["select"])
+            case "feature_collection":
+                dataset = (
+                    ee.Image()
+                    .float()
+                    .paint(
+                        ee.FeatureCollection(index_config["gee_path"]),
+                        index_config["select"],
+                    )
+                    .clip(self.roi)
+                )
+            case "algebraic":
+                image = self._cloudfree(index_config["gee_path"])
+                dataset = image.normalizedDifference(["B4", "B3"])
+            case _:
+                dataset = None
+
+        if not dataset:
+            raise Exception("Failed to generate dataset.")
+        if self.show and index_config.get("show"):
+            map.addLayer(dataset, index_config["viz"], index_config["name"])
+        logging.info(f"Generated index: {index_config['name']}")
+        return dataset
+
+    def zonal_mean_index(self, index_key):
+        index_config = self.indices[index_key]
+        dataset = self.generate_index(index_config)
+        # zm = self._zonal_mean(single, index_config.get('bandname') or 'constant')
+        out = dataset.reduceRegion(
+            **{
+                "reducer": ee.Reducer.mean(),
+                "geometry": self.roi,
+                "scale": 200,  # map scale
+            }
+        ).getInfo()
+        if index_config.get("bandname"):
+            return out[index_config.get("bandname")]
+        return out
+
+    def generate_composite_index_df(self, indices=[]):
+        data = {
+            "metric": indices,
+            "year": self.year,
+            "centroid": str(self.centroid),
+            "project_name": self.project_name,
+            "value": list(map(self.zonal_mean_index, indices)),
+            "area": self.roi.area().getInfo(),  # m^2
+            "geojson": str(self.roi.getInfo()),
+            # to-do: coefficient
+        }
+
+        logging.info("data", data)
+        df = pd.DataFrame(data)
+        return df
+
+
+def set_up_duckdb():
+    logging.info("set up duckdb")
+    # use `climatebase` db
+    if not os.getenv("motherduck_token"):
+        raise Exception(
+            "No motherduck token found. Please set the `motherduck_token` environment variable."
+        )
+    else:
+        con = duckdb.connect("md:climatebase")
+        con.sql("USE climatebase;")
+
+    # load extensions
+    con.sql("""INSTALL spatial; LOAD spatial;""")
+
+    return con
+
+
+def authenticate_gee(gee_service_account, gee_service_account_credentials_file):
+    logging.info("authenticate_gee")
+    # to-do: alert if dataset filter date nan
+    credentials = ee.ServiceAccountCredentials(
+        gee_service_account, gee_service_account_credentials_file
     )
+    ee.Initialize(credentials)
+
+
+def load_indices(indices_file):
+    # Read index configurations
+    with open(indices_file, "r") as stream:
+        try:
+            return yaml.safe_load(stream)
+        except yaml.YAMLError as e:
+            logging.error(e)
+            return None
+
+
+def create_dataframe(years, project_name):
+    dfs = []
+    logging.info(years)
+    indices = load_indices(INDICES_FILE)
+    for year in years:
+        logging.info(year)
+        ig = IndexGenerator(
+            centroid=LOCATION,
+            roi_radius=ROI_RADIUS,
+            year=year,
+            indices_file=INDICES_FILE,
+            project_name=project_name,
+        )
+        df = ig.generate_composite_index_df(list(indices.keys()))
+        dfs.append(df)
+    return pd.concat(dfs)
+
+
+# def preview_table():
+#     con.sql("FROM bioindicator;").show()
+
+# if __name__ == '__main__':
+
+
+# Map = geemap.Map()
+
+
+# # Create a cloud-free composite with custom parameters for cloud score threshold and percentile.
+# composite_cloudfree = ee.Algorithms.Landsat.simpleComposite(**{
+#   'collection': collection,
+#   'percentile': 75,
+#   'cloudScoreRange': 5
+# })
+
+# Map.addLayer(composite_cloudfree, {'bands': ['B4', 'B3', 'B2'], 'max': 128}, 'Custom TOA composite')
+# Map.centerObject(roi, 14)
+
+
+# ig = IndexGenerator(centroid=LOCATION, year=2015, indices_file=INDICES_FILE, project_name='Test Project', map=Map)
+# dataset = ig.generate_index(indices['Air'])
+
+# minMax = dataset.clip(roi).reduceRegion(
+#   geometry = roi,
+#   reducer = ee.Reducer.minMax(),
+#   scale= 3000,
+#   maxPixels= 10e3,
+# )
+
+
+# minMax.getInfo()
+def calculate_biodiversity_score(start_year, end_year, project_name):
+    years = []
+    for year in range(start_year, end_year):
+        row_exists = con.sql(
+            f"SELECT COUNT(1) FROM bioindicator WHERE (year = {year} AND project_name = '{project_name}')"
+        ).fetchall()[0][0]
+        if not row_exists:
+            years.append(year)
+
+    if len(years) > 0:
+        df = create_dataframe(years, project_name)
+        # con.sql('FROM df LIMIT 5').show()
+
+        # Write score table to `_temptable`
+        con.sql(
+            "CREATE OR REPLACE TABLE _temptable AS SELECT *, (value * area) AS score FROM (SELECT year, project_name, AVG(value) AS value, area  FROM df GROUP BY year, project_name, area ORDER BY project_name)"
+        )
+
+        # Create `bioindicator` table IF NOT EXISTS.
+        con.sql(
+            """
+            USE climatebase;
+            CREATE TABLE IF NOT EXISTS bioindicator (year BIGINT, project_name VARCHAR(255), value DOUBLE, area DOUBLE, score DOUBLE, CONSTRAINT unique_year_project_name UNIQUE (year, project_name));
+        """
+        )
+
+    return con.sql(
+        f"SELECT * FROM bioindicator WHERE (year > {start_year} AND year <= {end_year} AND project_name = '{project_name}')"
+    ).df()
+
+
+def view_all():
+    logging.info("view_all")
+    return con.sql(f"SELECT * FROM bioindicator").df()
+
+
+def push_to_md():
+    # UPSERT project record
+    con.sql(
+        """
+        INSERT INTO bioindicator FROM _temptable
+        ON CONFLICT (year, project_name) DO UPDATE SET value = excluded.value;
+    """
+    )
+    logging.info("upsert records into motherduck")
 
-    return fig
 
 with gr.Blocks() as demo:
+    con = set_up_duckdb()
+    authenticate_gee(GEE_SERVICE_ACCOUNT, GEE_SERVICE_ACCOUNT_CREDENTIALS_FILE)
+    # Create circle buffer over point
+    roi = ee.Geometry.Point(*LOCATION).buffer(ROI_RADIUS)
+
+    # # Load a raw Landsat ImageCollection for a single year.
+    # start_date = str(datetime.date(YEAR, 1, 1))
+    # end_date = str(datetime.date(YEAR, 12, 31))
+    # collection = (
+    #     ee.ImageCollection('LANDSAT/LC08/C02/T1')
+    #     .filterDate(start_date, end_date)
+    #     .filterBounds(roi)
+    # )
+
+    # indices = load_indices(INDICES_FILE)
+    # push_to_md(START_YEAR, END_YEAR, 'Test Project')
     with gr.Column():
+        # map = gr.Plot().style()
+        with gr.Row():
+            start_year = gr.Number(value=2017, label="Start Year", precision=0)
+            end_year = gr.Number(value=2022, label="End Year", precision=0)
+            project_name = gr.Textbox(label="Project Name")
+        # boroughs = gr.CheckboxGroup(choices=["Queens", "Brooklyn", "Manhattan", "Bronx", "Staten Island"], value=["Queens", "Brooklyn"], label="Select Methodology:")
+        # btn = gr.Button(value="Update Filter")
         with gr.Row():
-            min_price = gr.Number(value=250, label="Project Name")
-            max_price = gr.Number(value=1000, label="Project Description")
-        boroughs = gr.CheckboxGroup(choices=["Queens", "Brooklyn", "Manhattan", "Bronx", "Staten Island"], value=["Queens", "Brooklyn"], label="Select Methodology:")
-        btn = gr.Button(value="Update Filter")
-        btn = gr.Button(value="Save")
-        btn = gr.Button(value="Run")
-        map = gr.Plot().style()
-    demo.load(filter_map, [min_price, max_price, boroughs], map)
-    btn.click(filter_map, [min_price, max_price, boroughs], map)
+            calc_btn = gr.Button(value="Calculate!")
+            view_btn = gr.Button(value="View all")
+            save_btn = gr.Button(value="Save")
+        results_df = gr.Dataframe(
+            headers=["Year", "Project Name", "Score"],
+            datatype=["number", "str", "number"],
+            label="Biodiversity scores by year",
+        )
+    # demo.load(filter_map, [min_price, max_price, boroughs], map)
+    # btn.click(filter_map, [min_price, max_price, boroughs], map)
+    calc_btn.click(
+        calculate_biodiversity_score,
+        inputs=[start_year, end_year, project_name],
+        outputs=results_df,
+    )
+    view_btn.click(view_all, outputs=results_df)
+    save_btn.click(push_to_md)
 
 demo.launch()

indices.yaml

@@ -0,0 +1,169 @@
+---
+Water:
+  name: Water
+  roi: ''
+  gee_path: JRC/GSW1_1/GlobalSurfaceWater
+  gee_type: image
+  viz:
+    min: 0
+    max: 100
+    palette:
+    - ffffff
+    - ffbbbb
+    - 0000ff
+  bandname: occurrence
+  select: occurrence
+  show: true
+Protected:
+  name: Protected
+  roi: ''
+  gee_path: WCMC/WDPA/current/polygons
+  gee_type: feature_collection
+  viz:
+    palette:
+    - 2ed033
+    - 5aff05
+    - 67b9ff
+    - 5844ff
+    - 0a7618
+    - 2c05ff
+    min: 0
+    max: 1550000
+    opacity: 0.8
+  select: REP_AREA
+  bandname: constant
+  show: true
+Air:
+  name: Air
+  roi: ''
+  gee_path: COPERNICUS/S5P/OFFL/L3_AER_AI
+  gee_type: image_collection
+  viz:
+    min: -1
+    max: 2
+    palette:
+    - black
+    - blue
+    - purple
+    - cyan
+    - green
+    - yellow
+    - red
+  bandname: absorbing_aerosol_index
+  select: absorbing_aerosol_index
+  dates: false
+  show: false
+Soil:
+  name: Soil
+  roi: ''
+  gee_path: OpenLandMap/SOL/SOL_ORGANIC-CARBON_USDA-6A1C_M/v02
+  gee_type: image
+  viz:
+    bands:
+    - b200
+    min: 0
+    max: 12
+    palette:
+    - ffffa0
+    - f7fcb9
+    - d9f0a3
+    - addd8e
+    - 78c679
+    - 41ab5d
+    - '238443'
+    - 005b29
+    - 004b29
+    - 012b13
+    - 00120b
+  select: b0
+  bandname: b0
+  show: false
+Temperature:
+  name: Temperature
+  roi: ''
+  gee_path: MODIS/061/MYD21C1
+  gee_type: image_collection
+  viz:
+    min: 216
+    max: 348
+    palette:
+    - '040274'
+    - '040281'
+    - 0502a3
+    - 0502b8
+    - 0502ce
+    - 0502e6
+    - 0602ff
+    - 235cb1
+    - 307ef3
+    - 269db1
+    - 30c8e2
+    - 32d3ef
+    - 3be285
+    - 3ff38f
+    - 86e26f
+    - 3ae237
+    - b5e22e
+    - d6e21f
+    - fff705
+    - ffd611
+    - ffb613
+    - ff8b13
+    - ff6e08
+    - ff500d
+    - ff0000
+    - de0101
+    - c21301
+    - a71001
+    - '911003'
+  select: LST_Day
+  bandname: LST_Day
+  dates: true
+  show: true
+Habitat:
+  name: Habitat
+  roi: ''
+  gee_path: projects/sat-io/open-datasets/IUCN_HABITAT/iucn_habitatclassification_composite_lvl2_ver004
+  gee_type: image
+  viz: {}
+  bandname: comp_first
+  show: true
+NDVI:
+  name: NDVI
+  roi: ''
+  gee_path: LANDSAT/LC08/C02/T1
+  gee_type: algebraic
+  normalized_difference:
+  - B4
+  - B3
+  viz:
+    palette:
+    - "#d73027"
+    - "#f46d43"
+    - "#fdae61"
+    - "#fee08b"
+    - "#d9ef8b"
+    - "#a6d96a"
+    - "#66bd63"
+    - "#1a9850"
+  bandname: nd
+NDWI:
+  name: NDWI
+  roi: ''
+  gee_path: LANDSAT/LC08/C02/T1
+  gee_type: algebraic
+  normalized_difference:
+  - B3
+  - B5
+  viz:
+    palette:
+    - "#ece7f2"
+    - "#d0d1e6"
+    - "#a6bddb"
+    - "#74a9cf"
+    - "#3690c0"
+    - "#0570b0"
+    - "#045a8d"
+    - "#023858"
+  bandname: nd
+  show: true

requirements.txt

@@ -5,4 +5,3 @@ duckdb==0.8.1
 geemap
 segment-geospatial
 geojson
-yaml