Update app.py

app.py

@@ -1,186 +1,482 @@
-import re
-import os
+import param
 import panel as pn
-from io import StringIO
-from panel.io.mime_render import exec_with_return
-from llama_index import (
-    VectorStoreIndex,
-    SimpleDirectoryReader,
-    ServiceContext,
-    StorageContext,
-    load_index_from_storage,
-)
-from llama_index.chat_engine import ContextChatEngine
-from llama_index.embeddings import OpenAIEmbedding
-from llama_index.llms import OpenAI
-
-
-SYSTEM_PROMPT = (
-    "You are a data visualization pro and expert in HoloViz hvplot + holoviews. "
-    "Your primary goal is to assist the user in editing based on user requests using best practices. "
-    "Simply provide code in code fences (```python). You must have `hvplot_obj` as the last line of code. "
-    "Note, data columns are ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species'] and "
-    "hvplot is built on top of holoviews--anything you can do with holoviews, you can do "
-    "with hvplot. First try to use hvplot **kwargs instead of opts, e.g. `legend='top_right'` "
-    "instead of `opts(legend_position='top_right')`. If you need to use opts, you can use "
-    "concise version, e.g. `opts(xlabel='Petal Length')` vs `opts(hv.Opts(xlabel='Petal Length'))`"
+import numpy as np
+import pandas as pd
+import hvplot.pandas
+import geoviews as gv
+import holoviews as hv
+from holoviews.streams import Tap
+from bokeh.themes import Theme
+
+VAR_OPTIONS = {
+    "Maximum Air Temperature [F]": "max_temp_f",
+    "Minimum Air Temperature [F]": "min_temp_f",
+    "Maximum Dew Point [F]": "max_dewpoint_f",
+    "Minimum Dew Point [F]": "min_dewpoint_f",
+    "Daily Precipitation [inch]": "precip_in",
+    "Average Wind Speed [knots]": "avg_wind_speed_kts",
+    "Average Wind Direction [deg]": "avg_wind_drct",
+    "Minimum Relative Humidity [%]": "min_rh",
+    "Average Relative Humidity [%]": "avg_rh",
+    "Maximum Relative Humidity [%]": "max_rh",
+    "NCEI 1991-2020 Daily High Temperature Climatology [F]": "climo_high_f",
+    "NCEI 1991-2020 Daily Low Temperature Climatology [F]": "climo_low_f",
+    "NCEI 1991-2020 Daily Precipitation Climatology [inch]": "climo_precip_in",
+    "Reported Snowfall [inch]": "snow_in",
+    "Reported Snow Depth [inch]": "snowd_in",
+    "Minimum 'Feels Like' Temperature [F]": "min_feel",
+    "Average 'Feels Like' Temperature [F]": "avg_feel",
+    "Maximum 'Feels Like' Temperature [F]": "max_feel",
+    "Maximum sustained wind speed [knots]": "max_wind_speed_kts",
+    "Maximum wind gust [knots]": "max_wind_gust_kts",
+    "Daily Solar Radiation MJ/m2": "srad_mj",
+}
+VAR_OPTIONS_R = {v: k for k, v in VAR_OPTIONS.items()}
+NETWORKS_URL = "https://mesonet.agron.iastate.edu/sites/networks.php?network=_ALL_&format=csv&nohtml=on"
+STATION_URL_FMT = (
+    "https://mesonet.agron.iastate.edu/cgi-bin/request/daily.py?network={network}&stations={station}"
+    "&year1=1928&month1=1&day1=1&year2=2023&month2=12&day2=31&var={var}&na=blank&format=csv"
 )
+DARK_RED = "#FF5555"
+DARK_BLUE = "#5588FF"
+XTICKS = [
+    (1, "JAN"),
+    (31, "FEB"),
+    (59, "MAR"),
+    (90, "APR"),
+    (120, "MAY"),
+    (151, "JUN"),
+    (181, "JUL"),
+    (212, "AUG"),
+    (243, "SEP"),
+    (273, "OCT"),
+    (304, "NOV"),
+    (334, "DEC"),
+]
 
-USER_CONTENT_FORMAT = """
-Request:
-{content}
+THEME_JSON = {
+    "attrs": {
+        "figure": {
+            "background_fill_color": "#1b1e23",
+            "border_fill_color": "#1b1e23",
+            "outline_line_alpha": 0,
+        },
+        "Grid": {
+            "grid_line_color": "#808080",
+            "grid_line_alpha": 0.1,
+        },
+        "Axis": {
+            # tick color and alpha
+            "major_tick_line_color": "#4d4f51",
+            "minor_tick_line_alpha": 0,
+            # tick labels
+            "major_label_text_font": "Courier New",
+            "major_label_text_color": "#808080",
+            "major_label_text_align": "left",
+            "major_label_text_font_size": "0.95em",
+            "major_label_text_font_style": "normal",
+            # axis labels
+            "axis_label_text_font": "Courier New",
+            "axis_label_text_font_style": "normal",
+            "axis_label_text_font_size": "1.15em",
+            "axis_label_text_color": "lightgrey",
+            "axis_line_color": "#4d4f51",
+        },
+        "Legend": {
+            "spacing": 8,
+            "glyph_width": 15,
+            "label_standoff": 8,
+            "label_text_color": "#808080",
+            "label_text_font": "Courier New",
+            "label_text_font_size": "0.95em",
+            "label_text_font_style": "bold",
+            "border_line_alpha": 0,
+            "background_fill_alpha": 0.25,
+            "background_fill_color": "#1b1e23",
+        },
+        "BaseColorBar": {
+            # axis labels
+            "title_text_color": "lightgrey",
+            "title_text_font": "Courier New",
+            "title_text_font_size": "0.95em",
+            "title_text_font_style": "normal",
+            # tick labels
+            "major_label_text_color": "#808080",
+            "major_label_text_font": "Courier New",
+            "major_label_text_font_size": "0.95em",
+            "major_label_text_font_style": "normal",
+            "background_fill_color": "#1b1e23",
+            "major_tick_line_alpha": 0,
+            "bar_line_alpha": 0,
+        },
+        "Title": {
+            "text_font": "Courier New",
+            "text_font_style": "normal",
+            "text_color": "lightgrey",
+        },
+    }
+}
+theme = Theme(json=THEME_JSON)
 
-Code:
-```python
-{code}
-```
-""".strip()
+hv.renderer("bokeh").theme = theme
+pn.extension(throttled=True)
 
-DEFAULT_HVPLOT = """
-import hvplot.pandas
-from bokeh.sampledata.iris import flowers
-
-hvplot_obj = flowers.hvplot(x='petal_length', y='petal_width', by='species', kind='scatter')
-hvplot_obj
-""".strip()
-
-
-def exception_handler(exc):
-    if retries.value == 0:
-        chat_interface.send(f"Can't figure this out: {exc}", respond=False)
-        return
-    chat_interface.send(f"Fix this error:\n```python\n{exc}\n```")
-    retries.value = retries.value - 1
-
-
-def init_llm(event):
-    api_key = event.new
-    if not api_key:
-        api_key = os.environ.get("OPENAI_API_KEY")
-    if not api_key:
-        return
-    pn.state.cache["llm"] = OpenAI(api_key=api_key)
-
-
-def create_chat_engine(llm):
-    try:
-        storage_context = StorageContext.from_defaults(persist_dir="persisted/")
-        index = load_index_from_storage(storage_context=storage_context)
-    except Exception as exc:
-        embed_model = OpenAIEmbedding()
-        service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)
-        documents = SimpleDirectoryReader(
-            input_dir="hvplot_docs", required_exts=[".md"], recursive=True
-        ).load_data()
-        index = VectorStoreIndex.from_documents(
-            documents, service_context=service_context, show_progress=True
+class ClimateApp(pn.viewable.Viewer):
+    network = param.Selector(default="WA_ASOS")
+    station = param.Selector(default="SEA")
+    year = param.Integer(default=2023, bounds=(1928, 2023))
+    year_range = param.Range(default=(1990, 2020), bounds=(1928, 2023))
+    var = param.Selector(default="max_temp_f", objects=sorted(VAR_OPTIONS.values()))
+    stat = param.Selector(default="Mean", objects=["Mean", "Median"])
+
+    def __init__(self, **params):
+        super().__init__(**params)
+        pn.state.onload(self._onload)
+
+    def _onload(self):
+        self._networks_df = self._get_networks_df()
+        networks = sorted(self._networks_df["iem_network"].unique())
+        self.param["network"].objects = networks
+
+        network_select = pn.widgets.AutocompleteInput.from_param(
+            self.param.network, min_characters=0, case_sensitive=False
+        )
+        station_select = pn.widgets.AutocompleteInput.from_param(
+            self.param.station, min_characters=0, case_sensitive=False
+        )
+        var_select = pn.widgets.Select.from_param(self.param.var, options=VAR_OPTIONS)
+        year_slider = pn.widgets.IntSlider.from_param(self.param.year)
+        year_range_slider = pn.widgets.RangeSlider.from_param(self.param.year_range)
+        stat_select = pn.widgets.RadioButtonGroup.from_param(self.param.stat, sizing_mode="stretch_width")
+        self._sidebar = [
+            network_select,
+            station_select,
+            var_select,
+            year_slider,
+            year_range_slider,
+            stat_select,
+        ]
+
+        network_points = self._networks_df.hvplot.points(
+            "lon",
+            "lat",
+            legend=False,
+            cmap="category10",
+            color="iem_network",
+            hover_cols=["stid", "station_name", "iem_network"],
+            size=10,
+            geo=True,
+        ).opts(
+            "Points",
+            fill_alpha=0,
+            responsive=True,
+            tools=["tap", "hover"],
+            active_tools=["wheel_zoom"],
+        )
+
+        tap = Tap(source=network_points)
+        pn.bind(self._update_station, x=tap.param.x, y=tap.param.y, watch=True)
+        network_pane = pn.pane.HoloViews(
+            network_points * gv.tile_sources.CartoDark(),
+            sizing_mode="stretch_both",
+            max_height=625,
+        )
+        self._station_pane = pn.pane.HoloViews(sizing_mode="stretch_width", height=450)
+        main_tabs = pn.Tabs(
+            ("Climatology Plot", self._station_pane), ("Map Select", network_pane)
+        )
+        self._main = [self._station_pane]
+
+        self._update_var_station_dependents()
+        self._update_stations()
+        self._update_station_pane()
+
+    @pn.cache
+    def _get_networks_df(self):
+        networks_df = pd.read_csv(NETWORKS_URL)
+        return networks_df
+
+    @pn.depends("network", watch=True)
+    def _update_stations(self):
+        network_df_subset = self._networks_df.loc[
+            self._networks_df["iem_network"] == self.network,
+            ["stid", "station_name"],
+        ]
+        names = sorted(network_df_subset["station_name"].unique())
+        stids = sorted(network_df_subset["stid"].unique())
+        self.param["station"].objects = names + stids
+
+    def _update_station(self, x, y):
+        if x is None or y is None:
+            return
+
+        def haversine_vectorized(lon1, lat1, lon2, lat2):
+            R = 6371  # Radius of the Earth in kilometers
+            dlat = np.radians(lat2 - lat1)
+            dlon = np.radians(lon2 - lon1)
+            a = (
+                np.sin(dlat / 2.0) ** 2
+                + np.cos(np.radians(lat1))
+                * np.cos(np.radians(lat2))
+                * np.sin(dlon / 2.0) ** 2
+            )
+            c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
+            return R * c
+
+        distances = haversine_vectorized(
+            self._networks_df["lon"].values, self._networks_df["lat"].values, x, y
+        )
+
+        min_distance_index = np.argmin(distances)
+
+        closest_row = self._networks_df.iloc[min_distance_index]
+        with param.parameterized.batch_call_watchers(self):
+            self.network = closest_row["iem_network"]
+            self.station = closest_row["stid"]
+
+    @pn.cache
+    def _get_station_df(self, station, var):
+        if station in self._networks_df["station_name"].unique():
+            station = self._networks_df.loc[
+                self._networks_df["station_name"] == station, "stid"
+            ].iloc[0]
+            if station.startswith("K"):
+                station = station.lstrip("K")
+        station_url = STATION_URL_FMT.format(
+            network=self.network, station=station, var=var
+        )
+        station_df = (
+            pd.read_csv(
+                station_url,
+                parse_dates=True,
+                index_col="day",
+            )
+            .drop(columns=["station"])
+            .astype("float16")
+            .assign(
+                dayofyear=lambda df: df.index.dayofyear,
+                year=lambda df: df.index.year,
+            )
+            .dropna()
+        )
+        return station_df
+
+    @pn.depends("var", "station", watch=True)
+    def _update_var_station_dependents(self):
+        try:
+            self._station_pane.loading = True
+            self._station_df = self._get_station_df(self.station, self.var).dropna()
+            if len(self._station_df) == 0:
+                return
+
+            year_range_min = self._station_df["year"].min()
+            year_range_max = self._station_df["year"].max()
+            if self.year_range[0] < year_range_min:
+                self.year_range = (year_range_min, self.year_range[1])
+            if self.year_range[1] > year_range_max:
+                self.year_range = (self.year_range[0], year_range_max)
+
+            self.param["year_range"].bounds = (year_range_min, year_range_max)
+
+            self.param["year"].bounds = (year_range_min, year_range_max)
+            if self.year < year_range_min:
+                self.year = year_range_min
+            if self.year > year_range_max:
+                self.year = year_range_max
+        finally:
+            self._station_pane.loading = False
+
+    @pn.depends("var", "station", "year", "year_range", "stat", watch=True)
+    def _update_station_pane(self):
+        if len(self._station_df) == 0:
+            return
+
+        try:
+            self._station_pane.loading = True
+            df = self._station_df
+            if self.station not in self._networks_df["station_name"].unique():
+                station_name = self._networks_df.loc[
+                    self._networks_df["stid"] == self.station, "station_name"
+                ].iloc[0]
+            else:
+                station_name = self.station
+
+            # get average and year
+            df_avg = (
+                df.loc[df["year"].between(*self.year_range)].groupby("dayofyear").mean()
+            )
+            df_year = df[df.year == self.year]
+            if self.stat == "Mean":
+                df_year_avg = df_year[self.var].mean()
+            else:
+                df_year_avg = df_year[self.var].median()
+            df_year_max = df_year[self.var].max()
+            df_year_min = df_year[self.var].min()
+
+            # preprocess below/above
+            df_above = df_year[["dayofyear", self.var]].merge(
+                df_avg.reset_index()[["dayofyear", self.var]],
+                on="dayofyear",
+                suffixes=("_year", "_avg"),
+            )
+            df_above[self.var] = df_above[f"{self.var}_avg"]
+            df_above[self.var] = df_above.loc[
+                df_above[f"{self.var}_year"] >= df_above[f"{self.var}_avg"],
+                f"{self.var}_year",
+            ]
+
+            df_below = df_year[["dayofyear", self.var]].merge(
+                df_avg.reset_index()[["dayofyear", self.var]],
+                on="dayofyear",
+                suffixes=("_year", "_avg"),
+            )
+            df_below[self.var] = df_below[f"{self.var}_avg"]
+            df_below[self.var] = df_below.loc[
+                df_below[f"{self.var}_year"] < df_below[f"{self.var}_avg"],
+                f"{self.var}_year",
+            ]
+
+            days_above = df_above.loc[
+                df_above[f"{self.var}_year"] >= df_above[f"{self.var}_avg"]
+            ].shape[0]
+            days_below = df_below.loc[
+                df_below[f"{self.var}_year"] < df_below[f"{self.var}_avg"]
+            ].shape[0]
+
+            # create plot elements
+            plot_kwargs = {
+                "x": "dayofyear",
+                "y": self.var,
+                "responsive": True,
+                "legend": False,
+            }
+            plot = df.hvplot(
+                by="year",
+                color="grey",
+                alpha=0.02,
+                hover=False,
+                **plot_kwargs,
+            )
+            plot_year = (
+                df_year.hvplot(color="black", hover="vline", **plot_kwargs)
+                .opts(alpha=0.2)
+                .redim.label(**{"dayofyear": "Julian Day", self.var: str(self.year)})
+            )
+            plot_avg = df_avg.hvplot(color="grey", **plot_kwargs).redim.label(
+                **{"dayofyear": "Julian Day", self.var: "Average"}
+            )
+
+            plot_year_avg = hv.HLine(df_year_avg).opts(
+                line_color="lightgrey", line_dash="dashed", line_width=0.5
+            )
+            plot_year_max = hv.HLine(df_year_max).opts(
+                line_color=DARK_RED, line_dash="dashed", line_width=0.5
+            )
+            plot_year_min = hv.HLine(df_year_min).opts(
+                line_color=DARK_BLUE, line_dash="dashed", line_width=0.5
+            )
+
+            text_year_opts = {
+                "text_align": "right",
+                "text_baseline": "bottom",
+                "text_alpha": 0.8,
+            }
+            text_year_label = "AVERAGE" if self.stat == "Mean" else "MEDIAN"
+            text_year_avg = hv.Text(
+                360, df_year_avg + 3, f"{text_year_label} {df_year_avg:.0f}", fontsize=8
+            ).opts(
+                text_color="lightgrey",
+                **text_year_opts,
+            )
+            text_year_max = hv.Text(
+                360, df_year_max + 3, f"MAX {df_year_max:.0f}", fontsize=8
+            ).opts(
+                text_color=DARK_RED,
+                **text_year_opts,
+            )
+            text_year_min = hv.Text(
+                360, df_year_min + 3, f"MIN {df_year_min:.0f}", fontsize=8
+            ).opts(
+                text_color=DARK_BLUE,
+                **text_year_opts,
+            )
+
+            area_kwargs = {"fill_alpha": 0.2, "line_alpha": 0.8}
+            plot_above = df_above.hvplot.area(
+                x="dayofyear", y=f"{self.var}_avg", y2=self.var, hover=False
+            ).opts(line_color=DARK_RED, fill_color=DARK_RED, **area_kwargs)
+            plot_below = df_below.hvplot.area(
+                x="dayofyear", y=f"{self.var}_avg", y2=self.var, hover=False
+            ).opts(line_color=DARK_BLUE, fill_color=DARK_BLUE, **area_kwargs)
+
+            text_x = 25
+            text_y = df_year[self.var].max() + 10
+            text_days_above = hv.Text(text_x, text_y, f"{days_above}", fontsize=14).opts(
+                text_align="right",
+                text_baseline="bottom",
+                text_color=DARK_RED,
+                text_alpha=0.8,
+            )
+            text_days_below = hv.Text(text_x, text_y, f"{days_below}", fontsize=14).opts(
+                text_align="right",
+                text_baseline="top",
+                text_color=DARK_BLUE,
+                text_alpha=0.8,
+            )
+            text_above = hv.Text(text_x + 3, text_y, "DAYS ABOVE", fontsize=7).opts(
+                text_align="left",
+                text_baseline="bottom",
+                text_color="lightgrey",
+                text_alpha=0.8,
+            )
+            text_below = hv.Text(text_x + 3, text_y, "DAYS BELOW", fontsize=7).opts(
+                text_align="left",
+                text_baseline="top",
+                text_color="lightgrey",
+                text_alpha=0.8,
+            )
+
+            # overlay everything and save
+            station_overlay = (
+                plot
+                * plot_year
+                * plot_avg
+                * plot_year_avg
+                * plot_year_max
+                * plot_year_min
+                * text_year_avg
+                * text_year_max
+                * text_year_min
+                * plot_above
+                * plot_below
+                * text_days_above
+                * text_days_below
+                * text_above
+                * text_below
+            ).opts(
+                xlabel="TIME OF YEAR",
+                ylabel=VAR_OPTIONS_R[self.var],
+                title=f"{station_name} {self.year} vs AVERAGE ({self.year_range[0]}-{self.year_range[1]})",
+                gridstyle={"ygrid_line_alpha": 0},
+                xticks=XTICKS,
+                show_grid=True,
+                fontscale=1.18,
+                padding=(0, (0, 0.3))
+            )
+            self._station_pane.object = station_overlay
+        finally:
+            self._station_pane.loading = False
+
+    def __panel__(self):
+        return pn.template.FastListTemplate(
+            sidebar=self._sidebar,
+            main=self._main,
+            theme="dark",
+            theme_toggle=False,
+            main_layout=None,
+            title="Select Year vs Average Comparison",
+            accent="#2F4F4F",
         )
-        index.storage_context.persist("persisted/")
-
-    retriever = index.as_retriever()
-    chat_engine = ContextChatEngine.from_defaults(
-        system_prompt=SYSTEM_PROMPT,
-        retriever=retriever,
-        verbose=True,
-    )
-    return chat_engine
-
-
-def callback(content: str, user: str, instance: pn.chat.ChatInterface):
-    if "llm" not in pn.state.cache:
-        yield "Need to set OpenAI API key first"
-        return
-
-    if "engine" not in pn.state.cache:
-        engine = pn.state.cache["engine"] = create_chat_engine(pn.state.cache["llm"])
-    else:
-        engine = pn.state.cache["engine"]
-
-    # new user contents
-    user_content = USER_CONTENT_FORMAT.format(
-        content=content, code=code_editor.value
-    )
-
-    # send user content to chat engine
-    agent_response = engine.stream_chat(user_content)
-
-    message = None
-    for chunk in agent_response.response_gen:
-        message = instance.stream(chunk, message=message, user="OpenAI")
-
-    # extract code
-    llm_matches = re.findall(r"```python\n(.*)\n```", message.object, re.DOTALL)
-    if llm_matches:
-        llm_code = llm_matches[0]
-        if llm_code.splitlines()[-1].strip() != "hvplot_obj":
-            llm_code += "\nhvplot_obj"
-        code_editor.value = llm_code
-    retries.value = 2
-
-
-def update_plot(event):
-    with StringIO() as buf:
-        hvplot_pane.object = exec_with_return(event.new, stderr=buf)
-        buf.seek(0)
-        errors = buf.read()
-        if errors:
-            exception_handler(errors)
-
-pn.extension("codeeditor", sizing_mode="stretch_width", exception_handler=exception_handler)
-
-
-# instantiate widgets and panes
-api_key_input = pn.widgets.PasswordInput(
-    placeholder=(
-        "Currently subsidized by Andrew, "
-        "but you can also pass your own OpenAI API Key"
-    )
-)
-chat_interface = pn.chat.ChatInterface(
-    callback=callback,
-    show_clear=False,
-    show_undo=False,
-    show_button_name=False,
-    message_params=dict(
-        show_reaction_icons=False,
-        show_copy_icon=False,
-    ),
-    height=650,
-    callback_exception="verbose",
-)
-hvplot_pane = pn.pane.HoloViews(
-    exec_with_return(DEFAULT_HVPLOT),
-    sizing_mode="stretch_both",
-)
-code_editor = pn.widgets.CodeEditor(
-    value=DEFAULT_HVPLOT,
-    language="python",
-    sizing_mode="stretch_both",
-)
-retries = pn.widgets.IntInput(value=2, visible=False)
-error = pn.widgets.StaticText(visible=False)
-
-# watch for code changes
-api_key_input.param.watch(init_llm, "value")
-code_editor.param.watch(update_plot, "value")
-api_key_input.param.trigger("value")
-
-# lay them out
-tabs = pn.Tabs(
-    ("Plot", hvplot_pane),
-    ("Code", code_editor),
-)
 
-sidebar = [api_key_input, chat_interface]
-main = [tabs]
-template = pn.template.FastListTemplate(
-    sidebar=sidebar,
-    main=main,
-    sidebar_width=600,
-    main_layout=None,
-    accent_base_color="#fd7000",
-    header_background="#fd7000",
-    title="Chat with Plot"
-)
 
-template.servable()
+ClimateApp().servable()