Repository: rstudio/vetiver-python
Pull request #203 (2023-11-13 20:51:47)

FEAT: quarto monitoring dashboard template
Adds a monitoring dashboard template for a VetiverModel object/deployment.


Patch:
diff --git a/docs/_quarto.yml b/docs/_quarto.yml
index 8d67843..f975780 100644
--- a/docs/_quarto.yml
+++ b/docs/_quarto.yml
@@ -24,7 +24,7 @@ website:
         file: reference/index.qmd
       - text: "Advanced Usage"
         menu:
-          - custom_elements.qmd
+          - custom_code.qmd
       - text: "Changelog"
         file: changelog.md
       - text: "Learn more"
@@ -59,7 +59,7 @@ quartodoc:
         - VetiverModel
         - vetiver_pin_write
         - vetiver_create_prototype
-        - model_card.model_card
+        - templates.model_card
 
     - title: Deploy
       desc: ""
@@ -72,7 +72,7 @@ quartodoc:
         - predict
         - write_app
         - prepare_docker
-        - write_docker.write_docker
+        - write_docker
         - deploy_rsconnect
 
     - title: Monitor
@@ -81,6 +81,7 @@ quartodoc:
         - compute_metrics
         - pin_metrics
         - plot_metrics
+        - templates.monitoring_dashboard
 
     - title: Model Handlers
       desc: ""
diff --git a/docs/changelog.md b/docs/changelog.md
index e826dd1..8c50bc2 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -3,19 +3,10 @@
 
 For full details, view the [commit logs](https://github.com/rstudio/vetiver-python/commits/).
 
-## v0.2.3
-## What's Changed
-
-[**Full Changelog**](https://github.com/rstudio/vetiver-python/compare/v0.2.2...v0.2.3)
-
-* ENH: add `root_path` if user in Posit Workbench in [GH191](https://github.com/rstudio/vetiver-python/pull/191)
-
-
 ## v0.2.2
 ### What's Changed
 
-[**Full Changelog**](https://github.com/rstudio/vetiver-python/compare/v0.2.1...v0.2.2)
-
+**Full Changelog**: https://github.com/rstudio/vetiver-python/compare/v0.2.1...v0.2.2
 * DOCS: add square logo in [GH173](https://github.com/rstudio/vetiver-python/pull/173)
 * DOCS: add API structure documentation in [GH176](https://github.com/rstudio/vetiver-python/pull/176)
 * DOCS: update docs to look less like vetiver.rstudio.com in [GH178](https://github.com/rstudio/vetiver-python/pull/178)
@@ -30,7 +21,7 @@ For full details, view the [commit logs](https://github.com/rstudio/vetiver-pyth
 ## v0.2.1
 ### What's Changed
 
-[**Full Changelog**](https://github.com/rstudio/vetiver-python/compare/v0.2.0...v0.2.1)
+**Full Changelog**: https://github.com/rstudio/vetiver-python/compare/v0.2.0...v0.2.1
 
 * DOC: add changelog in [GH151](https://github.com/rstudio/vetiver-python/pull/151)
 * FIX: None type handling for `python_version` in metadata [GH149](https://github.com/rstudio/)vetiver-python/pull/149
diff --git a/setup.cfg b/setup.cfg
index ef3a541..62cd44f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -31,7 +31,7 @@ install_requires =
     nest-asyncio
     requests
     pins>=0.7.1
-    rsconnect-python>=1.11.0
+    rsconnect-python>=1.8.0
     plotly
     pip-tools
     httpx
diff --git a/vetiver/__init__.py b/vetiver/__init__.py
index 0aef0f5..1c85eed 100644
--- a/vetiver/__init__.py
+++ b/vetiver/__init__.py
@@ -24,7 +24,7 @@ from .handlers.spacy import SpacyHandler  # noqa
 from .helpers import api_data_to_frame  # noqa
 from .rsconnect import deploy_rsconnect  # noqa
 from .monitor import compute_metrics, pin_metrics, plot_metrics, _rolling_df  # noqa
-from .model_card import model_card  # noqa
+from .templates import model_card, monitoring_dashboard  # noqa
 from .types import create_prototype, Prototype  # noqa
 
 __author__ = "Isabel Zimmerman <isabel.zimmerman@posit.co>"
diff --git a/vetiver/model_card.py b/vetiver/model_card.py
deleted file mode 100644
index 5a41a8a..0000000
--- a/vetiver/model_card.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from importlib_resources import files as _files
-import shutil
-
-
-def model_card(path="."):
-    """
-    Create a model card for documentation
-
-    Parameters
-    ----------
-    path : str
-        Path to save model card
-
-    Notes
-    -----
-    This model card is generated as a Quarto document. For more info on
-    Quarto, visit https://quarto.org/
-    """
-    src_path = _files("vetiver") / "templates/model_card.qmd"
-
-    return shutil.copy(src=src_path, dst=path)
diff --git a/vetiver/rsconnect.py b/vetiver/rsconnect.py
index 7c410d2..08f8dad 100644
--- a/vetiver/rsconnect.py
+++ b/vetiver/rsconnect.py
@@ -19,6 +19,7 @@ def deploy_rsconnect(
     app_id: int = None,
     title: str = None,
     python: str = None,
+    conda_mode: bool = False,
     force_generate: bool = False,
     log_callback: typing.Callable = None,
     image: str = None,
@@ -45,6 +46,8 @@ def deploy_rsconnect(
         Optional title for the deploy.
     python : str
         Optional name of a Python executable
+    conda_mode : bool
+        Use conda to build an environment.yml
     force_generate : bool
         Force generating requirements.txt or environment.yml
     log_callback : typing.Callable
@@ -108,7 +111,7 @@ def deploy_rsconnect(
             app_id=app_id,
             title=title,
             python=python,
-            conda_mode=False,
+            conda_mode=conda_mode,
             force_generate=force_generate,
             log_callback=log_callback,
             image=image,
diff --git a/vetiver/server.py b/vetiver/server.py
index c5069f0..9260efc 100644
--- a/vetiver/server.py
+++ b/vetiver/server.py
@@ -373,8 +373,4 @@ def vetiver_endpoint(url: str = "http://127.0.0.1:8000/predict") -> str:
     >>> import vetiver
     >>> endpoint = vetiver.vetiver_endpoint(url='http://127.0.0.1:8000/predict')
     """
-    # remove trailing backslash, if it exists
-    if url[-1] == "/":
-        url = url[:-1]
-
     return url
diff --git a/vetiver/templates.py b/vetiver/templates.py
new file mode 100644
index 0000000..aa74183
--- /dev/null
+++ b/vetiver/templates.py
@@ -0,0 +1,42 @@
+from importlib_resources import files as _files
+import shutil
+from pathlib import Path
+
+
+def model_card(path="."):
+    """
+    Create a model card for documentation
+
+    Parameters
+    ----------
+    path : str
+        Path to save model card
+
+    Notes
+    -----
+    This model card is generated as a Quarto document. For more info on
+    Quarto, visit https://quarto.org/
+    """
+    src_path = _files("vetiver") / "templates/model_card.qmd"
+
+    return shutil.copy(src=src_path, dst=path)
+
+
+def monitoring_dashboard(path: str = "."):
+    """
+    Generate a monitoring dashboard template
+
+    Parameters
+    ----------
+    path : str
+        Path to save monitoring dashboard
+
+    Notes
+    -----
+    This model card is generated as a Quarto document. For more info on
+    Quarto, visit https://quarto.org/
+    """
+    p = Path(path)
+    src_path = p / _files("vetiver") / "templates" / "monitoring_dashboard.qmd"
+
+    return shutil.copy(src=src_path, dst=path)
diff --git a/vetiver/templates/monitoring_dashboard.qmd b/vetiver/templates/monitoring_dashboard.qmd
new file mode 100644
index 0000000..6f61048
--- /dev/null
+++ b/vetiver/templates/monitoring_dashboard.qmd
@@ -0,0 +1,136 @@
+---
+title: "Monitoring dashboard"
+format:
+  dashboard:
+    orientation: columns
+logo: https://github.com/rstudio/vetiver-python/blob/main/docs/figures/logo.png?raw=true
+output: asis
+jupyter: python3
+---
+
+```{python}
+#| include: false
+#| tags: [parameters]
+
+# import model and metadata
+import pins
+from IPython.display import display, Markdown, IFrame
+from datetime import datetime, timedelta
+import pandas as pd
+import plotly.express as px
+from sklearn import metrics
+from vetiver import VetiverModel, compute_metrics, plot_metrics
+from sklearn.metrics import recall_score, accuracy_score
+
+raw = "https://colorado.rstudio.com/rsc"
+paths = {"chicago-model-python": "chicago-model-python/"}
+board = pins.board_url(raw, paths, allow_pickle_read=True)
+v = VetiverModel.from_pin(board, "chicago-model-python")
+v_meta = board.pin_meta("chicago-model-python")
+days_old = datetime.today() - datetime.strptime(v_meta.created, "%Y%m%dT%H%M%SZ")
+```
+
+```{python}
+## the next few lines are an example model, here is a place to
+## add any code you need to import new data and make predictions
+
+# import new data to track performance over time
+raw = "https://colorado.rstudio.com/rsc"
+paths = {"new-data": "inspections-new-data/"}
+board = pins.board_url(raw, paths, allow_pickle_read=True)
+inspections_new = board.pin_read("new-data")
+
+# make predictions
+inspections_new["preds"] = v.model.predict(
+    inspections_new.drop(columns=["results", "aka_name", "inspection_date"])
+)
+
+# map results
+inspections_new["preds"] = inspections_new["preds"].map({"PASS": 0, "FAIL": 1})
+inspections_new["results"] = inspections_new["results"].map({"PASS": 0, "FAIL": 1})
+```
+
+# Model info
+
+## Column
+### Row {height="33%"}
+::: {.valuebox}
+`{python} v.description`
+
+`{python} v.model_name`
+:::
+
+::: {.valuebox}
+Model age
+
+`{python} days_old.days` days old
+:::
+
+### Row
+
+Model details
+
+- This model has the prototype:
+
+```
+`{python} v.prototype.construct().schema().get("properties")`
+```
+
+- The model was created by ...
+
+# Model metrics
+
+## Column
+```{python}
+import itables
+
+td = timedelta(weeks = 4)
+metric_set = [accuracy_score, recall_score]
+
+metrics_df = compute_metrics(
+    data = inspections_new,
+    date_var = "inspection_date",
+    period = td,
+    metric_set = metric_set,
+    truth = "results",
+    estimate = "preds"
+  )
+itables.show(metrics_df)
+```
+
+```{python}
+plot_metrics(metrics_df).show()
+```
+
+## Column {.sidebar}
+
+This tab is used to see model performance over time. In this context, _performance_ is the statistical properties of the model, eg, accuracy and recall.
+
+You can add custom information and metrics here.
+
+# Explore validation data
+
+```{python}
+fig = px.histogram(inspections_new, x = "facility_type")
+fig.show()
+```
+
+## Column {.sidebar}
+
+Write your own code to make visualizations or tables with the new validation data, and/or the new predictions.
+
+
+# API visual documentation
+
+## Column
+
+```{python}
+from IPython.display import IFrame
+IFrame('https://colorado.posit.co/rsc/chicago-inspections-python', width=750, height=350)
+```
+---
+
+
+## Column {.sidebar}
+
+Interact directly with your model via its visual documentation, and get `curl` examples.


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index aa4cec2..2c59f4b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -63,7 +63,6 @@ jobs:
           docker-compose up --build -d
           pip freeze > requirements.txt
           make dev
-          cat requirements.txt
         env:
           RSC_LICENSE: ${{ secrets.RSC_LICENSE }}
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
diff --git a/vetiver/tests/snapshots/monitoring_dashboard.qmd b/vetiver/tests/snapshots/monitoring_dashboard.qmd
new file mode 100644
index 0000000..6f61048
--- /dev/null
+++ b/vetiver/tests/snapshots/monitoring_dashboard.qmd
@@ -0,0 +1,136 @@
+---
+title: "Monitoring dashboard"
+format:
+  dashboard:
+    orientation: columns
+logo: https://github.com/rstudio/vetiver-python/blob/main/docs/figures/logo.png?raw=true
+output: asis
+jupyter: python3
+---
+
+```{python}
+#| include: false
+#| tags: [parameters]
+
+# import model and metadata
+import pins
+from IPython.display import display, Markdown, IFrame
+from datetime import datetime, timedelta
+import pandas as pd
+import plotly.express as px
+from sklearn import metrics
+from vetiver import VetiverModel, compute_metrics, plot_metrics
+from sklearn.metrics import recall_score, accuracy_score
+
+raw = "https://colorado.rstudio.com/rsc"
+paths = {"chicago-model-python": "chicago-model-python/"}
+board = pins.board_url(raw, paths, allow_pickle_read=True)
+v = VetiverModel.from_pin(board, "chicago-model-python")
+v_meta = board.pin_meta("chicago-model-python")
+days_old = datetime.today() - datetime.strptime(v_meta.created, "%Y%m%dT%H%M%SZ")
+```
+
+```{python}
+## the next few lines are an example model, here is a place to
+## add any code you need to import new data and make predictions
+
+# import new data to track performance over time
+raw = "https://colorado.rstudio.com/rsc"
+paths = {"new-data": "inspections-new-data/"}
+board = pins.board_url(raw, paths, allow_pickle_read=True)
+inspections_new = board.pin_read("new-data")
+
+# make predictions
+inspections_new["preds"] = v.model.predict(
+    inspections_new.drop(columns=["results", "aka_name", "inspection_date"])
+)
+
+# map results
+inspections_new["preds"] = inspections_new["preds"].map({"PASS": 0, "FAIL": 1})
+inspections_new["results"] = inspections_new["results"].map({"PASS": 0, "FAIL": 1})
+```
+
+# Model info
+
+## Column
+### Row {height="33%"}
+::: {.valuebox}
+`{python} v.description`
+
+`{python} v.model_name`
+:::
+
+::: {.valuebox}
+Model age
+
+`{python} days_old.days` days old
+:::
+
+### Row
+
+Model details
+
+- This model has the prototype:
+
+```
+`{python} v.prototype.construct().schema().get("properties")`
+```
+
+- The model was created by ...
+
+# Model metrics
+
+## Column
+```{python}
+import itables
+
+td = timedelta(weeks = 4)
+metric_set = [accuracy_score, recall_score]
+
+metrics_df = compute_metrics(
+    data = inspections_new,
+    date_var = "inspection_date",
+    period = td,
+    metric_set = metric_set,
+    truth = "results",
+    estimate = "preds"
+  )
+itables.show(metrics_df)
+```
+
+```{python}
+plot_metrics(metrics_df).show()
+```
+
+## Column {.sidebar}
+
+This tab is used to see model performance over time. In this context, _performance_ is the statistical properties of the model, eg, accuracy and recall.
+
+You can add custom information and metrics here.
+
+# Explore validation data
+
+```{python}
+fig = px.histogram(inspections_new, x = "facility_type")
+fig.show()
+```
+
+## Column {.sidebar}
+
+Write your own code to make visualizations or tables with the new validation data, and/or the new predictions.
+
+
+# API visual documentation
+
+## Column
+
+```{python}
+from IPython.display import IFrame
+IFrame('https://colorado.posit.co/rsc/chicago-inspections-python', width=750, height=350)
+```
+---
+
+
+## Column {.sidebar}
+
+Interact directly with your model via its visual documentation, and get `curl` examples.
diff --git a/vetiver/tests/test_server.py b/vetiver/tests/test_server.py
index 1992a3c..1f7e107 100644
--- a/vetiver/tests/test_server.py
+++ b/vetiver/tests/test_server.py
@@ -4,7 +4,6 @@ from vetiver import (
     VetiverAPI,
     vetiver_create_prototype,
     InvalidPTypeError,
-    vetiver_endpoint,
 )
 from pydantic import BaseModel, conint
 from fastapi.testclient import TestClient
@@ -118,10 +117,3 @@ def test_complex_prototype(complex_prototype_model):
 
     with pytest.raises(InvalidPTypeError):
         vetiver_create_prototype(response.json())
-
-
-def test_vetiver_endpoint():
-    url_raw = "http://127.0.0.1:8000/predict/"
-    url = vetiver_endpoint(url_raw)
-
-    assert url == "http://127.0.0.1:8000/predict"
diff --git a/vetiver/tests/test_templates.py b/vetiver/tests/test_templates.py
new file mode 100644
index 0000000..65fd876
--- /dev/null
+++ b/vetiver/tests/test_templates.py
@@ -0,0 +1,12 @@
+import vetiver
+from tempfile import TemporaryDirectory
+from pathlib import Path
+
+
+def test_monitoring_dashboard(snapshot):
+    with TemporaryDirectory() as tempdir:
+        file_path = Path(tempdir, "monitoring_dashboard.qmd")
+        vetiver.monitoring_dashboard(path=file_path)
+        snapshot.snapshot_dir = "./vetiver/tests/snapshots"
+        contents = open(file_path).read()
+        snapshot.assert_match(contents, "monitoring_dashboard.qmd")