Repository: mozilla/PRESC (language: python)

Issue:
Add a unified entrypoint to run the report
We need a better interface to run the report from, and to remove any dependency on the [example dataset/model](https://github.com/mozilla/PRESC/blob/master/presc/report/setup_report.py) we are currently using in the report.

As a part of this, we need a way to share the required inputs (data/model/config) between the notebooks. Looks like the simplest approach would be to serialize and reload separately in each notebook.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 88e3db7b2..52a789f3a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -74,7 +74,11 @@ instance/
 docs/_build/
 
 # Jupyter Book artifacts
-presc/report/_build/
+_build/
+
+# PRESC artifacts
+_context_store.db
+presc_report/
 
 # PyBuilder
 target/
diff --git a/docs/REPORT.md b/docs/REPORT.md
index bfa730844..b97144bb3 100644
--- a/docs/REPORT.md
+++ b/docs/REPORT.md
@@ -1,20 +1,11 @@
 # Report output
 
 The report generated by PRESC is built using [Jupyter Book](https://jupyterbook.org/intro.html).
-Report source is maintained under [`presc/report`](../presc/report/).
-
-To build the report, run:
-
-```bash
-# Hide all the code cells.
-# Currently this has to be done by manually adding tags to each notebook cell.
-python presc/report/add_tags.py
-
-# Build
-jupyter-book build presc/report
-```
-
-The rendered report can be viewed at
-[`presc/report/_build/html/index.html`](../presc/report/_build/html/index.html).
+Report source is maintained under
+[`presc/report/resources`](../presc/report/resources).
 
+An example of how to run the report using the Python API
+can be found in the
+[`sample_report.py`](../examples/report/sample_report.py)
+script.
 
diff --git a/environment.yml b/environment.yml
index edc4638c5..f99460f58 100644
--- a/environment.yml
+++ b/environment.yml
@@ -16,6 +16,7 @@ dependencies:
   - pre-commit=2.6.0
   - pytest=5.4.3
   - pytest-cov=2.10.0
+  - pyyaml=5.4.1
   - twine=3.2.0
   - imbalanced-learn=0.7.0
   - pandas-profiling=2.8.0 
diff --git a/presc/report/setup_report.py b/examples/report/sample_report.py
similarity index 63%
rename from presc/report/setup_report.py
rename to examples/report/sample_report.py
index f30640619..a9f4ccdc0 100644
--- a/presc/report/setup_report.py
+++ b/examples/report/sample_report.py
@@ -1,26 +1,25 @@
-"""Prepare the inputs to the PRESC report."""
+"""Example script demonstrating how to run the PRESC report."""
 
 import pandas as pd
 
 from presc.dataset import Dataset
 from presc.model import ClassificationModel
+from presc.report.runner import ReportRunner
 
 from sklearn.pipeline import Pipeline
 from sklearn.preprocessing import StandardScaler
 from sklearn.svm import SVC
 from sklearn.model_selection import ShuffleSplit
 
-# Better quality plots
-from IPython.display import set_matplotlib_formats
-
-import yaml
 import sys
+from pathlib import Path
 
-set_matplotlib_formats("svg")
+THIS_DIR = Path(__file__).parent
+DATASET_DIR = THIS_DIR / ".." / ".." / "datasets" / "winequality.csv"
 
 # Load the dataset.
 
-df = pd.read_csv("../../datasets/winequality.csv")
+df = pd.read_csv(DATASET_DIR)
 df = df.drop(columns=["quality"])
 dataset = Dataset(df, label_col="recommend")
 
@@ -34,9 +33,12 @@
 model = Pipeline([("scaler", StandardScaler()), ("clf", SVC(class_weight="balanced"))])
 cm = ClassificationModel(model, train_dataset, retrain_now=True)
 
-config_filename = "./report_config.yml"
+config_filename = None
 if len(sys.argv) == 2:
     config_filename = sys.argv[1]
 
-with open(config_filename) as f:
-    config = yaml.load(f, Loader=yaml.FullLoader)
+presc_report = ReportRunner(config_filepath=config_filename)
+presc_report.run(model=cm, test_dataset=test_dataset)
+
+print(f"The report is available at {presc_report.report_html}")
+presc_report.open()
diff --git a/presc/report/README.md b/presc/report/README.md
new file mode 100644
index 000000000..9ec79c20c
--- /dev/null
+++ b/presc/report/README.md
@@ -0,0 +1,4 @@
+### Note:
+
+Prior to committing changes to the report notebook, make sure they have all the
+necessary tags by running the [`add_tags.py`](../../scripts/add_tags.py) script.
diff --git a/presc/report/report_config.yml b/presc/report/default_config.yml
similarity index 100%
rename from presc/report/report_config.yml
rename to presc/report/default_config.yml
diff --git a/presc/report/_config.yml b/presc/report/resources/_config.yml
similarity index 93%
rename from presc/report/_config.yml
rename to presc/report/resources/_config.yml
index b4fabd0da..f4e194900 100644
--- a/presc/report/_config.yml
+++ b/presc/report/resources/_config.yml
@@ -1,5 +1,6 @@
 title: PRESC Report
 author: ""
+only_build_toc_files: true
 
 # Force re-execution of notebooks on each build.
 execute:
diff --git a/presc/report/spatial_distribution.ipynb b/presc/report/resources/_spatial_distribution.ipynb
similarity index 100%
rename from presc/report/spatial_distribution.ipynb
rename to presc/report/resources/_spatial_distribution.ipynb
diff --git a/presc/report/_toc.yml b/presc/report/resources/_toc.yml
similarity index 100%
rename from presc/report/_toc.yml
rename to presc/report/resources/_toc.yml
diff --git a/presc/report/conditional_distribution.ipynb b/presc/report/resources/conditional_distribution.ipynb
similarity index 79%
rename from presc/report/conditional_distribution.ipynb
rename to presc/report/resources/conditional_distribution.ipynb
index 06645bee9..f868f9bc9 100644
--- a/presc/report/conditional_distribution.ipynb
+++ b/presc/report/resources/conditional_distribution.ipynb
@@ -19,7 +19,12 @@
    },
    "outputs": [],
    "source": [
-    "from presc.evaluations.conditional_distribution import ConditionalDistribution"
+    "from presc.evaluations.conditional_distribution import ConditionalDistribution\n",
+    "from presc.report.runner import Context\n",
+    "\n",
+    "# Better quality plots\n",
+    "from IPython.display import set_matplotlib_formats\n",
+    "set_matplotlib_formats(\"svg\")"
    ]
   },
   {
@@ -32,9 +37,11 @@
    },
    "outputs": [],
    "source": [
-    "# TOOD: how to share state between notebooks\n",
-    "\n",
-    "%run setup_report.py"
+    "# Load inputs from data store in cwd.\n",
+    "ctx = Context()\n",
+    "test_dataset = ctx.test_dataset\n",
+    "cm = ctx.model\n",
+    "config = ctx.config"
    ]
   },
   {
@@ -60,7 +67,7 @@
    },
    "outputs": [],
    "source": [
-    "e.display(colnames=[\"pH\", \"residual sugar\"])"
+    "e.display()"
    ]
   }
  ],
diff --git a/presc/report/conditional_metric.ipynb b/presc/report/resources/conditional_metric.ipynb
similarity index 81%
rename from presc/report/conditional_metric.ipynb
rename to presc/report/resources/conditional_metric.ipynb
index 6a6cd98cc..f5c85d50a 100644
--- a/presc/report/conditional_metric.ipynb
+++ b/presc/report/resources/conditional_metric.ipynb
@@ -19,7 +19,12 @@
    },
    "outputs": [],
    "source": [
-    "from presc.evaluations.conditional_metric import ConditionalMetric"
+    "from presc.evaluations.conditional_metric import ConditionalMetric\n",
+    "from presc.report.runner import Context\n",
+    "\n",
+    "# Better quality plots\n",
+    "from IPython.display import set_matplotlib_formats\n",
+    "set_matplotlib_formats(\"svg\")"
    ]
   },
   {
@@ -32,9 +37,11 @@
    },
    "outputs": [],
    "source": [
-    "# TOOD: how to share state between notebooks\n",
-    "\n",
-    "%run setup_report.py"
+    "# Load inputs from data store in cwd.\n",
+    "ctx = Context()\n",
+    "test_dataset = ctx.test_dataset\n",
+    "cm = ctx.model\n",
+    "config = ctx.config"
    ]
   },
   {
diff --git a/presc/report/landing.ipynb b/presc/report/resources/landing.ipynb
similarity index 76%
rename from presc/report/landing.ipynb
rename to presc/report/resources/landing.ipynb
index 631e4a010..5af677c73 100644
--- a/presc/report/landing.ipynb
+++ b/presc/report/resources/landing.ipynb
@@ -17,7 +17,13 @@
    },
    "outputs": [],
    "source": [
-    "from sklearn.metrics import classification_report, plot_confusion_matrix"
+    "import yaml\n",
+    "from sklearn.metrics import classification_report, plot_confusion_matrix\n",
+    "from presc.report.runner import Context\n",
+    "\n",
+    "# Better quality plots\n",
+    "from IPython.display import set_matplotlib_formats\n",
+    "set_matplotlib_formats(\"svg\")"
    ]
   },
   {
@@ -30,16 +36,11 @@
    },
    "outputs": [],
    "source": [
-    "# Load the inputs to the report.\n",
-    "#\n",
-    "# This creates:\n",
-    "# - the Dataset instance `dataset`\n",
-    "# - the ClassificationModel instance `cm`\n",
-    "# - a dict of config options `config`.\n",
-    "#\n",
-    "# Imports are also all managed from here.\n",
-    "\n",
-    "%run setup_report.py"
+    "# Load inputs from data store in cwd.\n",
+    "ctx = Context()\n",
+    "test_dataset = ctx.test_dataset\n",
+    "dataset = test_dataset\n",
+    "cm = ctx.model"
    ]
   },
   {
@@ -86,10 +87,10 @@
    },
    "outputs": [],
    "source": [
-    "print(f\"Training set size: {train_dataset.size:,}\")\n",
+    "#print(f\"Training set size: {train_dataset.size:,}\")\n",
     "print(\n",
     "    f\"Test set size: {test_dataset.size:,}\" +\n",
-    "    f\" ({test_dataset.size / train_dataset.size:.1%})\"\n",
+    "    f\" ({test_dataset.size / dataset.size:.1%})\"\n",
     ")"
    ]
   },
@@ -145,6 +146,26 @@
    "source": [
     "_ = plot_confusion_matrix(cm.classifier, test_dataset.features, test_dataset.labels)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "remove-input"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "print(yaml.dump(ctx.config))"
+   ]
   }
  ],
  "metadata": {
diff --git a/presc/report/references.bib b/presc/report/resources/references.bib
similarity index 100%
rename from presc/report/references.bib
rename to presc/report/resources/references.bib
diff --git a/presc/report/runner.py b/presc/report/runner.py
new file mode 100644
index 000000000..e3471a0aa
--- /dev/null
+++ b/presc/report/runner.py
@@ -0,0 +1,296 @@
+from pathlib import Path
+import shelve
+from tempfile import TemporaryDirectory
+import shutil
+import subprocess
+import warnings
+import webbrowser
+
+import yaml
+
+from presc.utils import PrescError
+
+# Path to the report source dir
+REPORT_SOURCE_PATH = Path(__file__).parent / "resources"
+REPORT_OUTPUT_DIR = "presc_report"
+REPORT_EXECUTION_DIR = "_exec"
+REPORT_MAIN_PAGE = "report.html"
+SPHINX_INDEX_PAGE = Path("_build") / "html" / "index.html"
+JB_CLEAN_LOG = "jupyterbook_clean.log"
+JB_BUILD_LOG = "jupyterbook_build.log"
+# Path to the store for the inputs to the report, relative to the execution dir
+# for the report.
+CONTEXT_STORE_BASENAME = "_context_store"
+# Path to the default config file
+DEFAULT_CONFIG_PATH = Path(__file__).parent / "default_config.yml"
+
+
+def load_config(config_filepath=None):
+    """Loads the configuration from the given file.
+
+    config_filepath: path to a YAML file listing PRESC config options
+
+    Returns the config values as a dict.
+    """
+    if not config_filepath:
+        config_filepath = DEFAULT_CONFIG_PATH
+    try:
+        with open(config_filepath) as f:
+            config = yaml.load(f, Loader=yaml.FullLoader)
+        return config
+    except IOError as e:
+        msg = f"Error opening config file {config_filepath}"
+        raise PrescError(msg) from e
+    except yaml.YAMLError as e:
+        msg = f"Error parsing config file {config_filepath}"
+        raise PrescError(msg) from e
+
+
+class ReportRunner:
+    """Main entrypoint to run the PRESC report for the given modeling inputs.
+
+    The report is written to <output_path>/presc_report. If this dir already
+    exists, it will be overwritten.
+
+    To generate the report:
+    ```
+    pr = ReportRunner()
+    pr.run(...)
+    ```
+
+    The path to the report is accessible as `pr.report_html` and will open in
+    the default browser by calling `pr.open()`
+
+    output_path: path to the dir where the report will be written. If not
+        specified, defaults to the current working dir.
+    execution_path: path from which the report is built. If not specified, a
+        temporary dir is used.
+    config_filepath: path to a YAML file listing config options to govern the
+        report behaviour. If not specified, the default config is used.
+    """
+
+    def __init__(self, output_path=".", execution_path=None, config_filepath=None):
+        # The main config options, as a dict.
+        self.config = load_config(config_filepath)
+        # Path where the report output is written.
+        # Outputs are nested in a subdir.
+        self.output_path = Path(output_path) / REPORT_OUTPUT_DIR
+        self.output_path.mkdir(parents=True, exist_ok=True)
+        # Path where the report is built from.
+        # The report source files are copied here, and the model inputs are
+        # written to a data store.
+        # If missing, a temp dir will be used on execution.
+        self.execution_path = None
+        if execution_path is not None:
+            self.execution_path = Path(execution_path) / REPORT_EXECUTION_DIR
+
+        # Build artifacts:
+        # The main entry page for the report.
+        self.report_main_page = self.output_path / SPHINX_INDEX_PAGE
+        # Log files for jupyter-book execution.
+        self.jb_clean_log = self.output_path / JB_CLEAN_LOG
+        self.jb_build_log = self.output_path / JB_BUILD_LOG
+        # The main page will be linked to the top-level output dir, if possible.
+        self._linked_main_page = self.output_path / REPORT_MAIN_PAGE
+        # Cache the process results from running jupyter-book commands for
+        # debugging.
+        self._jb_clean_result = None
+        self._jb_build_result = None
+
+    def _presc_artifacts(self):
+        """List of paths to remove from the top-level output dir on clean."""
+        return [self._linked_main_page, self.jb_clean_log, self.jb_build_log]
+
+    def run(self, model, test_dataset, config_filepath=None, clean=True):
+        """Runs the PRESC report for the given modeling inputs.
+
+        The report is written to <output_path>/presc_report. If this dir already
+        exists, it will be overwritten.
+
+        model: a pre-trained ClassificationModel instance to evaluate
+        test_dataset: a test Dataset instance used to evaluate model performance
+        config_filepath: path to a YAML file listing config options to govern
+            the report behaviour. If specified, this overrides the current
+            config.
+        clean: should previous outputs be cleaned? Default: True
+        """
+        if config_filepath:
+            config = load_config(config_filepath)
+        else:
+            config = self.config
+
+        if clean:
+            self.clean()
+
+        tmpdir = None
+        exec_path = None
+        if self.execution_path:
+            # If using a user-defined execution path, need to make sure
+            # it doesn't exist for `shutil.copytree` to work.
+            # Note that this will only remove the nested subdir, not the actual
+            # user-specified dir.
+            if self.execution_path.exists():
+                shutil.rmtree(self.execution_path)
+            exec_path = self.execution_path
+        else:
+            # Create a temp dir to run the build from.
+            # We set up the temp dir here rather than using jupyter-book's
+            # `run_in_temp` option so that we have access to the temp path.
+            tmpdir = TemporaryDirectory()
+            exec_path = Path(tmpdir.name) / REPORT_EXECUTION_DIR
+
+        # Copy the report source files to the execution dir and
+        # execute from there. The data store for the inputs is saved to
+        # the same dir. That way, since the notebooks' working dir on execution
+        # is set to where they are located by jupyter-book, they can find
+        # the data store without needed to know the calling path.
+        try:
+            shutil.copytree(REPORT_SOURCE_PATH, exec_path)
+        except shutil.Error as e:
+            msg = f"Failed to copy report source to execution dir {exec_path}"
+            raise PrescError(msg) from e
+        # Write the inputs to the data store.
+        ctx = Context(store_dir=exec_path)
+        ctx.store_inputs(model=model, test_dataset=test_dataset, config=config)
+
+        # Build the report.
+        self._run_jb_build(exec_path)
+        if tmpdir:
+            tmpdir.cleanup()
+
+        # The build should have created index.html at the `report_main_page`
+        # path.
+        if self.report_main_page.exists():
+            # Symlink the main page to the top level for convenience.
+            try:
+                main_page_target = self.report_main_page.relative_to(
+                    self._linked_main_page.parent
+                )
+                self._linked_main_page.symlink_to(main_page_target)
+            except OSError:
+                pass
+        else:
+            msg = f"The expected report main page {self.report_main_page} does not appear to exist."
+            msg += " There may have been an error generating the report."
+            msg += f" Output is written to {self.jb_build_log}"
+            warnings.warn(msg)
+
+    @property
+    def report_html(self):
+        """The main page of the HTML report."""
+        # Return symlink, if available, for a more user-friendly experience.
+        if self._linked_main_page.exists():
+            # Resolve to an absolute path up to the symlink.
+            report_path = (
+                self._linked_main_page.parent.resolve() / self._linked_main_page.name
+            )
+        else:
+            report_path = self.report_main_page.resolve()
+        if not report_path.exists():
+            msg = "Report file does not appear to exist."
+            msg += " Make sure the report has already been built."
+            raise AttributeError(msg)
+
+        return str(report_path)
+
+    def open(self):
+        """Open the report in the default web browser."""
+        webbrowser.open_new_tab(f"file://{self.report_html}")
+
+    def clean(self):
+        """Remove artifacts from a previous run, if any."""
+        for p in self._presc_artifacts():
+            try:
+                p.unlink()
+            except FileNotFoundError:
+                pass
+        self._run_jb_clean()
+
+    def _run_jb_clean(self):
+        """Run `jupyter-book clean`."""
+        with open(self.jb_clean_log, "w") as outfile:
+            result = subprocess.run(
+                ["jupyter-book", "clean", self.output_path],
+                stdout=outfile,
+                stderr=subprocess.STDOUT,
+            )
+        if result.returncode > 0:
+            msg = f"`jupyter-book clean {self.output_path} did not succeed."
+            msg += f" Output is written to {self.jb_clean_log}"
+            warnings.warn(msg)
+
+        self._jb_clean_result = result
+
+    def _run_jb_build(self, input_path):
+        """Run `jupyter-book build` on the given path."""
+        with open(self.jb_build_log, "w") as outfile:
+            result = subprocess.run(
+                [
+                    "jupyter-book",
+                    "build",
+                    "--path-output",
+                    str(self.output_path.resolve()),
+                    str(input_path.resolve()),
+                ],
+                stdout=outfile,
+                stderr=subprocess.STDOUT,
+            )
+        if result.returncode > 0:
+            msg = f"`jupyter-book build {input_path} did not succeed."
+            msg += f" Output is written to {self.jb_build_log}"
+            warnings.warn(msg)
+
+        self._jb_build_result = result
+
+
+class Context:
+    """Persistent data store for sharing report inputs across notebooks.
+
+    Note that the store implementation does not support concurrent access. It is
+    up to the caller to ensure that multiple instances each have a unique store
+    location.
+
+    store_dir: the dir to contain the data store, implemented as one or more
+        database files. If not specified, defaults to the current working dir.
+    """
+
+    def __init__(self, store_dir="."):
+        store_path = Path(store_dir) / CONTEXT_STORE_BASENAME
+        self._store_path = str(store_path.resolve())
+
+    def store_inputs(self, model=None, test_dataset=None, config=None):
+        """Write the report inputs to the data store.
+
+        Any existing values will be overwritten.
+
+        model: a ClassificationModel instance
+        test_dataset: a Dataset instance
+        config: a dict of config options
+        """
+        with shelve.open(self._store_path) as ctx:
+            if model:
+                ctx["model"] = model
+            if test_dataset:
+                ctx["test_dataset"] = test_dataset
+            if config:
+                ctx["config"] = config
+
+    def _get(self, key):
+        try:
+            with shelve.open(self._store_path, flag="r") as ctx:
+                val = ctx[key]
+            return val
+        except KeyError:
+            raise PrescError(f"Could not find stored value for '{key}'")
+
+    @property
+    def model(self):
+        return self._get("model")
+
+    @property
+    def test_dataset(self):
+        return self._get("test_dataset")
+
+    @property
+    def config(self):
+        return self._get("config")
diff --git a/presc/report/add_tags.py b/scripts/add_tags.py
similarity index 89%
rename from presc/report/add_tags.py
rename to scripts/add_tags.py
index 74764aca8..b063bc14c 100644
--- a/presc/report/add_tags.py
+++ b/scripts/add_tags.py
@@ -4,7 +4,7 @@
 from pathlib import Path
 
 TAGS_ALL_CELLS = ["remove-input"]
-REPORT_DIR = Path(__file__).parent
+REPORT_DIR = Path(__file__).parent / ".." / "presc" / "report" / "resources"
 
 notebooks = REPORT_DIR.glob("*.ipynb")
 


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 3f9b51fa8..ec6aecff7 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,6 +2,14 @@
 
 import pytest
 from pandas import read_pickle
+from sklearn.pipeline import Pipeline
+from sklearn.linear_model import LogisticRegression
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer, make_column_selector
+from numpy import arange
+
+from presc.model import ClassificationModel
+from presc.dataset import Dataset
 
 DATASET_DF_PATH = Path(__file__).parent / "fixtures" / "dataset.pkl"
 
@@ -9,3 +17,42 @@
 @pytest.fixture
 def dataset_df():
     return read_pickle(DATASET_DF_PATH)
+
+
+@pytest.fixture
+def in_test_set():
+    # Deterministic train/test split for testing
+    # This is used with the synthetic `dataset_df` with 100 rows
+    rows = arange(100)
+    return (rows < 10) | (rows >= 90)
+
+
+@pytest.fixture
+def train_dataset(dataset_df, in_test_set):
+    return Dataset(dataset_df[~in_test_set], label_col="label")
+
+
+@pytest.fixture
+def test_dataset(dataset_df, in_test_set):
+    return Dataset(dataset_df[in_test_set], label_col="label")
+
+
+@pytest.fixture
+def dataset_transform():
+    return ColumnTransformer(
+        [
+            ("scaler", StandardScaler(), make_column_selector(dtype_include="number")),
+            ("encoder", OneHotEncoder(), make_column_selector(dtype_include=object)),
+        ],
+        remainder="passthrough",
+    )
+
+
+@pytest.fixture
+def pipeline_classifier(dataset_transform):
+    return Pipeline([("scaler", dataset_transform), ("clf", LogisticRegression())])
+
+
+@pytest.fixture
+def classification_model(pipeline_classifier, train_dataset):
+    return ClassificationModel(pipeline_classifier, train_dataset, retrain_now=True)
diff --git a/tests/test_model.py b/tests/test_model.py
index e47712305..e4095fa9a 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -1,56 +1,18 @@
 import pytest
 from sklearn.pipeline import Pipeline
-from sklearn.linear_model import LogisticRegression, RidgeClassifier
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer, make_column_selector
+from sklearn.linear_model import RidgeClassifier
 from sklearn.exceptions import NotFittedError
-from numpy import arange
 from pandas import Series, DataFrame
 
 from presc.model import ClassificationModel
-from presc.dataset import Dataset
 from presc.utils import PrescError
 
 
-@pytest.fixture
-def in_test_set():
-    # Deterministic train/test split for testing
-    # This is used with the synthetic `dataset_df` with 100 rows
-    rows = arange(100)
-    return (rows < 10) | (rows >= 90)
-
-
-@pytest.fixture
-def train_dataset(dataset_df, in_test_set):
-    return Dataset(dataset_df[~in_test_set], label_col="label")
-
-
-@pytest.fixture
-def test_dataset(dataset_df, in_test_set):
-    return Dataset(dataset_df[in_test_set], label_col="label")
-
-
 def pipeline_is_trained(pl):
     # Quick check that the classifier has fitted params
     return hasattr(pl.named_steps["clf"], "coef_")
 
 
-@pytest.fixture
-def dataset_transform():
-    return ColumnTransformer(
-        [
-            ("scaler", StandardScaler(), make_column_selector(dtype_include="number")),
-            ("encoder", OneHotEncoder(), make_column_selector(dtype_include=object)),
-        ],
-        remainder="passthrough",
-    )
-
-
-@pytest.fixture
-def pipeline_classifier(dataset_transform):
-    return Pipeline([("scaler", dataset_transform), ("clf", LogisticRegression())])
-
-
 @pytest.fixture
 def pipeline_classifier_noprobs(dataset_transform):
     # Example of a classifier that doesn't expose `predict_proba`
diff --git a/tests/test_runner.py b/tests/test_runner.py
new file mode 100644
index 000000000..32784b48c
--- /dev/null
+++ b/tests/test_runner.py
@@ -0,0 +1,305 @@
+import os
+from subprocess import CompletedProcess
+from pathlib import Path
+import shutil
+
+from pandas.testing import assert_frame_equal
+import pytest
+from sklearn.pipeline import Pipeline
+import yaml
+
+from presc.report.runner import Context, ReportRunner, load_config
+from presc.utils import PrescError
+
+
+@pytest.fixture
+def webbrowser_patched(monkeypatch):
+    import webbrowser
+
+    def _print_url(url, *args, **kwargs):
+        print(url)
+
+    monkeypatch.setattr(webbrowser, "open", _print_url)
+    monkeypatch.setattr(webbrowser, "open_new", _print_url)
+    monkeypatch.setattr(webbrowser, "open_new_tab", _print_url)
+
+
+def test_context(tmp_path, test_dataset, classification_model):
+    ctx = Context(store_dir=tmp_path)
+    ctx.store_inputs(test_dataset=test_dataset)
+
+    # Data store was created.
+    # The number of files and extensions are platform-dependent.
+    ctx_store_paths = [p for p in tmp_path.iterdir() if p.stem == "_context_store"]
+    assert len(ctx_store_paths) > 0
+
+    # A new context instance should load from the same data store.
+    ctx2 = Context(store_dir=tmp_path)
+    td2 = ctx2.test_dataset
+    assert_frame_equal(td2.df, test_dataset.df)
+    with pytest.raises(PrescError):
+        ctx2.model
+
+    # Adding additional data to the store.
+    config = {"bins": 10}
+    ctx2.store_inputs(config=config, model=classification_model)
+
+    assert ctx2.config == config
+    assert isinstance(ctx2.model.classifier, Pipeline)
+    assert_frame_equal(ctx.test_dataset.df, test_dataset.df)
+
+    assert ctx.config == config
+    assert isinstance(ctx.model.classifier, Pipeline)
+    assert_frame_equal(ctx.test_dataset.df, test_dataset.df)
+
+
+def test_config(tmp_path):
+    dummy_config = {"a": 1, "b": {"c": ["x", "y", "z"], "d": True}}
+    yaml_file = tmp_path / "config.yaml"
+    with open(yaml_file, "w") as f:
+        yaml.dump(dummy_config, f)
+    not_yaml_file = tmp_path / "config.txt"
+    with open(not_yaml_file, "w") as f:
+        f.write("\t" + str(dummy_config))
+
+    loaded_config = load_config(yaml_file)
+    assert loaded_config == dummy_config
+    with pytest.raises(PrescError):
+        load_config(not_yaml_file)
+    with pytest.raises(PrescError):
+        load_config(tmp_path / "xyz.yaml")
+
+    default_config = load_config()
+    assert isinstance(default_config, dict)
+
+
+def test_report_runner(tmp_path):
+    # Check paths are initialized correctly by the runner
+    os.chdir(tmp_path)
+    rr = ReportRunner()
+    assert str(rr.output_path.parent.resolve()) == os.getcwd()
+    assert rr.output_path.exists()
+    assert rr.execution_path is None
+    assert isinstance(rr.config, dict)
+
+    out_path = tmp_path / "abc" / "out"
+    exec_path = tmp_path / "exec"
+    conf_path = tmp_path / "conf.yaml"
+    dummy_conf = {"a": 1}
+    with open(conf_path, "w") as f:
+        yaml.dump(dummy_conf, f)
+
+    rr = ReportRunner(
+        output_path=out_path, execution_path=exec_path, config_filepath=conf_path
+    )
+    assert rr.output_path.parent == out_path
+    assert rr.output_path.exists()
+    assert rr.execution_path.parent == exec_path
+    assert rr.config == dummy_conf
+
+
+def test_run_report(
+    tmp_path, classification_model, test_dataset, webbrowser_patched, capsys
+):
+    out_path_run = tmp_path / "test_run"
+    exec_path_run = tmp_path / "test_exec"
+    rr = ReportRunner(output_path=out_path_run, execution_path=exec_path_run)
+    # Run a report on the test data. This will take ~10 seconds
+    rr.run(model=classification_model, test_dataset=test_dataset)
+
+    # Check top-level output files exist and paths resolve
+    assert isinstance(rr._jb_build_result, CompletedProcess)
+    assert rr._jb_build_result.returncode == 0
+    with open(rr.jb_build_log) as f:
+        build_log = f.read()
+    assert build_log.startswith("Running Jupyter-Book")
+    assert "Finished generating HTML" in build_log
+    assert "error" not in build_log.lower()
+    assert "failed" not in build_log.lower()
+
+    assert isinstance(rr._jb_clean_result, CompletedProcess)
+    assert rr._jb_clean_result.returncode == 0
+    # File should be empty as the output folder did not contain
+    # a previous `_build` dir.
+    assert rr.jb_clean_log.exists()
+
+    assert rr.report_main_page.exists()
+    # Link may not exist as it is platform-dependent.
+    if rr._linked_main_page.exists():
+        assert rr._linked_main_page.resolve() == rr.report_main_page.resolve()
+        report_path = Path(rr.report_html)
+        assert rr._linked_main_page.parent.resolve() == report_path.parent
+        assert rr._linked_main_page.name == report_path.name
+    else:
+        assert rr.report_html == str(rr.report_main_page.resolve())
+
+    # Opening in the browser: check the URL that is passed.
+    capsys.readouterr()
+    rr.open()
+    url = capsys.readouterr().out.strip()
+    assert url.startswith("file://")
+    assert url.endswith(rr.report_html)
+
+    # Check execution dir and output dir contains relevant files.
+    assert len(list(rr.execution_path.glob("_context_store*"))) > 0
+    assert (rr.execution_path / "_config.yml").exists()
+    with open(rr.execution_path / "_toc.yml") as f:
+        toc = [x.strip() for x in f.readlines()]
+        notebooks = [x[8:] for x in toc if x.startswith("- file: ")]
+    for nb in notebooks:
+        assert (rr.execution_path / f"{nb}.ipynb").exists()
+        assert (rr.report_main_page.parent / f"{nb}.html").exists()
+
+    # Test cleaning on existing report output.
+    rr.clean()
+    assert not rr.jb_build_log.exists()
+    assert not rr._linked_main_page.exists()
+    assert not rr.report_main_page.parent.exists()
+    assert isinstance(rr._jb_clean_result, CompletedProcess)
+    assert rr._jb_clean_result.returncode == 0
+    with open(rr.jb_clean_log) as f:
+        clean_log = f.read()
+    assert "Your _build dir" in clean_log
+    assert "error" not in clean_log.lower()
+    assert "failed" not in clean_log.lower()
+
+    # Test rerunning report with the same runner.
+    # Since the execution dir already exists, this tests that is get cleaned
+    # successfully prior to running.
+    rr.run(model=classification_model, test_dataset=test_dataset, clean=False)
+    assert rr.report_main_page.parent.exists()
+    assert rr._jb_build_result.returncode == 0
+
+
+def test_run_report_tmp_exec_dir(tmp_path, classification_model, test_dataset):
+    out_path_run = tmp_path / "test_run"
+    rr = ReportRunner(output_path=out_path_run)
+    # Run a report using the using default temp execution dir.
+    # This will take ~10 seconds
+    rr.run(model=classification_model, test_dataset=test_dataset)
+
+    # Just check that it worked.
+    assert rr.report_main_page.exists()
+    assert rr._jb_build_result.returncode == 0
+
+
+def test_run_report_error_notebook(tmp_path, pipeline_classifier, test_dataset):
+    # Error encountered while running the notebooks
+    out_path_run = tmp_path / "test_run"
+    exec_path_run = tmp_path / "test_exec"
+    rr = ReportRunner(output_path=out_path_run, execution_path=exec_path_run)
+    # pipeline_classifier is not a valid ClassificationModel instance
+    rr.run(model=pipeline_classifier, test_dataset=test_dataset)
+
+    # jupyter-book build job succeeded even though notebooks didn't
+    assert rr._jb_build_result.returncode == 0
+    with open(rr.jb_build_log) as f:
+        build_log = f.read()
+    assert build_log.startswith("Running Jupyter-Book")
+    assert "Finished generating HTML" in build_log
+    # Log file mentions failure
+    assert "failed" in build_log.lower()
+
+    # Report was created
+    assert rr.report_main_page.exists()
+    # Execution dir and output dir contains relevant files.
+    assert len(list(rr.execution_path.glob("_context_store*"))) > 0
+    with open(rr.execution_path / "_toc.yml") as f:
+        toc = [x.strip() for x in f.readlines()]
+        notebooks = [x[8:] for x in toc if x.startswith("- file: ")]
+    for nb in notebooks:
+        assert (rr.execution_path / f"{nb}.ipynb").exists()
+        assert (rr.report_main_page.parent / f"{nb}.html").exists()
+    # Error message shows in notebooks
+    with open(rr.report_main_page.parent / "landing.html") as f:
+        landing_html = f.read()
+    assert "AttributeError" in landing_html
+
+
+def test_run_report_error_build(
+    tmp_path, classification_model, test_dataset, monkeypatch
+):
+    # Missing report page: jupyter-book build job fails
+    from presc.report import runner
+
+    mock_report_source = tmp_path / "mock_report_source"
+    shutil.copytree(runner.REPORT_SOURCE_PATH, mock_report_source)
+    with open(mock_report_source / "_toc.yml", "a") as f:
+        f.write("- file: missing_notebook\n")
+    monkeypatch.setattr(runner, "REPORT_SOURCE_PATH", mock_report_source)
+
+    out_path_run = tmp_path / "test_run"
+    rr = ReportRunner(output_path=out_path_run)
+    # run() function generates warnings
+    with pytest.warns(UserWarning) as warning_records:
+        rr.run(model=classification_model, test_dataset=test_dataset)
+    assert len(warning_records) == 2
+    # Warning from build function
+    first_warning = warning_records[0].message.args[0]
+    assert "jupyter-book build" in first_warning
+    assert "did not succeed" in first_warning
+    assert str(rr.jb_build_log) in first_warning
+    # Warning from run function
+    second_warning = warning_records[1].message.args[0]
+    assert "expected report main page" in second_warning
+    assert "error generating" in second_warning
+    assert str(rr.jb_build_log) in second_warning
+
+    # Build artifacts mention error
+    assert rr._jb_build_result.returncode > 0
+    with open(rr.jb_build_log) as f:
+        build_log = f.read()
+    assert build_log.startswith("Running Jupyter-Book")
+    assert "Finished generating HTML" not in build_log
+    assert "error in building" in build_log.lower()
+    assert "RuntimeError" in build_log
+
+    # Report was not produced
+    assert not rr.report_main_page.exists()
+    assert not rr._linked_main_page.exists()
+    with pytest.raises(AttributeError):
+        rr.report_html
+    with pytest.raises(AttributeError):
+        rr.open()
+
+
+def test_run_report_error_config(tmp_path, classification_model, test_dataset):
+    # Passing in config file to run() overrides default config.
+    # Fake config causes error.
+    out_path_run = tmp_path / "test_run"
+    exec_path_run = tmp_path / "test_exec"
+    fake_config_file = tmp_path / "fake_config.yaml"
+    with open(fake_config_file, "w") as f:
+        yaml.dump({"fake_option": 123}, f)
+
+    rr = ReportRunner(output_path=out_path_run, execution_path=exec_path_run)
+    rr.run(
+        model=classification_model,
+        test_dataset=test_dataset,
+        config_filepath=fake_config_file,
+    )
+
+    # jupyter-book build job succeeded even though notebooks didn't
+    assert rr._jb_build_result.returncode == 0
+    with open(rr.jb_build_log) as f:
+        build_log = f.read()
+    assert build_log.startswith("Running Jupyter-Book")
+    assert "Finished generating HTML" in build_log
+    # Log file mentions failure
+    assert "failed" in build_log.lower()
+
+    # Report was created
+    assert rr.report_main_page.exists()
+    # Execution dir and output dir contains relevant files.
+    assert len(list(rr.execution_path.glob("_context_store*"))) > 0
+    with open(rr.execution_path / "_toc.yml") as f:
+        toc = [x.strip() for x in f.readlines()]
+        notebooks = [x[8:] for x in toc if x.startswith("- file: ")]
+    for nb in notebooks:
+        assert (rr.execution_path / f"{nb}.ipynb").exists()
+        assert (rr.report_main_page.parent / f"{nb}.html").exists()
+    # Error message shows in notebooks
+    with open(rr.report_main_page.parent / "conditional_metric.html") as f:
+        eval_html = f.read()
+    assert "Error" in eval_html
+    assert "Traceback" in eval_html