Repository: pvlib/pvanalytics

Issue:
stale_values_diff and interpolation_diff : option to label all values
Consider adding an option to label all values in a stale / interpolation window as stale, rather than only labelling the end value in a window.

Discussion:

Patch:
diff --git a/docs/api.rst b/docs/api.rst
index c51f3cd..04935e2 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -94,6 +94,18 @@ spacing, time-shifts, and time zone validation.
 
    quality.time.spacing
 
+Utilities
+---------
+
+The :py:mod:`quality.util` module contains general-purpose/utility
+functions for building your own quality checks.
+
+.. autosummary::
+   :toctree: generated/
+
+   quality.util.check_limits
+   quality.util.daily_min
+
 Weather
 -------
 
diff --git a/pvanalytics/quality/gaps.py b/pvanalytics/quality/gaps.py
index a08bf03..61d78e9 100644
--- a/pvanalytics/quality/gaps.py
+++ b/pvanalytics/quality/gaps.py
@@ -35,13 +35,29 @@ def _all_close_to_first(x, rtol=1e-5, atol=1e-8):
     return np.allclose(a=x, b=x[0], rtol=rtol, atol=atol)
 
 
-def stale_values_diff(x, window=3, rtol=1e-5, atol=1e-8):
+def _backfill_window(endpoints, window):
+    # propagate Trues in `endpoints` back `window` periods.  This
+    # makes Trues fill the entire window, rather than just marking the
+    # right endpoint of each window.
+    #
+    # `endpoints` must be the output of Series.rolling with `label='right'`
+    flags = endpoints
+    while window > 0:
+        window = window - 1
+        flags = flags | endpoints.shift(-window).fillna(False)
+    return flags
+
+
+def stale_values_diff(x, window=3, rtol=1e-5, atol=1e-8, label_all=False):
     """Identify stale values in the data.
 
     For a window of length N, the last value (index N-1) is considered
     stale if all values in the window are close to the first value
     (index 0).
 
+    Parameters `rtol` and `atol` have the same meaning as in
+    :py:func:`numpy.allclose`.
+
     Parameters
     ----------
     x : Series
@@ -53,9 +69,9 @@ def stale_values_diff(x, window=3, rtol=1e-5, atol=1e-8):
         relative tolerance for detecting a change in data values
     atol : float, default 1e-8
         absolute tolerance for detecting a change in data values
-
-    Parameters rtol and atol have the same meaning as in
-    numpy.allclose
+    label_all : bool, default False
+        Whether to label all values in the window. If False, then only
+        the right endpoint of the window is labeled.
 
     Returns
     -------
@@ -84,16 +100,21 @@ def stale_values_diff(x, window=3, rtol=1e-5, atol=1e-8):
         raw=True,
         kwargs={'rtol': rtol, 'atol': atol}
     ).fillna(False).astype(bool)
+    if label_all:
+        return _backfill_window(flags, window)
     return flags
 
 
-def interpolation_diff(x, window=3, rtol=1e-5, atol=1e-8):
+def interpolation_diff(x, window=3, rtol=1e-5, atol=1e-8, label_all=False):
     """Identify sequences which appear to be linear.
 
     Sequences are linear if the first difference appears to be
     constant.  For a window of length N, the last value (index N-1) is
     flagged if all values in the window appear to be a line segment.
 
+    Parameters `rtol` and `atol` have the same meaning as in
+    :py:func:`numpy.allclose`.
+
     Parameters
     ----------
     x : Series
@@ -105,6 +126,9 @@ def interpolation_diff(x, window=3, rtol=1e-5, atol=1e-8):
         tolerance relative to max(abs(x.diff()) for detecting a change
     atol : float, default 1e-8
         absolute tolerance for detecting a change in first difference
+    label_all : bool, default False
+        Whether to label all values in the window. If False, then only the
+        right endpoint of the window is labeled.
 
     Returns
     -------
@@ -135,6 +159,8 @@ def interpolation_diff(x, window=3, rtol=1e-5, atol=1e-8):
         rtol=rtol,
         atol=atol
     )
+    if label_all:
+        return _backfill_window(flags, window)
     return flags
 
 
diff --git a/pvanalytics/quality/util.py b/pvanalytics/quality/util.py
index a0fab27..fcc618d 100644
--- a/pvanalytics/quality/util.py
+++ b/pvanalytics/quality/util.py
@@ -58,3 +58,37 @@ def check_limits(val, lower_bound=None, upper_bound=None,
         return ub_op(val, upper_bound)
     else:
         raise ValueError('must provide either upper or lower bound')
+
+
+def daily_min(series, minimum, inclusive=False):
+    """Return True for data on days when the day's minimum exceeds `minimum`.
+
+    Parameters
+    ----------
+    series : Series
+        A Datetimeindexed series of floats.
+    minimum : float
+        The smallest acceptable value for the daily minimum.
+    inclusive : boolean, default False
+        Use greater than or equal to when comparing daily minimums from
+        `series` to `minimum`.
+
+    Returns
+    -------
+    Series
+        True for values on days where the minimum value recorded on
+        that day is greater than (or equal to) `minimum`.
+
+    Notes
+    -----
+    This function is derived from code in the pvfleets_qa_analysis
+    project under the terms of the 3-clause BSD license. Copyright (c)
+    2020 Alliance for Sustainable Energy, LLC.
+
+    """
+    greater_than = np.greater
+    if inclusive:
+        greater_than = np.greater_equal
+    dailymin = series.resample('D').min()
+    flags = greater_than(dailymin, minimum)
+    return flags.reindex(index=series.index, method='ffill', fill_value=False)


Test patch:
diff --git a/pvanalytics/tests/quality/test_gaps.py b/pvanalytics/tests/quality/test_gaps.py
index c4ce645..036225c 100644
--- a/pvanalytics/tests/quality/test_gaps.py
+++ b/pvanalytics/tests/quality/test_gaps.py
@@ -118,6 +118,17 @@ def test_stale_values_diff_raises_error(stale_data):
         gaps.stale_values_diff(stale_data, window=1)
 
 
+def test_stale_values_diff_label_all(stale_data):
+    """When label_all is True the full window is marked stale"""
+    assert_series_equal(
+        pd.Series([False, True, True, True, True,
+                   True, True, True, False, False]),
+        gaps.stale_values_diff(
+            stale_data, window=4, label_all=True
+        )
+    )
+
+
 @pytest.fixture
 def interpolated_data():
     """A series that contains linear interpolation.
@@ -136,6 +147,17 @@ def interpolated_data():
     return pd.Series(data=data)
 
 
+def test_interpolation_diff_label_all(interpolated_data):
+    """When label_all is True the full window is marked interpoated"""
+    assert_series_equal(
+        gaps.interpolation_diff(interpolated_data, window=3, label_all=True),
+        pd.Series([False, False, False, False, False,
+                   True, True, True, False, False,
+                   False, True, True, True, True, True,
+                   False])
+    )
+
+
 def test_interpolation_diff(interpolated_data):
     """Interpolation is detected correclty.
 
diff --git a/pvanalytics/tests/quality/test_util.py b/pvanalytics/tests/quality/test_util.py
index ae00a65..82a06c2 100644
--- a/pvanalytics/tests/quality/test_util.py
+++ b/pvanalytics/tests/quality/test_util.py
@@ -43,3 +43,56 @@ def test_check_limits():
 
     with pytest.raises(ValueError):
         util.check_limits(val=data)
+
+
+@pytest.fixture
+def ten_days():
+    """A ten day index (not localized) at ten minute frequency."""
+    return pd.date_range(
+        start='01/01/2020',
+        end='01/10/2020 23:59',
+        freq='10T'
+    )
+
+
+def test_daily_min_all_below(ten_days):
+    """If every data point is below the minimum then all values are
+    flagged False."""
+    data = pd.Series(1, index=ten_days)
+    assert_series_equal(
+        pd.Series(False, index=ten_days),
+        util.daily_min(data, minimum=2)
+    )
+
+
+def test_daily_min_some_values_above(ten_days):
+    """If there are some values above `maximum` on one day, all data is
+    flagged False."""
+    data = pd.Series(1, index=ten_days)
+    data.iloc[0:10] = 10
+    data.iloc[100:150] = 10
+    data.iloc[500:710] = 3
+    assert_series_equal(
+        pd.Series(False, index=ten_days),
+        util.daily_min(data, minimum=2)
+    )
+
+
+def test_daily_min_one_day_fails(ten_days):
+    """daily_maxmin flags day with all values above maximum True."""
+    data = pd.Series(2, index=ten_days)
+    data.loc['01/02/2020'] = 5
+    expected = pd.Series(False, index=ten_days)
+    expected['01/02/2020'] = True
+    assert_series_equal(
+        expected,
+        util.daily_min(data, minimum=3)
+    )
+
+
+def test_daily_min_exclusice(ten_days):
+    """Check that inclusive=False makes the comparison greater than or
+    equal."""
+    data = pd.Series(2, index=ten_days)
+    assert not util.daily_min(data, minimum=2, inclusive=False).any()
+    assert util.daily_min(data, minimum=2, inclusive=True).all()