Repository: sandialabs/pvOps (language: python)

Issue:
Add irradiance_anomalies and inverter clipping filters



Discussion / hints:

Fix patch:
diff --git a/docs/modules.rst b/docs/modules.rst
index 116f720..8304c2e 100644
--- a/docs/modules.rst
+++ b/docs/modules.rst
@@ -6,3 +6,4 @@ pvops
 
    text
    text2time
+   timeseries
diff --git a/docs/timeseries.rst b/docs/timeseries.rst
new file mode 100644
index 0000000..110776d
--- /dev/null
+++ b/docs/timeseries.rst
@@ -0,0 +1,10 @@
+timeseries package
+=================
+
+timeseries.preprocess module
+-----------------------------
+
+.. automodule:: timeseries.preprocess
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/examples/example_data/example_metadata2.csv b/examples/example_data/example_metadata2.csv
index f919d70..2fd743f 100644
--- a/examples/example_data/example_metadata2.csv
+++ b/examples/example_data/example_metadata2.csv
@@ -1,3 +1,3 @@
-randid,DC_Size_kW,COD
-R23,2500,10/20/2013
-R27,475,10/21/2017
+randid,DC_Size_kW,COD,latitude,longitude
+R23,2500,10/20/2013,-80,-35
+R27,475,10/21/2017,-81,-36
diff --git a/pvops/timeseries/__init__.py b/pvops/timeseries/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pvops/timeseries/preprocess.py b/pvops/timeseries/preprocess.py
new file mode 100644
index 0000000..8baf3d9
--- /dev/null
+++ b/pvops/timeseries/preprocess.py
@@ -0,0 +1,231 @@
+# A set of preprocessing methods, both based on photovoltaic-specific physics and data quality methods.
+
+import pvlib
+import pvanalytics
+from timezonefinder import TimezoneFinder
+import pandas as pd
+
+
+def prod_irradiance_filter(prod_df, prod_col_dict, meta_df, meta_col_dict,
+                           drop=True, irradiance_type='ghi', csi_max=1.1
+                           ):
+    """
+    Filter rows of production data frame according to performance and data quality.
+
+    THIS METHOD IS CURRENTLY IN DEVELOPMENT.
+
+    Parameters
+
+    ----------
+    prod_df: DataFrame
+        A data frame corresponding to production data.
+
+    prod_df_col_dict: dict of {str : str}
+        A dictionary that contains the column names associated with the production data,
+        which consist of at least:
+
+        - **timestamp** (*string*), should be assigned to associated time-stamp
+        column name in prod_df
+        - **siteid** (*string*), should be assigned to site-ID column name in prod_df
+        - **irradiance** (*string*), should be assigned to associated irradiance column name in prod_df
+        - **clearsky_irr** (*string*), should be assigned to clearsky irradiance column name in prod_df
+
+    meta_df: DataFrame
+        A data frame corresponding to site metadata.
+        At the least, the columns in meta_col_dict be present.
+
+    meta_col_dict: dict of {str : str}
+        A dictionary that contains the column names relevant for the meta-data
+
+        - **siteid** (*string*), should be assigned to site-ID column name
+        - **latitude** (*string*), should be assigned to column name corresponding to site's latitude
+        - **longitude** (*string*), should be assigned to column name corresponding to site's longitude
+
+    irradiance_type : str
+        A string description of the irradiance_type which was passed in prod_df. 
+        Options: `ghi`, `dni`, `dhi`.
+        In future, `poa` may be a feature.
+
+    csi_max: int
+        A pvanalytics parameter of maximum ratio of measured to clearsky (clearsky index).
+
+    Returns
+
+    -------
+    prod_df: DataFrame
+        A dataframe with new **clearsky_irr** column. If drop=True, a filtered prod_df according to clearsky.
+    clearsky_mask : series
+        Returns True for each value where the clearsky index is less than or equal to csi_mask
+    """
+
+    prod_df = prod_df.copy()
+    meta_df = meta_df.copy()
+
+    irr_name = prod_col_dict["irradiance"]
+    clearsky_irr_name = prod_col_dict["clearsky_irr"]
+
+    individual_sites = set(meta_df[meta_col_dict['siteid']].tolist())
+
+    for site in individual_sites:
+
+        # Get site-specific meta data
+        # site_meta_data= meta_df[meta_df[meta_col_dict['siteid']] == site]
+        site_meta_mask = meta_df.loc[:, meta_col_dict["siteid"]] == site
+        site_prod_mask = prod_df.loc[:, prod_col_dict["siteid"]] == site
+
+        # Save times in object
+        prod_times = prod_df.loc[site_prod_mask,
+                                 prod_col_dict['timestamp']].tolist()
+
+        # Extract site's position
+        latitude = meta_df.loc[site_meta_mask, meta_col_dict['latitude']].tolist()[
+            0]
+        longitude = meta_df.loc[site_meta_mask, meta_col_dict['longitude']].tolist()[
+            0]
+
+        # Derive
+        tf = TimezoneFinder()
+        derived_timezone = tf.timezone_at(lng=longitude, lat=latitude)
+
+        # Define Location object
+        # Altitude is not passed because it's not available usually. Fortunately, a clearsky
+        # model exists which does not use altitude.
+        loc = pvlib.location.Location(latitude, longitude, tz=derived_timezone)
+        times = pd.DatetimeIndex(
+            data=prod_times,
+            tz=loc.tz,
+        )
+        # Derive clearsky values
+        cs = loc.get_clearsky(times, model='haurwitz')
+        # Localize timestamps
+        cs.index = cs.index.tz_localize(None)
+
+        if irradiance_type == 'poa':
+
+            raise ValueError(
+                "POA is currently not configured because it requires `surface_tilt` and `surface_azimuth`, \
+                a trait which is not usually in the meta data.")
+            # Establish solarposition
+            # solpos = pvlib.solarposition.get_solarposition(prod_times,
+            #                                                latitude, longitude)
+
+            # # Returns dataframe with columns:
+            # # 'poa_global', 'poa_direct', 'poa_diffuse', 'poa_sky_diffuse', 'poa_ground_diffuse'
+            # cs_POA_irradiance = pvlib.irradiance.get_total_irradiance(
+            #     surface_tilt=20,
+            #     surface_azimuth=180,
+            #     dni=cs['dni'],
+            #     ghi=cs['ghi'],
+            #     dhi=cs['dhi'],
+            #     solar_zenith=solpos['apparent_zenith'].tolist(),
+            #     solar_azimuth=solpos['azimuth'])
+
+            # df = pd.merge(df, POA_irradiance, how="inner", left_index=True, right_index=True)
+
+        elif irradiance_type in ['dni', 'ghi', 'dhi']:
+            prod_df[clearsky_irr_name] = cs[irradiance_type]
+
+        else:
+            raise ValueError(
+                "Incorrect value passed to `irradiance_type`. Expected ['dni','ghi', or 'dhi']")
+
+    mask_series = pvanalytics.quality.irradiance.clearsky_limits(
+        prod_df[irr_name], prod_df[clearsky_irr_name], csi_max=csi_max)
+
+    prod_df['mask'] = mask_series
+
+    if not drop:
+        return prod_df, mask_series
+
+    if drop:
+        prod_df = prod_df[prod_df['mask'] == False]
+        prod_df.drop(columns=['mask'], inplace=True)
+        return prod_df, mask_series
+
+
+def prod_inverter_clipping_filter(prod_df, prod_col_dict, meta_df, meta_col_dict, model, **kwargs):
+    """
+    Filter rows of production data frame according to performance and data quality
+
+    Parameters
+
+    ----------
+    prod_df: DataFrame
+        A data frame corresponding to production data.
+
+    prod_df_col_dict: dict of {str : str}
+        A dictionary that contains the column names associated with the production data,
+        which consist of at least:
+
+        - **timestamp** (*string*), should be assigned to associated time-stamp
+        column name in prod_df
+        - **siteid** (*string*), should be assigned to site-ID column name in prod_df
+        - **power** (*string*), should be assigned to associated power production column name in prod_df
+
+    meta_df: DataFrame
+        A data frame corresponding to site metadata.
+        At the least, the columns in meta_col_dict be present.
+
+    meta_col_dict: dict of {str : str}
+        A dictionary that contains the column names relevant for the meta-data
+
+        - **siteid** (*string*), should be assigned to site-ID column name
+        - **latitude** (*string*), should be assigned to column name corresponding to site's latitude
+        - **longitude** (*string*), should be assigned to column name corresponding to site's longitude
+
+    model: str
+        A string distinguishing the inverter clipping detection model programmed in pvanalytics. Available options: ['geometric', 'threshold', 'levels']
+
+    kwargs:
+        Extra parameters passed to the relevant pvanalytics model. If none passed, defaults are used.
+
+    Returns
+
+    -------
+    prod_df: DataFrame
+        If drop=True, a filtered dataframe with clipping periods removed is returned.
+    """
+
+    prod_df = prod_df.copy()
+    meta_df = meta_df.copy()
+
+    individual_sites = set(meta_df[meta_col_dict['siteid']].tolist())
+
+    for site in individual_sites:
+
+        site_prod_mask = prod_df.loc[:, prod_col_dict["siteid"]] == site
+        ac_power = prod_df.loc[site_prod_mask, prod_col_dict["power"]]
+
+        if len(ac_power) == 0:
+            # If no rows exist for this company, skip it.
+            continue
+
+        if model == 'geometric':
+            window = kwargs.get('window')
+            slope_max = kwargs.get('slope_max') or 0.2
+            freq = kwargs.get('freq')  # Optional
+            tracking = kwargs.get('tracking') or False
+            prod_df.loc[site_prod_mask, "mask"] = pvanalytics.features.clipping.geometric(
+                ac_power, window=window, slope_max=slope_max, freq=freq, tracking=tracking)
+
+        elif model == 'threshold':
+            slope_max = kwargs.get('slope_max') or 0.0035
+            power_min = kwargs.get('power_min') or 0.75
+            power_quantile = kwargs.get('power_quantile') or 0.995
+            freq = kwargs.get('freq')  # Optional
+            prod_df.loc[site_prod_mask, "mask"] = pvanalytics.features.clipping.threshold(
+                ac_power, slope_max=slope_max, power_min=power_min, power_quantile=power_quantile, freq=freq)
+
+        elif model == 'levels':
+            window = kwargs.get('window') or 4
+            fraction_in_window = kwargs.get('fraction_in_window') or 0.75
+            rtol = kwargs.get('rtol') or 0.005
+            levels = kwargs.get('levels') or 2
+            prod_df.loc[site_prod_mask, "mask"] = pvanalytics.features.clipping.levels(
+                ac_power, window=window, fraction_in_window=fraction_in_window, rtol=rtol, levels=levels)
+
+        else:
+            raise ValueError(
+                "Invalid value passed to parameter `calculation`. Expected a value in ['geometric', 'threshold', 'levels']")
+
+    return prod_df
diff --git a/requirements-min.txt b/requirements-min.txt
index 826c3e4..68ef7ae 100644
--- a/requirements-min.txt
+++ b/requirements-min.txt
@@ -7,5 +7,8 @@ datefinder
 matplotlib
 seaborn
 plotly
-gensim>=4.0.0
+gensim
 networkx
+pvlib
+pvanalytics
+timezonefinder
diff --git a/requirements.txt b/requirements.txt
index 826c3e4..68ef7ae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,5 +7,8 @@ datefinder
 matplotlib
 seaborn
 plotly
-gensim>=4.0.0
+gensim
 networkx
+pvlib
+pvanalytics
+timezonefinder
diff --git a/setup.py b/setup.py
index 3c1a62d..adb427f 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,10 @@
     'seaborn',
     'plotly',
     'gensim',
-    'networkx'
+    'networkx',
+    'pvlib',
+    'pvanalytics',
+    'timezonefinder'
 ]
 
 DOCS_REQUIRE = [


Test patch:
diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml
index 36bcbce..25fa8c5 100644
--- a/.github/workflows/lint-and-test.yml
+++ b/.github/workflows/lint-and-test.yml
@@ -52,4 +52,4 @@ jobs:
     - name: Lint with flake8
       run: |
         pip install flake8
-        flake8 . --count --statistics --show-source --ignore=E402,E203,E266,E501,W503,F403,F401,E402,W291,E302,W391,W292,F405,E722,W504,E121,E125
+        flake8 . --count --statistics --show-source --ignore=E402,E203,E266,E501,W503,F403,F401,E402,W291,E302,W391,W292,F405,E722,W504,E121,E125,E712
diff --git a/pvops/tests/test_timeseries.py b/pvops/tests/test_timeseries.py
new file mode 100644
index 0000000..f8b9395
--- /dev/null
+++ b/pvops/tests/test_timeseries.py
@@ -0,0 +1,106 @@
+import os
+import sys
+import pandas as pd
+
+timeseries_directory = os.path.join("pvops")
+sys.path.append(timeseries_directory)
+
+from timeseries import preprocess as tprep
+from text2time import preprocess as t2tprep
+
+# Define csv paths
+datadir = os.path.join('examples', 'example_data')
+example_OMpath = os.path.join(datadir, 'example_om_data2.csv')
+example_prodpath = os.path.join(datadir, 'example_perf_data.csv')
+example_metapath = os.path.join(datadir, 'example_metadata2.csv')
+
+# Assigning dictionaries to connect pvops variables with User's column names
+# Format for dictionaries is {pvops variable: user-specific column names}
+prod_col_dict = {'siteid': 'randid',
+                 'timestamp': 'Date',
+                 'power': 'AC_POWER',
+                 'energyprod': 'Energy',
+                 'irradiance': 'POAirradiance',
+                 # user's name choice for new column (baseline expected energy defined by user or calculated based on IEC)
+                 'baseline': 'IEC_pstep',
+                 # user's name choice for new column (System DC-size, extracted from meta-data)
+                 'dcsize': 'dcsize',
+                 'compared': 'Compared',  # user's name choice for new column
+                 'energy_pstep': 'Energy_pstep',  # user's name choice for new column
+                 'clearsky_irr': 'clearsky_irr'  # user's name choice for new column
+                 }
+
+om_col_dict = {'siteid': 'randid',
+               'datestart': 'date_start',
+               'dateend': 'date_end',
+               'workID': 'WONumber',
+               'worktype': 'WOType',
+               'asset': 'Asset',
+               # user's name choice for new column (Repair Duration)
+               'eventdur': 'EventDur',
+               # user's name choice for new column (Month when an event begins)
+               'modatestart': 'MonthStart',
+               'agedatestart': 'AgeStart'}  # user's name choice for new column (Age of system when event begins)
+
+metad_col_dict = {'siteid': 'randid',
+                  'dcsize': 'DC_Size_kW',
+                  'COD': 'COD',
+                  'latitude': 'latitude',
+                  'longitude': 'longitude'}
+
+
+def test_prod_irradiance_filter():
+
+    prod_df = pd.read_csv(example_prodpath)
+    meta_df = pd.read_csv(example_metapath)
+
+    prod_df = t2tprep.prod_date_convert(prod_df, prod_col_dict)
+    prod_df.index = prod_df[prod_col_dict['timestamp']]
+    prod_df['randid'] = 'R27'
+
+    # Data is missing in the middle of this example, so only going to pass
+    # The first set of rows
+    prod_df = prod_df.iloc[0:200]
+
+    prod_df_out, mask_series = tprep.prod_irradiance_filter(prod_df, prod_col_dict,
+                                                            meta_df, metad_col_dict)
+
+    true_detection_irradiance = [0, 44]
+    assert sum(mask_series) in true_detection_irradiance
+
+
+def test_prod_inverter_clipping_filter():
+
+    prod_df = pd.read_csv(example_prodpath)
+    meta_df = pd.read_csv(example_metapath)
+
+    prod_df = t2tprep.prod_date_convert(prod_df, prod_col_dict)
+    prod_df.index = prod_df[prod_col_dict['timestamp']]
+    prod_df['randid'] = 'R27'
+
+    # Data is missing in the middle of this example, so only going to pass
+    # The first set of rows
+    prod_df = prod_df.iloc[0:200]
+
+    geometric = tprep.prod_inverter_clipping_filter(prod_df,
+                                                    prod_col_dict,
+                                                    meta_df, metad_col_dict,
+                                                    model='geometric')
+
+    threshold = tprep.prod_inverter_clipping_filter(prod_df,
+                                                    prod_col_dict,
+                                                    meta_df, metad_col_dict,
+                                                    model='threshold')
+
+    levels = tprep.prod_inverter_clipping_filter(prod_df,
+                                                 prod_col_dict,
+                                                 meta_df, metad_col_dict,
+                                                 model='levels')
+
+    true_detection_geometric = 0
+    true_detection_threshold = 0
+    true_detection_levels = 183
+
+    assert sum(geometric['mask']) == true_detection_geometric
+    assert sum(threshold['mask']) == true_detection_threshold
+    assert sum(levels['mask']) == true_detection_levels