Repository: petrobras/ccp (language: python)

Issue:
Implement module for data treatment
When we get process data it is often the case that temperature, pressure and other measurements are not following some criteria that enable us to calculate the compressor performance with more certainty. 
One example of this is the fluctuation that we have on measurements, meaning that the compressor is not operating in a steady state. We should implement a function to filter results that follow a certain criterion for the fluctuation for example, maybe with limits that can be customized so that we are not as restrictive as factory performance test standards.


Discussion / hints:

Fix patch:
diff --git a/ccp/data_io/__init__.py b/ccp/data_io/__init__.py
index 3ff1c8c..8992c63 100644
--- a/ccp/data_io/__init__.py
+++ b/ccp/data_io/__init__.py
@@ -4,3 +4,4 @@
 before beeing loaded.
 
 """
+from .processing import fluctuation, fluctuation_data, mean_data, filter_data
diff --git a/ccp/data_io/processing.py b/ccp/data_io/processing.py
new file mode 100644
index 0000000..8802081
--- /dev/null
+++ b/ccp/data_io/processing.py
@@ -0,0 +1,190 @@
+"""Data processing functions for ccp.
+"""
+import pandas as pd
+
+
+def fluctuation(x):
+    """Calculate fluctuation of x.
+
+    Fluctuation as defined in the ASME PTC 10-1997 is the highest reading minus
+    the lowest reading divided by the mean of the readings.
+
+    Parameters
+    ----------
+    x : array-like
+        Array of values.
+
+    Returns
+    -------
+    float
+        Fluctuation of x.
+
+    Examples
+    --------
+    >>> fluctuation([1, 2, 3, 4, 5])
+    80.0
+    >>> fluctuation([1, 1, 1, 1, 1])
+    0.0
+    """
+    if x.mean() == 0:
+        return 100
+    else:
+        return 100 * (x.max() - x.min()) / x.mean()
+
+
+def fluctuation_data(df, window=3):
+    """Calculate fluctuation of dataframe columns.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Dataframe with data to be filtered.
+    window : int, optional
+        Window size for rolling calculation. The default is 3.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Dataframe with fluctuation values.
+
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
+    >>> fluctuation_data(df)
+         a    b
+    0  0.0  0.0
+    1  0.0  0.0
+    2  0.0  0.0
+    >>> df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
+    >>> fluctuation_data(df, window=2)
+            a    b
+    0     0.0  0.0
+    1  1000.0  0.0
+    2  1000.0  0.0
+    """
+    fluctuation_df = (
+        df.apply(pd.to_numeric)
+        .rolling(
+            window=window,
+        )
+        .apply(fluctuation)
+        .fillna(0.0)
+    )
+    fluctuation_df = fluctuation_df[window - 1 :]
+    fluctuation_df.reset_index(drop=True, inplace=True)
+    return fluctuation_df
+
+
+def mean_data(df, window=3):
+    """Calculate the mean of dataframe columns.
+
+    The mean is calculated using a rolling window.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Dataframe with data to be filtered.
+    window : int, optional
+        Window size for rolling calculation. The default is 3.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Dataframe with mean values.
+
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
+    >>> fluctuation_data(df)
+         a    b
+    0  0.0  0.0
+    1  0.0  0.0
+    2  0.0  0.0
+    >>> df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
+    >>> fluctuation_data(df, window=2)
+            a    b
+    0     0.0  0.0
+    1  1000.0  0.0
+    2  1000.0  0.0
+    """
+    mean_df = (
+        df.apply(pd.to_numeric)
+        .rolling(
+            window=window,
+        )
+        .mean()
+    )
+    mean_df = mean_df[window - 1 :]
+    mean_df.reset_index(drop=True, inplace=True)
+    return mean_df
+
+
+def filter_data(
+    df,
+    window=3,
+    data_type=None,
+    temperature_fluctuation=0.5,
+    pressure_fluctuation=2,
+    speed_fluctuation=0.5,
+):
+    """Filter data according to fluctuation values.
+
+    This function filters the data according to the fluctuation values of the
+    columns and returns the mean value for set of values defined by the window size.
+    Default values for maximum fluctuation are based on ASME PTC 10-1997.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Dataframe with data to be filtered.
+    window : int, optional
+        Window size for rolling calculation, meaning how many rolls will be used
+        to calculate the fluctuation.
+        The default is 3.
+    data_type : dict
+        Dictionary with data types for each column.
+        Values for data_type can be: "pressure", "temperature", "speed".
+    temperature_fluctuation : float, optional
+        Maximum fluctuation for temperature data.
+        The default is 0.5.
+    pressure_fluctuation : float, optional
+        Maximum fluctuation for pressure data.
+        The default is 2.
+    speed_fluctuation : float, optional
+        Maximum fluctuation for speed data.
+        The default is 0.5.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Filtered dataframe.
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> df = pd.DataFrame({'a': [1, 2, 3, 4, 4.01, 4.02], 'b': [4, 5, 6, 6.01, 6.02, 6.03]})
+    >>> data_type = {'a': 'pressure', 'b': 'temperature'}
+    >>> filter_data(df, window=3, data_type=data_type)
+    """
+    fluctuation_df = fluctuation_data(df, window=window)
+    mean_df = mean_data(df, window=window)
+    # filter mean_df based on fluctuation_df max values
+    for column in fluctuation_df.columns:
+        if data_type[column] == "pressure":
+            max_fluctuation = pressure_fluctuation
+        elif data_type[column] == "temperature":
+            max_fluctuation = temperature_fluctuation
+        elif data_type[column] == "speed":
+            max_fluctuation = speed_fluctuation
+        else:
+            raise ValueError(
+                f"Invalid data type for column {column}. "
+                "Valid data types are: pressure, temperature and speed."
+            )
+        mean_df.loc[fluctuation_df[column] > max_fluctuation, column] = None
+    mean_df = mean_df.dropna()
+    return mean_df


Test patch:
diff --git a/ccp/tests/test_io.py b/ccp/tests/test_io.py
index 2b82c52..2785115 100644
--- a/ccp/tests/test_io.py
+++ b/ccp/tests/test_io.py
@@ -1,6 +1,9 @@
+import numpy as np
+import pandas as pd
 import pytest
 from pathlib import Path
 from numpy.testing import assert_allclose
+from pandas.testing import assert_frame_equal
 import ccp
 
 Q_ = ccp.Q_
@@ -50,3 +53,39 @@ def test_impeller_lp_sec1_from_csv(impeller_lp_sec1_from_csv):
     assert_allclose(p0.head.to("kJ/kg"), 82.7824137)
     assert_allclose(p0.eff, 0.789412, rtol=1e-5)
     assert_allclose(p0.power.to("kW"), 1431.03994)
+
+
+def test_fluctuation():
+    assert_allclose(ccp.data_io.fluctuation(np.array([1, 2, 3, 4, 5])), 133.333333)
+    assert_allclose(ccp.data_io.fluctuation(np.array([1, 1, 1, 1, 1])), 0.0)
+
+
+def test_fluctuation_data():
+    df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [4, 5, 6, 7]})
+    assert_frame_equal(
+        ccp.data_io.fluctuation_data(df),
+        pd.DataFrame(
+            {"a": [100.0, 66.66666666666667], "b": [40.0, 33.333333333333336]}
+        ),
+    )
+
+
+def test_mean_data():
+    df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [4, 5, 6, 7]})
+    assert_frame_equal(
+        ccp.data_io.mean_data(df),
+        pd.DataFrame({"a": [2.0, 3.0], "b": [5.0, 6.0]}),
+    )
+
+
+def test_filter_data():
+    df = pd.DataFrame(
+        {
+            "a": [1, 2, 3, 4, 4.01, 4.02, 5, 6.01, 6.02, 6.04, 6.05],
+            "b": [4, 5, 6, 7, 7.01, 7.02, 8, 9, 10, 11, 12],
+        }
+    )
+    assert_frame_equal(
+        ccp.data_io.filter_data(df, data_type={"a": "pressure", "b": "temperature"}),
+        pd.DataFrame(index=pd.Index([3]), data={"a": [4.01], "b": [7.01]}),
+    )