Repository: ARM-DOE/ACT

Issue:
Ability to handle time_bounds
A number of instruments have different methods for averaging to a timestamp, some have the timestamp at the end of the averaging interval and some as the start.  Two examples are the ARM ECOR and EBBR systems at the SGP site and include the time_bounds variable in the dataset. 

It would be beneficial to have a generic function that could update these times to a standard timeframe.  I.e. default to adjusting the times so that it's a standard where the timestamp is at the start of the averaging interval.


Discussion:
Yes this would be helpful. ADI has some code to do this I believe. Do we have any suggestions on how to interpolate the data?
@kenkehoe ADI has it in a database but all the information we have should be in the time_bounds variable in the file.  I think we just need to set a standard that we are going to have the sample time mean the start of the averaging period and have a function to adjust the time accordingly.  I.e. like with the EBBR which has a sample time at the end of the averaging period.  I think this is more of an adjustment than an averaging

Resolving patch:
diff --git a/act/utils/__init__.py b/act/utils/__init__.py
--- a/act/utils/__init__.py
+++ b/act/utils/__init__.py
@@ -29,6 +29,7 @@
             'numpy_to_arm_date',
             'reduce_time_ranges',
             'date_parser',
+            'adjust_timestamp'
         ],
         'geo_utils': [
             'add_solar_variable',
diff --git a/act/utils/datetime_utils.py b/act/utils/datetime_utils.py
--- a/act/utils/datetime_utils.py
+++ b/act/utils/datetime_utils.py
@@ -218,3 +218,56 @@ def date_parser(date_string, output_format='%Y%m%d', return_datetime=False):
             pass
     fmt_strings = ', '.join(date_fmts)
     raise ValueError('Invalid Date format, please use one of these formats ' + fmt_strings)
+
+
+def adjust_timestamp(ds, time_bounds='time_bounds', align='left', offset=None):
+    """
+    Will adjust the timestamp based on the time_bounds or other information
+    so that the timestamp aligns with user preference.
+
+    Will work to adjust the times based on the time_bounds variable
+    but if it's not available will rely on the user supplied input
+
+    Parameters
+    ----------
+    ds : Xarray Dataset
+        Dataset to adjust
+    time_bounds : str
+        Name of the time_bounds variable
+    align : str
+        Alignment of the time when using time_bounds.
+            left: Sets timestamp to start of  sample interval
+            right: Sets timestamp to end of sample interval
+            center: Sets timestamp to middle of sample interval
+    offset : int
+        Time in seconds to offset the timestamp.  This overrides
+        the time_bounds variable and can be positive or negative.
+        Required to be in seconds
+
+    Returns
+    -------
+    ds : Xarray DataSet
+        Adjusted DataSet
+
+    """
+
+    if time_bounds in ds and offset is None:
+        time_bounds = ds[time_bounds].values
+        if align == 'left':
+            time_start = [np.datetime64(t[0]) for t in time_bounds]
+        elif align == 'right':
+            time_start = [np.datetime64(t[1]) for t in time_bounds]
+        elif align == 'center':
+            time_start = [np.datetime64(t[0]) + (np.datetime64(t[0]) - np.datetime64(t[1])) / 2. for t in time_bounds]
+        else:
+            raise ValueError('Align should be set to one of [left, right, middle]')
+
+    elif offset is not None:
+        time = ds['time'].values
+        time_start = [t + np.timedelta64(offset, 's') for t in time]
+    else:
+        raise ValueError('time_bounds variable is not available')
+
+    ds = ds.assign_coords({'time': time_start})
+
+    return ds
diff --git a/examples/workflows/plot_merged_product.py b/examples/workflows/plot_merged_product.py
new file mode 100644
--- /dev/null
+++ b/examples/workflows/plot_merged_product.py
@@ -0,0 +1,85 @@
+"""
+Merge multiple datasets
+-----------------------
+
+Example to merge multiple data products into one using ACT.
+Shows how to adjust the timestamp if the timestamps are at
+different part of the sample interval (left, right, center).
+Also shows how to apply QC information, merge and resample
+data using xarray and plot/write out the results.
+
+"""
+
+import act
+import xarray as xr
+import matplotlib.pyplot as plt
+
+# Set data files
+# An alternative to this is to download data from the
+# ARM Data Webservice as shown in the discovery plot_neon.py example
+ebbr_file = act.tests.sample_files.EXAMPLE_EBBR3
+ecor_file = act.tests.sample_files.EXAMPLE_ECOR
+sebs_file = act.tests.sample_files.EXAMPLE_SEBS
+
+# Read data into datasets
+ds_ebbr = act.io.armfiles.read_netcdf(ebbr_file)
+ds_ecor = act.io.armfiles.read_netcdf(ecor_file)
+ds_sebs = act.io.armfiles.read_netcdf(sebs_file)
+
+# Check for ARM DQRs and add them to the QC variables
+ds_ebbr = act.qc.arm.add_dqr_to_qc(ds_ebbr)
+ds_ecor = act.qc.arm.add_dqr_to_qc(ds_ecor)
+ds_sebs = act.qc.arm.add_dqr_to_qc(ds_sebs)
+
+# The ECOR and EBBR have different definitions of latent heat
+# flux and what is positive vs negative.  Check out the ARM
+# Handbooks for more information
+ds_ecor['lv_e'].values = ds_ecor['lv_e'].values * -1.
+
+# For example purposes, let's rename the ecor latent heat flux
+ds_ecor = ds_ecor.rename({'lv_e': 'latent_heat_flux_ecor'})
+ds_ecor['latent_heat_flux_ecor'].attrs['ancillary_variables'] = 'qc_latent_heat_flux_ecor'
+ds_ecor = ds_ecor.rename({'qc_lv_e': 'qc_latent_heat_flux_ecor'})
+
+# Also going to Switch some QC for example purposes
+qc = ds_ecor['qc_latent_heat_flux_ecor'].values
+qc[10:20] = 2
+ds_ecor['qc_latent_heat_flux_ecor'].values = qc
+
+# There is a difference in how these timestamps are defined
+# The EBBR is at the end of the sampling interval and the
+# ECOR is at the beginning.  Knowing this, we can shift the
+# EBBR timestampes by 30 minutes to coincide with the ECOR
+ds_ebbr = act.utils.datetime_utils.adjust_timestamp(ds_ebbr, offset=-30 * 60)
+
+# Now, we can merge all these objects into one product
+ds = xr.merge([ds_ecor, ds_ebbr, ds_sebs], compat='override')
+
+# Apply the QC information to set all flagged data to missing/NaN
+ds.qcfilter.datafilter(del_qc_var=False, rm_assessments=['Bad', 'Incorrect', 'Indeterminate', 'Suspect'])
+
+# Plot up data from the merged object for each of the instruments
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(3,))
+display.plot('latent_heat_flux_ecor', label='ECOR', subplot_index=(0,))
+display.plot('latent_heat_flux', label='EBBR', subplot_index=(0,))
+plt.legend()
+display.plot('surface_soil_heat_flux_1', label='SEBS', subplot_index=(1,))
+
+# Plot out the QC information that was modified as well
+display.qc_flag_block_plot('latent_heat_flux_ecor', subplot_index=(2,))
+plt.show()
+
+# Resample the data to 1 hour mean
+# Check out the xarray documentation for more information
+# on the resample function. Options include mean, median,
+# max, min, sum, nearest, and more.
+ds = ds.resample(time='H').mean(keep_attrs=True)
+
+# Plot up data from the hourly merged object for ECOR and EBBR
+display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(1,))
+display.plot('latent_heat_flux_ecor', label='ECOR', subplot_index=(0,))
+display.plot('latent_heat_flux', label='EBBR', subplot_index=(0,))
+plt.show()
+
+# Write data out to netcdf
+ds.to_netcdf('./sgpecor_ebbr_sebs.nc')


Test patch:
diff --git a/act/tests/__init__.py b/act/tests/__init__.py
--- a/act/tests/__init__.py
+++ b/act/tests/__init__.py
@@ -21,6 +21,7 @@
             'EXAMPLE_DLPPI',
             'EXAMPLE_EBBR1',
             'EXAMPLE_EBBR2',
+            'EXAMPLE_EBBR3',
             'EXAMPLE_IRTSST',
             'EXAMPLE_LCL1',
             'EXAMPLE_MET1',
@@ -53,6 +54,8 @@
             'EXAMPLE_SP2B',
             'EXAMPLE_MET_YAML',
             'EXAMPLE_CLOUDPHASE'
+            'EXAMPLE_ECOR',
+            'EXAMPLE_SEBS',
         ]
     },
 )
diff --git a/act/tests/data/sgp30ebbrE13.b1.20190601.000000.nc b/act/tests/data/sgp30ebbrE13.b1.20190601.000000.nc
new file mode 100644
Binary files /dev/null and b/act/tests/data/sgp30ebbrE13.b1.20190601.000000.nc differ
diff --git a/act/tests/data/sgp30ecorE14.b1.20190601.000000.cdf b/act/tests/data/sgp30ecorE14.b1.20190601.000000.cdf
new file mode 100644
Binary files /dev/null and b/act/tests/data/sgp30ecorE14.b1.20190601.000000.cdf differ
diff --git a/act/tests/data/sgpsebsE14.b1.20190601.000000.cdf b/act/tests/data/sgpsebsE14.b1.20190601.000000.cdf
new file mode 100644
Binary files /dev/null and b/act/tests/data/sgpsebsE14.b1.20190601.000000.cdf differ
diff --git a/act/tests/sample_files.py b/act/tests/sample_files.py
--- a/act/tests/sample_files.py
+++ b/act/tests/sample_files.py
@@ -28,8 +28,6 @@
 EXAMPLE_AOSMET = os.path.join(DATA_PATH, 'maraosmetM1.a1.20180201.000000.nc')
 EXAMPLE_DLPPI = os.path.join(DATA_PATH, 'sgpdlppiC1.b1.20191015.120023.cdf')
 EXAMPLE_DLPPI_MULTI = os.path.join(DATA_PATH, 'sgpdlppiC1.b1.20191015.*.cdf')
-EXAMPLE_EBBR1 = os.path.join(DATA_PATH, 'sgp30ebbrE32.b1.20191125.000000.nc')
-EXAMPLE_EBBR2 = os.path.join(DATA_PATH, 'sgp30ebbrE32.b1.20191130.000000.nc')
 EXAMPLE_BRS = os.path.join(DATA_PATH, 'sgpbrsC1.b1.20190705.000000.cdf')
 EXAMPLE_AERI = os.path.join(DATA_PATH, 'sgpaerich1C1.b1.20190501.000342.nc')
 EXAMPLE_IRTSST = os.path.join(DATA_PATH, 'marirtsstM1.b1.20190320.000000.nc')
@@ -63,3 +61,8 @@
 EXAMPLE_NEON_VARIABLE = os.path.join(DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.variables.20221201T110553Z.csv')
 EXAMPLE_NEON_POSITION = os.path.join(DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.sensor_positions.20221107T205629Z.csv')
 EXAMPLE_DOD = os.path.join(DATA_PATH, 'vdis.b1')
+EXAMPLE_EBBR1 = os.path.join(DATA_PATH, 'sgp30ebbrE32.b1.20191125.000000.nc')
+EXAMPLE_EBBR2 = os.path.join(DATA_PATH, 'sgp30ebbrE32.b1.20191130.000000.nc')
+EXAMPLE_EBBR3 = os.path.join(DATA_PATH, 'sgp30ebbrE13.b1.20190601.000000.nc')
+EXAMPLE_ECOR = os.path.join(DATA_PATH, 'sgp30ecorE14.b1.20190601.000000.cdf')
+EXAMPLE_SEBS = os.path.join(DATA_PATH, 'sgpsebsE14.b1.20190601.000000.cdf')
diff --git a/act/tests/test_utils.py b/act/tests/test_utils.py
--- a/act/tests/test_utils.py
+++ b/act/tests/test_utils.py
@@ -650,3 +650,13 @@ def test_date_parser_minute_second():
 
     output_format = parsed_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
     assert output_format == '2020-01-01T12:00:00.000Z'
+
+
+def test_adjust_timestamp():
+    file = act.tests.sample_files.EXAMPLE_EBBR1
+    ds = act.io.armfiles.read_netcdf(file)
+    ds = act.utils.datetime_utils.adjust_timestamp(ds)
+    assert ds['time'].values[0] == np.datetime64('2019-11-24T23:30:00.000000000')
+
+    ds = act.utils.datetime_utils.adjust_timestamp(ds, offset=-60 * 60)
+    assert ds['time'].values[0] == np.datetime64('2019-11-24T22:30:00.000000000')