Repository: ARM-DOE/ACT

Issue:
Make the NOAA PSL reader more generic
The current reader only works with wind profiler data from the CTD site (see https://github.com/ARM-DOE/ACT/blob/main/act/io/noaapsl.py#L36)... we need to make this more generic to support other sites, and other date types from this site since the files are similar in structure and the name of the reader is "NOAA PSL" which covers a wide range of data


Discussion:

Resolving patch:
diff --git a/act/io/__init__.py b/act/io/__init__.py
--- a/act/io/__init__.py
+++ b/act/io/__init__.py
@@ -14,5 +14,5 @@
     read_gml_ozone,
     read_gml_radiation,
 )
-from .noaapsl import read_psl_wind_profiler
+from .noaapsl import read_psl_wind_profiler, read_psl_wind_profiler_temperature
 from .pysp2 import read_hk_file, read_sp2, read_sp2_dat
diff --git a/act/io/noaapsl.py b/act/io/noaapsl.py
--- a/act/io/noaapsl.py
+++ b/act/io/noaapsl.py
@@ -2,8 +2,9 @@
 Modules for reading in NOAA PSL data.
 """
 
-import datetime as dt
+from datetime import datetime
 
+import fsspec
 import numpy as np
 import pandas as pd
 import xarray as xr
@@ -61,7 +62,7 @@ def read_psl_wind_profiler(filename, transpose=True):
         date_str = list(filter(None, date_str[0].split(' ')))
         date_str = list(map(int, date_str))
         # Datetime not taking into account the utc offset yet
-        time = dt.datetime(
+        time = datetime(
             2000 + date_str[0],
             date_str[1],
             date_str[2],
@@ -148,3 +149,147 @@ def read_psl_wind_profiler(filename, transpose=True):
         obj_hi = obj_hi.transpose()
 
     return obj_low, obj_hi
+
+
+def read_psl_wind_profiler_temperature(filepath):
+    """
+    Returns `xarray.Dataset` with stored data and metadata from a user-defined
+    NOAA PSL wind profiler temperature file.
+
+    Parameters
+    ----------
+    filename : str
+        Name of file(s) to read.
+
+    Return
+    ------
+    ds :  Xarray.dataset
+        Standard Xarray dataset with the data
+
+    """
+
+    # Open the file, read in the lines as a list, and return that list
+    file = fsspec.open(filepath).open()
+    lines = file.readlines()
+    newlist = [x.decode().rstrip()[1:] for x in lines][1:]
+
+    # 1 - site
+    site = newlist[0]
+
+    # 2 - datetype
+    datatype, _, version = filter_list(newlist[1].split(' '))
+
+    # 3 - station lat, lon, elevation
+    latitude, longitude, elevation = filter_list(newlist[2].split('  ')).astype(float)
+
+    # 4 - year, month, day, hour, minute, second, utc
+    time = parse_date_line(newlist[3])
+
+    # 5 - Consensus averaging time, number of beams, number of range gates
+    consensus_average_time, number_of_beams, number_of_range_gates = filter_list(
+        newlist[4].split('  ')
+    ).astype(int)
+
+    # 7 - number of coherent integrations, number of spectral averages, pulse width, indder pulse period
+    (
+        number_coherent_integrations,
+        number_spectral_averages,
+        pulse_width,
+        inner_pulse_period,
+    ) = filter_list(newlist[6].split(' ')).astype(int)
+
+    # 8 - full-scale doppler value, delay to first gate, number of gates, spacing of gates
+    full_scale_doppler, delay_first_gate, number_of_gates, spacing_of_gates = filter_list(
+        newlist[7].split(' ')
+    ).astype(float)
+
+    # 9 - beam azimuth (degrees clockwise from north)
+    beam_azimuth, beam_elevation = filter_list(newlist[8].split(' ')).astype(float)
+
+    # Read in the data table section using pandas
+    df = pd.read_csv(filepath, skiprows=10, delim_whitespace=True)
+
+    # Only read in the number of rows for a given set of gates
+    df = df.iloc[: int(number_of_gates)]
+
+    # Nan values are encoded as 999999 - let's reflect that
+    df = df.replace(999999.0, np.nan)
+
+    # Ensure the height array is stored as a float
+    df['HT'] = df.HT.astype(float)
+
+    # Set the height as an index
+    df = df.set_index('HT')
+
+    # Rename the count and snr columns more usefully
+    df = df.rename(
+        columns={
+            'CNT': 'CNT_T',
+            'CNT.1': 'CNT_Tc',
+            'CNT.2': 'CNT_W',
+            'SNR': 'SNR_T',
+            'SNR.1': 'SNR_Tc',
+            'SNR.2': 'SNR_W',
+        }
+    )
+
+    # Convert to an xaray dataset
+    ds = df.to_xarray()
+
+    # Add attributes to variables
+    # Height
+    ds['HT'].attrs['long_name'] = 'height_above_ground'
+    ds['HT'].attrs['units'] = 'km'
+
+    # Temperature
+    ds['T'].attrs['long_name'] = 'average_uncorrected_RASS_temperature'
+    ds['T'].attrs['units'] = 'degC'
+    ds['Tc'].attrs['long_name'] = 'average_corrected_RASS_temperature'
+    ds['Tc'].attrs['units'] = 'degC'
+
+    # Vertical motion (w)
+    ds['W'].attrs['long_name'] = 'average_vertical_wind'
+    ds['W'].attrs['units'] = 'm/s'
+
+    # Add time to our dataset
+    ds['time'] = time
+
+    # Add in our additional attributes
+    ds.attrs['site_identifier'] = site
+    ds.attrs['latitude'] = latitude
+    ds.attrs['longitude'] = longitude
+    ds.attrs['elevation'] = elevation
+    ds.attrs['beam_azimuth'] = beam_azimuth
+    ds.attrs['revision_number'] = version
+    ds.attrs[
+        'data_description'
+    ] = 'https://psl.noaa.gov/data/obs/data/view_data_type_info.php?SiteID=ctd&DataOperationalID=5855&OperationalID=2371'
+    ds.attrs['consensus_average_time'] = consensus_average_time
+    ds.attrs['number_of_beams'] = int(number_of_beams)
+    ds.attrs['number_of_gates'] = int(number_of_gates)
+    ds.attrs['number_of_range_gates'] = int(number_of_range_gates)
+    ds.attrs['number_spectral_averages'] = int(number_spectral_averages)
+    ds.attrs['pulse_width'] = pulse_width
+    ds.attrs['inner_pulse_period'] = inner_pulse_period
+    ds.attrs['full_scale_doppler_value'] = full_scale_doppler
+    ds.attrs['spacing_of_gates'] = spacing_of_gates
+
+    return ds
+
+
+def filter_list(list_of_strings):
+    """
+    Parses a list of strings, remove empty strings, and return a numpy array
+    """
+    return np.array(list(filter(None, list_of_strings)))
+
+
+def parse_date_line(list_of_strings):
+    """
+    Parses the date line in PSL files
+    """
+    year, month, day, hour, minute, second, utc_offset = filter_list(
+        list_of_strings.split(' ')
+    ).astype(int)
+    year += 2000
+    return datetime(year, month, day, hour, minute, second)


Test patch:
diff --git a/act/tests/__init__.py b/act/tests/__init__.py
--- a/act/tests/__init__.py
+++ b/act/tests/__init__.py
@@ -15,6 +15,8 @@
     EXAMPLE_DLPPI,
     EXAMPLE_EBBR1,
     EXAMPLE_EBBR2,
+    EXAMPLE_HK,
+    EXAMPLE_INI,
     EXAMPLE_IRTSST,
     EXAMPLE_LCL1,
     EXAMPLE_MET1,
@@ -23,22 +25,21 @@
     EXAMPLE_MET_TEST1,
     EXAMPLE_MET_TEST2,
     EXAMPLE_MET_WILDCARD,
+    EXAMPLE_MET_YAML,
     EXAMPLE_METE40,
     EXAMPLE_MFRSR,
     EXAMPLE_MPL_1SAMPLE,
     EXAMPLE_NAV,
     EXAMPLE_NOAA_PSL,
+    EXAMPLE_NOAA_PSL_TEMPERATURE,
     EXAMPLE_RL1,
     EXAMPLE_SIGMA_MPLV5,
     EXAMPLE_SIRS,
     EXAMPLE_SONDE1,
     EXAMPLE_SONDE_WILDCARD,
+    EXAMPLE_SP2B,
     EXAMPLE_STAMP_WILDCARD,
     EXAMPLE_SURFSPECALB1MLAWER,
     EXAMPLE_TWP_SONDE_20060121,
     EXAMPLE_IRT25m20s,
-    EXAMPLE_HK,
-    EXAMPLE_INI,
-    EXAMPLE_SP2B,
-    EXAMPLE_MET_YAML
 )
diff --git a/act/tests/data/ctd22187.00t.txt b/act/tests/data/ctd22187.00t.txt
new file mode 100644
--- /dev/null
+++ b/act/tests/data/ctd22187.00t.txt
@@ -0,0 +1,37 @@
+
+ CTD
+ RASS    rev 5.1
+  34.66  -87.35    600
+  22 07 06 00 00 01   0
+  35  1  25
+ 23:46 (3.0)
+  10 28 417 20
+ 409.6  4000 25 417
+  45 90.0
+    HT        T       Tc        W     QC_T    QC_Tc     QC_W      CNT      CNT      CNT      SNR      SNR      SNR
+ 0.120     33.2   999999   999999      0.0      9.0      9.0       46       22       17      -14      -12       22
+ 0.182     32.9     45.0   999999      0.0      7.0      9.0       46       23       23       -8       -6      -10
+ 0.245     32.5     45.0   999999      0.0      7.0      9.0       46       23       22       -6       -4       -8
+ 0.307     32.2   999999   999999      0.0      9.0      9.0       46       22       22       -6      -12      -10
+ 0.370     31.9   999999   999999      0.0      9.0      9.0       46       22       23       -7       -6      -12
+ 0.432     31.7     34.1   999999      0.0      0.0      9.0       46       23       23      -10      -12      -17
+ 0.495     31.4     33.6   999999      0.0      0.0      9.0       46       23       23      -12      -13      -14
+ 0.557     30.8   999999   999999      0.0      9.0      9.0       46       22       23      -12      -11      -16
+ 0.619     30.2     32.2   999999      0.0      0.0      9.0       46       23       23      -12      -17      -17
+ 0.682     29.6     31.2   999999      0.0      0.0      9.0       46       23       23      -12      -19      -16
+ 0.744     29.1     30.9   999999      0.0      0.0      9.0       46       23       23      -13      -20      -17
+ 0.807     28.7     30.5   999999      0.0      0.0      9.0       46       23       23      -16      -22      -17
+ 0.869     28.0     29.2   999999      0.0      0.0      9.0       46       23       23      -20      -25      -18
+ 0.932     27.1     28.1   999999      0.0      0.0      9.0       46       23       23      -20      -24      -17
+ 0.994     26.5     27.6   999999      0.0      0.0      9.0       46       23       23      -20      -22      -16
+ 1.056     26.2     27.3   999999      0.0      0.0      9.0       46       23       23      -21      -21      -16
+ 1.119     25.8     36.0   999999      0.0      7.0      9.0       46       23       23      -23      -24      -17
+ 1.181     25.6   999999   999999      0.0      9.0      9.0       44       22       23      -27      -25      -16
+ 1.244     24.7   999999   999999      0.0      9.0      9.0       33       18       23      -31      -30        0
+ 1.306   999999   999999   999999      9.0      9.0      9.0       15       13       23      -34      -34      -16
+ 1.369   999999   999999   999999      9.0      9.0      9.0       11        8       23      -35      -35      -16
+ 1.431   999999   999999   999999      9.0      9.0      9.0        8       10       23      -36      -36      -16
+ 1.494   999999   999999   999999      9.0      9.0      9.0        8        9       23      -35      -35       -2
+ 1.556   999999   999999   999999      9.0      9.0      9.0        8        9       23      -36      -36      -17
+ 1.618   999999   999999   999999      9.0      9.0      9.0        9        7       23      -36      -37      -17
+$
diff --git a/act/tests/sample_files.py b/act/tests/sample_files.py
--- a/act/tests/sample_files.py
+++ b/act/tests/sample_files.py
@@ -50,11 +50,9 @@
 EXAMPLE_MET_TEST2 = os.path.join(DATA_PATH, 'sgpmet_test_time.nc')
 EXAMPLE_STAMP_WILDCARD = os.path.join(DATA_PATH, 'sgpstamp*202001*.nc')
 EXAMPLE_NOAA_PSL = os.path.join(DATA_PATH, 'ctd21125.15w')
-EXAMPLE_SP2B = os.path.join(
-    DATA_PATH, 'mosaossp2M1.00.20191216.130601.raw.20191216x193.sp2b')
-EXAMPLE_INI = os.path.join(
-    DATA_PATH, 'mosaossp2M1.00.20191216.000601.raw.20191216000000.ini')
-EXAMPLE_HK = os.path.join(
-    DATA_PATH, 'mosaossp2auxM1.00.20191217.010801.raw.20191216000000.hk')
+EXAMPLE_NOAA_PSL_TEMPERATURE = os.path.join(DATA_PATH, 'ctd22187.00t.txt')
+EXAMPLE_SP2B = os.path.join(DATA_PATH, 'mosaossp2M1.00.20191216.130601.raw.20191216x193.sp2b')
+EXAMPLE_INI = os.path.join(DATA_PATH, 'mosaossp2M1.00.20191216.000601.raw.20191216000000.ini')
+EXAMPLE_HK = os.path.join(DATA_PATH, 'mosaossp2auxM1.00.20191217.010801.raw.20191216000000.hk')
 EXAMPLE_MET_YAML = os.path.join(DATA_PATH, 'sgpmetE13.b1.yaml')
 EXAMPLE_CLOUDPHASE = os.path.join(DATA_PATH, 'nsacloudphaseC1.c1.20180601.000000.nc')
diff --git a/act/tests/test_io.py b/act/tests/test_io.py
--- a/act/tests/test_io.py
+++ b/act/tests/test_io.py
@@ -7,7 +7,7 @@
 
 import act
 import act.tests.sample_files as sample_files
-from act.io.noaagml import read_gml
+from act.io import read_gml, read_psl_wind_profiler_temperature
 
 
 def test_io():
@@ -34,19 +34,48 @@ def test_io():
 
 def test_keep_variables():
 
-    var_names = ['temp_mean', 'rh_mean', 'wdir_vec_mean', 'tbrg_precip_total_corr',
-                 'atmos_pressure', 'wspd_vec_mean', 'pwd_pw_code_inst', 'pwd_pw_code_15min',
-                 'pwd_mean_vis_10min', 'logger_temp', 'pwd_precip_rate_mean_1min',
-                 'pwd_cumul_snow', 'pwd_mean_vis_1min', 'pwd_pw_code_1hr', 'org_precip_rate_mean',
-                 'tbrg_precip_total', 'pwd_cumul_rain']
+    var_names = [
+        'temp_mean',
+        'rh_mean',
+        'wdir_vec_mean',
+        'tbrg_precip_total_corr',
+        'atmos_pressure',
+        'wspd_vec_mean',
+        'pwd_pw_code_inst',
+        'pwd_pw_code_15min',
+        'pwd_mean_vis_10min',
+        'logger_temp',
+        'pwd_precip_rate_mean_1min',
+        'pwd_cumul_snow',
+        'pwd_mean_vis_1min',
+        'pwd_pw_code_1hr',
+        'org_precip_rate_mean',
+        'tbrg_precip_total',
+        'pwd_cumul_rain',
+    ]
     var_names = var_names + ['qc_' + ii for ii in var_names]
     drop_variables = act.io.armfiles.keep_variables_to_drop_variables(
-        act.tests.EXAMPLE_MET1, var_names)
+        act.tests.EXAMPLE_MET1, var_names
+    )
 
     expected_drop_variables = [
-        'wdir_vec_std', 'base_time', 'alt', 'qc_wspd_arith_mean', 'pwd_err_code', 'logger_volt',
-        'temp_std', 'lon', 'qc_logger_volt', 'time_offset', 'wspd_arith_mean', 'lat', 'vapor_pressure_std',
-        'vapor_pressure_mean', 'rh_std', 'qc_vapor_pressure_mean']
+        'wdir_vec_std',
+        'base_time',
+        'alt',
+        'qc_wspd_arith_mean',
+        'pwd_err_code',
+        'logger_volt',
+        'temp_std',
+        'lon',
+        'qc_logger_volt',
+        'time_offset',
+        'wspd_arith_mean',
+        'lat',
+        'vapor_pressure_std',
+        'vapor_pressure_mean',
+        'rh_std',
+        'qc_vapor_pressure_mean',
+    ]
     assert drop_variables.sort() == expected_drop_variables.sort()
 
     ds_object = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_MET1, keep_variables='temp_mean')
@@ -54,15 +83,17 @@ def test_keep_variables():
     del ds_object
 
     var_names = ['temp_mean', 'qc_temp_mean']
-    ds_object = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_MET1, keep_variables=var_names,
-                                            drop_variables='nonsense')
+    ds_object = act.io.armfiles.read_netcdf(
+        act.tests.EXAMPLE_MET1, keep_variables=var_names, drop_variables='nonsense'
+    )
     assert list(ds_object.data_vars).sort() == var_names.sort()
     del ds_object
 
     var_names = ['temp_mean', 'qc_temp_mean', 'alt', 'lat', 'lon']
-    ds_object = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_MET_WILDCARD, keep_variables=var_names,
-                                            drop_variables=['lon'])
-    var_names = list(set(var_names) - set(['lon']))
+    ds_object = act.io.armfiles.read_netcdf(
+        act.tests.EXAMPLE_MET_WILDCARD, keep_variables=var_names, drop_variables=['lon']
+    )
+    var_names = list(set(var_names) - {'lon'})
     assert list(ds_object.data_vars).sort() == var_names.sort()
     del ds_object
 
@@ -88,8 +119,17 @@ def test_io_mfdataset():
     met_ds.load()
     var_name = 'temp_mean'
     qc_var_name = 'qc_' + var_name
-    attr_names = ['long_name', 'units', 'flag_masks', 'flag_meanings', 'flag_assessments',
-                  'fail_min', 'fail_max', 'fail_delta', 'standard_name']
+    attr_names = [
+        'long_name',
+        'units',
+        'flag_masks',
+        'flag_meanings',
+        'flag_assessments',
+        'fail_min',
+        'fail_max',
+        'fail_delta',
+        'standard_name',
+    ]
     assert var_name in met_ds.variables.keys()
     assert qc_var_name in met_ds.variables.keys()
     assert sorted(attr_names) == sorted(list(met_ds[qc_var_name].attrs.keys()))
@@ -464,3 +504,11 @@ def test_read_psl_wind_profiler():
     assert test_obj_low['SPD'].shape == (49, 4)
     assert test_obj_hi['SPD'].shape == (50, 3)
     test_obj_low.close()
+
+
+def test_read_psl_wind_profiler_temperature():
+    ds = read_psl_wind_profiler_temperature(act.tests.EXAMPLE_NOAA_PSL_TEMPERATURE)
+
+    ds.attrs['site_identifier'] == 'CTD'
+    ds.attrs['elevation'] = 600.0
+    ds.T.values[0] == 33.2