Repository: Unidata/siphon

Issue:
Add time to upper air data
Current the returned data does not have a column for time, since this isn't provided as a column in the output--but it *is* in the text header. It would be nice to parse this and add as a column so that we have a self-contained dataset. It's probably also needed when we try to handle requesting and returning data for multiple times.

Discussion:

Patch:
diff --git a/siphon/simplewebservice/iastate.py b/siphon/simplewebservice/iastate.py
index f31d8cfa..cb931967 100644
--- a/siphon/simplewebservice/iastate.py
+++ b/siphon/simplewebservice/iastate.py
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 """Read upper air data from the IA State archives."""
 
+from datetime import datetime
 import json
 import warnings
 
@@ -64,9 +65,8 @@ class IAStateUpperAir(HTTPEndPoint):
 
         """
         json_data = self._get_data_raw(time, site_id)
-
         data = {}
-        for pt in json_data:
+        for pt in json_data['profiles'][0]['profile']:
             for field in ('drct', 'dwpc', 'hght', 'pres', 'sknt', 'tmpc'):
                 data.setdefault(field, []).append(np.nan if pt[field] is None else pt[field])
 
@@ -90,6 +90,10 @@ class IAStateUpperAir(HTTPEndPoint):
         df = df.dropna(subset=('temperature', 'dewpoint', 'direction', 'speed',
                                'u_wind', 'v_wind'), how='all').reset_index(drop=True)
 
+        df['station'] = json_data['profiles'][0]['station']
+        df['time'] = datetime.strptime(json_data['profiles'][0]['valid'],
+                                       '%Y-%m-%dT%H:%M:%SZ')
+
         # Add unit dictionary
         df.units = {'pressure': 'hPa',
                     'height': 'meter',
@@ -98,8 +102,9 @@ class IAStateUpperAir(HTTPEndPoint):
                     'direction': 'degrees',
                     'speed': 'knot',
                     'u_wind': 'knot',
-                    'v_wind': 'knot'}
-
+                    'v_wind': 'knot',
+                    'station': None,
+                    'time': None}
         return df
 
     def _get_data_raw(self, time, site_id):
@@ -119,10 +124,10 @@ class IAStateUpperAir(HTTPEndPoint):
         """
         path = ('raob.py?ts={time:%Y%m%d%H}00&station={stid}').format(time=time, stid=site_id)
         resp = self.get_path(path)
-        json_data = json.loads(resp.text)['profiles'][0]['profile']
+        json_data = json.loads(resp.text)
 
         # See if the return is valid, but has no data
-        if not json_data:
+        if not json_data['profiles'][0]['profile']:
             raise ValueError('No data available for {time:%Y-%m-%d %HZ} '
                              'for station {stid}.'.format(time=time, stid=site_id))
         return json_data
diff --git a/siphon/simplewebservice/wyoming.py b/siphon/simplewebservice/wyoming.py
index e2d9e4b7..48d0907f 100644
--- a/siphon/simplewebservice/wyoming.py
+++ b/siphon/simplewebservice/wyoming.py
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 """Read upper air data from the Wyoming archives."""
 
+from datetime import datetime
 from io import StringIO
 import warnings
 
@@ -69,8 +70,10 @@ class WyomingUpperAir(HTTPEndPoint):
 
         """
         raw_data = self._get_data_raw(time, site_id, region)
+        soup = BeautifulSoup(raw_data, 'html.parser')
+        tabular_data = StringIO(soup.find_all('pre')[0].contents[0])
         col_names = ['pressure', 'height', 'temperature', 'dewpoint', 'direction', 'speed']
-        df = pd.read_fwf(raw_data, skiprows=5, usecols=[0, 1, 2, 3, 6, 7], names=col_names)
+        df = pd.read_fwf(tabular_data, skiprows=5, usecols=[0, 1, 2, 3, 6, 7], names=col_names)
         df['u_wind'], df['v_wind'] = get_wind_components(df['speed'],
                                                          np.deg2rad(df['direction']))
 
@@ -78,6 +81,23 @@ class WyomingUpperAir(HTTPEndPoint):
         df = df.dropna(subset=('temperature', 'dewpoint', 'direction', 'speed',
                                'u_wind', 'v_wind'), how='all').reset_index(drop=True)
 
+        # Parse metadata
+        meta_data = soup.find_all('pre')[1].contents[0]
+        lines = meta_data.splitlines()
+        station = lines[1].split(':')[1].strip()
+        station_number = int(lines[2].split(':')[1].strip())
+        sounding_time = datetime.strptime(lines[3].split(':')[1].strip(), '%y%m%d/%H%M')
+        latitude = float(lines[4].split(':')[1].strip())
+        longitude = float(lines[5].split(':')[1].strip())
+        elevation = float(lines[6].split(':')[1].strip())
+
+        df['station'] = station
+        df['station_number'] = station_number
+        df['time'] = sounding_time
+        df['latitude'] = latitude
+        df['longitude'] = longitude
+        df['elevation'] = elevation
+
         # Add unit dictionary
         df.units = {'pressure': 'hPa',
                     'height': 'meter',
@@ -86,7 +106,13 @@ class WyomingUpperAir(HTTPEndPoint):
                     'direction': 'degrees',
                     'speed': 'knot',
                     'u_wind': 'knot',
-                    'v_wind': 'knot'}
+                    'v_wind': 'knot',
+                    'station': None,
+                    'station_number': None,
+                    'time': None,
+                    'latitude': 'degrees',
+                    'longitude': 'degrees',
+                    'elevation': 'meter'}
         return df
 
     def _get_data_raw(self, time, site_id, region='naconf'):
@@ -103,7 +129,7 @@ class WyomingUpperAir(HTTPEndPoint):
 
         Returns
         -------
-        a file-like object from which to read the data
+        text of the server response
 
         """
         path = ('?region={region}&TYPE=TEXT%3ALIST'
@@ -118,5 +144,4 @@ class WyomingUpperAir(HTTPEndPoint):
                 'for station {stid}.'.format(time=time, region=region,
                                              stid=site_id))
 
-        soup = BeautifulSoup(resp.text, 'html.parser')
-        return StringIO(soup.find_all('pre')[0].contents[0])
+        return resp.text


Test patch:
diff --git a/siphon/tests/test_iastate.py b/siphon/tests/test_iastate.py
index c5d038f6..ca7448d2 100644
--- a/siphon/tests/test_iastate.py
+++ b/siphon/tests/test_iastate.py
@@ -19,6 +19,9 @@ def test_iastate():
     """Test that we are properly parsing data from the Iowa State archive."""
     df = IAStateUpperAir.request_data(datetime(1999, 5, 4, 0), 'OUN')
 
+    assert(df['time'][0] == datetime(1999, 5, 4, 0))
+    assert(df['station'][0] == 'KOUN')
+
     assert_almost_equal(df['pressure'][6], 872.7, 2)
     assert_almost_equal(df['height'][6], 1172.0, 2)
     assert_almost_equal(df['temperature'][6], 18.2, 2)
@@ -36,6 +39,8 @@ def test_iastate():
     assert(df.units['v_wind'] == 'knot')
     assert(df.units['speed'] == 'knot')
     assert(df.units['direction'] == 'degrees')
+    assert(df.units['station'] is None)
+    assert(df.units['time'] is None)
 
 
 @recorder.use_cassette('iastate_high_alt_sounding')
@@ -43,6 +48,9 @@ def test_high_alt_iastate():
     """Test Iowa State data that starts at pressure less than 925 hPa."""
     df = IAStateUpperAir.request_data(datetime(2010, 12, 9, 12), 'BOI')
 
+    assert(df['time'][0] == datetime(2010, 12, 9, 12))
+    assert(df['station'][0] == 'KBOI')
+
     assert_almost_equal(df['pressure'][0], 919.0, 2)
     assert_almost_equal(df['height'][0], 871.0, 2)
     assert_almost_equal(df['temperature'][0], -0.1, 2)
@@ -51,3 +59,14 @@ def test_high_alt_iastate():
     assert_almost_equal(df['v_wind'][0], 1.500, 2)
     assert_almost_equal(df['speed'][0], 3.0, 1)
     assert_almost_equal(df['direction'][0], 240.0, 1)
+
+    assert(df.units['pressure'] == 'hPa')
+    assert(df.units['height'] == 'meter')
+    assert(df.units['temperature'] == 'degC')
+    assert(df.units['dewpoint'] == 'degC')
+    assert(df.units['u_wind'] == 'knot')
+    assert(df.units['v_wind'] == 'knot')
+    assert(df.units['speed'] == 'knot')
+    assert(df.units['direction'] == 'degrees')
+    assert(df.units['station'] is None)
+    assert(df.units['time'] is None)
diff --git a/siphon/tests/test_wyoming.py b/siphon/tests/test_wyoming.py
index 9675cd28..01ba59ff 100644
--- a/siphon/tests/test_wyoming.py
+++ b/siphon/tests/test_wyoming.py
@@ -19,6 +19,13 @@ def test_wyoming():
     """Test that we are properly parsing data from the Wyoming archive."""
     df = WyomingUpperAir.request_data(datetime(1999, 5, 4, 0), 'OUN')
 
+    assert(df['time'][0] == datetime(1999, 5, 4, 0))
+    assert(df['station'][0] == 'OUN')
+    assert(df['station_number'][0] == 72357)
+    assert(df['latitude'][0] == 35.18)
+    assert(df['longitude'][0] == -97.44)
+    assert(df['elevation'][0] == 345.0)
+
     assert_almost_equal(df['pressure'][5], 867.9, 2)
     assert_almost_equal(df['height'][5], 1219., 2)
     assert_almost_equal(df['temperature'][5], 17.4, 2)
@@ -36,6 +43,12 @@ def test_wyoming():
     assert(df.units['v_wind'] == 'knot')
     assert(df.units['speed'] == 'knot')
     assert(df.units['direction'] == 'degrees')
+    assert(df.units['latitude'] == 'degrees')
+    assert(df.units['longitude'] == 'degrees')
+    assert(df.units['elevation'] == 'meter')
+    assert(df.units['station'] is None)
+    assert(df.units['station_number'] is None)
+    assert(df.units['time'] is None)
 
 
 @recorder.use_cassette('wyoming_high_alt_sounding')
@@ -43,6 +56,13 @@ def test_high_alt_wyoming():
     """Test Wyoming data that starts at pressure less than 925 hPa."""
     df = WyomingUpperAir.request_data(datetime(2010, 12, 9, 12), 'BOI')
 
+    assert(df['time'][0] == datetime(2010, 12, 9, 12))
+    assert(df['station'][0] == 'BOI')
+    assert(df['station_number'][0] == 72681)
+    assert(df['latitude'][0] == 43.56)
+    assert(df['longitude'][0] == -116.21)
+    assert(df['elevation'][0] == 874.0)
+
     assert_almost_equal(df['pressure'][2], 890.0, 2)
     assert_almost_equal(df['height'][2], 1133., 2)
     assert_almost_equal(df['temperature'][2], 5.4, 2)
@@ -51,3 +71,18 @@ def test_high_alt_wyoming():
     assert_almost_equal(df['v_wind'][2], 5.99, 2)
     assert_almost_equal(df['speed'][2], 6.0, 1)
     assert_almost_equal(df['direction'][2], 176.0, 1)
+
+    assert(df.units['pressure'] == 'hPa')
+    assert(df.units['height'] == 'meter')
+    assert(df.units['temperature'] == 'degC')
+    assert(df.units['dewpoint'] == 'degC')
+    assert(df.units['u_wind'] == 'knot')
+    assert(df.units['v_wind'] == 'knot')
+    assert(df.units['speed'] == 'knot')
+    assert(df.units['direction'] == 'degrees')
+    assert(df.units['latitude'] == 'degrees')
+    assert(df.units['longitude'] == 'degrees')
+    assert(df.units['elevation'] == 'meter')
+    assert(df.units['station'] is None)
+    assert(df.units['station_number'] is None)
+    assert(df.units['time'] is None)