Repository: Unidata/siphon
Pull request #177 (2018-01-09 15:23:22)

Adds acis.py for ACIS Web Services functionality
Adds acis.py, which provides a function for connecting to ACIS Web Services. This function is Python 3 compatible, but will need to use urllib2 to become Python2.7 compatible.


Patch:
diff --git a/examples/acis/Basic_Overview.py b/examples/acis/Basic_Overview.py
new file mode 100644
index 00000000..4ccfc61c
--- /dev/null
+++ b/examples/acis/Basic_Overview.py
@@ -0,0 +1,95 @@
+"""
+=============================
+Basic ACIS Web Services Usage
+=============================
+
+Siphon's simplewebservice support also includes the ability to query the
+Regional Climate Centers' ACIS data servers. ACIS data provides daily records
+for most station networks in the U.S. and is updated hourly.
+
+In this example we will be querying the service for 20 years of temperature data
+from Denver International Airport.
+"""
+
+import matplotlib.pyplot as plt
+
+from siphon.simplewebservice.acis import acis_request
+
+###########################################
+# First, we need to assemble a dictionary containing the information we want.
+# For this example we want the average temperature at Denver International (KDEN)
+# from January 1, 1997 to December 1, 2017. While we could get the daily data,
+# we will instead request the monthly averages, which the remote service will
+# find for us.
+parameters = {'sid': 'KDEN', 'sdate': '19970101', 'edate': '20171231', 'elems': [
+    {'name': 'avgt', 'interval': 'mly', 'duration': 'mly', 'reduce': 'mean'}]}
+
+###########################################
+# These parameters are used to specify what kind of data we want. We are
+# formatting this as a dictionary, the acis_request function will handle the
+# conversion of this into a JSON string for us!
+#
+# As we explain how this dictionary is formatted, feel free to follow along
+# using the API documentation here :http://www.rcc-acis.org/docs_webservices.html
+#
+# The first section of the parameters dictionary is focused on the station and
+# period of interest. We have a 'sid' element where the airport identifier is,
+# and sdate/edate which correspond to the starting and ending dates of the
+# period of interest.
+#
+# The 'elems' list contains individual dictionaries of elements (variables) of
+# interest. In this example we are requesting the average monthly temperature.
+# If we also wanted the minimum temperature, we would simply add an additional
+# dictionary to the 'elems' list.
+#
+# Now that we have assembled our dictionary, we need to decide what type of
+# request we are making. You can request meta data (information about the
+# station), station data (data from an individual station), data from multiple
+# stations, or even images of pre-prepared data.
+#
+# In this case we are interested in a single station, so we will be using the
+# method set aside for this called, 'StnData'.
+
+method = 'StnData'
+
+###########################################
+# Now that we have our request information ready, we can call the acis_request
+# function and recieve our data!
+
+myData = acis_request(method, parameters)
+
+###########################################
+# The data is also returned in a dictionary format, decoded from a JSON string.
+
+print(myData)
+
+###########################################
+# We can see there are two parts to this data: The metadata, and the data. The
+# metadata can be useful in mapping the observations (We'll do this in a later
+# example).
+#
+# To wrap this example up, we are going to do a simple line graph of this 30
+# year temperature data using MatPlotLib! Notice that the data is decoded as
+# a string, so you should convert those back into numbers before use.
+#
+# *Note: Missing data is recorded as M!
+
+stnName = myData['meta']['name']
+
+avgt = []
+dates = []
+for obs in myData['data']:
+    if obs[0].endswith('01'):
+        dates.append(obs[0])
+    else:
+        dates.append('')
+    avgt.append(float(obs[1]))
+
+X = list(range(len(avgt)))
+
+plt.title(stnName)
+plt.ylabel('Average Temperature (F)')
+plt.plot(X, avgt)
+plt.xticks(X, dates, rotation=45)
+
+plt.show()
diff --git a/examples/acis/Mapping_Example.py b/examples/acis/Mapping_Example.py
new file mode 100644
index 00000000..9038d87c
--- /dev/null
+++ b/examples/acis/Mapping_Example.py
@@ -0,0 +1,149 @@
+"""
+===============================
+Multi Station Calls and Mapping
+===============================
+
+In this example we will be using Siphon's simplewebservice support to query
+ACIS Web Services for multiple stations. We will plot precipitation
+values recorded in Colorado and Wyoming during the 2013 flooding event.
+"""
+
+import cartopy.crs as ccrs
+import cartopy.feature as feat
+import matplotlib.pyplot as plt
+
+from siphon.simplewebservice.acis import acis_request
+
+###########################################
+# First, we need to assemble a dictionary containing the information we want.
+# In this example we want multiple station information, which indicates we
+# need a MultiStnData call. Our event period spans from September 9 through
+# September 12, 2013. We know we are interested in precipitation totals,
+# but we are also going to take advantage of the long-term record in ACIS
+# and ask it to return what the departure from normal precipitation was on
+# this day.
+
+parameters = {'state': 'co', 'sdate': '20130909', 'edate': '20130912', 'elems': [
+    {'name': 'pcpn', 'interval': 'dly'},
+    {'name': 'pcpn', 'interval': 'dly', 'normal': 'departure'}]}
+
+method = 'MultiStnData'
+###########################################
+# In this case, rather than using station ID's, we are able to specify a new
+# parameter called 'state'. If we were interested in other states, we could just
+# add another to the list like this: 'co,wy'. Also notice how we are getting
+# both the precipitation and departure from normal within one variable. We'll
+# see how this changes the final data dictionary. Now let's make our call and
+# review our data.
+
+myData = acis_request(method, parameters)
+
+print(myData)
+
+###########################################
+# MultiStnData calls take longer to return than single stations, especially when
+# you request multiple states. We can see the data is divided by station, with
+# each station having it's own meta and data components. This time we also have
+# multiple values in each data list. Each value corresponds to the variable we
+# requested, in the order we requested it. So in this case, we have the
+# precipitation value, followed by the departure from normal value. Before we
+# plot this information, we need to add up the precipitation sums. But rather
+# than doing it in Python, let's make another ACIS call that prepares this for
+# us.
+
+parameters = {'state': 'co', 'sdate': '20130909', 'edate': '20130912', 'elems': [
+    {'name': 'pcpn', 'interval': 'dly', 'smry': 'sum', 'smry_only': 1},
+    {'name': 'pcpn', 'interval': 'dly', 'smry': 'sum', 'smry_only': 1, 'normal': 'departure'}]}
+
+myData = acis_request(method, parameters)
+
+print(myData)
+
+###########################################
+# First of all, we have two new components to our elements: 'smry' and 'smry_only'.
+# 'smry' allows us to summarize the data over our time period. There are a few
+# options for this, including being able to count the number of records exceeding
+# a threshold (something we will explore in the next example). The other parameter,
+# 'smry_only', allows us to only return the summary value and not the intermediate
+# data.
+#
+# Now let's look at how our data has changed. Rather than having a just a 'meta'
+# and 'data' component, we have a new one called 'smry'. As you've guessed,
+# this contains our summary information (also in the order we requested it).
+# By specifying 'smry_only', there is no 'data' component. If we also wanted
+# all 4 days of data, we would simply remove that parameter.
+#
+# To wrap up this example, we will finally plot our precipitation sums and
+# departures onto a map using CartoPy. To do this we will utilize
+# the meta data that is provided with each station's data. Within the metadata
+# is a 'll' element that contains the latitude and longitude, which is perfect
+# for plotting!
+#
+# One final thing to note is that not all stations have location information.
+# Stations from the ThreadEx network cover general areas, and thus aren't
+# packaged with precise latitudes and longitudes. We will skip them by
+# identifying their network ID of 9 in the ACIS metadata. Don't worry about
+# lost information though! These summarize stations that already exist within
+# their areas!
+
+lat = []
+lon = []
+pcpn = []
+pcpn_dep = []
+
+for stn in myData['data']:
+    # Skip threaded stations! They have no lat/lons
+    if stn['meta']['sids'][-1].endswith('9'):
+        continue
+    # Skip stations with missing data
+    if stn['smry'][0] == 'M' or stn['smry'][1] == 'M':
+        continue
+
+    lat.append(stn['meta']['ll'][1])
+    lon.append(stn['meta']['ll'][0])
+    pcpn.append(float(stn['smry'][0]))
+    pcpn_dep.append(float(stn['smry'][1]))
+###########################################
+# Now we setup our map and plot the data! We are going to plot the station
+# locations with a '+' symbol and label them with the precipitation value.
+# We will use the departures to set the departure from normal values where:
+# * Departure < 0 is Red
+# * Departure > 0 is Green
+# * Departure > 2 is Magenta
+#
+# This should help us visualize where the precipitation event was strongest!
+
+proj = ccrs.LambertConformal(central_longitude=-105, central_latitude=0,
+                             standard_parallels=[35])
+
+fig = plt.figure(figsize=(20, 10))
+ax = fig.add_subplot(1, 1, 1, projection=proj)
+
+state_boundaries = feat.NaturalEarthFeature(category='cultural',
+                                            name='admin_1_states_provinces_lines',
+                                            scale='110m', facecolor='none')
+
+ax.add_feature(feat.LAND, zorder=-1)
+ax.add_feature(feat.OCEAN, zorder=-1)
+ax.add_feature(feat.LAKES, zorder=-1)
+ax.coastlines(resolution='110m', zorder=2, color='black')
+ax.add_feature(state_boundaries, edgecolor='black')
+ax.add_feature(feat.BORDERS, linewidth=2, edgecolor='black')
+
+# Set plot bounds
+ax.set_extent((-109.9, -101.8, 36.5, 41.3))
+
+# Plot each station, labeling based on departure
+for stn in range(len(pcpn)):
+    if pcpn_dep[stn] >= 0 and pcpn_dep[stn] < 2:
+        ax.plot(lon[stn], lat[stn], 'g+', markersize=7, transform=ccrs.Geodetic())
+        ax.text(lon[stn], lat[stn], pcpn[stn], transform=ccrs.Geodetic())
+    elif pcpn_dep[stn] >= 2:
+        ax.plot(lon[stn], lat[stn], 'm+', markersize=7, transform=ccrs.Geodetic())
+        ax.text(lon[stn], lat[stn], pcpn[stn], transform=ccrs.Geodetic())
+    elif pcpn_dep[stn] < 0:
+        ax.plot(lon[stn], lat[stn], 'r+', markersize=7, transform=ccrs.Geodetic())
+        ax.text(lon[stn], lat[stn], pcpn[stn], transform=ccrs.Geodetic())
+ax.plot(pcpn)
+
+plt.show()
diff --git a/examples/acis/readme.txt b/examples/acis/readme.txt
new file mode 100644
index 00000000..bf110757
--- /dev/null
+++ b/examples/acis/readme.txt
@@ -0,0 +1,27 @@
+.. _acis_examples:
+
+ACIS Web Services
+-----------------
+
+Examples of using Siphon's simplewebservice support to access and use data from
+the Applied Climate Information System (ACIS) Web Services API provided by the
+Regional Climate Centers.
+
+The ACIS Web Service API provides daily records for stations from the following
+networks:
+* WBAN
+* COOP
+* FAA
+* WMO
+* ICAO
+* GHCN
+* NWSLI/SHEF
+* ThreadEx
+* CoCoRahs
+* California Multi-station Index
+
+Data is updated hourly and documentation on the API is available at
+http://www.rcc-acis.org/docs_webservices.html
+
+A useful web tool for constructing query parameters is available here:
+http://builder.rcc-acis.org/
diff --git a/siphon/simplewebservice/acis.py b/siphon/simplewebservice/acis.py
new file mode 100644
index 00000000..07dbe221
--- /dev/null
+++ b/siphon/simplewebservice/acis.py
@@ -0,0 +1,65 @@
+"""Requests data from the ACIS Web Services API."""
+
+import requests
+
+from ..http_util import create_http_session
+
+
+def acis_request(method, params):
+    """Request data from the ACIS Web Services API.
+
+    Makes a request from the ACIS Web Services API for data
+    based on a given method (StnMeta,StnData,MultiStnData,GridData,General)
+    and parameters string. Information about the parameters can be obtained at:
+    http://www.rcc-acis.org/docs_webservices.html
+
+    If a connection to the API fails, then it will raise an exception. Some bad
+    calls will also return empty dictionaries.
+
+    ACIS Web Services is a distributed system! A call to the main URL can be
+    delivered to any climate center running a public instance of the service.
+    This makes the calls efficient, but also occasionaly results in failed
+    calls when a server you are directed to is having problems. Generally,
+    reconnecting after waiting a few seconds will resolve a problem. If problems
+    are persistent, contact ACIS developers at the High Plains Regional Climate
+    Center or Northeast Regional Climate Center who will look into server
+    issues.
+
+    Parameters
+    ----------
+    method : str
+        The Web Services request method (StnMeta, StnData, MultiStnData, GridData, General)
+    params : dict
+        A JSON array of parameters (See Web Services API)
+
+    Returns
+    -------
+    A dictionary of data based on the JSON parameters
+
+    Raises
+    ------
+    :class: `ACIS_API_Exception`
+        When the API is unable to establish a connection or returns
+        unparsable data.
+
+    """
+    base_url = 'http://data.rcc-acis.org/'  # ACIS Web API URL
+
+    timeout = 300 if method == 'MultiStnData' else 60
+
+    try:
+        response = create_http_session().post(base_url + method, json=params, timeout=timeout)
+        return response.json()
+    except requests.exceptions.Timeout:
+        raise AcisApiException('Connection Timeout')
+    except requests.exceptions.TooManyRedirects:
+        raise AcisApiException('Bad URL. Check your ACIS connection method string.')
+    except ValueError:
+        raise AcisApiException('No data returned! The ACIS parameter dictionary'
+                               'may be incorrectly formatted')
+
+
+class AcisApiException(Exception):
+    """This class handles exceptions raised by the acis_request function."""
+
+    pass


Test patch:
diff --git a/siphon/tests/fixtures/acis_request b/siphon/tests/fixtures/acis_request
new file mode 100644
index 00000000..9b0979e1
--- /dev/null
+++ b/siphon/tests/fixtures/acis_request
@@ -0,0 +1,148 @@
+interactions:
+- request:
+    body: '{"sids": "KLNK"}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      Content-Length: ['16']
+      Content-Type: [application/json]
+      User-Agent: [Siphon (0.6.0+10.g7d00b16.dirty)]
+    method: POST
+    uri: http://data.rcc-acis.org/StnMeta
+  response:
+    body: {string: '{"meta":[
+
+        {"name": "LINCOLN AIRPORT", "ll": [-96.7475, 40.8508], "sids": ["14939 1",
+        "254795 2", "LNK 3", "72551 4", "KLNK 5", "USW00014939 6", "LNK 7"], "state":
+        "NE", "elev": 1190.0, "uid": 12527}]}
+
+        '}
+    headers:
+      Access-Control-Allow-Origin: ['*']
+      Connection: [keep-alive]
+      Content-Type: [application/json]
+      Date: ['Fri, 19 Jan 2018 19:04:12 GMT']
+      Server: [nginx/1.12.2]
+    status: {code: 200, message: OK}
+- request:
+    body: '{"sid": "klnk", "elems": [{"name": "avgt", "interval": "dly"}, {"name":
+      "mint", "interval": "dly"}], "date": "20000101"}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      Content-Length: ['120']
+      Content-Type: [application/json]
+      User-Agent: [Siphon (0.6.0+10.g7d00b16.dirty)]
+    method: POST
+    uri: http://data.rcc-acis.org/StnData
+  response:
+    body: {string: '{"meta":{"uid": 12527, "ll": [-96.7475, 40.8508], "sids": ["14939
+        1", "254795 2", "LNK 3", "72551 4", "KLNK 5", "USW00014939 6", "LNK 7"], "state":
+        "NE", "elev": 1190.0, "name": "LINCOLN AIRPORT"},
+
+        "data":[["2000-01-01","37.5","26"]]}
+
+        '}
+    headers:
+      Access-Control-Allow-Origin: ['*']
+      Connection: [keep-alive]
+      Content-Type: [application/json]
+      Date: ['Fri, 19 Jan 2018 19:04:14 GMT']
+      Server: [nginx/1.12.2]
+    status: {code: 200, message: OK}
+- request:
+    body: '{"sids": "klnk,kgso", "elems": [{"name": "avgt", "interval": "dly"}, {"name":
+      "mint", "interval": "dly"}], "date": "20000101"}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      Content-Length: ['126']
+      Content-Type: [application/json]
+      User-Agent: [Siphon (0.6.0+10.g7d00b16.dirty)]
+    method: POST
+    uri: http://data.rcc-acis.org/MultiStnData
+  response:
+    body: {string: '{"data":[
+
+        {"meta":{"uid": 12527, "ll": [-96.7475, 40.8508], "sids": ["14939 1", "254795
+        2", "LNK 3", "72551 4", "KLNK 5", "USW00014939 6", "LNK 7"], "state": "NE",
+        "elev": 1190.0, "name": "LINCOLN AIRPORT"},"data":["37.5","26"]},
+
+        {"meta":{"uid": 13284, "ll": [-79.9432, 36.0969], "sids": ["13723 1", "313630
+        2", "GSO 3", "72317 4", "KGSO 5", "USW00013723 6", "GSO 7"], "state": "NC",
+        "elev": 890.0, "name": "GREENSBORO  AP"},"data":["49.0","37"]}]}
+
+        '}
+    headers:
+      Access-Control-Allow-Origin: ['*']
+      Connection: [keep-alive]
+      Content-Type: [application/json]
+      Date: ['Fri, 19 Jan 2018 19:04:14 GMT']
+      Server: [nginx/1.12.2]
+    status: {code: 200, message: OK}
+- request:
+    body: '{"loc": "-95.36, 29.76", "sdate": "2000-01", "edate": "2000-07", "grid":
+      "3", "elems": [{"name": "maxt", "interval": "mly", "reduce": "max", "smry":
+      "max"}]}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      Content-Length: ['157']
+      Content-Type: [application/json]
+      User-Agent: [Siphon (0.6.0+10.g7d00b16.dirty)]
+    method: POST
+    uri: http://data.rcc-acis.org/GridData
+  response:
+    body: {string: '{"data":[["2000-01",81],
+
+        ["2000-02",82],
+
+        ["2000-03",87],
+
+        ["2000-04",89],
+
+        ["2000-05",93],
+
+        ["2000-06",94],
+
+        ["2000-07",103]],
+
+        "smry":[103]}
+
+        '}
+    headers:
+      Access-Control-Allow-Origin: ['*']
+      Connection: [keep-alive]
+      Content-Type: [application/json]
+      Date: ['Fri, 19 Jan 2018 19:04:15 GMT']
+      Server: [nginx/1.12.2]
+    status: {code: 200, message: OK}
+- request:
+    body: '{"state": "ne"}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      Content-Length: ['15']
+      Content-Type: [application/json]
+      User-Agent: [Siphon (0.6.0+10.g7d00b16.dirty)]
+    method: POST
+    uri: http://data.rcc-acis.org/General/state
+  response:
+    body: {string: '{"meta":[
+
+        {"id":"NE","name":"Nebraska"}]}
+
+        '}
+    headers:
+      Access-Control-Allow-Origin: ['*']
+      Connection: [keep-alive]
+      Content-Type: [application/json]
+      Date: ['Fri, 19 Jan 2018 19:04:15 GMT']
+      Server: [nginx/1.12.2]
+    status: {code: 200, message: OK}
+version: 1
diff --git a/siphon/tests/test_acis.py b/siphon/tests/test_acis.py
new file mode 100644
index 00000000..a6d115b5
--- /dev/null
+++ b/siphon/tests/test_acis.py
@@ -0,0 +1,54 @@
+"""Test ACIS Web Services API Access."""
+from siphon.simplewebservice.acis import acis_request
+from siphon.testing import get_recorder
+
+recorder = get_recorder(__file__)
+
+
+@recorder.use_cassette('acis_request')
+def test_acis_metadata():
+    """Testing ACIS MetaData request."""
+    data = acis_request('StnMeta', {'sids': 'KLNK'})
+
+    assert data['meta'][0]['uid'] == 12527
+
+
+def test_acis_stndata():
+    """Testing ACIS Station Data request."""
+    data = acis_request('StnData', {'sid': 'klnk', 'elems': [
+                        {'name': 'avgt', 'interval': 'dly'},
+                        {'name': 'mint', 'interval': 'dly'}], 'date': '20000101'})
+
+    assert data['meta']['uid'] == 12527
+    assert data['data'][0][0] == '2000-01-01'
+    assert data['data'][0][1] == '37.5'
+    assert data['data'][0][2] == '26'
+
+
+def test_acis_multistn():
+    """Testing ACIS Multi Station Data request."""
+    data = acis_request('MultiStnData', {'sids': 'klnk,kgso', 'elems': [
+                        {'name': 'avgt', 'interval': 'dly'},
+                        {'name': 'mint', 'interval': 'dly'}], 'date': '20000101'})
+
+    assert data['data'][0]['meta']['uid'] == 12527
+    assert data['data'][0]['data'][0] == '37.5'
+    assert data['data'][1]['meta']['uid'] == 13284
+    assert data['data'][1]['data'][0] == '49.0'
+
+
+def test_acis_griddata():
+    """Testing ACIS Gridded Data request."""
+    data = acis_request('GridData',
+                        {'loc': '-95.36, 29.76', 'sdate': '2000-01', 'edate': '2000-07',
+                         'grid': '3', 'elems': [{'name': 'maxt', 'interval': 'mly',
+                                                 'reduce': 'max', 'smry': 'max'}]})
+
+    assert data['data'][0][1] == 81
+
+
+def test_acis_general():
+    """Testing ACIS General request."""
+    data = acis_request('General/state', {'state': 'ne'})
+
+    assert data['meta'][0]['name'] == 'Nebraska'