Repository: ioos/compliance-checker

Issue:
Redirect Issue
Some THREDD instances are behind a reverse-proxy and do some 301 redirecting when copy-pasting the OPeNDAP URL into input for Compliance-Checker. It would be good to have CChecker recognize this and get the forwarded address. I.e. follow-redirects.



Discussion:
Do you happen to have any URIs I could test this against?

http://data.ioos.us/thredds/dodsC/deployments/drudnick/sp049-20160405T1844/sp049-20160405T1844.nc3.nc

Returns a 301

Unable to reproduce with `netCDF4==1.2.4` on Python 3.4.3 and `netCDF4==1.2.1` on Python 2.7.5.

```
(cchecker)➜  compliance-checker git:(master) compliance-checker -t cf -c strict http://data.ioos.us/thredds/dodsC/deployments/drudnick/sp049-20160405T1844/sp049-20160405T1844.nc3.nc
Could not load ncei-timeseries = cc_plugin_ncei.ncei_timeseries:NCEITimeSeries : 'module' object has no attribute 'NCEITimeSeries'
Could not load acdd:1.1 = compliance_checker.acdd:ACDD1_1Check : 'module' object has no attribute 'ACDD1_1Check'
Could not load acdd:1.3 = compliance_checker.acdd:ACDD1_3Check : 'module' object has no attribute 'ACDD1_3Check'
Could not load acdd = compliance_checker.acdd:ACDD1_1Check : 'module' object has no attribute 'ACDD1_1Check'
Running Compliance Checker on the dataset from: http://data.ioos.us/thredds/dodsC/deployments/drudnick/sp049-20160405T1844/sp049-20160405T1844.nc3.nc
Traceback (most recent call last):
  File "/Users/lcampbell/.virtualenvs/cchecker/bin/compliance-checker", line 9, in <module>
    load_entry_point('compliance-checker', 'console_scripts', 'compliance-checker')()
  File "/Users/lcampbell/Documents/Dev/code/compliance-checker/cchecker.py", line 41, in main
    args.format)
  File "/Users/lcampbell/Documents/Dev/code/compliance-checker/compliance_checker/runner.py", line 33, in run_checker
    ds = cs.load_dataset(ds_loc)
  File "/Users/lcampbell/Documents/Dev/code/compliance-checker/compliance_checker/suite.py", line 472, in load_dataset
    raise Exception("Could not understand response code %s and content-type %s" % (rhead.status_code, rhead.headers.get('content-type', 'none')))
Exception: Could not understand response code 301 and content-type text/html
```

netcdf: 4.3.3.1_4

Hmmm, now I'm getting issues.  Doesn't look like a NetCDF issue, but maybe a requests issue instead.


Resolving patch:
diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py
--- a/compliance_checker/suite.py
+++ b/compliance_checker/suite.py
@@ -269,11 +269,10 @@ def get_points(self, groups, limit):
         score_list = []
         score_only_list = []
 
-        for v in range(len(groups)):
-            score_list.append([groups[v].name, groups[v].weight, groups[v].value,
-                               groups[v].children])
-            if groups[v].weight >= limit:
-                score_only_list.append(groups[v].value)
+        for g in groups:
+            score_list.append([g.name, g.weight, g.value, g.children])
+            if g.weight >= limit:
+                score_only_list.append(g.value)
 
         points = [x[0] for x in score_only_list]
         out_of = [x[1] for x in score_only_list]
@@ -447,6 +446,21 @@ def weight_func(r):
             if res.children:
                 self.reasoning_routine(res.children, indent + 1, False)
 
+    def process_doc(doc):
+        """
+        Attempt to parse an xml string conforming to either an SOS or SensorML
+        dataset and return the results
+        """
+        xml_doc = ET.fromstring(str(doc))
+        if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities":
+            ds = SensorObservationService(ds_str, xml=str(doc))
+
+        elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML":
+            ds = SensorML(xml_doc)
+        else:
+            raise ValueError("Unrecognized XML root element: %s" % xml_doc.tag)
+        return ds
+
     def load_dataset(self, ds_str):
         """
         Helper method to load a dataset or SOS GC/DS url.
@@ -457,7 +471,7 @@ def load_dataset(self, ds_str):
         doc = None
         pr = urlparse(ds_str)
         if pr.netloc:       # looks like a remote url
-            rhead = requests.head(ds_str)
+            rhead = requests.head(ds_str, allow_redirects=True)
 
             # if we get a 400 here, it's likely a Dataset openable OpenDAP url
             if rhead.status_code == 400:
@@ -476,13 +490,14 @@ def is_binary_string(bts):
                 # http://stackoverflow.com/a/7392391/84732
                 if sys.version_info >= (3, ):
                     join_str = ''
-                    textchars = join_str.join(map(chr, [7, 8, 9, 10, 12, 13, 27] + list(range(0x20, 0x100)))).encode()
-                    #textchars = textchars.encode()
+                    textchars = join_str.join(map(chr, [7, 8, 9, 10, 12, 13, 27]
+                                                  + list(range(0x20, 0x100)))).encode()
                 else:
                     # because of `unicode_literals` import, we need to convert
                     # to a Py2 string/bytes
                     join_str = str('')
-                    textchars = join_str.join(map(chr, [7, 8, 9, 10, 12, 13, 27] + list(range(0x20, 0x100))))
+                    textchars = join_str.join(map(chr, [7, 8, 9, 10, 12, 13, 27]
+                                                  + list(range(0x20, 0x100))))
                 return bool(bts.translate(None, textchars))
 
             with open(ds_str, 'rb') as f:
@@ -495,14 +510,7 @@ def is_binary_string(bts):
                     doc = "".join(f.readlines())
 
         if doc is not None:
-            xml_doc = ET.fromstring(str(doc))
-            if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities":
-                ds = SensorObservationService(ds_str, xml=str(doc))
-
-            elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML":
-                ds = SensorML(xml_doc)
-            else:
-                raise Exception("Unrecognized XML root element: %s" % xml_doc.tag)
+            ds = process_doc(doc)
         else:
             # no doc? try the dataset constructor
             ds = Dataset(ds_str)


Test patch: