Repository: ioos/compliance-checker

Issue:
Checker returns overall "pass" result and no stderr output when exceptions encountered
[This file](http://thredds.aodn.org.au/thredds/catalog/IMOS/eMII/checker_test/misc/catalog.html?dataset=eMII/checker_test/misc/cf_checker_exception.nc) appears to pass all CF checks, but causes an exception in the `check_flags` method (separate issue). The checker output looks like this:

```
Running Compliance Checker on the dataset from: cf_checker_exception.nc
The following exceptions occured during the cf checker (possibly indicate compliance checker issues):
cf.check_flags: 'unicode' object has no attribute 'dtype'


--------------------------------------------------------------------------------
                     The dataset scored 65 out of 65 points                     
                              during the cf check                               
--------------------------------------------------------------------------------
All tests passed!
```

Of this, only the first line is sent to stderr. 

Obviously we can fix the code so this particular exception doesn't occur, but others may still crop up. In general, if any exceptions occur during checks, I think it would make sense to 
- report these to stderr; and
- return an overall "fail" result (i.e. non-zero exit value).

Probably best to use a specific exit value other than 1, which is used when the file has failed a correctly running check.



Discussion:

Resolving patch:
diff --git a/cchecker.py b/cchecker.py
--- a/cchecker.py
+++ b/cchecker.py
@@ -18,18 +18,21 @@ def main():
     args.test = args.test or ['acdd']
 
     return_values = []
+    had_errors = []
     for dataset in args.dataset_location:
         if args.format != 'json':
             print("Running Compliance Checker on the dataset from: %s" % dataset, file=sys.stderr)
-        return_value = ComplianceChecker.run_checker(args.dataset_location[0],
+        return_value, errors = ComplianceChecker.run_checker(args.dataset_location[0],
                                       args.test,
                                       args.verbose,
                                       args.criteria,
                                       args.output,
                                       args.format)
         return_values.append(return_value)
+        had_errors.append(errors)
 
-
+    if any(had_errors):
+        return 2
     if all(return_values):
         return 0
     return 1
diff --git a/compliance_checker/runner.py b/compliance_checker/runner.py
--- a/compliance_checker/runner.py
+++ b/compliance_checker/runner.py
@@ -1,4 +1,5 @@
 import traceback
+import sys
 
 from StringIO import StringIO
 from compliance_checker.acdd import ACDDBaseCheck
@@ -64,7 +65,9 @@ def run_checker(cls, ds_loc, checker_names, verbose, criteria, output_filename='
         else:
             raise TypeError('Invalid format %s' % output_format)
 
-        return cs.passtree(groups, limit)
+        errors_occurred = cls.check_errors(score_groups, verbose)
+
+        return cs.passtree(groups, limit), errors_occurred
 
     @classmethod
     def stdout_output(cls, cs, score_groups, verbose, limit):
@@ -79,16 +82,6 @@ def stdout_output(cls, cs, score_groups, verbose, limit):
         '''
         for checker, rpair in score_groups.iteritems():
             groups, errors = rpair
-
-            if len(errors):
-                print "The following exceptions occured during the %s checker (possibly indicate compliance checker issues):" % checker
-
-                for check_name, epair in errors.iteritems():
-                    print "%s.%s: %s" % (checker, check_name, epair[0].message)
-                    if verbose > 0:
-                        traceback.print_tb(epair[1].tb_next.tb_next)    # skip first two as they are noise from the running itself @TODO search for check_name
-                        print
-
             score_list, points, out_of = cs.standard_output(limit, checker, groups)
             if not verbose:
                 cs.non_verbose_output_generation(score_list, groups, limit, points, out_of)
@@ -140,3 +133,25 @@ def json_output(cls, cs, score_groups, output_filename, ds_loc):
 
         return groups
 
+    @classmethod
+    def check_errors(cls, score_groups, verbose):
+        '''
+        Reports any errors (exceptions) that occurred during checking to stderr.
+        Goes to verbose function if called by user.
+
+        @param score_groups List of results
+        @param verbose      Integer value for verbosity level
+        '''
+        errors_occurred = False
+        for checker, rpair in score_groups.iteritems():
+            groups, errors = rpair
+            if len(errors):
+                errors_occurred = True
+                print >>sys.stderr, "WARNING: The following exceptions occured during the %s checker (possibly indicate compliance checker issues):" % checker
+                for check_name, epair in errors.iteritems():
+                    print >>sys.stderr, "%s.%s: %s" % (checker, check_name, epair[0].message)
+                    if verbose > 0:
+                        traceback.print_tb(epair[1].tb_next.tb_next)    # skip first two as they are noise from the running itself @TODO search for check_name
+                        print >>sys.stderr
+
+        return errors_occurred


Test patch: