Repository: ioos/compliance-checker
Pull request #540 (2017-11-17 09:03:05)

Added new supported dataset/file type: GenericFile
The `GenericFile` is a file that is only checked from "outside". The contents are not read but the file must exist. This enables checks of directory/file names and the file size. Potentially other file metadata obtained from the file system (such as creation time) could also be checked.

STFC CEDA is building new checkers that use this file type.


Patch:
diff --git a/compliance_checker/base.py b/compliance_checker/base.py
index e68f56e..f057c3b 100644
--- a/compliance_checker/base.py
+++ b/compliance_checker/base.py
@@ -24,6 +24,18 @@ def get_namespaces():
     return ns
 
 
+# Simple class for Generic File type (default to this if file not recognised)
+class GenericFile(object):
+    """
+    Simple class for any file. Has same path lookup as netCDF4.Dataset.
+    """
+    def __init__(self, fpath):
+        self.fpath = fpath
+
+    def filepath(self):
+        return self.fpath
+
+
 class BaseCheck(object):
     HIGH   = 3
     MEDIUM = 2
diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py
index 414668b..663ac90 100644
--- a/compliance_checker/suite.py
+++ b/compliance_checker/suite.py
@@ -4,6 +4,7 @@ Compliance Checker suite runner
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import os
 import sys
 import subprocess
 import inspect
@@ -11,7 +12,7 @@ import itertools
 from operator import itemgetter
 from netCDF4 import Dataset
 from lxml import etree as ET
-from compliance_checker.base import fix_return_value, Result
+from compliance_checker.base import fix_return_value, Result, GenericFile
 from owslib.sos import SensorObservationService
 from owslib.swe.sensor.sml import SensorML
 from compliance_checker.protocols import opendap, netcdf, cdl
@@ -556,6 +557,10 @@ class CheckSuite(object):
         if netcdf.is_netcdf(ds_str):
             return Dataset(ds_str)
 
+        # Assume this is just a Generic File if it exists
+        if os.path.isfile(ds_str):
+            return GenericFile(ds_str)
+
         raise ValueError("File is an unknown format")
 
     def scores(self, raw_scores):


Test patch:
diff --git a/compliance_checker/tests/data/non-comp/empty.file b/compliance_checker/tests/data/non-comp/empty.file
new file mode 100644
index 0000000..190a180
--- /dev/null
+++ b/compliance_checker/tests/data/non-comp/empty.file
@@ -0,0 +1,1 @@
+123
diff --git a/compliance_checker/tests/test_base.py b/compliance_checker/tests/test_base.py
index 66dd8da..550f64a 100644
--- a/compliance_checker/tests/test_base.py
+++ b/compliance_checker/tests/test_base.py
@@ -81,3 +81,18 @@ class TestBase(TestCase):
         self.ds.dummy = 'dummy'
         base.attr_check(attr, self.ds, priority, rv3)
         assert rv3[0] == base.Result(priority, True, 'dummy', [])
+
+
+class TestGenericFile(TestCase):
+    '''
+    Tests the GenericFile class.
+    '''
+
+    def test_create_GenericFile_success(self):
+        path = '/tmp/test.txt'
+        gf = base.GenericFile(path)
+        self.assertEqual(gf.filepath(), path)
+
+    def test_create_GenericFile_failure(self):
+        gf = base.GenericFile("will not match")
+        self.assertNotEqual(gf.filepath(), "do NOT MATCH") 
diff --git a/compliance_checker/tests/test_suite.py b/compliance_checker/tests/test_suite.py
index f756ade..724225e 100644
--- a/compliance_checker/tests/test_suite.py
+++ b/compliance_checker/tests/test_suite.py
@@ -1,6 +1,6 @@
 from pkg_resources import resource_filename
 from compliance_checker.suite import CheckSuite
-from compliance_checker.base import Result, BaseCheck
+from compliance_checker.base import Result, BaseCheck, GenericFile
 import numpy as np
 import unittest
 import os
@@ -11,6 +11,7 @@ static_files = {
     'bad_data_type'              : resource_filename('compliance_checker', 'tests/data/bad_data_type.nc'),
     'test_cdl'                   : resource_filename('compliance_checker', 'tests/data/test_cdl.cdl'),
     'test_cdl_nc'                : resource_filename('compliance_checker', 'tests/data/test_cdl_nc_file.nc'),
+    'empty'                    : resource_filename('compliance_checker', 'tests/data/non-comp/empty.file'),
 }
 
 
@@ -131,3 +132,9 @@ class TestSuite(unittest.TestCase):
         # Ok the scores should be equal!
         self.assertEqual(nc_points, cdl_points)
         self.assertEqual(nc_out_of, cdl_out_of)
+
+    def test_load_local_dataset_GenericFile(self):
+        cs = CheckSuite()
+        resp = cs.load_local_dataset(static_files['empty'])
+        assert isinstance(resp, GenericFile) ==  True
+        
diff --git a/test_requirements.txt b/test_requirements.txt
index d6e1072..5a0c471 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -1,2 +1,3 @@
 flake8
 pytest>=2.9.0
+httpretty