Repository: ioos/compliance-checker
Pull request #731 (2020-02-09 00:16:36)

generate netCDF4 file from cdl file by default and corresponding tests
* generate netCDF4 file from cdl with ncgen: When a CDL file is provided, currently, a netCDF classic file is created by ncgen. However, a CDL file might contain netCDF4 features such as attributes/variables of type int64 (indicated by `LL` after numbers). Therefore, the `ncgen`-call that creates a netCDF file was modified to create a netCDF4 instead of a netCDF-classic file. If this does not work, we try to create a netCDF Classic file. If that does not work, we exit the program.
* added cdl file with netcdf4 features for testing
* two new test for netCDF4 file generation


Patch:
diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py
index 489a28d..5e6b952 100644
--- a/compliance_checker/suite.py
+++ b/compliance_checker/suite.py
@@ -695,9 +695,22 @@ class CheckSuite(object):
             ds_str = cdl_path.replace('.cdl', '.nc')
         else:
             ds_str = cdl_path + '.nc'
-        returncode = subprocess.call(['ncgen', '-o', ds_str, cdl_path])
-        if returncode != 0:
-            print("Error generating dataset from .cdl")
+
+        # generate netCDF-4 file
+        iostat = subprocess.run(['ncgen', '-k', 'nc4', '-o', ds_str, cdl_path], stderr=subprocess.PIPE)
+        if iostat.returncode != 0:
+          # if not successfull, create netCDF classic file
+          print('netCDF-4 file could not be generated from cdl file with ' +
+                'message:')
+          print(iostat.stderr.decode())
+          print('Trying to create netCDF Classic file instead.')
+          iostat = subprocess.run(['ncgen', '-k', 'nc3', '-o', ds_str, cdl_path], stderr=subprocess.PIPE)
+          if iostat.returncode != 0:
+            # Exit program if neither a netCDF Classic nor a netCDF-4 file
+            # could be created.
+            print('netCDF Classic file could not be generated from cdl file' +
+                  'with message:')
+            print(iostat.stderr.decode())
             sys.exit(1)
         return ds_str
 


Test patch:
diff --git a/compliance_checker/tests/data/test_cdl_nc4_file.cdl b/compliance_checker/tests/data/test_cdl_nc4_file.cdl
new file mode 100644
index 0000000..cc7b990
--- /dev/null
+++ b/compliance_checker/tests/data/test_cdl_nc4_file.cdl
@@ -0,0 +1,40 @@
+netcdf example_with_netcdf4_feature {
+dimensions:
+	time = UNLIMITED ; // (2 currently)
+	z = 3 ;
+	lat = 10 ;
+	lon = 10 ;
+variables:
+	float time(time) ;
+		time:units = "seconds since 2000-06-21 00:00:00" ;
+		time:axis = "T" ;
+		time:standard_name = "time" ;
+	float tas(time, z, lat, lon) ;
+		tas:_FillValue = -999.f ;
+		tas:standard_name = "air_temperature" ;
+		tas:units = "degree_C" ;
+	int64 mask(time, z, lat, lon) ;
+		mask:_FillValue = -999LL ;
+		mask:long_name = "a mask" ;
+		mask:units = "1" ;
+	double z(z) ;
+		z:units = "m" ;
+		z:axis = "Z" ;
+		z:standard_name = "height" ;
+		z:positive = "up" ;
+	double lon(lon) ;
+		lon:standard_name = "longitude" ;
+		lon:units = "degrees_east" ;
+		lon:axis = "X" ;
+	double lat(lat) ;
+		lat:standard_name = "latitude" ;
+		lat:units = "degrees_north" ;
+		lat:axis = "Y" ;
+
+// global attributes:
+		:title = "A test file" ;
+		:history = "2020-02-04T18:00 manually create by Daniel Neumann, DKRZ" ;
+		:global_att_of_type_int = 2 ;
+		:global_att_of_type_int64 = 2LL ;
+		:Conventions = "CF-1.7" ;
+}
diff --git a/compliance_checker/tests/test_suite.py b/compliance_checker/tests/test_suite.py
index 3b0429d..19e606d 100644
--- a/compliance_checker/tests/test_suite.py
+++ b/compliance_checker/tests/test_suite.py
@@ -13,7 +13,8 @@ static_files = {
     'test_cdl'     : resource_filename('compliance_checker', 'tests/data/test_cdl.cdl'),
     'test_cdl_nc'  : resource_filename('compliance_checker', 'tests/data/test_cdl_nc_file.nc'),
     'empty'        : resource_filename('compliance_checker', 'tests/data/non-comp/empty.file'),
-    'ru07'         : resource_filename('compliance_checker', 'tests/data/ru07-20130824T170228_rt0.nc')
+    'ru07'         : resource_filename('compliance_checker', 'tests/data/ru07-20130824T170228_rt0.nc'),
+    'netCDF4'      : resource_filename('compliance_checker', 'tests/data/test_cdl_nc4_file.cdl')
 }
 
 
@@ -59,6 +60,18 @@ class TestSuite(unittest.TestCase):
             # This asserts that print is able to generate all of the unicode output
             self.cs.standard_output_generation(groups, limit, points, out_of, checker)
 
+    def test_generate_dataset_netCDF4(self):
+        """
+        Tests that suite.generate_dataset works with cdl file with netCDF4
+        features.
+        """
+        # create netCDF4 file
+        ds_name = self.cs.generate_dataset(static_files['netCDF4'])
+        # check if correct name is return
+        assert ds_name == static_files['netCDF4'].replace('.cdl', '.nc')
+        # check if netCDF4 file was created
+        assert os.path.isfile(static_files['netCDF4'].replace('.cdl', '.nc'))
+
     def test_skip_checks(self):
         """Tests that checks are properly skipped when specified"""
         ds = self.cs.load_dataset(static_files['2dim'])
@@ -184,3 +197,18 @@ class TestSuite(unittest.TestCase):
                                                         groups)
         assert all_passed < out_of
 
+    def test_netCDF4_features(self):
+        """
+        Check if a proper netCDF4 file with netCDF4-datatypes is created.
+        """
+        # create and open dataset
+        ds = self.cs.load_dataset(static_files['netCDF4'])
+        # check if netCDF type of global attributes is correct
+        assert isinstance(ds.global_att_of_type_int, np.int32)
+        # check if netCDF4 type of global attributes is correct
+        assert isinstance(ds.global_att_of_type_int64, np.int64)
+        # check if netCDF type of variable is correct
+        assert ds['tas'].dtype is np.dtype('float32')
+        # check if netCDF4 type of variable is correct
+        assert ds['mask'].dtype is np.dtype('int64')
+