Repository: Unidata/netcdf-c (c)

Issue:
enable compact storage as an additional storage option for netCDF-4 vars
HDF5 supports a "compact" storage option. It helps a lot with parallel I/O performance, but may only be used for small variables (< 64 MB).

See #1554 for prototype implementation by @gsjaardema, who suggested this feature.

This can be accomplished without API changes. Currently we have two constants for nc_def_var_chunking():

```
/** In HDF5 files you can set storage for each variable to be either
 * contiguous or chunked, with nc_def_var_chunking().  This define is
 * used there. */
/**@{*/
#define NC_CHUNKED    0
#define NC_CONTIGUOUS 1
/**@}*/
```

By adding NC_COMPACT as a third constant, we can enable compact storage for users.

This feature is backward compatible to HDF5 1.8.x.



Pull request description:
enable compact storage for netcdf-4 vars
Fixes #1568 

@gsjaardema thanks very much for the suggestion and prototype implementation.

This PR includes a test and a method of setting (and inquiring about) compact storage, with the existing API.

This PR is instead of #1535 


Patch:
diff --git a/include/nc4internal.h b/include/nc4internal.h
index 2a1b3558b..157e6008c 100644
--- a/include/nc4internal.h
+++ b/include/nc4internal.h
@@ -205,6 +205,7 @@ typedef struct NC_VAR_INFO
     void *fill_value;
     size_t *chunksizes;
     nc_bool_t contiguous;        /**< True if variable is stored contiguously in HDF5 file */
+    nc_bool_t compact;           /**< True if variable is in comact storage in HDF5 file */
     int parallel_access;         /**< Type of parallel access for I/O on variable (collective or independent) */
     nc_bool_t dimscale;          /**< True if var is a dimscale */
     nc_bool_t *dimscale_attached;  /**< Array of flags that are true if dimscale is attached for that dim index */
diff --git a/include/netcdf.h b/include/netcdf.h
index aa908dd61..cc8370f2b 100644
--- a/include/netcdf.h
+++ b/include/netcdf.h
@@ -293,6 +293,7 @@ NOTE: The NC_MAX_DIMS, NC_MAX_ATTRS, and NC_MAX_VARS limits
 /**@{*/
 #define NC_CHUNKED    0
 #define NC_CONTIGUOUS 1
+#define NC_COMPACT    2
 /**@}*/
 
 /** In HDF5 files you can set check-summing for each variable.
diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c
index bafa5389c..8029ad660 100644
--- a/libdispatch/dvar.c
+++ b/libdispatch/dvar.c
@@ -452,9 +452,25 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
 /**
    Define chunking parameters for a variable
 
-   The function nc_def_var_chunking sets the chunking parameters for a
-   variable in a netCDF-4 file. It can set the chunk sizes to get chunked
-   storage, or it can set the contiguous flag to get contiguous storage.
+   The function nc_def_var_chunking sets the storage and, optionally,
+   the chunking parameters for a variable in a netCDF-4 file.
+
+   The storage may be set to NC_CONTIGUOUS, NC_COMPACT, or NC_CHUNKED.
+
+   Contiguous storage means the variable is stored as one block of
+   data in the file.
+
+   Compact storage means the variable is stored in the header record
+   of the file. This can have large performance benefits on HPC system
+   running many processors. Compact storage is only available for
+   variables whose data are 64 KB or less. Attempting to turn on
+   compact storage for a variable that is too large will result in the
+   ::NC_EVARSIZE error.
+
+   Chunked storage means the data are stored as chunks, of
+   user-configurable size. Chunked storage is required for variable
+   with one or more unlimted dimensions, or variable which use
+   compression.
 
    The total size of a chunk must be less than 4 GiB. That is, the
    product of all chunksizes and the size of the data (or the size of
@@ -467,20 +483,21 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    Note that this does not work for scalar variables. Only non-scalar
    variables can have chunking.
 
-   @param[in] ncid NetCDF ID, from a previous call to nc_open or
-   nc_create.
+   @param ncid NetCDF ID, from a previous call to nc_open() or
+   nc_create().
 
-   @param[in] varid Variable ID.
+   @param varid Variable ID.
 
-   @param[in] storage If ::NC_CONTIGUOUS, then contiguous storage is used
-   for this variable. Variables with one or more unlimited dimensions
-   cannot use contiguous storage. If contiguous storage is turned on, the
-   chunksizes parameter is ignored. If ::NC_CHUNKED, then chunked storage
-   is used for this variable. Chunk sizes may be specified with the
-   chunksizes parameter or default sizes will be used if that parameter
-   is NULL.
+   @param storage If ::NC_CONTIGUOUS or ::NC_COMPACT, then contiguous
+   or compact storage is used for this variable. Variables with one or
+   more unlimited dimensions cannot use contiguous or compact
+   storage. If contiguous or compact storage is turned on, the
+   chunksizes parameter is ignored. If ::NC_CHUNKED, then chunked
+   storage is used for this variable. Chunk sizes may be specified
+   with the chunksizes parameter or default sizes will be used if that
+   parameter is NULL.
 
-   @param[in] chunksizesp A pointer to an array list of chunk sizes. The
+   @param chunksizesp A pointer to an array list of chunk sizes. The
    array must have one chunksize for each dimension of the variable. If
    ::NC_CONTIGUOUS storage is set, then the chunksizes parameter is
    ignored.
@@ -500,6 +517,10 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    @return ::NC_EBADCHUNK Returns if the chunk size specified for a
    variable is larger than the length of the dimensions associated with
    variable.
+   @return ::NC_EVARSIZE Compact storage attempted for variable bigger
+   than 64 KB.
+   @return ::NC_EINVAL Attempt to set contiguous or compact storage
+   for var with one or more unlimited dimensions.
 
    @section nc_def_var_chunking_example Example
 
@@ -539,6 +560,7 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    if (chunksize[d] != chunksize_in[d]) ERR;
    if (storage_in != NC_CHUNKED) ERR;
    @endcode
+   @author Ed Hartnett, Dennis Heimbigner
 */
 int
 nc_def_var_chunking(int ncid, int varid, int storage,
diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c
index 38362d0c7..577f03fe6 100644
--- a/libhdf5/hdf5open.c
+++ b/libhdf5/hdf5open.c
@@ -1088,8 +1088,10 @@ static int get_chunking_info(hid_t propid, NC_VAR_INFO_T *var)
         for (d = 0; d < var->ndims; d++)
             var->chunksizes[d] = chunksize[d];
     }
-    else if (layout == H5D_CONTIGUOUS || layout == H5D_COMPACT)
+    else if (layout == H5D_CONTIGUOUS)
         var->contiguous = NC_TRUE;
+    else if (layout == H5D_COMPACT)
+        var->compact = NC_TRUE;
 
     return NC_NOERR;
 }
diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index 78f0cad89..076238821 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -22,6 +22,9 @@
  * order. */
 #define NC_TEMP_NAME "_netcdf4_temporary_variable_name_for_rename"
 
+/** Number of bytes in 64 KB. */
+#define SIXTY_FOUR_KB (65536)
+
 #ifdef LOGGING
 /**
  * Report the chunksizes selected for a variable.
@@ -707,41 +710,65 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
         var->contiguous = NC_FALSE;
     }
 
-    /* Does the user want a contiguous dataset? Not so fast! Make sure
-     * that there are no unlimited dimensions, and no filters in use
-     * for this data. */
-    if (contiguous && *contiguous)
+    /* Handle storage settings. */
+    if (contiguous)
     {
-        if (var->deflate || var->fletcher32 || var->shuffle)
-            return NC_EINVAL;
-
-        for (d = 0; d < var->ndims; d++)
-            if (var->dim[d]->unlimited)
+        /* Does the user want a contiguous or compact dataset? Not so
+         * fast! Make sure that there are no unlimited dimensions, and
+         * no filters in use for this data. */
+        if (*contiguous)
+        {
+            if (var->deflate || var->fletcher32 || var->shuffle)
                 return NC_EINVAL;
-        var->contiguous = NC_TRUE;
-    }
 
-    /* Chunksizes anyone? */
-    if (contiguous && *contiguous == NC_CHUNKED)
-    {
-        var->contiguous = NC_FALSE;
+            for (d = 0; d < var->ndims; d++)
+                if (var->dim[d]->unlimited)
+                    return NC_EINVAL;
+        }
 
-        /* If the user provided chunksizes, check that they are not too
-         * big, and that their total size of chunk is less than 4 GB. */
-        if (chunksizes)
+        /* Handle chunked storage settings. */
+        if (*contiguous == NC_CHUNKED)
         {
+            var->contiguous = NC_FALSE;
 
-            if ((retval = check_chunksizes(grp, var, chunksizes)))
-                return retval;
+            /* If the user provided chunksizes, check that they are not too
+             * big, and that their total size of chunk is less than 4 GB. */
+            if (chunksizes)
+            {
+                /* Check the chunksizes for validity. */
+                if ((retval = check_chunksizes(grp, var, chunksizes)))
+                    return retval;
 
-            /* Ensure chunksize is smaller than dimension size */
-            for (d = 0; d < var->ndims; d++)
-                if(!var->dim[d]->unlimited && var->dim[d]->len > 0 && chunksizes[d] > var->dim[d]->len)
-                    return NC_EBADCHUNK;
+                /* Ensure chunksize is smaller than dimension size */
+                for (d = 0; d < var->ndims; d++)
+                    if (!var->dim[d]->unlimited && var->dim[d]->len > 0 &&
+                        chunksizes[d] > var->dim[d]->len)
+                        return NC_EBADCHUNK;
+
+                /* Set the chunksizes for this variable. */
+                for (d = 0; d < var->ndims; d++)
+                    var->chunksizes[d] = chunksizes[d];
+            }
+        }
+        else if (*contiguous == NC_CONTIGUOUS)
+        {
+            var->contiguous = NC_TRUE;
+        }
+        else if (*contiguous == NC_COMPACT)
+        {
+            size_t ndata = 1;
 
-            /* Set the chunksizes for this variable. */
+            /* Find the number of elements in the data. */
             for (d = 0; d < var->ndims; d++)
-                var->chunksizes[d] = chunksizes[d];
+                ndata *= var->dim[d]->len;
+
+            /* Ensure var is small enough to fit in compact
+             * storage. It must be <= 64 KB. */
+            if (ndata * var->type_info->size > SIXTY_FOUR_KB)
+                return NC_EVARSIZE;
+
+            var->contiguous = NC_FALSE;
+            var->compact = NC_TRUE;
         }
     }
 
diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c
index fca9533d4..7075551df 100644
--- a/libhdf5/nc4hdf.c
+++ b/libhdf5/nc4hdf.c
@@ -994,11 +994,17 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
             }
         }
 
+        /* Set the var storage to contiguous, compact, or chunked. */
         if (var->contiguous)
         {
             if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
                 BAIL(NC_EHDFERR);
         }
+        else if (var->compact)
+        {
+            if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
+                BAIL(NC_EHDFERR);
+        }
         else
         {
             if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
@@ -1106,9 +1112,11 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
     int d;
     int retval;
 
-    /* Nothing to be done. */
-    if (var->contiguous)
+    /* Nothing to be done for contiguous or compact data. */
+    if (var->contiguous || var->compact)
         return NC_NOERR;
+
+    /* No cache adjusting for parallel builds. */
 #ifdef USE_PARALLEL4
     return NC_NOERR;
 #endif
diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c
index b953bc821..de3c5d21e 100644
--- a/libsrc4/nc4var.c
+++ b/libsrc4/nc4var.c
@@ -187,16 +187,26 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
     if (nattsp)
         *nattsp = ncindexcount(var->att);
 
-    /* Chunking stuff. */
-    if (!var->contiguous && chunksizesp)
+    /* Did the user want the chunksizes? */
+    if (!var->contiguous && !var->compact && chunksizesp)
+    {
         for (d = 0; d < var->ndims; d++)
         {
             chunksizesp[d] = var->chunksizes[d];
             LOG((4, "chunksizesp[%d]=%d", d, chunksizesp[d]));
         }
+    }
 
+    /* Did the user inquire about the storage? */
     if (contiguousp)
-        *contiguousp = var->contiguous ? NC_CONTIGUOUS : NC_CHUNKED;
+    {
+        if (var->contiguous)
+            *contiguousp = NC_CONTIGUOUS;
+        else if (var->compact)
+            *contiguousp = NC_COMPACT;
+        else
+            *contiguousp = NC_CHUNKED;
+    }
 
     /* Filter stuff. */
     if (deflatep)


Test patch:
diff --git a/nc_test4/tst_vars4.c b/nc_test4/tst_vars4.c
index 5d8712b42..38c2de271 100644
--- a/nc_test4/tst_vars4.c
+++ b/nc_test4/tst_vars4.c
@@ -10,220 +10,268 @@
 #include "err_macros.h"
 
 #define FILE_NAME "tst_vars4.nc"
-#define NDIMS2 2
+#define NDIM2 2
 #define NUM_VARS 1
 #define Y_NAME "y"
 #define X_NAME "x"
+#define Z_NAME "z"
 #define VAR_NAME Y_NAME
 #define XDIM_LEN 2
 #define YDIM_LEN 5
+#define ZDIM_LEN 8193
 #define CLAIR "Clair"
 #define JAMIE "Jamie"
 
 int
 main(int argc, char **argv)
 {
-   printf("\n*** Testing netcdf-4 variable functions, even more.\n");
-   printf("**** testing Jeff's dimension problem...");
-   {
-      int varid, ncid, dims[NDIMS2], dims_in[NDIMS2];
-      int ndims, nvars, ngatts, unlimdimid, natts;
-      char name_in[NC_MAX_NAME + 1];
-      nc_type type_in;
-      size_t len_in;
-
-      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
-      if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
-      if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
-      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
-      if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
-      if (nvars != NUM_VARS || ndims != NDIMS2 || ngatts != 0 || unlimdimid != -1) ERR;
-      if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
-      if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIMS2 ||
-	  dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR;
-      if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR;
-      if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR;
-      if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR;
-      if (strcmp(name_in, Y_NAME)) ERR;
-      if (len_in != YDIM_LEN) ERR;
-      if (nc_close(ncid)) ERR;
-
-      /* Open the file and check. */
-      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
-      if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
-      if (nvars != NUM_VARS || ndims != NDIMS2 || ngatts != 0 || unlimdimid != -1) ERR;
-      if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
-      if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIMS2 ||
-	  dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR;
-      if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR;
-      if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR;
-      if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR;
-      if (strcmp(name_in, Y_NAME)) ERR;
-      if (len_in != YDIM_LEN) ERR;
-      if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
-   printf("**** testing chunking turned on by fletcher...");
-   {
-      int varid, ncid, dims[NDIMS2];
-      int storage_in;
-      size_t chunksizes_in[NDIMS2];
-
-      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
-      if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
-      if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
-      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
-      if (nc_def_var_fletcher32(ncid, varid, NC_FLETCHER32)) ERR;
-      if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
-      if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
-      if (nc_close(ncid)) ERR;
-
-      /* Open the file and check. */
-      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
-      if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
-      if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
-      if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
-   printf("**** testing chunking turned on by shuffle...");
-   {
-      int varid, ncid, dims[NDIMS2];
-      int storage_in;
-      size_t chunksizes_in[NDIMS2];
-
-      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
-      if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
-      if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
-      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
-      if (nc_def_var_deflate(ncid, varid, NC_SHUFFLE, 0, 0)) ERR;
-      if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
-      if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
-      if (nc_close(ncid)) ERR;
-
-      /* Open the file and check. */
-      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
-      if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
-      if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
-      if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
+    printf("\n*** Testing netcdf-4 variable functions, even more.\n");
+    printf("**** testing Jeff's dimension problem...");
+    {
+        int varid, ncid, dims[NDIM2], dims_in[NDIM2];
+        int ndims, nvars, ngatts, unlimdimid, natts;
+        char name_in[NC_MAX_NAME + 1];
+        nc_type type_in;
+        size_t len_in;
+
+        if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
+        if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
+        if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
+        if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
+        if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
+        if (nvars != NUM_VARS || ndims != NDIM2 || ngatts != 0 || unlimdimid != -1) ERR;
+        if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
+        if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIM2 ||
+            dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR;
+        if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR;
+        if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR;
+        if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR;
+        if (strcmp(name_in, Y_NAME)) ERR;
+        if (len_in != YDIM_LEN) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check. */
+        if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
+        if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
+        if (nvars != NUM_VARS || ndims != NDIM2 || ngatts != 0 || unlimdimid != -1) ERR;
+        if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
+        if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIM2 ||
+            dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR;
+        if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR;
+        if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR;
+        if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR;
+        if (strcmp(name_in, Y_NAME)) ERR;
+        if (len_in != YDIM_LEN) ERR;
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
+    printf("**** testing chunking turned on by fletcher...");
+    {
+        int varid, ncid, dims[NDIM2];
+        int storage_in;
+        size_t chunksizes_in[NDIM2];
+
+        if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
+        if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
+        if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
+        if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
+        if (nc_def_var_fletcher32(ncid, varid, NC_FLETCHER32)) ERR;
+        if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
+        if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check. */
+        if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
+        if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
+        if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
+    printf("**** testing chunking turned on by shuffle...");
+    {
+        int varid, ncid, dims[NDIM2];
+        int storage_in;
+        size_t chunksizes_in[NDIM2];
+
+        if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
+        if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
+        if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
+        if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
+        if (nc_def_var_deflate(ncid, varid, NC_SHUFFLE, 0, 0)) ERR;
+        if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
+        if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check. */
+        if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
+        if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
+        if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
 #define DIM_NAME "Distance_from_Mayo"
 #define VAR_NAME_2 "Rocky_Road_to_Dublin"
 #define NDIMS1 1
 #define NUM_RECORDS 3
-   printf("**** testing extending var along unlimited dim with no coord var...");
-   {
-      int varid, ncid, dimid;
-      int ndims, nvars, natts, unlimdimid;
-      size_t dim_len_in, index;
-      int data = TEST_VAL_42;
-
-      /* Create the test file with one var, one unlimited dim. */
-      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
-      if (nc_def_dim(ncid, DIM_NAME, NC_UNLIMITED, &dimid)) ERR;
-      if (nc_def_var(ncid, VAR_NAME_2, NC_INT, NDIMS1, &dimid, &varid)) ERR;
-
-      /* Write some records. */
-      for (index = 0; index < NUM_RECORDS; index++)
-         if (nc_put_var1_int(ncid, varid, &index, &data)) ERR;
-      if (nc_close(ncid)) ERR;
-
-      /* Open the file and check. */
-      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
-      if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
-      if (ndims != 1 || nvars != 1 || natts != 0 || unlimdimid != 0) ERR;
-      if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR;
-      if (dim_len_in != NUM_RECORDS) ERR;
-
-      /* Now add more records. */
-      for (index = 3; index < NUM_RECORDS * 2; index++)
-         if (nc_put_var1_int(ncid, varid, &index, &data)) ERR;
-      if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR;
-
-      if (dim_len_in != NUM_RECORDS * 2) ERR;
-
-      /* Close the file. */
-      if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
-   printf("**** testing type creation and destruction for atomic types...");
-   {
-      int varid1, varid2, ncid;
-      int ndims, nvars, natts, unlimdimid;
-
-      /* Create the test file with two scalar vars. */
-      /* nc_set_log_level(4); */
-      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
-      if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR;
-      if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR;
-      if (nc_close(ncid)) ERR;
-
-      /* Open the file and check. */
-      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
-      if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
-      if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR;
-      if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
-   printf("**** testing scalar big endian vars...");
-   {
-      int varid1, varid2, ncid;
-      int ndims, nvars, natts, unlimdimid;
-      int test_val = TEST_VAL_42;
-      int test_val2 = TEST_VAL_42 * 2;
-      int data_in;
-
-      /* Create the test file with two scalar vars. */
-      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
-      if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR;
-      if (nc_def_var_endian(ncid, varid1, NC_ENDIAN_BIG)) ERR;
-      if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR;
-      if (nc_def_var_endian(ncid, varid2, NC_ENDIAN_BIG)) ERR;
-      if (nc_enddef(ncid)) ERR;
-      if (nc_put_var(ncid, varid1, &test_val)) ERR;
-      if (nc_put_var(ncid, varid2, &test_val2)) ERR;
-      if (nc_close(ncid)) ERR;
-
-      /* Open the file and check. */
-      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
-      if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
-      if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR;
-      if (nc_get_var(ncid, varid1, &data_in)) ERR;
-      if (data_in != TEST_VAL_42) ERR;
-      if (nc_get_var(ncid, varid2, &data_in)) ERR;
-      if (data_in != TEST_VAL_42 * 2) ERR;
-      if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
-   printf("**** testing scalar big endian vars...");
-   {
-       int ncid, enumid;
-       int bigid, littleid;
-       int endian_in;
-       /* Note: if no zero valued enum, then causes ncdump error */
-       int econst0 = 0;
-       int econst1 = 1;
-
-       if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
-
-       if (nc_def_enum(ncid, NC_INT, "enum_t", &enumid)) ERR;
-       if (nc_insert_enum(ncid, enumid, "econst0", &econst0)) ERR;
-       if (nc_insert_enum(ncid, enumid, "econst1", &econst1)) ERR;
-
-       if (nc_def_var(ncid, "little", enumid, 0, NULL, &littleid)) ERR;
-       if (nc_def_var(ncid, "big", enumid, 0, NULL, &bigid)) ERR;
-
-       if (nc_def_var_endian(ncid, littleid, NC_ENDIAN_LITTLE) != NC_EINVAL) ERR;
-       if (nc_def_var_endian(ncid, bigid, NC_ENDIAN_BIG) != NC_EINVAL) ERR;
-
-       /* Note that it is important to set endian ness before testing it */
-       if (nc_inq_var_endian(ncid, littleid, &endian_in)) ERR;
-       if (endian_in) ERR;
-       if (nc_inq_var_endian(ncid, bigid, &endian_in)) ERR;
-       if (endian_in) ERR;
-
-       if (nc_close(ncid)) ERR;
-   }
-   SUMMARIZE_ERR;
-   FINAL_RESULTS;
+    printf("**** testing extending var along unlimited dim with no coord var...");
+    {
+        int varid, ncid, dimid;
+        int ndims, nvars, natts, unlimdimid;
+        size_t dim_len_in, index;
+        int data = TEST_VAL_42;
+
+        /* Create the test file with one var, one unlimited dim. */
+        if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
+        if (nc_def_dim(ncid, DIM_NAME, NC_UNLIMITED, &dimid)) ERR;
+        if (nc_def_var(ncid, VAR_NAME_2, NC_INT, NDIMS1, &dimid, &varid)) ERR;
+
+        /* Write some records. */
+        for (index = 0; index < NUM_RECORDS; index++)
+            if (nc_put_var1_int(ncid, varid, &index, &data)) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check. */
+        if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
+        if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
+        if (ndims != 1 || nvars != 1 || natts != 0 || unlimdimid != 0) ERR;
+        if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR;
+        if (dim_len_in != NUM_RECORDS) ERR;
+
+        /* Now add more records. */
+        for (index = 3; index < NUM_RECORDS * 2; index++)
+            if (nc_put_var1_int(ncid, varid, &index, &data)) ERR;
+        if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR;
+
+        if (dim_len_in != NUM_RECORDS * 2) ERR;
+
+        /* Close the file. */
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
+    printf("**** testing type creation and destruction for atomic types...");
+    {
+        int varid1, varid2, ncid;
+        int ndims, nvars, natts, unlimdimid;
+
+        /* Create the test file with two scalar vars. */
+        /* nc_set_log_level(4); */
+        if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
+        if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR;
+        if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check. */
+        if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
+        if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
+        if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR;
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
+    printf("**** testing scalar big endian vars...");
+    {
+        int varid1, varid2, ncid;
+        int ndims, nvars, natts, unlimdimid;
+        int test_val = TEST_VAL_42;
+        int test_val2 = TEST_VAL_42 * 2;
+        int data_in;
+
+        /* Create the test file with two scalar vars. */
+        if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
+        if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR;
+        if (nc_def_var_endian(ncid, varid1, NC_ENDIAN_BIG)) ERR;
+        if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR;
+        if (nc_def_var_endian(ncid, varid2, NC_ENDIAN_BIG)) ERR;
+        if (nc_enddef(ncid)) ERR;
+        if (nc_put_var(ncid, varid1, &test_val)) ERR;
+        if (nc_put_var(ncid, varid2, &test_val2)) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check. */
+        if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
+        if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
+        if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR;
+        if (nc_get_var(ncid, varid1, &data_in)) ERR;
+        if (data_in != TEST_VAL_42) ERR;
+        if (nc_get_var(ncid, varid2, &data_in)) ERR;
+        if (data_in != TEST_VAL_42 * 2) ERR;
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
+    printf("**** testing scalar big endian vars...");
+    {
+        int ncid, enumid;
+        int bigid, littleid;
+        int endian_in;
+        /* Note: if no zero valued enum, then causes ncdump error */
+        int econst0 = 0;
+        int econst1 = 1;
+
+        if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
+
+        if (nc_def_enum(ncid, NC_INT, "enum_t", &enumid)) ERR;
+        if (nc_insert_enum(ncid, enumid, "econst0", &econst0)) ERR;
+        if (nc_insert_enum(ncid, enumid, "econst1", &econst1)) ERR;
+
+        if (nc_def_var(ncid, "little", enumid, 0, NULL, &littleid)) ERR;
+        if (nc_def_var(ncid, "big", enumid, 0, NULL, &bigid)) ERR;
+
+        if (nc_def_var_endian(ncid, littleid, NC_ENDIAN_LITTLE) != NC_EINVAL) ERR;
+        if (nc_def_var_endian(ncid, bigid, NC_ENDIAN_BIG) != NC_EINVAL) ERR;
+
+        /* Note that it is important to set endian ness before testing it */
+        if (nc_inq_var_endian(ncid, littleid, &endian_in)) ERR;
+        if (endian_in) ERR;
+        if (nc_inq_var_endian(ncid, bigid, &endian_in)) ERR;
+        if (endian_in) ERR;
+
+        if (nc_close(ncid)) ERR;
+    }
+    SUMMARIZE_ERR;
+    printf("**** testing compact storage...");
+    {
+        int ncid, dimid[NDIM2], varid, varid2;
+        int data[XDIM_LEN];
+        int x;
+
+        /* Create some data. */
+        for (x = 0; x < XDIM_LEN; x++)
+            data[x] = x;
+
+        /* Create a file with one var with compact storage. */
+        if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
+
+        /* Define dims. */
+        if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dimid[0])) ERR;
+        if (nc_def_dim(ncid, Z_NAME, ZDIM_LEN, &dimid[1])) ERR;
+
+        /* Define vars. */
+        if (nc_def_var(ncid, Y_NAME, NC_INT, 1, dimid, &varid)) ERR;
+        if (nc_def_var_chunking(ncid, varid, NC_COMPACT, NULL)) ERR;
+        if (nc_def_var(ncid, CLAIR, NC_INT, NDIM2, dimid, &varid2)) ERR;
+        /* This won't work, the var is too big for compact! */
+        if (nc_def_var_chunking(ncid, varid2, NC_COMPACT, NULL) != NC_EVARSIZE) ERR;
+
+        /* Write data. */
+        if (nc_put_var_int(ncid, varid, data)) ERR;
+
+        /* Close file. */
+        if (nc_close(ncid)) ERR;
+
+        /* Open the file and check it. */
+        {
+            int ndims, nvars;
+            int storage_in;
+
+            if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
+            if (nc_inq(ncid, &ndims, &nvars, NULL, NULL)) ERR;
+            if (ndims != 2 || nvars != 2) ERR;
+            if (nc_inq_var_chunking(ncid, varid, &storage_in, NULL)) ERR;
+            if (storage_in != NC_COMPACT) ERR;
+            if (nc_inq_var_chunking(ncid, varid2, &storage_in, NULL)) ERR;
+            if (storage_in != NC_CONTIGUOUS) ERR;
+            if (nc_close(ncid)) ERR;
+        }
+    }
+    SUMMARIZE_ERR;
+    FINAL_RESULTS;
 }