Repository: Unidata/netcdf-c (c)

Issue:
--enable-benchmark build sometimes fails on nc_perf/tst_mem1.c for HDF5-1.12.x
With test program tst_mem1.c

This program is based on code provided by @jswhit and checks memory use of netCDF when repeatedly opening and closing a file.

There is a certain number of open/close cycles (currently 10) during which memory use is expected to fluctuate, then it must settle down to a constant number. Here's the code:
```

            if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
            if (nc_close(ncid)) ERR;
            getrusage(RUSAGE_SELF, &r_usage);
            /* if (!(idx % 100)) */
            /*     printf("Memory usage: %ld kilobytes\n",r_usage.ru_maxrss); */

            /* Memory usage goes up in the first couple of opens, but
             * should then remain steady. Check that it does not
             * change after some number of iterations; the expected number is 10. */
#if 1
            if (!my_rss || idx < 10)
                my_rss = r_usage.ru_maxrss;
            else
                if (my_rss != r_usage.ru_maxrss) ERR;
#else

```

This has been failing for me occasionally. I'm going to raise the 10 to 100 and see if that fixes the issue.

Pull request:
Turn off tst_mem1.c test as part of benchmarks
Fixes #2057 

Turn off tst_mem1 as a benchmark test. It does not work reliably on very busy systems.

Patch:
diff --git a/configure.ac b/configure.ac
index 12f721271..a3e88630e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1256,18 +1256,6 @@ if test "x$enable_hdf5" = xyes; then
 
    AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_all_coll_metadata_ops H5Literate])
 
-   # Check to see if HDF5 library has H5Literate (HDF5 1.8.x, 1.10.x)
-   if test "x$ac_cv_func_H5Literate" = xno; then
-      AC_MSG_CHECKING([for H5Literate macro])
-      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "H5public.h"],
-      [[#if !(H5Literate_vers == 1)
-      # error
-      #endif]
-      ])], [], [AC_MSG_RESULT(no); AC_MSG_ERROR([HDF5 was not built with API compatibility version v18. Please recompile your libhdf5 install using '--with-default-api-version=v18'.])])
-      AC_MSG_RESULT(yes)
-   fi
-
-
    # Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
    if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
       AC_DEFINE([HDF5_HAS_COLL_METADATA_OPS], [1], [if true, use collective metadata ops in parallel netCDF-4])
diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c
index 374118096..44d66f322 100644
--- a/libhdf5/hdf5open.c
+++ b/libhdf5/hdf5open.c
@@ -2556,7 +2556,12 @@ oinfo_list_add(user_data_t *udata, const hdf5_obj_info_t *oinfo)
  * @author Ed Hartnett
  */
 static int
-read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info,
+read_hdf5_obj(hid_t grpid, const char *name,
+#if H5_VERSION_GE(1,12,0)
+	      const H5L_info2_t *info,
+#else
+	      const H5L_info_t *info,
+#endif
               void *_op_data)
 {
     /* Pointer to user data for callback */
diff --git a/nc_perf/Makefile.am b/nc_perf/Makefile.am
index 6516d7b64..6617be764 100644
--- a/nc_perf/Makefile.am
+++ b/nc_perf/Makefile.am
@@ -35,9 +35,10 @@ tst_knmi_SOURCES = tst_knmi.c tst_utils.c
 tst_wrf_reads_SOURCES = tst_wrf_reads.c tst_utils.c
 tst_bm_rando_SOURCES = tst_bm_rando.c tst_utils.c
 
-TESTS = tst_ar4_3d tst_create_files tst_files3 tst_mem tst_mem1	\
-run_knmi_bm.sh tst_wrf_reads tst_attsperf perftest.sh		\
-run_tst_chunks.sh run_bm_elena.sh tst_bm_rando
+# Removing tst_mem1 because it sometimes fails on very busy system.
+TESTS = tst_ar4_3d tst_create_files tst_files3 tst_mem run_knmi_bm.sh	\
+tst_wrf_reads tst_attsperf perftest.sh run_tst_chunks.sh		\
+run_bm_elena.sh tst_bm_rando
 
 run_bm_elena.log: tst_create_files.log
 
diff --git a/nc_perf/tst_mem1.c b/nc_perf/tst_mem1.c
index ad473ad50..5e935f1e9 100644
--- a/nc_perf/tst_mem1.c
+++ b/nc_perf/tst_mem1.c
@@ -6,6 +6,10 @@
    suggest by Jeff Whitaker. See
    https://github.com/Unidata/netcdf-c/issues/1575.
 
+   This was removed from the list of tests run in this
+   directory. Sometimes it fails on very busy systems even though
+   everything is fine. Ed - 8/9/21
+
    Ed Hartnett 2/9/20
 */
 
@@ -19,7 +23,7 @@
 #endif
 
 #define FILE_NAME "tst_mem1.nc"
-#define NUM_FILE_OPENS 10000
+#define NUM_FILE_OPENS 100
 
 int main()
 {


Test patch:
diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am
index d26cad304..d5c514a0f 100644
--- a/ncdap_test/Makefile.am
+++ b/ncdap_test/Makefile.am
@@ -32,10 +32,8 @@ check_PROGRAMS += t_dap3a test_cvt3 test_vara
 TESTS += t_dap3a test_cvt3 test_vara
 if BUILD_UTILITIES
 TESTS += tst_ncdap3.sh
-if ENABLE_DAP_REMOTE_TESTS
 TESTS += testpathcvt.sh
 endif
-endif
 
 # remote tests are optional
 # because the server may be down or inaccessible
diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am
index cfa4bd840..3825de3fa 100644
--- a/nczarr_test/Makefile.am
+++ b/nczarr_test/Makefile.am
@@ -68,16 +68,13 @@ endif
 if BUILD_BENCHMARKS
 if BUILD_UTILITIES
 
-UTILSRC = bm_utils.c timer_utils.c tst_utils.c bm_utils.h bm_timer.h
+UTILSRC = bm_utils.c timer_utils.c tst_utils.c
 
 bm_chunks3_SOURCES = bm_chunks3.c ${UTILSRC}
 
 check_PROGRAMS += bm_chunks3
 
-# The perf tests need modernization
-if AX_IGNORE
 TESTS += run_perf_chunks1.sh
-endif
 
 endif # BUILD_BENCHMARKS
 
diff --git a/nczarr_test/bm_timer.h b/nczarr_test/bm_timer.h
deleted file mode 100644
index f538fda9a..000000000
--- a/nczarr_test/bm_timer.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* This is part of the netCDF package. Copyright 2005-2018 University
-   Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
-   conditions of use.
-*/
-
-#ifndef BM_TIMER_H
-#define BM_TIMER_H 1
-
-#undef NOREPEAT
-#undef NOCONTIG
-
-/*
- * The following timing macros can be used by including the necessary
- * declarations with
- *
- *     TIMING_DECLS(seconds)
- *
- * and surrounding sections of code to be timed with the "statements"
- *
- *     TIMING_START
- *     [code to be timed goes here]
- *     TIMING_END(seconds)
- *
- * The macros assume the user has stored a description of what is
- * being timed in a 100-char string time_mess, and has included
- * <sys/times.h> and <sys/resource.h>.  The timing message printed by
- * TIMING_END is not terminated by a new-line, to permit appending
- * additional text to that line, so user must at least printf("\n")
- * after that.
- */
-
-#define TIMING_DECLS(seconds)						       \
-	long TMreps;		/* counts repetitions of timed code */ \
-	long TMrepeats;		/* repetitions needed to exceed 0.1 second */ \
-	Nanotime bnano,enano,delta;	/* start/stop times in nanoseconds */ \
-        double seconds; \
-	NCT_inittimer();
-
-#ifndef NOREPEAT
-#define TIMING_START \
-	TMrepeats = 1; \
-	do { \
-	    NCT_marktime(&bnano); \
-	    for(TMreps=0; TMreps < TMrepeats; TMreps++) {
-
-#define TIMING_END(seconds)				\
-            } \
-	    NCT_marktime(&enano); \
-	    NCT_elapsedtime(&bnano,&enano,&delta); \
-	    TMrepeats *= 2; \
-	} while (NCT_nanoseconds(delta) < 100000000 ); \
-	seconds = ((double)NCT_nanoseconds(delta)) / (1000000000.0 * TMreps); \
-
-#else /*NOREPEAT*/
-
-#define TIMING_START \
-	do { \
-fprintf(stderr,"TIME_START\n"); \
-	    NCT_marktime(&bnano); \
-	    {
-
-#define TIMING_END(time_mess,seconds) \
-	    } \
-	    NCT_marktime(&enano); \
-	    NCT_elapsedtime(&bnano,&enano,&delta); \
-        } while (0); \
-fprintf(stderr,"TIME_END\n"); \
-	seconds = ((double)NCT_nanoseconds(delta)) / (1000000000.0 * TMreps); \
-	printf("%-45.45s %#08.6F sec", time_mess, seconds);
-
-#endif /*NOREPEAT*/
-
-#endif /*BM_TIMER_H*/
diff --git a/nczarr_test/bm_utils.c b/nczarr_test/bm_utils.c
index f37ba81c5..0395f5149 100644
--- a/nczarr_test/bm_utils.c
+++ b/nczarr_test/bm_utils.c
@@ -41,7 +41,7 @@ static struct option options[] = {
 {NULL, 0, NULL, 0}
 };
 
-struct BMOptions bmoptions;
+struct Options bmoptions;
 
 static char* dumpintlist(struct IntList* list);
 static int parseintlist(const char* s0, struct IntList* list);
@@ -76,7 +76,7 @@ nc4_timeval_subtract (struct timeval *result, struct timeval *x, struct timeval*
 #endif
 
 int
-nc4_buildpath(struct BMOptions* o, char** pathp)
+nc4_buildpath(struct Options* o, char** pathp)
 {
     char* path = NULL;
     size_t len;
@@ -99,15 +99,15 @@ fprintf(stderr,"buildpath: file=%s\n",path);
 }
 
 EXTERNL int
-bm_getoptions(int* argcp, char*** argvp, struct BMOptions* opt)
+getoptions(int* argcp, char*** argvp, struct Options* opt)
 {
     int stat = NC_NOERR;
     int tag;
     int argc = *argcp;
     char** argv = *argvp;
-    const char* nczarr_s3_test_url = "https://stratus.ucar.edu/unidata-netcdf-zarr-testing";
+    const char* nczarr_s3_test_url = ncz_gets3testurl();
 
-    memset((void*)opt,0,sizeof(struct BMOptions));
+    memset((void*)opt,0,sizeof(struct Options));
     if(argc <= 1) return NC_NOERR;
     while ((tag = getopt_long_only(argc, argv, "", options, NULL)) >= 0) {
 #ifdef DEBUG
@@ -145,13 +145,13 @@ fprintf(stderr,"arg=%s value=%s\n",argv[optind-1],optarg);
 	        opt->pathtemplate = strdup(optarg);
 	        break;
 	    case OPT_FORMAT:
-		if(strcasecmp(optarg,"file")==0) {
+		if(strcasecmp(optarg,"file")==0)
 		    opt->format = NC_FORMATX_NCZARR;
 		    opt->impl = NCZM_FILE;
-		}  else if(strcasecmp(optarg,"zip")==0) {
+		else if(strcasecmp(optarg,"zip")==0)
 		    opt->format = NC_FORMATX_NCZARR;
 		    opt->impl = NCZM_ZIP;
-		} else if(strcasecmp(optarg,"s3")==0) {
+		else if(strcasecmp(optarg,"s3")==0) {
 		    opt->format = NC_FORMATX_NCZARR;
 		    opt->impl = NCZM_S3;
 		} else 
@@ -198,7 +198,7 @@ fprintf(stderr,"arg=%s value=%s\n",argv[optind-1],optarg);
 	    case NCZM_S3:
 		ptlen = strlen(nczarr_s3_test_url) + strlen("/netcdf-c/%s.%s#mode=nczarr,s3");
 	        opt->pathtemplate = (char*)calloc(1,ptlen+1);
-		if(opt->pathtemplate == NULL) return NC_ENOMEM;
+		if(opt->template == NULL) return NC_ENOMEM;
 		strlcat(opt->pathtemplate,nczarr_s3_test_url,ptlen+1);
 		strlcat(opt->pathtemplate,"/netcdf-c/%s.%s#mode=nczarr,s3",ptlen+1);
 		break;
@@ -234,7 +234,7 @@ fprintf(stderr,"arg=%s value=%s\n",argv[optind-1],optarg);
 }
 
 EXTERNL void
-clearoptions(struct BMOptions* o)
+clearoptions(struct Options* o)
 {
     nullfree(o->pathtemplate);
     nullfree(o->filename);
@@ -243,16 +243,15 @@ clearoptions(struct BMOptions* o)
 }
 
 EXTERNL const char*
-formatname(const struct BMOptions* o)
+formatname(const struct Options* o)
 {
     switch (o->format) {
     case NC_FORMATX_NCZARR:
 	switch (o->impl) {
 	case NCZM_S3: return "s3";
+	case NCZM_NC4: return "nz4";
 	case NCZM_FILE: return "file";
 	case NCZM_ZIP: return "zip";
-	default: abort();
-	}
 	break;
     case NC_FORMATX_NC4: /* fall thru */
 	return "nz4";
@@ -264,17 +263,18 @@ formatname(const struct BMOptions* o)
 
 
 EXTERNL void
-reportoptions(struct BMOptions* o)
+reportoptions(struct Options* o)
 {
     fprintf(stderr,"--format=%d\n",o->format);
     fprintf(stderr,"--pathtemplate=%s\n",o->pathtemplate);
     fprintf(stderr,"--filename=%s\n",o->filename);
+    if(o->iss3) fprintf(stderr,"--s3\n");
     fprintf(stderr,"--X=%d\n",o->xvalue);
     fflush(stderr);
 }
 
 EXTERNL void
-reportmetaoptions(struct BMMeta* o)
+reportmetaoptions(struct Meta* o)
 {
     fprintf(stderr,"--treedepth=%d\n",o->treedepth);
     fprintf(stderr,"--ngroups=%d\n",o->ngroups);
diff --git a/nczarr_test/bm_utils.h b/nczarr_test/bm_utils.h
deleted file mode 100644
index 52103225a..000000000
--- a/nczarr_test/bm_utils.h
+++ /dev/null
@@ -1,95 +0,0 @@
-/*********************************************************************
- *   Copyright 2018, UCAR/Unidata
- *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
- *********************************************************************/
-
-#ifndef BM_UTILS_H
-#define BM_UTILS_H
-
-#include "config.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-#include "zincludes.h"
-
-#undef DEBUG
-
-/* Define the getopt tags */
-#define OPT_UNKNOWN 0
-#define OPT_TREEDEPTH 1
-#define OPT_NGROUPS 2
-#define OPT_NGROUPATTRS 3
-#define OPT_NDIMS 4
-#define OPT_NTYPES 5
-#define OPT_NVARS 6
-#define OPT_VARRANK 7
-#define OPT_NVARATTRS 8
-#define OPT_FORMAT 9
-#define OPT_PATH 10
-#define OPT_FILE 11
-#define OPT_X 12
-#define OPT_DEBUG 13
-#define OPT_DIMS 14
-#define OPT_CHUNKS 15
-#define OPT_CACHEFACTOR 16
-#define OPT_CACHESIZE 17
-#define OPT_DEFLATELEVEL 18
-#define OPT_WDEBUG 19
-#define OPT_URL 20
-#define OPT_READONLY 21
-#define OPT_WRITEONLY 22
-
-#define X_OPT_MATCH 1
-
-extern struct BMOptions {
-    int format;
-    char* filename;
-    char* url;
-    char* path; /* as sent to e.g. nc_create() */
-    char* pathtemplate;
-    int xvalue;
-    NCZM_IMPL impl;
-    int debug;
-    int wdebug;
-    int readonly;
-    int writeonly;
-    struct BMMeta {
-	int treedepth;
-	int ngroups;
-	int ngroupattrs;
-	int ndims;
-	int ntypes;
-	int nvars;
-	int varrank;
-	int nvarattrs;
-	size_t cachefactor;
-	size_t cachesize;
-	struct IntList {
-	    int count;
-   	    size_t* list;
-	} dims;
-	struct IntList chunks;
-	int deflatelevel;
-    } meta;
-    struct X {
-	int sync;
-    } x;
-} bmoptions;
-
-#define NCCHECK(expr) nccheck((expr),__LINE__)
-
-EXTERNL int bm_buildpath(struct BMOptions*);
-EXTERNL int bm_getoptions(int* argcp, char*** argvp, struct BMOptions*);
-EXTERNL void bm_clearoptions(struct BMOptions*);
-EXTERNL const char* formatname(const struct BMOptions*);
-EXTERNL void bm_reportoptions(struct BMOptions* o);
-EXTERNL void bm_reportmetaoptions(struct BMMeta* o);
-EXTERNL const char* bm_printvector(int rank, const size_t* vec);
-EXTERNL const char* bm_printvectorp(int rank, const ptrdiff_t* vec);
-EXTERNL const char* bm_varasprint(int rank, const size_t* start, const size_t* edges, const ptrdiff_t* stride);
-
-#include "ut_test.h"
-
-#endif /*BM_UTILS_H*/