Repository: Unidata/netcdf-c
Pull request #2708 (2023-06-10 21:04:22)

Provide a single option to disable all network access and testing.
Add the option "--disable-network-access" (automake) or "-DENABLE_NETWORK_ACCESS=OFF" (cmake).
When disabled, this option transitively disables all network access capabilities and testing.
If set, this option implies the following:
* --disable-dap
* --disable-byterange
* --disable-s3

This PR answers a request for a feature from Ed Hartnett.

## Misc. Other changes
* Take the opportunity to clean up some old, unused options; e.g. --enable-multifilters.
* Fix bug in using S3 urls.


Patch:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6630a1635..9460f47d8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -523,6 +523,15 @@ if(ENABLE_HDF5 OR ENABLE_HDF4 OR ENABLE_NCZARR)
   SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE)
 endif()
 
+# enable|disable all forms of network access
+OPTION(ENABLE_REMOTE_FUNCTIONALITY "Enable|disable all forms remote data access (DAP, S3, etc)" ON)
+MESSAGE(">>> ENABLE_REMOTE_FUNCTIONALITY=${ENABLE_REMOTE_FUNCTIONALITY}")
+if(NOT ENABLE_REMOTE_FUNCTIONALITY)
+MESSAGE(WARNING "ENABLE_REMOTE_FUNCTIONALITY=NO => ENABLE_DAP[4]=NO")
+SET(ENABLE_DAP OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY=NO => ENABLE_DAP=NO" FORCE)
+SET(ENABLE_DAP4 OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY=NO => ENABLE_DAP4=NO" FORCE)
+ENDIF()
+
 IF(ENABLE_HDF4)
   SET(USE_HDF4 ON)
   # Check for include files, libraries.
@@ -622,9 +631,6 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
   SET(ENABLE_SET_LOG_LEVEL ON)
 ENDIF()
 
-# This has multiversion capability
-SET(ENABLE_MULTIFILTERS yes CACHE BOOL "")
-
 # Option to allow for strict null file padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information
 OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF)
@@ -633,12 +639,6 @@ IF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
   SET(USE_STRICT_NULL_BYTE_HEADER_PADDING ON CACHE BOOL "")
 ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
 
-# Option for building RPC
-#OPTION(ENABLE_RPC "Enable RPC Client and Server." OFF)
-#IF(ENABLE_RPC)
-#  SET(BUILD_RPC ON CACHE BOOL "")
-#ENDIF()
-
 # Note that szip management is tricky.
 # This is because we have three things to consider:
 # 1. is libsz available?
@@ -995,6 +995,12 @@ ELSE()
   SET(FOUND_CURL FALSE)
 ENDIF()
 
+# Start disabling if curl not found
+IF(NOT FOUND_CURL)
+  MESSAGE(WARNING "ENABLE_REMOTE_FUNCTIONALITY requires libcurl; disabling")
+  SET(ENABLE_REMOTE_FUNCTIONALITY OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY requires libcurl; disabling" FORCE)
+ENDIF()
+
 set (CMAKE_REQUIRED_INCLUDES ${CURL_INCLUDE_DIRS})
 # Check to see if we have libcurl 7.66 or later
 CHECK_C_SOURCE_COMPILES("
@@ -1078,6 +1084,11 @@ ENDIF()
 
 # Option to support byte-range reading of remote datasets
 OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." ON)
+if(NOT ENABLE_REMOTE_FUNCTIONALITY)
+MESSAGE(WARNING "ENABLE_REMOTE_FUNCTIONALITY=NO => ENABLE_BYTERANGE=NO")
+SET(ENABLE_BYTERANGE OFF CACHE BOOL "ENABLE_REMOTE_FUNCTIONALITY=NO => ENABLE_BYTERANGE=NO" FORCE)
+ENDIF()
+
 
 # Check for the math library so it can be explicitly linked.
 IF(NOT WIN32)
@@ -1098,8 +1109,14 @@ ENDIF()
 OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
 OPTION(ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF)
 OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
-SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
+if(NOT ENABLE_DAP)
+SET(ENABLE_DAP_REMOTE_TESTS OFF CACHE BOOL "" FORCE)
+SET(ENABLE_EXTERNAL_SERVER_TESTS OFF CACHE BOOL "" FORCE)
+SET(ENABLE_DAP_LONG_TESTS OFF CACHE BOOL "" FORCE)
+ENDIF()
+
+SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
 # See if we have zlib
@@ -1273,6 +1290,12 @@ OPTION(ENABLE_S3 "Enable S3 support." OFF)
 OPTION(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF)
 OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF)
 
+IF(NOT ENABLE_REMOTE_FUNCTIONALITY)
+  SET(ENABLE_S3 OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE)
+ENDIF()
+
 # Control S3 Testing: Multi-valued option
 SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none)  PUBLIC")
     SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC)  #
@@ -1290,6 +1313,15 @@ if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3)
 ENDIF()
 UNSET(ENABLE_NCZARR_S3)
 
+IF(NOT ENABLE_REMOTE_FUNCTIONALITY)
+  MESSAGE(WARNING "ENABLE_REMOTE_FUNCTIONALITY=NO => disable all s3 functionality")
+  SET(ENABLE_S3 OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
+  SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
+ENDIF()
+
 # Note we check for the library after checking for enable_s3
 # because for some reason this screws up if we unconditionally test for sdk
 # and it is not available. Fix someday
@@ -1332,29 +1364,6 @@ IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
 ENDIF()
 ENDIF()
 
-# Start disabling if curl not found
-IF(NOT FOUND_CURL)
-  IF(ENABLE_BYTERANGE)
-    MESSAGE(FATAL_ERROR "Byte-range support specified, CURL libraries are not found.")
-  ENDIF()
-
-  IF(ENABLE_DAP2 OR ENABLE_DAP4)
-    MESSAGE(WARNING "CURL libraries are not found; DAP support disabled")
-    SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE)
-    SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE)
-  ENDIF()
-
-  IF(ENABLE_HDF5_ROS3)
-    MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.")
-    SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
-  ENDIF()
-
-  IF(ENABLE_S3)
-    MESSAGE(FATAL_ERROR "S3 support specified, but CURL libraries are not found.")
-  ENDIF()
-ENDIF(NOT FOUND_CURL)
-
-
 OPTION(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON)
 SET(XMLPARSER "tinyxml2 (bundled)")
 
@@ -2569,7 +2578,6 @@ is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL)
 is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
 is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
-is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
 is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
 is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
 is_enabled(ENABLE_LOGGING HAS_LOGGING)
@@ -2579,6 +2587,7 @@ is_enabled(HAVE_SZ HAS_SZLIB_WRITE)
 is_enabled(HAVE_ZSTD HAS_ZSTD)
 is_enabled(HAVE_BLOSC HAS_BLOSC)
 is_enabled(HAVE_BZ2 HAS_BZ2)
+is_enabled(ENABLE_REMOTE_FUNCTIONALITY DO_REMOTE_FUNCTIONALITY)
 
 if(ENABLE_S3_INTERNAL)
 SET(WHICH_S3_SDK "internal")
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index f2268ea40..ef88feb4a 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.3 - TBD
 
+* Provide a single option to disable all network access and testing. See [Github #2708](https://github.com/Unidata/netcdf-c/pull/2708).
 * Fix some problems with earthdata authorization and data access. See [Github #2709](https://github.com/Unidata/netcdf-c/pull/2709).
 * Fix a race condition in some ncdump tests. See [Github #2682](https://github.com/Unidata/netcdf-c/pull/2682).
 * Fix a minor bug in reporting the use of szip. See [Github #2679](https://github.com/Unidata/netcdf-c/pull/2679).
diff --git a/configure.ac b/configure.ac
index d45a5d224..bb55945d2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -178,6 +178,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
 test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
 AC_MSG_RESULT($enable_pnetcdf)
 
+AC_MSG_CHECKING([whether any network access should be allowed])
+AC_ARG_ENABLE([remote-functionality], [AS_HELP_STRING([--enable-remote-functionality],
+              [enable|disable all forms of network access (default enabled)])])
+test "x$enable_remote_functionality" = xno || enable_remote_functionality=yes
+AC_MSG_RESULT($enable_remote_functionality)
+
 # We need curl for remote operations
 AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
 if test "x$found_curl" = "xyes" ; then
@@ -192,6 +198,11 @@ AC_ARG_ENABLE([dap],
 test "x$enable_dap" = xno || enable_dap=yes
 AC_MSG_RESULT($enable_dap)
 
+if test "x$enable_remote_functionality" = xno ; then
+  AC_MSG_WARN([All network access is disabled => DAP support disabled.])
+  enable_dap=no
+fi
+
 if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then
   AC_MSG_WARN([curl required for dap access. DAP support disabled.])
   enable_dap=no
@@ -641,7 +652,7 @@ AC_ARG_ENABLE([dap-auth-tests],
 test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no
 AC_MSG_RESULT($enable_dap_auth_tests)
 
-# dap must be enabled
+# dap must be enabled in order to use various other flags
 
 if test "x$enable_dap" = "xno" ; then
   AC_MSG_NOTICE([--disable-dap => --disable-dap-remote-tests --disable-auth-tests --disable-external-server-tests])
@@ -738,7 +749,6 @@ if test "x$have_zstd" = "xyes" ; then
    fi
 fi
 
-
 # See if we have libbz2
 AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no])
 if test "x$have_bz2" = "xyes" ; then
@@ -799,6 +809,11 @@ AC_ARG_ENABLE([s3],
 test "x$enable_s3" = xyes || enable_s3=no
 AC_MSG_RESULT($enable_s3)
 
+if test "x$enable_remote_functionality" = xno ; then
+  AC_MSG_WARN([--disable-remote-functionality => --disable-s3])
+  enable_s3=no
+fi
+
 # --enable-nczarr-s3 is synonym for --enable-s3 (but...)
 AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled])
 AC_ARG_ENABLE([nczarr-s3],
@@ -807,7 +822,7 @@ AC_ARG_ENABLE([nczarr-s3],
 AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)])
 
 # Set enable_s3 instead of enable_nczarr_s3
-if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then
+if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then
 enable_s3=yes # back compatibility
 fi
 unset enable_nczarr_s3
@@ -898,12 +913,6 @@ AC_ARG_WITH([chunk-cache-size-nczarr],
 AC_MSG_RESULT([$CHUNK_CACHE_SIZE_NCZARR])
 AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE_NCZARR], [$CHUNK_CACHE_SIZE_NCZARR], [default nczarr chunk cache size.])
 
-# This has multiversion capability
-AC_MSG_CHECKING([whether multi-filter support is enabled])
-has_multifilters=yes
-AC_MSG_RESULT($has_multifilters)
-AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled])
-
 # Check whether we want to enable strict null byte header padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information.
 AC_MSG_CHECKING([whether to enable strict null-byte header padding when reading (default off)])
@@ -1310,6 +1319,11 @@ AC_ARG_ENABLE([byterange],
 test "x$enable_byterange" = xno || enable_byterange=yes
 AC_MSG_RESULT($enable_byterange)
 
+if test "x$enable_remote_functionality" = xno ; then
+  AC_MSG_WARN([--disable-remote-functionality => --disable-byterange])
+  enable_byterange=no
+fi
+
 # Need curl for byte ranges
 if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then
   AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.])
@@ -1433,7 +1447,6 @@ AC_CHECK_SIZEOF(size_t)
 $SLEEPCMD
 AC_CHECK_SIZEOF(unsigned long long)
 
-
 if test "$ac_cv_sizeof_size_t" -lt "8" ; then
    if test "x${enable_cdf5}" = xyes ; then
       dnl unable to support CDF5, but --enable-cdf5 is explicitly set
@@ -1860,7 +1873,6 @@ AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
 AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => public
 AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes])
 AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
-AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 AM_CONDITIONAL(HAVE_DEFLATE, [test "x$have_deflate" = xyes])
 AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
 AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
@@ -1971,7 +1983,6 @@ AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
 AC_SUBST(DO_S3_TESTING,[$with_s3_testing])
 AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
-AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
 AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
 AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
 AC_SUBST(HAS_LOGGING,[$enable_logging])
@@ -1981,6 +1992,7 @@ AC_SUBST(HAS_SZLIB,[$have_sz])
 AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
 AC_SUBST(HAS_ZSTD,[$have_zstd])
 AC_SUBST(DO_LARGE_TESTS,[$enable_large_file_tests])
+AC_SUBST(DO_REMOTE_FUNCTIONALITY,[$enable_remote_functionality])
 
 if test "x$enable_s3_aws" = xyes ; then
 AC_SUBST(WHICH_S3_SDK,[aws-sdk-cpp])
@@ -2115,7 +2127,6 @@ AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes])
 AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes])
 AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes])
 AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
-AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
 AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
 AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes])
 AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes])
diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in
index e1f7fbe1a..8f24e759f 100644
--- a/include/netcdf_meta.h.in
+++ b/include/netcdf_meta.h.in
@@ -60,8 +60,8 @@
 #define NC_RELAX_COORD_BOUND 1 /*!< Always allow 0 counts in parallel I/O. */
 #define NC_DISPATCH_VERSION  @NC_DISPATCH_VERSION@ /*!< Dispatch table version. */
 #define NC_HAS_PAR_FILTERS   @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */
-#define NC_HAS_MULTIFILTERS  @NC_HAS_MULTIFILTERS@ /*!< Nczarr support. */
 #define NC_HAS_LOGGING       @NC_HAS_LOGGING@ /*!< Logging support. */
+#define NC_HAS_MULTIFILTERS  @NC_HAS_MULTIFILTERS@ /*!< Nczarr support. */
 #define NC_HAS_QUANTIZE      @NC_HAS_QUANTIZE@ /*!< Quantization support. */
 #define NC_HAS_ZSTD          @NC_HAS_ZSTD@ /*!< Zstd support. */
 #define NC_HAS_BENCHMARKS    @NC_HAS_BENCHMARKS@ /*!< Benchmarks. */
diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in
index 5d24d1bae..1315370c1 100644
--- a/libnetcdf.settings.in
+++ b/libnetcdf.settings.in
@@ -51,11 +51,9 @@ JNA Support:		@HAS_JNA@
 ERANGE Fill Support:	@HAS_ERANGE_FILL@
 Relaxed Boundary Check:	@RELAX_COORD_BOUND@
 
-Multi-Filter Support:	@HAS_MULTIFILTERS@
 Quantization:		@HAS_QUANTIZE@
 Logging:     		@HAS_LOGGING@
 SZIP Write Support:     @HAS_SZLIB_WRITE@
 Standard Filters:       @STD_FILTERS@
 ZSTD Support:           @HAS_ZSTD@
 Parallel Filters:       @HAS_PAR_FILTERS@
-
diff --git a/nc-config.cmake.in b/nc-config.cmake.in
index e9dda9999..8588665d2 100644
--- a/nc-config.cmake.in
+++ b/nc-config.cmake.in
@@ -116,12 +116,7 @@ else
     has_benchmarks="yes"
 fi
 
-has_multifilters="@HAS_MULTIFILTERS@"
-if [ -z "$has_multifilters" -o "$has_multifilters" = "OFF" -o "$has_multifilters" = "FALSE" ]; then
-    has_multifilters="no"
-else
-    has_multifilters="yes"
-fi
+has_multifilters="yes"
 
 has_zstd="@HAS_ZSTD"
 if [ -z "$has_zstd" -o "$has_zstd" = "OFF" -o "$has_zstd" = "FALSE" ]; then
diff --git a/nc-config.in b/nc-config.in
index 66913a1ac..ec338794a 100644
--- a/nc-config.in
+++ b/nc-config.in
@@ -35,7 +35,7 @@ has_parallel4="@HAS_PARALLEL4@"
 has_nczarr="@HAS_NCZARR@"
 has_zstd="@HAS_ZSTD@"
 has_benchmarks="@HAS_BENCHMARKS@"
-has_multifilters="@HAS_MULTIFILTERS@"
+has_multifilters="yes"
 has_quantize="@HAS_QUANTIZE@"
 has_stdfilters="@STD_FILTERS@"
 version="@PACKAGE_NAME@ @PACKAGE_VERSION@"


Test patch:
diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml
index e7cf2f052..bc084a401 100644
--- a/.github/workflows/run_tests_win_cygwin.yml
+++ b/.github/workflows/run_tests_win_cygwin.yml
@@ -49,7 +49,7 @@ jobs:
           --disable-static --enable-dap --disable-dap-remote-tests
           --enable-plugins ${{ matrix.plugin_dir_option }}
           --disable-nczarr --disable-nczarr-filters
-          --disable-nczarr-s3 --disable-nczarr-s3-tests
+          --disable-s3 --with-s3-testing=no
 
 
       - name: Look at config.log if error
diff --git a/ncdump/test_rcmerge.sh b/ncdump/test_rcmerge.sh
index fe1bd0488..6c0462be0 100755
--- a/ncdump/test_rcmerge.sh
+++ b/ncdump/test_rcmerge.sh
@@ -138,7 +138,4 @@ mergecase3
 # Test the .rc api
 rcapi1
 
-# Test the .rc api
-rcapi1
-
 resetrc