Repository: OPENDAP/libdap4
Pull request #283: Jhrg/xsputn part1
Author: login: jgallagher59701
Created: 2025-05-23T18:59:32Z | Merged: 2025-05-28T16:28:01Z

This PR mixes refactoring the aged D4StreamMarshaller and checked_ostream code with improvements to the code to support sending individual variables that are larger than (2^31)-1 bytes. The associated shows that variables as large as 2^33 bytes (8GB) can be sent, both using the single and multithreaded build modes of libdap4. There are a few more ways this code can be improved, mostly using shared_ptr<> objects to minimize memory copy operations. But this PR does not address those. It only enables transfers larger than (2^31)-1.

The most important changes to the code are in util.cc.  The next most important are in D4StreamMarshaller.cc/h.


Diff:
diff --git a/.gitignore b/.gitignore
index 7a75bdc8b..64621cbf6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -203,6 +203,7 @@ unit-tests/UInt64Test
 unit-tests/GridTest
 unit-tests/IsDap4ProjectedTest
 unit-tests/util_mitTest
+unit-tests/MarshallerFutureTest
 
 
 unit-tests/chunked-io/
diff --git a/D4StreamMarshaller.cc b/D4StreamMarshaller.cc
index 762f477c6..6e6911ef5 100644
--- a/D4StreamMarshaller.cc
+++ b/D4StreamMarshaller.cc
@@ -20,23 +20,19 @@
 //
 // You should have received a copy of the GNU Lesser General Public
 // License along with this library; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 //
 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
 
 #include "config.h"
 
-#include <byteswap.h>
 #include <cassert>
 #include <cstring>
 
 #include <iomanip>
-#include <iostream>
 #include <limits>
 #include <sstream>
 
-// #define DODS_DEBUG 1
-
 #ifdef HAVE_PTHREAD_H
 #include <pthread.h>
 #endif
@@ -53,88 +49,12 @@
 
 #include "DapIndent.h"
 #include "debug.h"
+#include "util.h"
 
 using namespace std;
 
 namespace libdap {
 
-#if 0
-// We decided to use int64_t to represent sizes of both arrays and strings,
-// So this code is not used. jhrg 10/4/13
-
-// From the Google protobuf library
-inline uint8_t* WriteVarint64ToArrayInline(uint64_t value, uint8_t* target) {
-  // Splitting into 32-bit pieces gives better performance on 32-bit
-  // processors.
-  uint32_t part0 = static_cast<uint32_t>(value      );
-  uint32_t part1 = static_cast<uint32_t>(value >> 28);
-  uint32_t part2 = static_cast<uint32_t>(value >> 56);
-
-  int size;
-
-  // Here we can't really optimize for small numbers, since the value is
-  // split into three parts.  Checking for numbers < 128, for instance,
-  // would require three comparisons, since you'd have to make sure part1
-  // and part2 are zero.  However, if the caller is using 64-bit integers,
-  // it is likely that they expect the numbers to often be very large, so
-  // we probably don't want to optimize for small numbers anyway.  Thus,
-  // we end up with a hard coded binary search tree...
-  if (part2 == 0) {
-    if (part1 == 0) {
-      if (part0 < (1 << 14)) {
-        if (part0 < (1 << 7)) {
-          size = 1; goto size1;
-        } else {
-          size = 2; goto size2;
-        }
-      } else {
-        if (part0 < (1 << 21)) {
-          size = 3; goto size3;
-        } else {
-          size = 4; goto size4;
-        }
-      }
-    } else {
-      if (part1 < (1 << 14)) {
-        if (part1 < (1 << 7)) {
-          size = 5; goto size5;
-        } else {
-          size = 6; goto size6;
-        }
-      } else {
-        if (part1 < (1 << 21)) {
-          size = 7; goto size7;
-        } else {
-          size = 8; goto size8;
-        }
-      }
-    }
-  } else {
-    if (part2 < (1 << 7)) {
-      size = 9; goto size9;
-    } else {
-      size = 10; goto size10;
-    }
-  }
-
-  // GOOGLE_LOG(FATAL) << "Can't get here.";
-
-  size10: target[9] = static_cast<uint8_t>((part2 >>  7) | 0x80);
-  size9 : target[8] = static_cast<uint8_t>((part2      ) | 0x80);
-  size8 : target[7] = static_cast<uint8_t>((part1 >> 21) | 0x80);
-  size7 : target[6] = static_cast<uint8_t>((part1 >> 14) | 0x80);
-  size6 : target[5] = static_cast<uint8_t>((part1 >>  7) | 0x80);
-  size5 : target[4] = static_cast<uint8_t>((part1      ) | 0x80);
-  size4 : target[3] = static_cast<uint8_t>((part0 >> 21) | 0x80);
-  size3 : target[2] = static_cast<uint8_t>((part0 >> 14) | 0x80);
-  size2 : target[1] = static_cast<uint8_t>((part0 >>  7) | 0x80);
-  size1 : target[0] = static_cast<uint8_t>((part0      ) | 0x80);
-
-  target[size-1] &= 0x7F;
-  return target + size;
-}
-#endif
-
 #if USE_XDR_FOR_IEEE754_ENCODING
 void D4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int width, Type type) {
     dods_uint64 size = num * width;
@@ -195,7 +115,8 @@ void D4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int widt
  * @param out Write to this stream object.
  * @param write_data If true, write data values. True by default
  */
-D4StreamMarshaller::D4StreamMarshaller(ostream &out, bool write_data) : d_out(out), d_write_data(write_data), tm(0) {
+D4StreamMarshaller::D4StreamMarshaller(ostream &out, bool write_data)
+    : d_out(out), d_write_data(write_data), tm(nullptr) {
     assert(sizeof(std::streamsize) >= sizeof(int64_t));
 
 #if USE_XDR_FOR_IEEE754_ENCODING
@@ -218,8 +139,9 @@ D4StreamMarshaller::~D4StreamMarshaller() {
 #if USE_XDR_FOR_IEEE754_ENCODING
     xdr_destroy(&d_scalar_sink);
 #endif
-
+#ifdef USE_POSIX_THREADS
     delete tm;
+#endif
 }
 
 /** Initialize the checksum buffer. This resets the checksum calculation.
@@ -263,7 +185,7 @@ void D4StreamMarshaller::put_checksum() {
  * zero has no effect on the checksum value.
  */
 void D4StreamMarshaller::checksum_update(const void *data, unsigned long len) {
-    d_checksum.AddData(reinterpret_cast<const uint8_t *>(data), len);
+    d_checksum.AddData(static_cast<const uint8_t *>(data), len);
 }
 
 void D4StreamMarshaller::put_byte(dods_byte val) {
@@ -461,7 +383,7 @@ void D4StreamMarshaller::put_uint64(dods_uint64 val) {
  * methods put_opaque_dap4(), ..., that need counts sent as prefixes to
  * their data handle it themselves.
  *
- * @param count How many elements follow.
+ * @param count The number of elements that will follow in the stream.
  */
 void D4StreamMarshaller::put_count(int64_t count) {
 #ifdef USE_POSIX_THREADS
@@ -474,45 +396,46 @@ void D4StreamMarshaller::put_str(const string &val) {
     checksum_update(val.c_str(), val.length());
 
     if (d_write_data) {
-        int64_t len = val.length();
+        const auto len = static_cast<int64_t>(val.length());
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 #endif
         d_out.write(reinterpret_cast<const char *>(&len), sizeof(int64_t));
-        d_out.write(val.data(), val.length());
+        d_out.write(val.data(), static_cast<std::streamsize>(val.length()));
     }
 }
 
 void D4StreamMarshaller::put_url(const string &val) { put_str(val); }
 
-void D4StreamMarshaller::put_opaque_dap4(const char *val, int64_t len) {
+void D4StreamMarshaller::put_opaque_dap4(const char *val, int64_t num_bytes) {
     assert(val);
-    assert(len >= 0);
+    assert(num_bytes >= 0);
 
-    checksum_update(val, len);
+    checksum_update(val, num_bytes);
 
     if (d_write_data) {
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 
-        d_out.write(reinterpret_cast<const char *>(&len), sizeof(int64_t));
+        d_out.write(reinterpret_cast<const char *>(&num_bytes), sizeof(int64_t));
 
-        char *byte_buf = new char[len];
-        memcpy(byte_buf, val, len);
+        char *byte_buf = new char[num_bytes];
+        memcpy(byte_buf, val, num_bytes);
 
         tm->increment_child_thread_count();
-        tm->start_thread(MarshallerThread::write_thread, d_out, byte_buf, len);
+        tm->start_thread(MarshallerThread::write_thread, d_out, byte_buf, num_bytes);
 #else
-        d_out.write(reinterpret_cast<const char *>(&len), sizeof(int64_t));
-        d_out.write(val, len);
+        d_out.write(reinterpret_cast<const char *>(&num_bytes), sizeof(int64_t));
+        segmented_write(d_out, val, num_bytes);
 #endif
     }
 }
 
 /**
  * @brief Write a fixed size vector
+ * @todo Add a new method that takes a unique_ptr<char> to transfer ownership here.
  * @param val Pointer to the data
- * @param num Number of bytes to write
+ * @param num_bytes Number of bytes to write
  */
 void D4StreamMarshaller::put_vector(char *val, int64_t num_bytes) {
     assert(val);
@@ -530,7 +453,7 @@ void D4StreamMarshaller::put_vector(char *val, int64_t num_bytes) {
         tm->increment_child_thread_count();
         tm->start_thread(MarshallerThread::write_thread, d_out, buf, num_bytes);
 #else
-        d_out.write(val, num_bytes);
+        segmented_write(d_out, val, num_bytes);
 #endif
     }
 }
@@ -540,44 +463,42 @@ void D4StreamMarshaller::put_vector(char *val, int64_t num_elem, int elem_size)
     assert(num_elem >= 0);
     assert(elem_size > 0);
 
-    int64_t bytes;
+    int64_t num_bytes = num_elem;
 
     switch (elem_size) {
     case 1:
-        assert(!"Don't call this method for bytes, use put_vector(val, bytes) instead");
-        bytes = num_elem;
         break;
     case 2:
         // Don't bother testing the sign bit
         assert(!(num_elem & 0x4000000000000000)); // 0x 40 00 --> 0100 0000
-        bytes = num_elem << 1;
+        num_bytes = num_elem << 1;
         break;
     case 4:
         assert(!(num_elem & 0x6000000000000000)); // 0x 60 00 --> 0110 0000
-        bytes = num_elem << 2;
+        num_bytes = num_elem << 2;
         break;
     case 8:
         assert(!(num_elem & 0x7000000000000000)); // 0111 0000
-        bytes = num_elem << 3;
+        num_bytes = num_elem << 3;
         break;
     default:
-        bytes = num_elem * elem_size;
+        num_bytes = num_elem * elem_size;
         break;
     }
 
-    checksum_update(val, bytes);
+    checksum_update(val, num_bytes);
 
     if (d_write_data) {
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 
-        char *buf = new char[bytes];
-        memcpy(buf, val, bytes);
+        char *buf = new char[num_bytes];
+        memcpy(buf, val, num_bytes);
 
         tm->increment_child_thread_count();
-        tm->start_thread(MarshallerThread::write_thread, d_out, buf, bytes);
+        tm->start_thread(MarshallerThread::write_thread, d_out, buf, num_bytes);
 #else
-        d_out.write(val, bytes);
+        segmented_write(d_out, val, num_bytes);
 #endif
     }
 }
@@ -585,11 +506,9 @@ void D4StreamMarshaller::put_vector(char *val, int64_t num_elem, int elem_size)
 /**
  * @brief Write a fixed size vector
  * @note This method and its companion for float64 exists in case we need to
- * support machine that do not use IEEE754 for their floating point representation.
+ * support machines that do not use IEEE754 for their floating point representation.
  * @param val Pointer to the data
- * @param num Number of elements
- * @param width Size of a single element
- * @param type DAP variable type; used to handle float32 and float64 types correctly
+ * @param num_elem Number of elements
  */
 void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem) {
 #if !USE_XDR_FOR_IEEE754_ENCODING
@@ -599,25 +518,25 @@ void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem) {
     assert(num_elem >= 0);
     // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
     // make sure that doesn't overflow a 63-bit integer (the max positive value in
-    // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
+    // a signed int64; use 1110 0000 0... (0xe000 ...) to mask for non-zero bits
     // to test that num can be multiplied by 4. A
     assert(!(num_elem & 0xe000000000000000));
 
-    num_elem = num_elem << 2; // num_elem is now the number of bytes
+    auto num_bytes = num_elem << 2; // num_elem is now the number of bytes
 
-    checksum_update(val, num_elem);
+    checksum_update(val, num_bytes);
 
     if (d_write_data) {
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 
-        char *buf = new char[num_elem];
-        memcpy(buf, val, num_elem);
+        char *buf = new char[num_bytes];
+        memcpy(buf, val, num_bytes);
 
         tm->increment_child_thread_count();
-        tm->start_thread(MarshallerThread::write_thread, d_out, buf, num_elem);
+        tm->start_thread(MarshallerThread::write_thread, d_out, buf, num_bytes);
 #else
-        d_out.write(val, num_elem);
+        segmented_write(d_out, val, num_bytes);
 #endif
     }
 
@@ -626,7 +545,7 @@ void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem) {
     assert(num_elem >= 0);
     // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
     // make sure that doesn't overflow a 63-bit integer (the max positive value in
-    // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
+    // a signed int64; use 1110 0000 0... (0xe000 ...) to mask for non-zero bits
     // to test that num can be multiplied by 4. A
     assert(!(num_elem & 0xe000000000000000));
 
@@ -657,11 +576,8 @@ void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem) {
 
 /**
  * @brief Write a fixed size vector of float64s
- *
  * @param val Pointer to the data
- * @param num Number of elements
- * @param width Size of a single element
- * @param type DAP variable type; used to handle float32 and float64 types correctly
+ * @param num_elem Number of elements
  */
 void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) {
 #if !USE_XDR_FOR_IEEE754_ENCODING
@@ -672,21 +588,21 @@ void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) {
     // See comment above
     assert(!(num_elem & 0xf000000000000000));
 
-    num_elem = num_elem << 3; // num_elem is now the number of bytes
+    auto num_bytes = num_elem << 3; // num_elem is now the number of bytes
 
-    checksum_update(val, num_elem);
+    checksum_update(val, num_bytes);
 
     if (d_write_data) {
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 
-        char *buf = new char[num_elem];
-        memcpy(buf, val, num_elem);
+        char *buf = new char[num_bytes];
+        memcpy(buf, val, num_bytes);
 
         tm->increment_child_thread_count();
-        tm->start_thread(MarshallerThread::write_thread, d_out, buf, num_elem);
+        tm->start_thread(MarshallerThread::write_thread, d_out, buf, num_bytes);
 #else
-        d_out.write(val, num_elem);
+        segmented_write(d_out, val, num_bytes);
 #endif
     }
 #else
@@ -694,7 +610,7 @@ void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) {
     assert(num_elem >= 0);
     // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
     // make sure that doesn't overflow a 63-bit integer (the max positive value in
-    // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
+    // a signed int64; use 1110 0000 0... (0xe000 ...) to mask for non-zero bits
     // to test that num can be multiplied by 4. A
     assert(!(num_elem & 0xe000000000000000));
 
@@ -723,9 +639,16 @@ void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) {
 #endif
 }
 
+/**
+ * @brief Write one part of a vector's contents.
+ *
+ * @param val Pointer to the part's values
+ * @param num The number of values in this part
+ * @param width The number of bytes per value
+ * @param type The DAP2 data type for each value
+ */
 void D4StreamMarshaller::put_vector_part(char *val, unsigned int num, int width, Type type) {
     assert(val);
-    assert(num >= 0);
     assert(width > 0);
 
     switch (type) {
@@ -757,7 +680,7 @@ void D4StreamMarshaller::put_vector_part(char *val, unsigned int num, int width,
         break;
 
     case dods_float64_c:
-        put_vector_float32(val, num);
+        put_vector_float64(val, num);
         break;
 
     case dods_str_c:
@@ -770,7 +693,8 @@ void D4StreamMarshaller::put_vector_part(char *val, unsigned int num, int width,
     case dods_opaque_c:
     case dods_structure_c:
     case dods_sequence_c:
-        throw InternalErr(__FILE__, __LINE__, "Array of String should not be passed to put_vector.");
+        throw InternalErr(__FILE__, __LINE__,
+                          "Array of Opaque, Structure or Sequence should not be passed to put_vector.");
 
     case dods_grid_c:
         throw InternalErr(__FILE__, __LINE__, "Grid is not part of DAP4.");
diff --git a/D4StreamMarshaller.h b/D4StreamMarshaller.h
index 7acceafe6..d946db253 100644
--- a/D4StreamMarshaller.h
+++ b/D4StreamMarshaller.h
@@ -21,15 +21,13 @@
 //
 // You should have received a copy of the GNU Lesser General Public
 // License along with this library; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 //
 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
 
 #ifndef I_D4StreamMarshaller_h
 #define I_D4StreamMarshaller_h 1
 
-#include <iostream>
-
 // By default, only support platforms that use IEEE754 for floating point values.
 // Hacked up code leftover from an older version of the class; largely untested.
 // jhrg 10/3/13
@@ -48,7 +46,6 @@
 #endif
 
 #include "crc.h"
-#include <stdint.h>
 
 #include "InternalErr.h"
 #include "Marshaller.h"
@@ -65,36 +62,37 @@ class MarshallerThread;
  * computing the checksum only.
  *
  * @note This class uses the Marshaller interface; it could be rewritten
- * to use far fewer methods since all of the put_*() methods take different
+ * to use far fewer methods since all the put_*() methods take different
  * types.
  */
 class D4StreamMarshaller : public Marshaller {
 
-private:
 #if USE_XDR_FOR_IEEE754_ENCODING
     XDR d_scalar_sink;
     char d_ieee754_buf[sizeof(dods_float64)]; // used to serialize a float or double
 #endif
 
     ostream &d_out;
-    bool d_write_data; // jhrg 1/27/12
+    bool d_write_data = true; // jhrg 1/27/12
 
     Crc32 d_checksum;
 
-    MarshallerThread *tm;
-
-    // These are private so they won't ever get used.
-    D4StreamMarshaller();
-    D4StreamMarshaller(const D4StreamMarshaller &);
-    D4StreamMarshaller &operator=(const D4StreamMarshaller &);
+    MarshallerThread *tm = nullptr;
 
 #if USE_XDR_FOR_IEEE754_ENCODING
     void m_serialize_reals(char *val, int64_t num, int width, Type type);
 #endif
 
 public:
-    D4StreamMarshaller(std::ostream &out, bool write_data = true);
-    virtual ~D4StreamMarshaller();
+    explicit D4StreamMarshaller(std::ostream &out, bool write_data = true);
+
+    D4StreamMarshaller() = delete;
+    D4StreamMarshaller(const D4StreamMarshaller &) = delete;
+    D4StreamMarshaller &operator=(const D4StreamMarshaller &) = delete;
+    D4StreamMarshaller(const D4StreamMarshaller &&) = delete;
+    D4StreamMarshaller &operator=(const D4StreamMarshaller &&) = delete;
+
+    ~D4StreamMarshaller() override;
 
     virtual void reset_checksum();
     virtual string get_checksum();
@@ -103,70 +101,58 @@ class D4StreamMarshaller : public Marshaller {
     virtual void put_checksum();
     virtual void put_count(int64_t count);
 
-    virtual void put_byte(dods_byte val);
+    void put_byte(dods_byte val) override;
     virtual void put_int8(dods_int8 val);
 
-    virtual void put_int16(dods_int16 val);
-    virtual void put_int32(dods_int32 val);
+    void put_int16(dods_int16 val) override;
+    void put_int32(dods_int32 val) override;
     // Added
     virtual void put_int64(dods_int64 val);
 
-    virtual void put_float32(dods_float32 val);
-    virtual void put_float64(dods_float64 val);
+    void put_float32(dods_float32 val) override;
+    void put_float64(dods_float64 val) override;
 
-    virtual void put_uint16(dods_uint16 val);
-    virtual void put_uint32(dods_uint32 val);
+    void put_uint16(dods_uint16 val) override;
+    void put_uint32(dods_uint32 val) override;
     // Added
     virtual void put_uint64(dods_uint64 val);
 
-    virtual void put_str(const string &val);
-    virtual void put_url(const string &val);
+    void put_str(const string &val) override;
+    void put_url(const string &val) override;
 
-    virtual void put_opaque(char *, unsigned int) {
+    /// @brief Not supported by DAP4
+    void put_opaque(char *, unsigned int) override {
         throw InternalErr(__FILE__, __LINE__, "Not implemented for DAP4; use put_opaque_dap4() instead.");
     }
 
-    virtual void put_opaque_dap4(const char *val, int64_t len);
+    virtual void put_opaque_dap4(const char *val, int64_t num_bytes);
 
-    // Never use put_int() to send length information in DAP4.
-    virtual void put_int(int) { throw InternalErr(__FILE__, __LINE__, "Not Implemented; use put_length_prefix."); }
+    /// @brief Not supported by DAP4
+    void put_int(int) override { throw InternalErr(__FILE__, __LINE__, "Not Implemented; use put_length_prefix."); }
 
     virtual void put_vector(char *val, int64_t num_bytes);
     virtual void put_vector(char *val, int64_t num_elem, int elem_size);
     virtual void put_vector_float32(char *val, int64_t num_elem);
     virtual void put_vector_float64(char *val, int64_t num_elem);
 
-    virtual void put_vector(char *, int, Vector &) {
+    /// @brief Not supported by DAP4
+    void put_vector(char *, int, Vector &) override {
         throw InternalErr(__FILE__, __LINE__, "Not Implemented; use other put_vector() versions.");
     }
-    virtual void put_vector(char *, int, int, Vector &) {
+    /// @brief Not supported by DAP4
+    void put_vector(char *, int, int, Vector &) override {
         throw InternalErr(__FILE__, __LINE__, "Not Implemented; use other put_vector() versions.");
     }
 
-    /**
-     * Prepare to send a single array/vector using a series of 'put' calls.
-     * In DAP4 this does nothing because arrays are serialized using the server's
-     * binary representation (i.e., using 'reader make right').
-     *
-     * @param num Ignored
-     * @see put_vector_part()
-     * @see put_vector_end()
-     */
-    virtual void put_vector_start(int /*num*/) {}
-
-    virtual void put_vector_part(char * /*val*/, unsigned int /*num*/, int /*width*/, Type /*type*/);
-
-    /**
-     * Close a vector when its values are written using put_vector_part().
-     * In DAP4 this does nothing because arrays are serialized using the server's
-     * binary representation (i.e., using 'reader make right').
-     *
-     * @see put_vector_start()
-     * @see put_vector_part()
-     */
-    virtual void put_vector_end() {}
-
-    virtual void dump(std::ostream &strm) const;
+    /// @brief Does nothing in DAP4
+    void put_vector_start(int /*num*/) override {}
+
+    void put_vector_part(char * /*val*/, unsigned int /*num*/, int /*width*/, Type /*type*/) override;
+
+    /// @brief Does nothing in DAP4
+    void put_vector_end() override {}
+
+    void dump(std::ostream &strm) const override;
 };
 
 } // namespace libdap
diff --git a/Marshaller.h b/Marshaller.h
index 1a480d972..6661412a3 100644
--- a/Marshaller.h
+++ b/Marshaller.h
@@ -34,7 +34,6 @@
 #define A_Marshaller_h 1
 
 #include <string>
-#include <vector>
 
 #include "DapObj.h"
 #include "InternalErr.h"
@@ -95,7 +94,7 @@ class Marshaller : public DapObj {
      */
     virtual void put_vector_end() { throw InternalErr(__FILE__, __LINE__, "Not Implemented yet"); } // = 0;
 
-    virtual void dump(std::ostream &strm) const = 0;
+    void dump(std::ostream &strm) const override = 0;
 };
 
 } // namespace libdap
diff --git a/MarshallerFuture.h b/MarshallerFuture.h
new file mode 100644
index 000000000..016ca1b87
--- /dev/null
+++ b/MarshallerFuture.h
@@ -0,0 +1,196 @@
+// -*- mode: c++; c-basic-offset:4 -*-
+
+// This file is part of libdap, A C++ implementation of the OPeNDAP Data
+// Access Protocol.
+
+// Copyright (c) 2025 OPeNDAP, Inc.
+// Author: James Gallagher <jgallagher@opendap.org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+//
+// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
+
+#ifndef MARSHALLERFUTURE_H_
+#define MARSHALLERFUTURE_H_
+
+#include <cstring>
+#include <future>
+#include <memory>
+#include <stdexcept>
+#include <unistd.h>
+
+namespace libdap {
+
+/**
+ * @brief Run output calls using a future
+ *
+ * This version of MarshallerThread uses futures to implement what would have
+ * been called 'double-buffered I/O' although this is only for writing. When
+ * libdap's D4 and XDRStreamMarshaller classes are used to send data, this class
+ * can be used to perform the data write() in a child thread that is managed
+ * using the future objects introduced in C++11.
+ *
+ * Given the nature of DAP2 and DAP4 data responses, it is not possible to interleave
+ * data for variables, so only one transmission is possible at any time. This
+ * means that only one 'output future' can be running and the future::get() can
+ * serve as a synchronization tool.
+ *
+ * This class is a replacement for the older MarshallerThread class that was
+ * implemented using threads. It is implemented as a header-only class.
+ */
+class MarshallerThread {
+    std::future<std::streampos> d_ostream_future;
+    std::future<ssize_t> d_fp_future;
+
+public:
+    MarshallerThread() = default;
+
+    ~MarshallerThread() {
+        if (d_ostream_future.valid()) {
+            try {
+                d_ostream_future.get();
+            } catch (...) {
+                // Suppress exceptions in destructor
+            }
+        }
+        if (d_fp_future.valid()) {
+            try {
+                d_fp_future.get();
+            } catch (...) {
+                // Suppress exceptions in destructor
+            }
+        }
+    }
+
+    /// @return The future associated with the ostream threads
+    auto get_ostream_future() { return d_ostream_future.get(); }
+    /// @return The future associated with the file descriptor threads
+    auto get_fp_future() { return d_fp_future.get(); }
+
+    /**
+     * @brief start a thread to send byte_buf out using the stream out
+     * @param out The ostream
+     * @param byte_buf The bytes (a const char *)
+     * @param num_bytes The number of byes
+     */
+    void start_thread(std::ostream &out, const char *byte_buf, std::streamsize num_bytes) {
+        if (d_ostream_future.valid()) {
+            d_ostream_future.get();
+        }
+
+        auto buffer_copy = std::make_unique<char[]>(num_bytes);
+        std::memcpy(buffer_copy.get(), byte_buf, num_bytes);
+
+        d_ostream_future = std::async(std::launch::async, [buffer = std::move(buffer_copy), &out, num_bytes]() {
+            out.write(buffer.get(), num_bytes);
+            if (out.fail()) {
+                throw std::runtime_error("Failed to write stream.");
+            }
+            return out.tellp();
+        });
+    }
+
+    /**
+     * @brief start a thread to send byte_buf out using the stream out
+     * @param out The ostream
+     * @param byte_buf The bytes (a std::shared_ptr<const char>)
+     * @param num_bytes The number of byes
+     */
+    void start_thread(std::ostream &out, std::shared_ptr<const char> byte_buf, std::streamsize num_bytes) {
+        if (d_ostream_future.valid()) {
+            d_ostream_future.get();
+        }
+
+        d_ostream_future = std::async(std::launch::async, [buffer = std::move(byte_buf), &out, num_bytes]() {
+            out.write(buffer.get(), num_bytes);
+            if (out.fail()) {
+                throw std::runtime_error("Failed to write stream.");
+            }
+            return out.tellp();
+        });
+    }
+
+    void start_thread_part(std::ostream &out, const char *byte_buf, std::streamsize num_bytes) {
+        if (d_ostream_future.valid()) {
+            d_ostream_future.get();
+        }
+
+        auto buffer_copy = std::make_unique<char[]>(num_bytes);
+        std::memcpy(buffer_copy.get(), byte_buf, num_bytes);
+
+        d_ostream_future = std::async(std::launch::async, [buffer = std::move(buffer_copy), &out, num_bytes]() {
+            if (num_bytes <= 4) {
+                throw std::runtime_error("Data too short for partial write.");
+            }
+            out.write(buffer.get() + 4, num_bytes - 4);
+            if (out.fail()) {
+                throw std::runtime_error("Failed to write partial stream.");
+            }
+            return out.tellp();
+        });
+    }
+
+    void start_thread_part(std::ostream &out, std::shared_ptr<const char> byte_buf, std::streamsize num_bytes) {
+        if (d_ostream_future.valid()) {
+            d_ostream_future.get();
+        }
+
+        d_ostream_future = std::async(std::launch::async, [buffer = std::move(byte_buf), &out, num_bytes]() {
+            if (num_bytes <= 4) {
+                throw std::runtime_error("Data too short for partial write.");
+            }
+            out.write(buffer.get() + 4, num_bytes - 4);
+            if (out.fail()) {
+                throw std::runtime_error("Failed to write partial stream.");
+            }
+            return out.tellp();
+        });
+    }
+
+    void start_thread(int fd, const char *byte_buf, std::streamsize num_bytes) {
+        if (d_fp_future.valid()) {
+            d_fp_future.get();
+        }
+
+        auto buffer_copy = std::make_unique<char[]>(num_bytes);
+        std::memcpy(buffer_copy.get(), byte_buf, num_bytes);
+
+        d_fp_future = std::async(std::launch::async, [buffer = std::move(buffer_copy), fd, num_bytes]() {
+            auto written = write(fd, buffer.get(), num_bytes);
+            if (written != num_bytes) {
+                throw std::runtime_error("Failed to write all data to fd.");
+            }
+            return written;
+        });
+    }
+
+    void start_thread(int fd, std::shared_ptr<const char> byte_buf, std::streamsize num_bytes) {
+        if (d_fp_future.valid()) {
+            d_fp_future.get();
+        }
+
+        d_fp_future = std::async(std::launch::async, [buffer = std::move(byte_buf), fd, num_bytes]() {
+            auto written = write(fd, buffer.get(), num_bytes);
+            if (written != num_bytes) {
+                throw std::runtime_error("Failed to write all data to fd.");
+            }
+            return written;
+        });
+    }
+};
+
+} // namespace libdap
+
+#endif
diff --git a/MarshallerThread.cc b/MarshallerThread.cc
index 969dbf260..84f8e1c09 100644
--- a/MarshallerThread.cc
+++ b/MarshallerThread.cc
@@ -43,6 +43,7 @@
 #include "InternalErr.h"
 #include "MarshallerThread.h"
 #include "debug.h"
+#include "util.h"
 
 using namespace libdap;
 using namespace std;
@@ -50,6 +51,11 @@ using namespace std;
 // Set TIMING to 1 to enable timing output
 #define TIMING 0
 
+// The segmented write function enables writing 2^32 bytes or more,
+// up to the limits of the machine. This switch only affects POSIX
+// THREADS builds of libdap. jhrg 5/22/25
+#define USE_SEGMENTED_WRITE 1
+
 #if TIMING
 bool MarshallerThread::print_time = false;
 
@@ -226,7 +232,11 @@ void *MarshallerThread::write_thread(void *arg) {
         if (bytes_written != args->d_num)
             return (void *)-1;
     } else {
+#if USE_SEGMENTED_WRITE
+        segmented_write(args->d_out, args->d_buf, args->d_num);
+#else
         args->d_out.write(args->d_buf, args->d_num);
+#endif
         if (args->d_out.fail()) {
             ostringstream oss;
             oss << "Could not write data: " << __FILE__ << ":" << __LINE__;
@@ -273,7 +283,11 @@ void *MarshallerThread::write_thread_part(void *arg) {
         if (bytes_written != args->d_num)
             return (void *)-1;
     } else {
+#if USE_SEGMENTED_WRITE
+        segmented_write(args->d_out, args->d_buf + 4, args->d_num);
+#else
         args->d_out.write(args->d_buf + 4, args->d_num);
+#endif
         if (args->d_out.fail()) {
             ostringstream oss;
             oss << "Could not write data: " << __FILE__ << ":" << __LINE__;
diff --git a/MarshallerThread.h b/MarshallerThread.h
index c10a68587..02515ede7 100644
--- a/MarshallerThread.h
+++ b/MarshallerThread.h
@@ -52,13 +52,12 @@ namespace libdap {
 class Locker {
 public:
     Locker() = delete;
+    Locker(const Locker &rhs) = delete;
     Locker(pthread_mutex_t &lock, pthread_cond_t &cond, int &count);
     virtual ~Locker();
 
 private:
     pthread_mutex_t &m_mutex;
-
-    Locker(const Locker &rhs);
 };
 
 /**
@@ -85,16 +84,16 @@ class ChildLocker {
 };
 
 /**
- * Implement a multi-threaded data transmission sub-system for libdap.
- * This class makes it fairly painless to send data using a child thread
- * so that the main thread can be used to read the next chunk of data
+ * Implement a multithreaded data transmission subsystem for libdap.
+ * This class makes it fairly painless to send data using a child thread.
+ * The main thread can be used to read the next chunk of data
  * while whatever has been read to this point is sent over the wire.
  *
  * This code is used by XDRStreamMarshaller and (soon) D4StreamMarshaller.
  */
 class MarshallerThread {
 private:
-    pthread_t d_thread = 0;
+    pthread_t d_thread;
     pthread_attr_t d_thread_attr;
 
     pthread_mutex_t d_out_mutex;
@@ -149,7 +148,7 @@ class MarshallerThread {
     void start_thread(void *(*thread)(void *arg), std::ostream &out, char *byte_buf, std::streamsize bytes_written);
     void start_thread(void *(*thread)(void *arg), int fd, char *byte_buf, std::streamsize bytes_written);
 
-    // These are static so they will have c-linkage - required because they
+    // These are static, so they will have c-linkage - required because they
     // are passed to pthread_create()
     static void *write_thread(void *arg);
     static void *write_thread_part(void *arg);
diff --git a/Vector.cc b/Vector.cc
index aad900f68..e2018909b 100644
--- a/Vector.cc
+++ b/Vector.cc
@@ -217,22 +217,27 @@ void Vector::m_delete_cardinal_data_buffer() {
     d_capacity_ll = 0;
 }
 
-/** Helper to reduce cut and paste in the virtual's.
- *
+const auto ERR_NEGATIVE_ELEMENTS = "Logic error: Vector::set_cardinal_values_internal() called with negative size!";
+const auto ERR_NULL_ARRAY = "Logic error: Vector::set_cardinal_values_internal() called with null source!";
+
+/**
+ * @brief Helper to reduce cut and paste in the virtual's.
+ * @tparam CardType
+ * @param src Source data array
+ * @param elements Number of elements in the source array
  */
-template <class CardType> void Vector::m_set_cardinal_values_internal(const CardType *fromArray, int64_t numElts) {
-    if (numElts < 0) {
-        throw InternalErr(__FILE__, __LINE__,
-                          "Logic error: Vector::set_cardinal_values_internal() called with negative numElts!");
+template <class CardType> void Vector::m_set_cardinal_values_internal(const CardType *src, int64_t elements) {
+    if (elements < 0) {
+        throw InternalErr(__FILE__, __LINE__, ERR_NEGATIVE_ELEMENTS);
     }
-    if (!fromArray) {
-        throw InternalErr(__FILE__, __LINE__,
-                          "Logic error: Vector::set_cardinal_values_internal() called with null fromArray!");
+    if (!src) {
+        throw InternalErr(__FILE__, __LINE__, ERR_NULL_ARRAY);
     }
-    set_length_ll(numElts);
-    m_create_cardinal_data_buffer_for_type(numElts);
+
+    set_length_ll(elements);
+    m_create_cardinal_data_buffer_for_type(elements);
     if (d_buf)
-        memcpy(d_buf, fromArray, numElts * sizeof(CardType));
+        memcpy(d_buf, src, elements * sizeof(CardType));
     set_read_p(true);
 }
 
@@ -1207,10 +1212,10 @@ void Vector::deserialize(D4StreamUnMarshaller &um, DMR &dmr) {
  For a Vector of Str (OPeNDAP Strings), this assumes \e val points to an
  array of C++ strings.
 
- This method should not be used for Structure, Sequence or Grid.
+ This method should not be used for Structure, Sequence, or Grid.
 
  @brief Reads data into the Vector buffer.
- @exception InternalErr Thrown if called for Structure, Sequence or
+ @exception InternalErr Thrown if called for Structure, Sequence, or
  Grid.
  @return The number of bytes used by the array.
  @param val A pointer to the input data.
@@ -1313,7 +1318,7 @@ int64_t Vector::val2buf_ll(void *val, bool reuse) {
     if (!val && length() == 0)
         return 0;
 
-    // I *think* this method has been mainly designed to be use by read which
+    // I *think* this method has been mainly designed to be used by read which
     // is implemented in the surrogate library. Passing NULL as a pointer to
     // this method will be an error of the creator of the surrogate library.
     // Even though I recognize the fact that some methods inside libdap++ can
@@ -1388,7 +1393,7 @@ int64_t Vector::val2buf_ll(void *val, bool reuse) {
  allocate memory if the handle @p val references NULL, otherwise it
  assumes the handle references enough storage for the data to be copied.
 
- Never call this method for constructor types Structure, Sequence or Grid.
+ Never call this method for constructor types Structure, Sequence, or Grid.
 
  When reading data out of a variable that has been constrained, this method
  assumes the N values/bytes of constrained data start at the beginning
diff --git a/Vector.h b/Vector.h
index 8b81fa34b..3281ef0b4 100644
--- a/Vector.h
+++ b/Vector.h
@@ -118,7 +118,7 @@ class Vector : public BaseType {
     bool m_is_cardinal_type() const;
     int64_t m_create_cardinal_data_buffer_for_type(int64_t num_elements);
     void m_delete_cardinal_data_buffer();
-    template <class CardType> void m_set_cardinal_values_internal(const CardType *fromArray, int64_t num_elements);
+    template <class CardType> void m_set_cardinal_values_internal(const CardType *src, int64_t num_elements);
 
     // This function copies the private members of Vector.
     void m_duplicate(const Vector &v);
diff --git a/XDRFileMarshaller.h b/XDRFileMarshaller.h
index 2383c5752..0e4f414ba 100644
--- a/XDRFileMarshaller.h
+++ b/XDRFileMarshaller.h
@@ -35,15 +35,10 @@
 
 #include "Marshaller.h"
 #include "XDRUtils.h"
-#if 0
-#include "Array.h"
-#include "Vector.h"
-#endif
 
 namespace libdap {
 
-/** @brief marshaller that knows how to marshall/serialize dap data objects
- * to a file using XDR
+/** @brief marshaller that knows how to marshall/serialize dap data objects to a file using XDR
  */
 class XDRFileMarshaller : public Marshaller {
 private:
diff --git a/XDRUtils.cc b/XDRUtils.cc
index 0bef7f27c..848af9a25 100644
--- a/XDRUtils.cc
+++ b/XDRUtils.cc
@@ -137,17 +137,17 @@ namespace libdap {
 xdrproc_t XDRUtils::xdr_coder(const Type &t) {
     switch (t) {
     case dods_int16_c:
-        return (xdrproc_t)XDR_INT16;
+        return reinterpret_cast<xdrproc_t>(XDR_INT16);
     case dods_uint16_c:
-        return (xdrproc_t)XDR_UINT16;
+        return reinterpret_cast<xdrproc_t>(XDR_UINT16);
     case dods_int32_c:
-        return (xdrproc_t)XDR_INT32;
+        return reinterpret_cast<xdrproc_t>(XDR_INT32);
     case dods_uint32_c:
-        return (xdrproc_t)XDR_UINT32;
+        return reinterpret_cast<xdrproc_t>(XDR_UINT32);
     case dods_float32_c:
-        return (xdrproc_t)XDR_FLOAT32;
+        return reinterpret_cast<xdrproc_t>(XDR_FLOAT32);
     case dods_float64_c:
-        return (xdrproc_t)XDR_FLOAT64;
+        return reinterpret_cast<xdrproc_t>(XDR_FLOAT64);
     case dods_byte_c:
     case dods_str_c:
     case dods_url_c:
diff --git a/chunked_ostream.cc b/chunked_ostream.cc
index d8e7eb8cd..a577aaf69 100644
--- a/chunked_ostream.cc
+++ b/chunked_ostream.cc
@@ -18,26 +18,22 @@
 //
 // You should have received a copy of the GNU Lesser General Public
 // License along with this library; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 //
 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
 //
-// Portions of this code were taken verbatim from  Josuttis,
+// Portions of this code were taken verbatim from Josuttis,
 // "The C++ Standard Library," p.672
 
 #include "config.h"
 
 #include <arpa/inet.h>
 
-#include <stdint.h>
-
 #include <streambuf>
 #include <string>
 
 #include <cstring>
 
-// #define DODS_DEBUG
-
 #include "chunked_ostream.h"
 #include "chunked_stream.h"
 #include "debug.h"
@@ -53,21 +49,21 @@ namespace libdap {
 std::streambuf::int_type chunked_outbuf::data_chunk() {
     DBG(cerr << "In chunked_outbuf::data_chunk" << endl);
 
-    int32_t num = pptr() - pbase(); // num needs to be signed for the call to pbump
+    const auto num = static_cast<int_type>(pptr() - pbase()); // num needs to be signed for the call to pbump
 
     // Since this is called by sync() (e.g., flush()), return 0 and do nothing
     // when there's no data to send.
     if (num == 0)
         return 0;
 
-    // here, write out the chunk headers: CHUNKTYPE and CHUNKSIZE
+    // Here, write out the chunk headers: CHUNKTYPE and CHUNKSIZE
     // as a 32-bit unsigned int. Here I assume that num is never
     // more than 2^24 because that was tested in the constructor
 
     // Trick: This method always writes CHUNK_DATA type chunks so
     // the chunk type is always 0x00, and given that num never has
-    // anything bigger than 24-bits, the high order byte is always
-    // 0x00. Of course bit-wise OR with 0x00 isn't going to do
+    // anything bigger than 24-bits, the high-order byte is always
+    // 0x00. Of course, bit-wise OR with 0x00 isn't going to do
     // much anyway... Here's the general idea all the same:
     //
     // unsigned int chunk_header = (unsigned int)num | CHUNK_type;
@@ -80,12 +76,12 @@ std::streambuf::int_type chunked_outbuf::data_chunk() {
     // network byte order for the header
     header = htonl(header);
 
-    d_os.write((const char *)&header, sizeof(int32_t));
+    d_os.write(reinterpret_cast<const char *>(&header), sizeof(int32_t));
 
-    // Should bad() throw an error?
-    // Are these functions fast or would the bits be faster?
     d_os.write(d_buffer, num);
-    if (d_os.eof() || d_os.bad())
+    if (d_os.bad())
+        throw InternalErr(__FILE__, __LINE__, "chunked_outbuf::data_chunk");
+    if (d_os.eof())
         return traits_type::eof();
 
     pbump(-num);
@@ -97,22 +93,23 @@ std::streambuf::int_type chunked_outbuf::data_chunk() {
  *
  * This is like calling flush_chunk(), but it sends a chunk header with a type of
  * CHUNK_END (instead of CHUNK_DATA). Whatever is in the buffer is written out, but
- * the stream is can be used to send more chunks.
+ * the stream can be used to send more chunks.
+ *
  * @note This is called by the chunked_outbuf destructor, so closing a stream using
- * chunked_outbuf always sends a CHUNK_END type chunk, even if it will have zero
- * bytes
+ * chunked_outbuf always sends a CHUNK_END type chunk, even if it has zero bytes.
+ *
  * @return EOF on error, otherwise the number of bytes sent in the chunk.
  */
 std::streambuf::int_type chunked_outbuf::end_chunk() {
     DBG(cerr << "In chunked_outbuf::end_chunk" << endl);
 
-    int32_t num = pptr() - pbase(); // num needs to be signed for the call to pbump
+    const auto num = static_cast<int_type>(pptr() - pbase());
 
     // write out the chunk headers: CHUNKTYPE and CHUNKSIZE
     // as a 32-bit unsigned int. Here I assume that num is never
     // more than 2^24 because that was tested in the constructor
 
-    uint32_t header = (uint32_t)num | CHUNK_END;
+    uint32_t header = num | CHUNK_END;
 
     // Add encoding of host's byte order. jhrg 11/24/13
     if (!d_big_endian)
@@ -124,11 +121,11 @@ std::streambuf::int_type chunked_outbuf::end_chunk() {
     // Write out the CHUNK_END header with the byte count.
     // This should be called infrequently, so it's probably not worth
     // optimizing away chunk_header
-    d_os.write((const char *)&header, sizeof(uint32_t));
+    d_os.write(reinterpret_cast<const char *>(&header), sizeof(uint32_t));
 
-    // Should bad() throw an error?
-    // Are these functions fast or would the bits be faster?
     d_os.write(d_buffer, num);
+    // It seems wrong to return 'eof' when 'bad()' is true, but that's the correct
+    // behavior for std::streambuf. jhrg 5/28/25
     if (d_os.eof() || d_os.bad())
         return traits_type::eof();
 
@@ -138,26 +135,26 @@ std::streambuf::int_type chunked_outbuf::end_chunk() {
 
 /**
  * @brief Send an error chunk
+ *
  * While building up the next chunk, send an error chunk, ignoring the data currently
  * write buffer. The buffer is left in a consistent state.
- * @param msg The error message to include in the error chunk
+ *
+ * @param m The error message to include in the error chunk
  * @return The number of characters ignored.
  */
 std::streambuf::int_type chunked_outbuf::err_chunk(const std::string &m) {
     DBG(cerr << "In chunked_outbuf::err_chunk" << endl);
     std::string msg = m;
 
-    // Figure out how many chars are in the buffer - these will be
-    // ignored.
-    int32_t num = pptr() - pbase(); // num needs to be signed for the call to pbump
+    const auto num = static_cast<int_type>(pptr() - pbase());
 
-    // write out the chunk headers: CHUNKTYPE and CHUNKSIZE
-    // as a 32-bit unsigned int. Here I assume that num is never
+    // Write out the chunk headers: CHUNKTYPE and CHUNKSIZE
+    // as a 32-bit unsigned int. We assume that num is never
     // more than 2^24 because that was tested in the constructor
-    if (msg.length() > 0x00FFFFFF)
+    if (msg.length() > 0x00'FF'FF'FF)
         msg = "Error message too long";
 
-    uint32_t header = (uint32_t)msg.length() | CHUNK_ERR;
+    uint32_t header = msg.length() | CHUNK_ERR;
 
     // Add encoding of host's byte order. jhrg 11/24/13
     if (!d_big_endian)
@@ -166,15 +163,16 @@ std::streambuf::int_type chunked_outbuf::err_chunk(const std::string &m) {
     // network byte order for the header
     header = htonl(header);
 
-    // Write out the CHUNK_END header with the byte count.
+    // Write out the CHUNK_ERR header with the byte count.
     // This should be called infrequently, so it's probably not worth
     // optimizing away chunk_header
-    d_os.write((const char *)&header, sizeof(uint32_t));
+    d_os.write(reinterpret_cast<const char *>(&header), sizeof(uint32_t));
 
     // Should bad() throw an error?
-    // Are these functions fast or would the bits be faster?
     d_os.write(msg.data(), msg.length());
-    if (d_os.eof() || d_os.bad())
+    if (d_os.bad())
+        throw InternalErr(__FILE__, __LINE__, "chunked_outbuf::err_chunk");
+    if (d_os.eof())
         return traits_type::eof();
 
     // Reset the buffer pointer, effectively ignoring what's in there now
@@ -208,8 +206,7 @@ std::streambuf::int_type chunked_outbuf::overflow(int c) {
     }
     // flush the buffer
     if (data_chunk() == traits_type::eof()) {
-        // Error
-        return traits_type::eof();
+        return traits_type::eof(); // Return EOF on error
     }
 
     return traits_type::not_eof(c);
@@ -239,15 +236,14 @@ std::streambuf::int_type chunked_outbuf::overflow(int c) {
 std::streamsize chunked_outbuf::xsputn(const char *s, std::streamsize num) {
     DBG(cerr << "In chunked_outbuf::xsputn: num: " << num << endl);
 
-    // if the current block of data will fit in the buffer, put it there.
+    // If the current block of data fits in the buffer, put it there,
     // else, there is at least a complete chunk between what's in the buffer
     // and what's in 's'; send a chunk header, the stuff in the buffer and
     // bytes from 's' to make a complete chunk. Then iterate over 's' sending
     // more chunks until there's less than a complete chunk left in 's'. Put
-    // the bytes remaining 's' in the buffer. Return the number of bytes sent
+    // the bytes remaining in 's' in the buffer. Return the number of bytes sent
     // or 0 if an error is encountered.
-
-    int32_t bytes_in_buffer = pptr() - pbase(); // num needs to be signed for the call to pbump
+    const auto bytes_in_buffer = static_cast<int_type>(pptr() - pbase());
 
     // Will num bytes fit in the buffer? The location of epptr() is one back from
     // the actual end of the buffer, so the next char written will trigger a write
@@ -270,7 +266,7 @@ std::streamsize chunked_outbuf::xsputn(const char *s, std::streamsize num) {
     // network byte order for the header
     header = htonl(header);
 
-    d_os.write((const char *)&header, sizeof(int32_t)); // Data chunk's CHUNK_TYPE is 0x00000000
+    d_os.write(reinterpret_cast<const char *>(&header), sizeof(int32_t)); // Data chunk's CHUNK_TYPE is 0x00000000
 
     // Reset the pptr() and epptr() now in case of an error exit. See the 'if'
     // at the end of this for the only code from here down that will modify the
@@ -278,32 +274,40 @@ std::streamsize chunked_outbuf::xsputn(const char *s, std::streamsize num) {
     setp(d_buffer, d_buffer + (d_buf_size - 1));
 
     d_os.write(d_buffer, bytes_in_buffer);
-    if (d_os.eof() || d_os.bad())
+    if (d_os.bad())
+        throw InternalErr(__FILE__, __LINE__, "chunked_outbuf::xsputn");
+    if (d_os.eof())
         return traits_type::not_eof(0);
 
+    // Send data right from 's' to avoid a needless copy operation. jhrg 5/22/5
     int bytes_to_fill_out_buffer = d_buf_size - bytes_in_buffer;
     d_os.write(s, bytes_to_fill_out_buffer);
-    if (d_os.eof() || d_os.bad())
+    if (d_os.bad())
+        throw InternalErr(__FILE__, __LINE__, "chunked_outbuf::xsputn");
+    if (d_os.eof())
         return traits_type::not_eof(0);
+
     s += bytes_to_fill_out_buffer;
-    uint32_t bytes_still_to_send = num - bytes_to_fill_out_buffer;
+    int bytes_still_to_send = num - bytes_to_fill_out_buffer;
 
     // Now send all the remaining data in s until the amount remaining doesn't
     // fill a complete chunk and buffer those data.
     while (bytes_still_to_send >= d_buf_size) {
-        // This is header for  a chunk of d_buf_size bytes; the size was set above
-        d_os.write((const char *)&header, sizeof(int32_t));
+        // This is the header for a chunk of d_buf_size bytes; the size was set above
+        d_os.write(reinterpret_cast<const char *>(&header), sizeof(int32_t));
         d_os.write(s, d_buf_size);
-        if (d_os.eof() || d_os.bad())
+        if (d_os.bad())
+            throw InternalErr(__FILE__, __LINE__, "chunked_outbuf::xsputn");
+        if (d_os.eof() /*|| d_os.bad()*/)
             return traits_type::not_eof(0);
+
         s += d_buf_size;
         bytes_still_to_send -= d_buf_size;
     }
 
     if (bytes_still_to_send > 0) {
-        // if the code is here, one or more chunks have been sent, the
-        // buffer is empty and there are < d_buf_size bytes to send. Buffer
-        // them.
+        // If the code is here, one or more chunks have been sent, the
+        // buffer is empty, and there are < d_buf_size bytes to send. Buffer them.
         memcpy(d_buffer, s, bytes_still_to_send);
         pbump(bytes_still_to_send);
     }
@@ -322,8 +326,7 @@ std::streambuf::int_type chunked_outbuf::sync() {
     DBG(cerr << "In chunked_outbuf::sync" << endl);
 
     if (data_chunk() == traits_type::eof()) {
-        // Error
-        return traits_type::not_eof(-1);
+        return traits_type::not_eof(-1); // returning -1 signals an error
     }
     return traits_type::not_eof(0);
 }
diff --git a/chunked_ostream.h b/chunked_ostream.h
index 688b2ec26..13ccf1722 100644
--- a/chunked_ostream.h
+++ b/chunked_ostream.h
@@ -22,7 +22,7 @@
 //
 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
 //
-// Portions of this code were taken verbatim from  Josuttis,
+// Portions of this code were taken verbatim from Josuttis,
 // "The C++ Standard Library," p.672
 
 #ifndef _chunkedostream_h
@@ -30,7 +30,6 @@
 
 #include "chunked_stream.h"
 
-#include <ostream>
 #include <stdexcept> // std::out_of_range
 #include <streambuf>
 
@@ -54,13 +53,13 @@ class chunked_outbuf : public std::streambuf {
     friend class chunked_ostream;
 
 protected:
-    std::ostream &d_os;      // Write stuff here
-    unsigned int d_buf_size; // Size of the data buffer
-    char *d_buffer;          // Data buffer
-    bool d_big_endian;
+    std::ostream &d_os;       // Write stuff here
+    int d_buf_size = 0;       // Size of the data buffer
+    char *d_buffer = nullptr; // Data buffer
+    bool d_big_endian = false;
 
 public:
-    chunked_outbuf(std::ostream &os, unsigned int buf_size) : d_os(os), d_buf_size(buf_size), d_buffer(0) {
+    chunked_outbuf(std::ostream &os, int buf_size) : d_os(os), d_buf_size(buf_size) {
         if (d_buf_size & CHUNK_TYPE_MASK)
             throw std::out_of_range(
                 "A chunked_outbuf (or chunked_ostream) was built using a buffer larger than 0x00ffffff");
@@ -73,7 +72,7 @@ class chunked_outbuf : public std::streambuf {
         setp(d_buffer, d_buffer + (buf_size - 1));
     }
 
-    virtual ~chunked_outbuf() {
+    ~chunked_outbuf() noexcept override {
         // call end_chunk() and not sync()
         end_chunk();
 
@@ -88,13 +87,13 @@ class chunked_outbuf : public std::streambuf {
 
     int_type err_chunk(const std::string &msg);
 
-    virtual std::streamsize xsputn(const char *s, std::streamsize num);
+    std::streamsize xsputn(const char *s, std::streamsize num) override;
     // Manipulate the buffer pointers using pbump() after filling the buffer
     // and then call data_chunk(). Leave remainder in buffer. Or copy logic
     // for data_chunk() into loop in this code.
 
-    virtual int_type overflow(int c);
-    virtual int_type sync();
+    int_type overflow(int c) override;
+    int_type sync() override;
 };
 
 /**
@@ -129,7 +128,7 @@ class chunked_ostream : public std::ostream {
      * @note The buffer size must not be more than 2^24 bytes (0x00ffffff)
      * @param buf_size The size of the buffer in bytes.
      */
-    chunked_ostream(std::ostream &os, unsigned int buf_size) : std::ostream(&d_cbuf), d_cbuf(os, buf_size) {}
+    chunked_ostream(std::ostream &os, int buf_size) : std::ostream(&d_cbuf), d_cbuf(os, buf_size) {}
 
     /**
      * @brief Send an end chunk.
@@ -155,7 +154,7 @@ class chunked_ostream : public std::ostream {
      * @brief Send an error message down the stream.
      * When called, this method dumps all the data currently in the buffer and
      * sends the error message text instead, using a chunk type of CHUNK_ERR. The
-     * write buffer is maintained, however, so the stream ibject can still be used.
+     * write buffer is maintained, so the stream object can still be used.
      * @param msg The error message text
      * @return The number of bytes 'dumped' from the write buffer.
      */
diff --git a/chunked_stream.h b/chunked_stream.h
index 406d8e22d..970c8134e 100644
--- a/chunked_stream.h
+++ b/chunked_stream.h
@@ -9,18 +9,18 @@
 #define CHUNK_STREAM_H_
 
 // These are the three chunk types
-#define CHUNK_DATA 0x00000000
-#define CHUNK_END 0x01000000
-#define CHUNK_ERR 0x02000000
+#define CHUNK_DATA 0x00'00'00'00
+#define CHUNK_END 0x01'00'00'00
+#define CHUNK_ERR 0x02'00'00'00
 
 // This is the bit in the chunk that indicates the byte-order of the data,
 // not the byte order of the chunk. The chunk is always in network byte order.
-#define CHUNK_LITTLE_ENDIAN 0x04000000
+#define CHUNK_LITTLE_ENDIAN 0x04'00'00'00
 
 // Chunk type mask masks off the low bytes and the little endian bit.
 // The three chunk types (DATA, END and ERR) are mutually exclusive.
-#define CHUNK_TYPE_MASK 0x03000000
-#define CHUNK_SIZE_MASK 0x00FFFFFF
+#define CHUNK_TYPE_MASK 0x03'00'00'00
+#define CHUNK_SIZE_MASK 0x00'FF'FF'FF
 
 #define CHUNK_SIZE 4096
 
diff --git a/tests/D4ResponseBuilder.cc b/tests/D4ResponseBuilder.cc
index f349feb10..c73244fae 100644
--- a/tests/D4ResponseBuilder.cc
+++ b/tests/D4ResponseBuilder.cc
@@ -65,7 +65,7 @@ D4ResponseBuilder::~D4ResponseBuilder() {
     // If an alarm was registered, delete it. The register code in SignalHandler
     // always deletes the old alarm handler object, so only the one returned by
     // remove_handler needs to be deleted at this point.
-    delete dynamic_cast<AlarmHandler *>(SignalHandler::instance()->remove_handler(SIGALRM));
+    delete dynamic_cast<AlarmHandler *>(libdap::SignalHandler::remove_handler(SIGALRM));
 }
 
 /** Set the dataset name, which is a string used to access the dataset
@@ -87,8 +87,11 @@ void D4ResponseBuilder::set_dataset_name(const string ds) { d_dataset = www2id(d
 */
 void D4ResponseBuilder::establish_timeout(ostream &stream) const {
     if (d_timeout > 0) {
-        SignalHandler *sh = SignalHandler::instance();
-        EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler(stream));
+#if 0
+          SignalHandler *sh = SignalHandler::instance();
+#endif
+
+        EventHandler *old_eh = libdap::SignalHandler::register_handler(SIGALRM, new AlarmHandler(stream));
         delete old_eh;
         alarm(d_timeout);
     }
@@ -103,7 +106,7 @@ void D4ResponseBuilder::remove_timeout() const { alarm(0); }
  *
  * @param out Write to this stream object
  * @param dmr This is the DMR to write
- * @param eval Use this ConstraintEvaluator
+
  * @param with_mime_headers If true, include the MIME headers in the response
  */
 void D4ResponseBuilder::send_dmr(ostream &out, DMR &dmr, bool with_mime_headers, bool constrained) {
diff --git a/tests/DMRTest.at b/tests/DMRTest.at
index 669c19179..5d11e25bd 100644
--- a/tests/DMRTest.at
+++ b/tests/DMRTest.at
@@ -222,7 +222,9 @@ m4_define([DMR_TRANS_FUNC_CE], [
     fe="$2"
     ce="$3"
     baseline=$abs_srcdir/dmr-testsuite/$WORD_ORDER/$4
-    
+
+    AS_IF([test -z "$at_verbose"], [echo "Command: dmr-test -x -t $input -f $fe -c $ce"])
+
     AS_IF([test -n "$baselines" -a x$baselines = xyes],
         [
         # // old version - sbl 9.4.19
diff --git a/tests/TestArray.cc b/tests/TestArray.cc
index 9347312ad..9c4ecece1 100644
--- a/tests/TestArray.cc
+++ b/tests/TestArray.cc
@@ -315,7 +315,7 @@ template <typename T, class C> void TestArray::m_cardinal_type_read_helper() {
         } else if (dimensions() == 2) {
             vector<T> tmp(length_ll());
             m_constrained_matrix<T, C>(tmp);
-            set_value_ll((T *)tmp.data(), length_ll());
+            set_value_ll(static_cast<T *>(tmp.data()), length_ll());
         } else {
             vector<T> tmp(length_ll());
             for (int64_t i = 0, end = length(); i < end; ++i) {
@@ -323,26 +323,20 @@ template <typename T, class C> void TestArray::m_cardinal_type_read_helper() {
                 tmp[i] = static_cast<C *>(var())->value();
                 var()->set_read_p(false); // pick up the next value
             }
-            set_value_ll((T *)tmp.data(), length_ll());
+            set_value_ll(static_cast<T *>(tmp.data()), length_ll());
         }
     } else {
         // read a value into the Array's prototype element
         var()->read();
         T value = static_cast<C *>(var())->value();
-        vector<T> tmp(length_ll());
-        for (int64_t i = 0, end = length_ll(); i < end; ++i) {
-            tmp[i] = value;
-        }
 
-#if 0
-        set_value(tmp, length_ll());
-#endif
-        // The following code inside #if 0 block always goes to set_value.
-        // May check why in the future. KY 2023-01-12
-#if 0
-        set_value_ll(tmp, length_ll());
-#endif
-        set_value_ll((T *)tmp.data(), length_ll());
+        // I tried a number of tricks to improve the time it takes to initialize this
+        // when the size is large (e.g., 4GB), but nothing improved on the performance
+        // of this. In fact, most were about 10% slower. This was measured on a 32 GB
+        // Intel Mac running OSX 15.5. jhrg 5/22/25
+        vector<T> vec(length_ll(), value);
+
+        set_value_ll(static_cast<T *>(vec.data()), length_ll());
     }
 }
 
diff --git a/tests/dmr-testsuite/test_array_2_large.xml b/tests/dmr-testsuite/test_array_2_large.xml
new file mode 100644
index 000000000..bb0044c75
--- /dev/null
+++ b/tests/dmr-testsuite/test_array_2_large.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- First in a series of very simple tests. jhrg 7/10/12 -->
+
+<Dataset name="test_array_2" dapVersion="4.0" dmrVersion="1.0" xml:base="file:dap4/test_array_2_large.xml"
+  xmlns="http://xml.opendap.org/ns/DAP/4.0#"
+  xmlns:dap="http://xml.opendap.org/ns/DAP/4.0#">
+
+  <dap:Int8 name="large">
+    <Dim size="3"/>
+    <Dim size="1024"/>
+    <Dim size="1024"/>
+    <Dim size="1024"/>
+  </dap:Int8>
+
+</Dataset>
diff --git a/tests/dmr-testsuite/test_array_2_large_vector.xml b/tests/dmr-testsuite/test_array_2_large_vector.xml
new file mode 100644
index 000000000..a69b8bfed
--- /dev/null
+++ b/tests/dmr-testsuite/test_array_2_large_vector.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- First in a series of very simple tests. jhrg 7/10/12 -->
+
+<Dataset name="test_array_2" dapVersion="4.0" dmrVersion="1.0" xml:base="file:dap4/test_array_2_large.xml"
+  xmlns="http://xml.opendap.org/ns/DAP/4.0#"
+  xmlns:dap="http://xml.opendap.org/ns/DAP/4.0#">
+
+  <dap:Int8 name="large_vector_2_30">
+    <Dim size="1073741824"/>
+    <!-- 1'073'741'824 == 2^30 == 0x3f'ff'ff'ff -->
+  </dap:Int8>
+
+  <dap:Int8 name="large_vector_2_31_1">
+    <Dim size="2147483647"/>
+    <!-- 2'147'483'647 == (2^31)-1 == 0x4f'ff'ff'fe -->
+  </dap:Int8>
+
+  <dap:Int8 name="large_vector_2_31">
+    <Dim size="2147483648"/>
+    <!-- 2'147'483'648 == (2^31) == 0x4f'ff'ff'ff -->
+  </dap:Int8>
+
+  <dap:Int8 name="large_vector_2_32">
+    <Dim size="4294967296"/>
+    <!-- 4'294'967'296 == (2^32) == 0xff'ff'ff'ff -->
+  </dap:Int8>
+
+  <dap:Int8 name="large_vector_2_33">
+    <Dim size="8589934592"/>
+    <!-- 8'589'934'592 == (2^33) == 0x00'00'00'01'ff'ff'ff'ff -->
+  </dap:Int8>
+
+</Dataset>
diff --git a/unit-tests/Makefile.am b/unit-tests/Makefile.am
index 33b8b772c..d3c8d4c1d 100644
--- a/unit-tests/Makefile.am
+++ b/unit-tests/Makefile.am
@@ -90,7 +90,8 @@ UNIT_TESTS = marshT arrayT attrTableT structT sequenceT ddsT dasT \
 	RCReaderTest SequenceTest SignalHandlerTest  MarshallerTest \
 	ServerFunctionsListUnitTest Int8Test Int16Test UInt16Test \
 	Int32Test UInt32Test Int64Test UInt64Test Float32Test Float64Test \
-	D4BaseTypeFactoryTest BaseTypeFactoryTest util_mitTest ErrorTest
+	D4BaseTypeFactoryTest BaseTypeFactoryTest util_mitTest ErrorTest \
+	MarshallerFutureTest
 
 # ErrorAssignmentTest
 
@@ -268,3 +269,5 @@ BigArrayTest_SOURCES = BigArrayTest.cc $(TEST_SRC)
 D4SequenceTest_LDADD = ../tests/libtest-types.a ../libdap.la $(AM_LDADD)
 
 IsDap4ProjectedTest_SOURCES = IsDap4ProjectedTest.cc
+
+MarshallerFutureTest_SOURCES = MarshallerFutureTest.cc
diff --git a/unit-tests/MarshallerFutureTest.cc b/unit-tests/MarshallerFutureTest.cc
new file mode 100644
index 000000000..517e67ae7
--- /dev/null
+++ b/unit-tests/MarshallerFutureTest.cc
@@ -0,0 +1,175 @@
+// -*- mode: c++; c-basic-offset:4 -*-
+
+// This file is part of libdap, A C++ implementation of the OPeNDAP Data
+// Access Protocol.
+
+// Copyright (c) 2025 OPeNDAP, Inc.
+// Author: James Gallagher <jgallagher@opendap.org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+//
+// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
+
+#include "config.h"
+
+#include <cppunit/extensions/HelperMacros.h>
+#include <fstream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <sys/fcntl.h>
+
+#include "../MarshallerFuture.h"
+#include "run_tests_cppunit.h"
+
+using namespace libdap;
+
+class MarshallerThreadSharedPtrTest : public CppUnit::TestFixture {
+    CPPUNIT_TEST_SUITE(MarshallerThreadSharedPtrTest);
+
+    CPPUNIT_TEST(test_write_to_stream);
+    CPPUNIT_TEST(test_write_to_stream_shared_ptr);
+
+    CPPUNIT_TEST(test_write_part_to_stream);
+    CPPUNIT_TEST(test_write_part_to_stream_shared_ptr);
+
+    CPPUNIT_TEST(test_write_to_file_descriptor);
+    CPPUNIT_TEST(test_write_to_file_descriptor_shared_ptr);
+
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    static void test_write_to_stream() {
+        std::ostringstream oss;
+
+        std::string test_data = "Shared pointer test";
+
+        // This block is used to force a call to the future get() method
+        {
+            MarshallerThread mt;
+            mt.start_thread(oss, test_data.data(), test_data.size());
+            std::cerr << "Bytes written: " << mt.get_ostream_future() << "\n";
+            std::cerr << "The data written: " << oss.str() << "\n";
+        }
+
+        CPPUNIT_ASSERT_EQUAL(test_data, oss.str());
+    }
+
+    static void test_write_to_stream_shared_ptr() {
+        std::ostringstream oss;
+
+        std::string test_data = "Shared pointer test";
+        std::size_t len = test_data.size();
+        std::shared_ptr<char> buf(new char[len], std::default_delete<char[]>());
+
+        std::memcpy(buf.get(), test_data.data(), test_data.size());
+
+        // This block is used to force a call to the future get() method
+        {
+            MarshallerThread mt;
+            mt.start_thread(oss, buf, test_data.size());
+            std::cerr << "Bytes written: " << mt.get_ostream_future() << "\n";
+            std::cerr << "The data written: " << oss.str() << "\n";
+        }
+
+        CPPUNIT_ASSERT_EQUAL(test_data, oss.str());
+    }
+
+    static void test_write_part_to_stream() {
+        std::ostringstream oss;
+
+        std::string test_data = "1234DATA";
+
+        {
+            MarshallerThread mt;
+            mt.start_thread_part(oss, test_data.data(), test_data.size());
+            std::cerr << "Bytes written: " << mt.get_ostream_future() << "\n";
+            std::cerr << "The data written: " << oss.str() << "\n";
+        }
+
+        CPPUNIT_ASSERT_EQUAL(std::string("DATA"), oss.str());
+    }
+
+    static void test_write_part_to_stream_shared_ptr() {
+        std::ostringstream oss;
+
+        std::string test_data = "1234DATA";
+        std::size_t len = test_data.size();
+        std::shared_ptr<char> buf(new char[len], std::default_delete<char[]>());
+        std::memcpy(buf.get(), test_data.data(), len);
+
+        {
+            MarshallerThread mt;
+            mt.start_thread_part(oss, buf, test_data.size());
+            std::cerr << "Bytes written: " << mt.get_ostream_future() << "\n";
+            std::cerr << "The data written: " << oss.str() << "\n";
+        }
+
+        CPPUNIT_ASSERT_EQUAL(std::string("DATA"), oss.str());
+    }
+
+    static void test_write_to_file_descriptor() {
+        std::string test_data = "Shared FD test";
+
+        const char *filename = "/tmp/marshaller_test_shared_ptr.txt";
+        int fd = creat(filename, 0644);
+        CPPUNIT_ASSERT(fd != -1);
+
+        {
+            MarshallerThread mt;
+            mt.start_thread(fd, test_data.data(), test_data.size());
+            std::cerr << "Bytes written: " << mt.get_fp_future() << "\n";
+        }
+
+        close(fd);
+
+        std::ifstream ifs(filename);
+        std::stringstream content;
+        content << ifs.rdbuf();
+        ifs.close();
+
+        CPPUNIT_ASSERT_EQUAL(test_data, content.str());
+    }
+
+    static void test_write_to_file_descriptor_shared_ptr() {
+        std::string test_data = "Shared FD test";
+        std::size_t len = test_data.size();
+        std::shared_ptr<char> buf(new char[len], std::default_delete<char[]>());
+        std::memcpy(buf.get(), test_data.data(), len);
+
+        const char *filename = "/tmp/marshaller_test_shared_ptr.txt";
+        int fd = creat(filename, 0644);
+        CPPUNIT_ASSERT(fd != -1);
+
+        {
+            MarshallerThread mt;
+            mt.start_thread(fd, buf, test_data.size());
+            std::cerr << "Bytes written: " << mt.get_fp_future() << "\n";
+        }
+
+        close(fd);
+
+        std::ifstream ifs(filename);
+        std::stringstream content;
+        content << ifs.rdbuf();
+        ifs.close();
+
+        CPPUNIT_ASSERT_EQUAL(test_data, content.str());
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MarshallerThreadSharedPtrTest);
+
+int main(int argc, char *argv[]) { return run_tests<MarshallerThreadSharedPtrTest>(argc, argv) ? 0 : 1; }
diff --git a/util.cc b/util.cc
index b588aa6e0..046fc1e4d 100644
--- a/util.cc
+++ b/util.cc
@@ -37,13 +37,13 @@
 #include <fstream>
 
 #include <cassert>
+#include <cctype>
 #include <climits>
 #include <cstdlib>
 #include <cstring>
 
-#include <ctype.h>
 #ifndef TM_IN_SYS_TIME
-#include <time.h>
+#include <ctime>
 #else
 #include <sys/time.h>
 #endif
@@ -449,15 +449,15 @@ double extract_double_value(BaseType *arg) {
  @param name The URL to process
  @return Returns a new string object that contains the pruned URL. */
 string prune_spaces(const string &name) {
-    // If the URL does not even have white space return.
-    if (name.find_first_of(' ') == name.npos)
+    // If the URL does not even have white space, return.
+    if (name.find_first_of(' ') == std::string::npos)
         return name;
     else {
         // Strip leading spaces from http://...
         unsigned int i = name.find_first_not_of(' ');
         string tmp_name = name.substr(i);
 
-        // Strip leading spaces from constraint part (following `?').
+        // Strip leading spaces from the constraint part (following `?').
         unsigned int j = tmp_name.find('?') + 1;
         i = tmp_name.find_first_not_of(' ', j);
         tmp_name.erase(j, i - j);
@@ -475,7 +475,7 @@ bool unique_names(vector<BaseType *> l, const string &var_name, const string &ty
 
     int nelem = 0;
     typedef std::vector<BaseType *>::const_iterator citer;
-    for (citer i = l.begin(); i != l.end(); i++) {
+    for (citer i = l.begin(); i != l.end(); ++i) {
         assert(*i);
         names[nelem++] = (*i)->name();
         DBG(cerr << "NAMES[" << nelem - 1 << "]=" << names[nelem - 1] << endl);
@@ -1233,4 +1233,70 @@ string open_temp_fstream(ofstream &f, const string &name_template, const string
     return string(name.data());
 }
 
+/** @name Segmented read and write functions. */
+//@{
+
+// The maximum number of bytes that can be safely written in a single call
+// to ostream::write() is typically limited by std::streamsize, which is
+// often a signed 32-bit integer (2^31) - 1.
+constexpr int64_t MAX_SEGMENT_SIZE = (static_cast<int64_t>(1) << 31) - 1;
+
+/**
+ * @brief Ensure that no more than (2^31)-1 bytes are ever written by ostream::write()
+ * @param out The ostream& where the bytes are to be written
+ * @param val The bytes to write
+ * @param num_bytes The total number of bytes to write
+ */
+void segmented_write(ostream &out, const char *val, int64_t num_bytes) {
+    int64_t bytes_remaining = num_bytes;
+    const char *current_ptr = val; // Use a mutable pointer to advance through the buffer
+
+    while (bytes_remaining > 0) {
+        // Determine the size of the current chunk to write
+        const auto current_chunk_size = static_cast<std::streamsize>(std::min(bytes_remaining, MAX_SEGMENT_SIZE));
+
+        // Call ostream::write() with the current segment
+        out.write(current_ptr, current_chunk_size);
+        if (out.fail()) {
+            throw InternalErr(__FILE__, __LINE__, "Error writing to a stream");
+        }
+
+        current_ptr += current_chunk_size;
+
+        bytes_remaining -= current_chunk_size;
+    }
+}
+
+/**
+ * @brief Reads a large number of bytes from an istream in segments, matching segmented_write.
+ * @param in The istream& from where the bytes are to be read.
+ * @param buffer The char* buffer where the bytes will be read into.
+ * @param num_bytes_to_read The total number of bytes to read.
+ */
+void segmented_read(std::istream &in, char *buffer, int64_t num_bytes_to_read) {
+    int64_t bytes_remaining = num_bytes_to_read;
+    char *current_ptr = buffer;
+
+    while (bytes_remaining > 0) {
+        // Determine the size of the current chunk to read
+        const auto current_chunk_size = static_cast<std::streamsize>(std::min(bytes_remaining, MAX_SEGMENT_SIZE));
+
+        in.read(current_ptr, current_chunk_size);
+
+        // Check if the read operation failed or didn't read the expected number of bytes
+        if (in.fail()) { // Failbit set, and not just end of file
+            throw InternalErr(__FILE__, __LINE__, "Error reading from a stream");
+        }
+
+        // Check if we reached EOF prematurely (i.e., Fewer bytes were read than expected)
+        if (in.gcount() < current_chunk_size) {
+            throw InternalErr(__FILE__, __LINE__, "Unexpected end of stream: Less bytes read than expected.");
+        }
+
+        current_ptr += current_chunk_size;
+        bytes_remaining -= current_chunk_size;
+    }
+}
+//@}
+
 } // namespace libdap
diff --git a/util.h b/util.h
index 7d55a72e9..bad5ae1a4 100644
--- a/util.h
+++ b/util.h
@@ -91,7 +91,7 @@ bool dir_exists(const string &dir);
 /** @name Integer to string conversion functions
    Fast, safe conversions from long to a character representation which gets
    appended to a string. This method will take a long value 'val' and it will
-   recursively divide it by 'base' in order to "extract" one by one the
+   recursively divide it by 'base' to "extract" one by one the
    digits which compose it; these digits will be <i>appended</i> to the
    string <tt>str_val</tt> which will become the textual representation of
    'val'. Please notice that the digits ``extracted'' from `val' will vary
@@ -114,8 +114,8 @@ bool dir_exists(const string &dir);
    operator+ does) without having to create a new string object s2 and then
    use string::operator+ between s1 and s2.
 
-   @return void. This method returns nothing however be aware that it will
-   throw and exception of type <tt>std::invalid_argument</tt> if the parameter
+   @return void. This method returns nothing, however, be aware that it will
+   throw a <tt>std::invalid_argument</tt> exception if the parameter
    base is not in the valid range. */
 //@{
 void append_long_to_string(long val, int base, string &str_val);
@@ -148,6 +148,9 @@ bool pathname_ok(const string &path, bool strict = true);
 string dap_version();
 string open_temp_fstream(ofstream &f, const string &name_template, const string &suffix = "");
 
+void segmented_write(ostream &out, const char *val, int64_t num_bytes);
+void segmented_read(std::istream &in, char *buffer, int64_t num_bytes_to_read);
+
 } // namespace libdap
 
 #endif


Reviews:
user: login: ndp-opendap
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T15:50:29Z
user: login: kyang2014
body: Before merging to the master,  I think you should  check BES with your branch and verify if the issue described in https://bugs.earthdata.nasa.gov/browse/HYRAX-1313 is still gone.  
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-27T15:59:18Z
user: login: jgallagher59701
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-27T23:00:21Z
user: login: jgallagher59701
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-27T23:11:11Z

Review comments:
diff_hunk: @@ -1233,4 +1233,70 @@ string open_temp_fstream(ofstream &f, const string &name_template, const string
     return string(name.data());
 }
 
+/** @name Segmented read and write functions. */
+//@{
+
+// The maximum number of bytes that can be safely written in a single call
+// to ostream::write() is typically limited by std::streamsize, which is
+// often a signed 32-bit integer (2^31) - 1.
+constexpr int64_t MAX_SEGMENT_SIZE = (static_cast<int64_t>(1) << 31) - 1;
+
+/**
+ * @brief Ensure that no more than (2^31)-1 bytes are ever written by ostream::write()
+ * @param out The ostream& where the bytes are to be written
+ * @param val The bytes to write
+ * @param num_bytes The total number of bytes to write
+ */
+void segmented_write(ostream &out, const char *val, int64_t num_bytes) {
+    int64_t bytes_remaining = num_bytes;
+    const char *current_ptr = val; // Use a mutable pointer to advance through the buffer
+
+    while (bytes_remaining > 0) {
+        // Determine the size of the current chunk to write
+        const auto current_chunk_size = static_cast<std::streamsize>(std::min(bytes_remaining, MAX_SEGMENT_SIZE));
+
+        // Call ostream::write() with the current segment
+        out.write(current_ptr, current_chunk_size);
+        if (out.fail()) {
+            throw InternalErr(__FILE__, __LINE__, "Error writing to a stream");
+        }
+
+        current_ptr += current_chunk_size;
+
+        bytes_remaining -= current_chunk_size;
+    }
+}
+
+/**
+ * @brief Reads a large number of bytes from an istream in segments, matching segmented_write.
+ * @param in The istream& from where the bytes are to be read.
+ * @param buffer The char* buffer where the bytes will be read into.
+ * @param num_bytes_to_read The total number of bytes to read.
+ */
+void segmented_read(std::istream &in, char *buffer, int64_t num_bytes_to_read) {
+    int64_t bytes_remaining = num_bytes_to_read;
+    char *current_ptr = buffer;
+
+    while (bytes_remaining > 0) {
+        // Determine the size of the current chunk to read
+        const auto current_chunk_size = static_cast<std::streamsize>(std::min(bytes_remaining, MAX_SEGMENT_SIZE));
+
+        in.read(current_ptr, current_chunk_size);
+
+        // Check if the read operation failed or didn't read the expected number of bytes
+        if (in.fail() && !in.eof()) { // Failbit set, and not just end of file
path: util.cc
user: login: kyang2014
body: I don't understand why you need to check if in.eof() is false here.  in.fail() should be sufficient to throw an error.  You also have another check after this line, these two checks should be sufficient, I think.
created_at: 2025-05-27T15:46:40Z
author_association: COLLABORATOR
diff_hunk: @@ -672,21 +588,21 @@ void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) {
     // See comment above
     assert(!(num_elem & 0xf000000000000000));
 
-    num_elem = num_elem << 3; // num_elem is now the number of bytes
+    auto num_bytes = num_elem << 3; // num_elem is now the number of bytes
 
-    checksum_update(val, num_elem);
+    checksum_update(val, num_bytes);
 
     if (d_write_data) {
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 
-        char *buf = new char[num_elem];
-        memcpy(buf, val, num_elem);
+        char *buf = new char[num_bytes];
+        memcpy(buf, val, num_bytes);
path: D4StreamMarshaller.cc
user: login: ndp-opendap
body: The [contract for memcpy](https://cplusplus.com/reference/cstring/memcpy/) says that the third parameter is an instance of `size_t`. On my Mac that's 64bit:
sizeof(size_t): 8
sizeof(streamsize): 8
But I think this may be risky.
created_at: 2025-05-27T15:49:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -672,21 +588,21 @@ void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) {
     // See comment above
     assert(!(num_elem & 0xf000000000000000));
 
-    num_elem = num_elem << 3; // num_elem is now the number of bytes
+    auto num_bytes = num_elem << 3; // num_elem is now the number of bytes
 
-    checksum_update(val, num_elem);
+    checksum_update(val, num_bytes);
 
     if (d_write_data) {
 #ifdef USE_POSIX_THREADS
         Locker lock(tm->get_mutex(), tm->get_cond(), tm->get_child_thread_count());
 
-        char *buf = new char[num_elem];
-        memcpy(buf, val, num_elem);
+        char *buf = new char[num_bytes];
+        memcpy(buf, val, num_bytes);
path: D4StreamMarshaller.cc
user: login: jgallagher59701
body: True. I made a ticket for this. There is a second problem with memcpy and that is it doubles the memory needed. In the MarshallerFuture code that will (hopefully) replace MarshallerThread, there is a shared_ptr method that will eliminate the copy. For now, this fix will work on the NASA machines.
created_at: 2025-05-27T23:00:21Z
author_association: MEMBER
diff_hunk: @@ -1233,4 +1233,70 @@ string open_temp_fstream(ofstream &f, const string &name_template, const string
     return string(name.data());
 }
 
+/** @name Segmented read and write functions. */
+//@{
+
+// The maximum number of bytes that can be safely written in a single call
+// to ostream::write() is typically limited by std::streamsize, which is
+// often a signed 32-bit integer (2^31) - 1.
+constexpr int64_t MAX_SEGMENT_SIZE = (static_cast<int64_t>(1) << 31) - 1;
+
+/**
+ * @brief Ensure that no more than (2^31)-1 bytes are ever written by ostream::write()
+ * @param out The ostream& where the bytes are to be written
+ * @param val The bytes to write
+ * @param num_bytes The total number of bytes to write
+ */
+void segmented_write(ostream &out, const char *val, int64_t num_bytes) {
+    int64_t bytes_remaining = num_bytes;
+    const char *current_ptr = val; // Use a mutable pointer to advance through the buffer
+
+    while (bytes_remaining > 0) {
+        // Determine the size of the current chunk to write
+        const auto current_chunk_size = static_cast<std::streamsize>(std::min(bytes_remaining, MAX_SEGMENT_SIZE));
+
+        // Call ostream::write() with the current segment
+        out.write(current_ptr, current_chunk_size);
+        if (out.fail()) {
+            throw InternalErr(__FILE__, __LINE__, "Error writing to a stream");
+        }
+
+        current_ptr += current_chunk_size;
+
+        bytes_remaining -= current_chunk_size;
+    }
+}
+
+/**
+ * @brief Reads a large number of bytes from an istream in segments, matching segmented_write.
+ * @param in The istream& from where the bytes are to be read.
+ * @param buffer The char* buffer where the bytes will be read into.
+ * @param num_bytes_to_read The total number of bytes to read.
+ */
+void segmented_read(std::istream &in, char *buffer, int64_t num_bytes_to_read) {
+    int64_t bytes_remaining = num_bytes_to_read;
+    char *current_ptr = buffer;
+
+    while (bytes_remaining > 0) {
+        // Determine the size of the current chunk to read
+        const auto current_chunk_size = static_cast<std::streamsize>(std::min(bytes_remaining, MAX_SEGMENT_SIZE));
+
+        in.read(current_ptr, current_chunk_size);
+
+        // Check if the read operation failed or didn't read the expected number of bytes
+        if (in.fail() && !in.eof()) { // Failbit set, and not just end of file
path: util.cc
user: login: jgallagher59701
body: Yes, I think this is correct. it doesn't hurt to test eof, but it is redundant.  Changed and pushed.
created_at: 2025-05-27T23:11:11Z
author_association: MEMBER

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-05-28T15:48:28Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=OPENDAP-libdap4&pullRequest=283) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [21 New issues](https://sonarcloud.io/project/issues?id=OPENDAP-libdap4&pullRequest=283&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [1 Accepted issue](https://sonarcloud.io/project/issues?id=OPENDAP-libdap4&pullRequest=283&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=OPENDAP-libdap4&pullRequest=283&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [43.3% Coverage on New Code](https://sonarcloud.io/component_measures?id=OPENDAP-libdap4&pullRequest=283&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [1.8% Duplication on New Code](https://sonarcloud.io/component_measures?id=OPENDAP-libdap4&pullRequest=283&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=OPENDAP-libdap4&pullRequest=283)


author_association: NONE