Repository: stephenberry/glaze (2026-06-10T01:21:57Z)

Issue:
[bug] Input Streaming with error_on_unknown_keys option set to false fails to refill the buffer when skipping values
The following snippet demonstrates the issue:
```cpp
#include <glaze/json.hpp>
#include <glaze/core/istream_buffer.hpp>
import std;
struct output {
	std::vector<int> fill;
	int data{10};
};
struct input {
	int data;
};
int main()
{
	output out{};
	out.fill.resize(2 << 14, 0);
	auto outResult = glz::write_json(out);
	if (!outResult) return 1;
	//use stringstream to make the test filesystem-independent
	std::stringstream inStream{ outResult.value() };
	input in;
	auto inResult = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(in, glz::basic_istream_buffer{ inStream });
	if (inResult) {
		std::cout << glz::format_error(inResult) << '\n';
		return 2;
	}
	std::cout << in.data << '\n';
	return 0;
}
```
I expect the output to be:
```
10
```
However, the actual output is:
```
unexpected_end
```
The reason is that in the process of skipping the `fill` property the end of initial data read from the stream is reached. Instead of refilling the input buffer, the process ends with an error.

Tested on Visual Studio 2026 v18.5.0 (cl.exe version 19.50.35729) with glaze version 7.7.1


Discussion:
Thanks for reporting this, should be fixed with the merge of #2611 



Patch:
diff --git a/include/glaze/json/skip.hpp b/include/glaze/json/skip.hpp
index a3efebacb7..6ecb1e5238 100644
--- a/include/glaze/json/skip.hpp
+++ b/include/glaze/json/skip.hpp
@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include "glaze/core/streaming_state.hpp"
 #include "glaze/util/parse.hpp"
 
 namespace glz
@@ -180,6 +181,128 @@ namespace glz
       }
    }
 
+   // Streaming-aware value skip. The fixed-position skip routines error with
+   // unexpected_end when a value extends past the current buffer, so when a
+   // streaming context is active we instead scan character by character,
+   // refilling the input buffer whenever it runs dry. String, escape, and
+   // nesting state is tracked across refills, so skipped values may be
+   // arbitrarily larger than the buffer (e.g. unknown keys holding large arrays).
+   // Skipped bytes are consumed and not validated, matching the non-validating
+   // skip path. After this returns, the caller's `end` may be stale; callers
+   // resynchronize iterators at their streaming refill checkpoints.
+   template <auto Opts>
+      requires(not Opts.comments)
+   void skip_value_streaming(is_context auto&& ctx, auto&& it, auto end) noexcept
+   {
+      const auto refill = [&]() -> bool {
+         const size_t consumed = static_cast<size_t>(it - ctx.stream.data());
+         const char* new_it;
+         const char* new_end;
+         ctx.stream.consume_and_refill(consumed, new_it, new_end);
+         it = new_it;
+         end = new_end;
+         return it < end;
+      };
+
+      // Skip leading whitespace, refilling as needed
+      while (true) {
+         while (it < end && whitespace_table[uint8_t(*it)]) {
+            ++it;
+         }
+         if (it < end) break;
+         if (!refill()) {
+            ctx.error = error_code::unexpected_end;
+            return;
+         }
+      }
+
+      const char open = *it;
+      if (open == '{' || open == '[') {
+         ++it;
+         size_t depth = 1;
+         bool in_string = false;
+         bool escaped = false;
+         while (depth) {
+            if (it == end) {
+               if (!refill()) {
+                  ctx.error = error_code::unexpected_end;
+                  return;
+               }
+               continue;
+            }
+            const char c = *it;
+            ++it;
+            if (in_string) {
+               if (escaped) {
+                  escaped = false;
+               }
+               else if (c == '\\') {
+                  escaped = true;
+               }
+               else if (c == '"') {
+                  in_string = false;
+               }
+            }
+            else {
+               switch (c) {
+               case '"':
+                  in_string = true;
+                  break;
+               case '{':
+               case '[':
+                  ++depth;
+                  break;
+               case '}':
+               case ']':
+                  --depth;
+                  break;
+               }
+            }
+         }
+      }
+      else if (open == '"') {
+         ++it;
+         bool escaped = false;
+         while (true) {
+            if (it == end) {
+               if (!refill()) {
+                  ctx.error = error_code::unexpected_end;
+                  return;
+               }
+               continue;
+            }
+            const char c = *it;
+            ++it;
+            if (escaped) {
+               escaped = false;
+            }
+            else if (c == '\\') {
+               escaped = true;
+            }
+            else if (c == '"') {
+               break;
+            }
+         }
+      }
+      else {
+         // Numbers and literals (true/false/null): scan until a delimiter.
+         // End of stream also terminates the value.
+         while (true) {
+            if (it == end) {
+               if (!refill()) {
+                  break;
+               }
+               continue;
+            }
+            const char c = *it;
+            if (whitespace_table[uint8_t(c)] || c == ',' || c == '}' || c == ']') {
+               break;
+            }
+            ++it;
+         }
+      }
+   }
+
    // parse_value is used for JSON pointer reading
    // we want the JSON pointer access to not care about trailing whitespace
    // so we use validate_skipped for precise validation and value skipping
@@ -202,6 +325,15 @@ namespace glz
    {
       using namespace glz::detail;
 
+      // Validating skips (parse_value/JSON pointer) require the full value in a
+      // contiguous buffer, so streaming refill is not applied there.
+      if constexpr (has_streaming_state<decltype(ctx)> && not check_validate_skipped(Opts)) {
+         if (ctx.stream.enabled()) {
+            skip_value_streaming<Opts>(ctx, it, end);
+            return;
+         }
+      }
+
       if constexpr (not check_validate_skipped(Opts)) {
          if constexpr (not check_ws_handled(Opts)) {
             if (skip_ws<Opts>(ctx, it, end)) {


Test patch:
diff --git a/tests/istream_buffer_test/istream_buffer_test.cpp b/tests/istream_buffer_test/istream_buffer_test.cpp
index 0045ff15a0..ef3d1eeefd 100644
--- a/tests/istream_buffer_test/istream_buffer_test.cpp
+++ b/tests/istream_buffer_test/istream_buffer_test.cpp
@@ -477,6 +477,121 @@ suite json_read_streaming_tests = [] {
    };
 };
 
+// Regression tests for https://github.com/stephenberry/glaze/issues/2609
+// Skipping unknown keys (error_on_unknown_keys = false) must refill the
+// streaming buffer when the skipped value is larger than the buffer.
+suite json_streaming_skip_unknown_tests = [] {
+   "skip unknown array larger than buffer"_test = [] {
+      std::string json = R"({"fill":[)";
+      for (int i = 0; i < 4096; ++i) {
+         json += "0,";
+      }
+      json.back() = ']'; // replace trailing comma
+      json += R"(,"id":7,"name":"after"})";
+      expect(json.size() > 1024u);
+
+      std::istringstream iss(json);
+      glz::istream_buffer<1024> buffer(iss);
+
+      Record r{};
+      auto ec = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(r, buffer);
+      expect(!ec) << glz::format_error(ec, json);
+      expect(r.id == 7);
+      expect(r.name == "after");
+   };
+
+   "skip unknown nested object larger than buffer"_test = [] {
+      std::string json = R"({"fill":{)";
+      for (int i = 0; i < 512; ++i) {
+         json += "\"k" + std::to_string(i) + R"(":{"a":[1,2,3],"b":"text"},)";
+      }
+      json.pop_back(); // remove trailing comma
+      json += R"(},"id":3,"name":"nested"})";
+      expect(json.size() > 1024u);
+
+      std::istringstream iss(json);
+      glz::istream_buffer<1024> buffer(iss);
+
+      Record r{};
+      auto ec = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(r, buffer);
+      expect(!ec) << glz::format_error(ec, json);
+      expect(r.id == 3);
+      expect(r.name == "nested");
+   };
+
+   "skip unknown string larger than buffer with escapes"_test = [] {
+      std::string json = R"({"fill":")";
+      for (int i = 0; i < 1024; ++i) {
+         json += R"(abc\"\\)";
+      }
+      json += R"(","id":9,"name":"escaped"})";
+      expect(json.size() > 1024u);
+
+      std::istringstream iss(json);
+      glz::istream_buffer<1024> buffer(iss);
+
+      Record r{};
+      auto ec = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(r, buffer);
+      expect(!ec) << glz::format_error(ec, json);
+      expect(r.id == 9);
+      expect(r.name == "escaped");
+   };
+
+   "skip unknown array with chunked stream arrival"_test = [] {
+      std::string json = R"({"fill":[)";
+      for (int i = 0; i < 4096; ++i) {
+         json += std::to_string(i) + ",";
+      }
+      json.back() = ']';
+      json += R"(,"id":1,"name":"chunked"})";
+
+      slow_stringbuf sbuf(json, 100); // data arrives 100 bytes at a time
+      std::istream stream(&sbuf);
+      glz::istream_buffer<1024> buffer(stream);
+
+      Record r{};
+      auto ec = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(r, buffer);
+      expect(!ec) << glz::format_error(ec, json);
+      expect(r.id == 1);
+      expect(r.name == "chunked");
+   };
+
+   "skip multiple unknown keys spanning refills"_test = [] {
+      std::string big_array = "[";
+      for (int i = 0; i < 2048; ++i) {
+         big_array += "1,";
+      }
+      big_array.back() = ']';
+
+      std::string json = R"({"a":)" + big_array + R"(,"id":5,"b":)" + big_array + R"(,"name":"multi","c":true})";
+
+      std::istringstream iss(json);
+      glz::istream_buffer<1024> buffer(iss);
+
+      Record r{};
+      auto ec = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(r, buffer);
+      expect(!ec) << glz::format_error(ec, json);
+      expect(r.id == 5);
+      expect(r.name == "multi");
+   };
+
+   "unterminated unknown value still errors"_test = [] {
+      std::string json = R"({"fill":[)";
+      for (int i = 0; i < 2048; ++i) {
+         json += "0,";
+      }
+      // never closed
+
+      std::istringstream iss(json);
+      glz::istream_buffer<1024> buffer(iss);
+
+      Record r{};
+      auto ec = glz::read_streaming<glz::opts{.error_on_unknown_keys = false}>(r, buffer);
+      expect(bool(ec));
+      expect(ec.ec == glz::error_code::unexpected_end);
+   };
+};
+
 suite json_roundtrip_tests = [] {
    "JSON roundtrip - write to ostream, read from istream"_test = [] {
       ComplexObj original{.id = 42,