Repository: daggaz/json-stream

Issue:
Stream writing from streamable_dict generator doesn't find anything to encode 
With json-stream 2.3.2, I'm trying to do some basic stream processing on an s3 object, but I've narrowed this down to this test case.

The `json.dumps` call doesn't seem to touch the generator created by process_secrets.  Neither first loop, nor second loop should have anything after them but the other static messages.

```import base64
import io
import json
import json_stream

from json_stream.dump import JSONStreamEncoder
from json_stream.dump import default as js_dump_default


def decrypt_secret(r):
    return r


@json_stream.streamable_dict
def process_secrets(cc):
    for uid, res in cc.items():
        res = json_stream.to_standard_types(res)
        if res['kind'] == 'Secret':
            res = decrypt_secret(res)
        yield uid, res


if __name__ == '__main__':
    b64_data = ('eyIxZDczMjVlYS02ZjYxLTRmN2YtOTNkZi0xZDJiMjI2YmZlNGEiOiB7ImtpbmQiOiAiRGVwbG95bWVudCJ9LCAiOTQxN2Y4NWMtN'
                'mJhNy00OTllLWE4NzctNzY3NDAzNzU3NDc1IjogeyJraW5kIjogIkRlcGxveW1lbnQifSwgImI1NWQ0YjU1LTE0MGMtNGU5Ni1iOD'
                'dkLWM5N2ZkMDlkZWEwMiI6IHsia2luZCI6ICJEZXBsb3ltZW50In0sICIxNjg3Y2IwMy00ZjMzLTQ3MWItYTI2YS0xZTMwYjA3YzM'
                '5ODIiOiB7ImtpbmQiOiAiRGVwbG95bWVudCJ9LCAiNTRkMDM0NGQtOGU0ZC00YjIzLWJhZWUtNjY0NjliOThlMTMyIjogeyJraW5k'
                'IjogIkRlcGxveW1lbnQifSwgIjA3ZTIzMDIzLTMxYjgtNDdmYi1hOWUxLTZmYmIwNzA3N2M2NiI6IHsia2luZCI6ICJEZXBsb3ltZ'
                'W50In0sICJhYzc5MDg4NS03N2RiLTRhZDYtODFiYi1mNmFlNTIxZTU5MDciOiB7ImtpbmQiOiAiRGVwbG95bWVudCJ9LCAiN2Y4Yj'
                'c2NmItYjQyYS00NTQzLThlYTAtOGMyYjY3OWIwNWFmIjogeyJraW5kIjogIkRlcGxveW1lbnQifSwgImJhZTViMDEwLWRjMzktNDg'
                '0MC04ZGQwLWUzNWFkYWU3Yjk5OSI6IHsia2luZCI6ICJEZXBsb3ltZW50In0sICI1NGE0YzBiZi0xZGIyLTQzYmUtYTk4MS00NjNl'
                'OTExN2FiY2UiOiB7ImtpbmQiOiAiRGVwbG95bWVudCJ9fQ==')
    data = base64.b64decode(b64_data)
    filename = '/tmp/foo.json'
    with open(filename, 'wb') as f:
        f.write(data)
    with open(filename, 'rb') as rf:
        loader = json_stream.load(rf, persistent=True)
        print(type(loader))
        z = process_secrets(loader)
        print('dumps', type(z))
        print(json.dumps(z, default=js_dump_default))
        print('first loop')
        for k, v in z.items():
            print(k, v)
        print('second loop')
        for k, v in z.items():
            print(k, v)
        print('fin')
```

Hints from issue comments:
@daggaz: I've narrowed it down to this case:
```python
import json
import json_stream


@json_stream.streamable_dict
def dict_generator():
    yield from (('a', 1), ('b', 2), ('c', 3))


if __name__ == '__main__':
    print(json.dumps(dict_generator()))
```

The output at the moment is `{}`.@daggaz: This is because `encoder_listencode_dict(...)` in `_json.c` shortcircuits empty dicts:

```c
    if (PyDict_GET_SIZE(dct) == 0)  /* Fast path */
        return _PyAccu_Accumulate(acc, empty_dict);
```

Because this is implemented at the C level with special C only dict APIs, it never calls into the python class, so there is no way to override it...@daggaz: Since we are hiding the underlying dict anyway, and hijacking it's `items()` (the API that the non-short-circuited path uses) we can just pop an item in that underlying dict to force the C code to take the long way round.

Mega hack, but it works!@daggaz: Here's something somewhat weird...

```python
import json
from io import StringIO

import json_stream


@json_stream.streamable_dict
def dict_generator():
    yield from (('a', 1), ('b', 2), ('c', 3))


if __name__ == '__main__':
    fp = StringIO()
    json.dump(dict_generator(), fp)
    print("dump:", fp.getvalue())
    print("dumps:", json.dumps(dict_generator()))
```

`json.dump(obj, fp)` and `json.dumps(obj)` behave...differently

Output:
```
dump: {"a": 1, "b": 2, "c": 3}
dumps: {}
```@daggaz: This is why the tests pass (they use `dump()` not `dumps()`)@daggaz: Added some test cases that fail on master using `dumps()`.

Solution patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8db3606..917d815 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,20 +1,22 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: Tests
+name: CI
 
 on:
+  # run tests on any push to any branch, version tags, and on pull requests
   push:
+    branches: ['**']
+    tags: ['v*']
   pull_request:
 
 jobs:
-  build:
-
+  # TEST JOB (multi-Python matrix)
+  test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v3
@@ -44,3 +46,55 @@ jobs:
         pip install json-stream-rs-tokenizer
         rm -f src/json_stream_rs_tokenizer.py
         pytest
+
+  # PUBLISH JOB (only on tag push)
+  publish:
+    # This job will only run if the "test" job succeeds
+    needs: test
+    runs-on: ubuntu-latest
+    
+    # Only run if event is a tag push matching "v*"
+    # e.g. refs/tags/v1.2.3
+    if: startsWith(github.ref, 'refs/tags/v')
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.13"
+
+    - name: Verify tag matches pyproject.toml
+      id: check_version
+      run: |
+        python <<EOF
+        import os, sys, tomllib
+        from pathlib import Path
+    
+        data = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))
+        version = data["project"]["version"]
+        
+        tag = os.environ.get("GITHUB_REF", "").split("/")[-1]
+        
+        if tag.lstrip("v") != version:
+            print(f"ERROR: Tag {tag} does not match pyproject.toml version {version}.")
+            sys.exit(1)
+    
+        print(f"Version check passed: tag {tag} matches pyproject.toml version {version}.")
+        with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
+            gh_out.write(f"version={version}\n")
+        EOF
+
+    - name: Build
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install build twine
+        python -m build
+
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/pyproject.toml b/pyproject.toml
index a066e5c..060adc5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,11 +18,12 @@ classifiers = [
     "Topic :: Software Development :: Libraries",
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 requires-python = ">=3.5,<4"
 dependencies = ["json-stream-rs-tokenizer >= 0.4.17"]
diff --git a/src/json_stream/writer.py b/src/json_stream/writer.py
index c2d27fc..a42b770 100644
--- a/src/json_stream/writer.py
+++ b/src/json_stream/writer.py
@@ -1,10 +1,11 @@
+import typing
 from collections import deque
 from functools import wraps
 
 
 class Streamable:
-    def __init__(self, iterable):
-        super().__init__()
+    def __init__(self, iterable, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._it = iter(iterable)
         self._cache = deque()
 
@@ -48,6 +49,9 @@ class StreamableDict(Streamable, dict):
         for lists, except items are provided by passed in
         generator. Generator must produce pairs of key/value
     """
+    def __init__(self, iterable):
+        super().__init__(iterable, {'__error__': 'json-stream error'})
+
     def items(self):
         return self
 


Test patch:
diff --git a/src/json_stream/tests/test_writer.py b/src/json_stream/tests/test_writer.py
index 24f2d79..5e15275 100644
--- a/src/json_stream/tests/test_writer.py
+++ b/src/json_stream/tests/test_writer.py
@@ -5,19 +5,19 @@ from unittest import TestCase
 from json_stream.writer import streamable_dict, streamable_list
 
 
-class TestWriter(TestCase):
+class BaseTestWriter:
+    def dump(self, o):
+        raise NotImplementedError()
+
     def test_writer_wrapper(self):
         def dict_content(n):
             yield "a  list", streamable_list(range(4))
 
         o = streamable_dict(dict_content(5))
-
-        buffer = StringIO()
-        json.dump(o, buffer)
-        self.assertEqual('{"a  list": [0, 1, 2, 3]}', buffer.getvalue())
+        result = self.dump(o)
+        self.assertEqual('{"a  list": [0, 1, 2, 3]}', result)
 
     def test_writer_decorator(self):
-
         @streamable_list
         def list_content(n):
             return range(n)
@@ -28,19 +28,27 @@ class TestWriter(TestCase):
 
         o = dict_content(5)
 
-        buffer = StringIO()
-        json.dump(o, buffer)
-        self.assertEqual('{"a  list": [0, 1, 2, 3, 4]}', buffer.getvalue())
+        result = self.dump(o)
+        self.assertEqual('{"a  list": [0, 1, 2, 3, 4]}', result)
 
     def test_writer_empty(self):
-
         @streamable_list
         def empty_list():
             for i in range(0):  # never yields
                 yield i
 
         o = empty_list()
+        result = self.dump(o)
+        self.assertEqual('[]', result)
+
 
+class TestWriterDump(BaseTestWriter, TestCase):
+    def dump(self, o):
         buffer = StringIO()
         json.dump(o, buffer)
-        self.assertEqual('[]', buffer.getvalue())
+        return buffer.getvalue()
+
+
+class TestWriterDumps(BaseTestWriter, TestCase):
+    def dump(self, o):
+        return json.dumps(o)