Repository: spesmilo/electrum

Issue:
`JsonPointerException` from `JsonDB.load_data` can contain secret keys, leaking into logs or TARS reports
> This report contains private keys. That is unacceptable and we need to fix it (separately). 

 _Originally posted by @SomberNight in [#10000](https://github.com/spesmilo/electrum/issues/10000#issuecomment-3023304038)_

-----

We should somehow sanitize or replace `JsonPointerException`, and not let it propagate out as-is. Clearly it can contain arbitrary parts of the WalletDB. Secret key material should not be part of exceptions, as exceptions get logged and optionally sent via TARS to the public issue tracker.

-----

Example TARS traceback:
```python
Traceback (most recent call last):
  File "/home/sander/projects/electrum/electrum/packages/jsonpointer.py", line 271, in walk
    return doc[part]
           ~~~^^^^^^
KeyError: 'a20f36c11b403a6c2db235822442c291a24e61d3e75dc545de75bc748ea9874e'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/sander/projects/electrum/electrum/electrum/util.py", line 1130, in run_with_except_hook
    run_original(*args2, **kwargs2)
  File "/usr/lib/python3.12/threading.py", line 1012, in run
    self._target(*self._args, **self._kwargs)
  File "/home/sander/projects/electrum/electrum/electrum/gui/qml/qedaemon.py", line 212, in load_wallet_task
    wallet = self.daemon.load_wallet(self._path, local_password, upgrade=True)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/electrum/daemon.py", line 475, in func_wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/electrum/daemon.py", line 486, in load_wallet
    wallet = self._load_wallet(path, password, upgrade=upgrade, config=self.config)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/electrum/util.py", line 509, in do_profile
    o = func(*args, **kw_args)
        ^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/electrum/daemon.py", line 518, in _load_wallet
    db = WalletDB(storage.read(), storage=storage, upgrade=upgrade)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/electrum/wallet_db.py", line 1292, in __init__
    JsonDB.__init__(self, s, storage=storage, encoder=MyEncoder, upgrader=partial(upgrade_wallet_db, do_upgrade=upgrade))
  File "/home/sander/projects/electrum/electrum/electrum/json_db.py", line 245, in __init__
    data = self.load_data(s)
           ^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/electrum/json_db.py", line 274, in load_data
    data = patch.apply(data)
           ^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/packages/jsonpatch.py", line 692, in apply
    obj = operation.apply(obj)
          ^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/packages/jsonpatch.py", line 334, in apply
    subobj, part = self.pointer.to_last(obj)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/packages/jsonpointer.py", line 180, in to_last
    doc = self.walk(doc, part)
          ^^^^^^^^^^^^^^^^^^^^
  File "/home/sander/projects/electrum/electrum/packages/jsonpointer.py", line 274, in walk
    raise JsonPointerException("member '%s' not found in %s" % (part, doc))
jsonpointer.JsonPointerException: member 'a20f36c11b403a6c2db235822442c291a24e61d3e75dc545de75bc748ea9874e' not found in {'106cd8985b12824e1ef14977312d6c8345d4806c9c212cb0b05024018b699611': {'funding_txid': None, 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548769, 'lockup_address': 'tb1qqzqt2gddvj2pcasmye327hfqk8uww3843jzlf9mlz529r4ezlzaswh6v5t', 'onchain_amount': 49295, 'preimage': None, 'prepay_hash': None, 'privkey': 'e6f22e718203df5182491b0b02070894ecc53e29301ef0b4ff366a1cbed9d938', 'receive_address': 'tb1qa0828sapv3stz7tc6malyxer0fukssrzcrjvet', 'redeem_script': '8201208763a91407a41388a31252335444ffc8f00996fd4488f7ea882102e21de506a9700bc4928bf04fae5927f741446f5633e72838eece3eed9e6fe99d677503a16845b1752103c42e08b45b869b6611143adb181ca7bb950fee5ef359a00c6b27d553fcd2218a68ac', 'spending_txid': None}, '33fb199ee3d6f656e6e5a8e81952b3a579d3a8cf8c5b543db2f3869bf4133fc0': {'funding_txid': 'caa7ba7ebcc2016966dd4ef3335e0487d3bc28b02dfa2056c807399619f2ee15', 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548776, 'lockup_address': 'tb1qv77hc44tswqhrjjelymrfczc02p97sgdu0k270ymugmrqq4f524s74h9lx', 'onchain_amount': 49295, 'preimage': None, 'prepay_hash': None, 'privkey': '501f4fece3c3f34067c5f2f07b314dc6fb20910933e059381577ddce428fe48b', 'receive_address': 'tb1qy0j6dlk22vnsjksyycl3dzmsse7gmgamqnasc7', 'redeem_script': '8201208763a91411b14e6bfdceab4d2c8654a25f4d8e6e588c2f02882103237ca4ac3ce276f62d454ec1e25ce0d6b7c3e6cbc924888fb0e59eaa760ba487677503a86845b17521035ac8cf19530a91f509f94d94043a1004170e54c408ef57c781078ca3ce06c6a668ac', 'spending_txid': None}, '48f0eb12b77a51df04b726e5d786ff82de8f61d9b40e3e6aef8a4675bb980db3': {'funding_txid': None, 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548769, 'lockup_address': 'tb1qfngmgx6c3jl0jllqk0fgy46r7e32uwszzvdz40ppj8xpqz5v3jrqmce2e0', 'onchain_amount': 49295, 'preimage': None, 'prepay_hash': None, 'privkey': 'cde4143e3997c6b4b98746c06130098cdb48988998d86da8fb7d30d6fc5d572f', 'receive_address': 'tb1qa0828sapv3stz7tc6malyxer0fukssrzcrjvet', 'redeem_script': '8201208763a91474c97e7b61a3f18f32f5c4fa6c3732e29e44466d882102864bc3355a9d01f464d698f9a7a67818ee7f9cd28d0cc6440675235c67518bab677503a16845b1752103cfe991bef97bc950b0665e326ec8f6700bdf6b48f98250d8e7d9f718a267c71768ac', 'spending_txid': None}, 'c662945dafe637e97939fd99e5728a8fa1985682d8166b69ebdc02fb6e0bcc5a': {'funding_txid': '0e2d724e4671b53065f3361d55c7ba56b862c5cfc5c361790b7ccb94e068a991', 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548770, 'lockup_address': 'tb1qumuht8ykck4y8zqj632w7783uq4h5af69zualxa3hm6k5y4zrw7sydpp0r', 'onchain_amount': 49295, 'preimage': 'c12ea2eb99122ba37be2a59a3a67aab382373d0aa1ac3c1aecc0a42d1e793694', 'prepay_hash': None, 'privkey': '917c62fb998cb21c954dbfc457fef9c178a91404c5b3e5715bde7163c1862106', 'receive_address': 'tb1qa0828sapv3stz7tc6malyxer0fukssrzcrjvet', 'redeem_script': '8201208763a914d118f6334efa6742810d1b3b1eed74a3c3a8d0aa8821025912a44a37a288332eb2d0ab502ba24ac265068cd626fd57ae9f15a0ae5660f8677503a26845b175210244212e72369ef73d9fc11ffadf32f41107f79928ea452daf528c87e47546c3cf68ac', 'spending_txid': '5c1c90eb420720f46fc3618c9573e2f9dfe6667f9b53cfa1e4d54ad010274310'}}

JsonPointerException: member 'a20f36c11b403a6c2db235822442c291a24e61d3e75dc545de75bc748ea9874e' not found in {'106cd8985b12824e1ef14977312d6c8345d4806c9c212cb0b05024018b699611': {'funding_txid': None, 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548769, 'lockup_address': 'tb1qqzqt2gddvj2pcasmye327hfqk8uww3843jzlf9mlz529r4ezlzaswh6v5t', 'onchain_amount': 49295, 'preimage': None, 'prepay_hash': None, 'privkey': 'e6f22e718203df5182491b0b02070894ecc53e29301ef0b4ff366a1cbed9d938', 'receive_address': 'tb1qa0828sapv3stz7tc6malyxer0fukssrzcrjvet', 'redeem_script': '8201208763a91407a41388a31252335444ffc8f00996fd4488f7ea882102e21de506a9700bc4928bf04fae5927f741446f5633e72838eece3eed9e6fe99d677503a16845b1752103c42e08b45b869b6611143adb181ca7bb950fee5ef359a00c6b27d553fcd2218a68ac', 'spending_txid': None}, '33fb199ee3d6f656e6e5a8e81952b3a579d3a8cf8c5b543db2f3869bf4133fc0': {'funding_txid': 'caa7ba7ebcc2016966dd4ef3335e0487d3bc28b02dfa2056c807399619f2ee15', 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548776, 'lockup_address': 'tb1qv77hc44tswqhrjjelymrfczc02p97sgdu0k270ymugmrqq4f524s74h9lx', 'onchain_amount': 49295, 'preimage': None, 'prepay_hash': None, 'privkey': '501f4fece3c3f34067c5f2f07b314dc6fb20910933e059381577ddce428fe48b', 'receive_address': 'tb1qy0j6dlk22vnsjksyycl3dzmsse7gmgamqnasc7', 'redeem_script': '8201208763a91411b14e6bfdceab4d2c8654a25f4d8e6e588c2f02882103237ca4ac3ce276f62d454ec1e25ce0d6b7c3e6cbc924888fb0e59eaa760ba487677503a86845b17521035ac8cf19530a91f509f94d94043a1004170e54c408ef57c781078ca3ce06c6a668ac', 'spending_txid': None}, '48f0eb12b77a51df04b726e5d786ff82de8f61d9b40e3e6aef8a4675bb980db3': {'funding_txid': None, 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548769, 'lockup_address': 'tb1qfngmgx6c3jl0jllqk0fgy46r7e32uwszzvdz40ppj8xpqz5v3jrqmce2e0', 'onchain_amount': 49295, 'preimage': None, 'prepay_hash': None, 'privkey': 'cde4143e3997c6b4b98746c06130098cdb48988998d86da8fb7d30d6fc5d572f', 'receive_address': 'tb1qa0828sapv3stz7tc6malyxer0fukssrzcrjvet', 'redeem_script': '8201208763a91474c97e7b61a3f18f32f5c4fa6c3732e29e44466d882102864bc3355a9d01f464d698f9a7a67818ee7f9cd28d0cc6440675235c67518bab677503a16845b1752103cfe991bef97bc950b0665e326ec8f6700bdf6b48f98250d8e7d9f718a267c71768ac', 'spending_txid': None}, 'c662945dafe637e97939fd99e5728a8fa1985682d8166b69ebdc02fb6e0bcc5a': {'funding_txid': '0e2d724e4671b53065f3361d55c7ba56b862c5cfc5c361790b7ccb94e068a991', 'is_redeemed': False, 'is_reverse': False, 'lightning_amount': 38273, 'locktime': 4548770, 'lockup_address': 'tb1qumuht8ykck4y8zqj632w7783uq4h5af69zualxa3hm6k5y4zrw7sydpp0r', 'onchain_amount': 49295, 'preimage': 'c12ea2eb99122ba37be2a59a3a67aab382373d0aa1ac3c1aecc0a42d1e793694', 'prepay_hash': None, 'privkey': '917c62fb998cb21c954dbfc457fef9c178a91404c5b3e5715bde7163c1862106', 'receive_address': 'tb1qa0828sapv3stz7tc6malyxer0fukssrzcrjvet', 'redeem_script': '8201208763a914d118f6334efa6742810d1b3b1eed74a3c3a8d0aa8821025912a44a37a288332eb2d0ab502ba24ac265068cd626fd57ae9f15a0ae5660f8677503a26845b175210244212e72369ef73d9fc11ffadf32f41107f79928ea452daf528c87e47546c3cf68ac', 'spending_txid': '5c1c90eb420720f46fc3618c9573e2f9dfe6667f9b53cfa1e4d54ad010274310'}}
```


Discussion:
We could in the `BaseCrashReporter` remove the content str of all exceptions that don't originate from Electrum code as we don't have control over the content. This would catch e.g. the content of the `JsonPointerException` as it originates from `electrum/packages/jsonpointer.py` instead of the `electrum/electrum/` dir. 
Do you think this i suitable (or overkill)?
Note that ideally we would not even *log* private keys.
I like to be able to ask users for logs. And I have logging enabled for my mainnet wallets.

Also, any change should be balanced with keeping tracebacks useful for debugging.

> remove the content str of all exceptions that don't originate from Electrum code as we don't have control over the content

What if there is a chain of exceptions, with the deepest exception originating from external code?
See e.g. first report of https://github.com/spesmilo/electrum/issues/8524

Note that besides electrum / electrum_ecc / electrum_aionostr, etc,
- there are the third-party packages
- there is the stdlib

Still, perhaps you are right and indeed a whitelist-based approach would be the truly prudent one.

-----

In general if we pass sensitive data to a function call, it might raise an exception containing that sensitive data.
So perhaps we could get away with a blacklist-based approach, where we try to carefully handle exceptions in places where function calls are potentially dealing with secret keys.
Ultimately it is us who have to be mindful of this, see e.g. https://github.com/spesmilo/electrum/commit/99647fc070ad1aa988c3fc5fa16476a71564fd3f

-----

btw one general observation re the wallet db and other json structs in general is that usually in case of json `dict`s, it is the `values` that might be sensitive, almost never the dict `keys`. And the `keys` along with redacted values often contain plenty information for debugging usage.

But in this example with `JsonPointerException`, the arg is a human-readable str, not a json object.

-----

So, perhaps, in this example, we could just re-wrap exceptions around https://github.com/spesmilo/electrum/blob/55c7c6c4df22bb857654b31bd03c9ce0710aa11d/electrum/json_db.py#L273-L274
but yeah, I am not so sure myself whether we should/could do some generic solution.



Patch:
diff --git a/electrum/json_db.py b/electrum/json_db.py
index 28d1ec141395..ef7742b16f60 100644
--- a/electrum/json_db.py
+++ b/electrum/json_db.py
@@ -28,14 +28,30 @@
 from typing import TYPE_CHECKING, Optional, Sequence, List, Union
 
 import jsonpatch
+import jsonpointer
 
 from . import util
-from .util import WalletFileException, profiler
+from .util import WalletFileException, profiler, sticky_property
 from .logging import Logger
 
 if TYPE_CHECKING:
     from .storage import WalletStorage
 
+
+# We monkeypatch exceptions in the jsonpatch package to ensure they do not contain secrets from the DB.
+# We often log exceptions and offer to send them to the crash reporter, so they must not contain secrets.
+jsonpointer.JsonPointerException.__str__ = lambda self: """(JPE) 'redacted'"""
+jsonpointer.JsonPointerException.__repr__ = lambda self: """<JsonPointerException 'redacted'>"""
+setattr(jsonpointer.JsonPointerException, '__cause__', sticky_property(None))
+setattr(jsonpointer.JsonPointerException, '__context__', sticky_property(None))
+setattr(jsonpointer.JsonPointerException, '__suppress_context__', sticky_property(True))
+jsonpatch.JsonPatchException.__str__ = lambda self: """(JPE) 'redacted'"""
+jsonpatch.JsonPatchException.__repr__ = lambda self: """<JsonPatchException 'redacted'>"""
+setattr(jsonpatch.JsonPatchException, '__cause__', sticky_property(None))
+setattr(jsonpatch.JsonPatchException, '__context__', sticky_property(None))
+setattr(jsonpatch.JsonPatchException, '__suppress_context__', sticky_property(True))
+
+
 def modifier(func):
     def wrapper(self, *args, **kwargs):
         with self.lock:
diff --git a/electrum/util.py b/electrum/util.py
index 7cb38faf859c..d266a6a5c1bd 100644
--- a/electrum/util.py
+++ b/electrum/util.py
@@ -2199,6 +2199,30 @@ def __get__(self, owner_self, owner_cls):
         return self.fget(owner_cls)
 
 
+def sticky_property(val):
+    """Creates a 'property' whose value cannot be changed and that cannot be deleted.
+    Attempts to change the value are silently ignored.
+
+    >>> class C: pass
+    ...
+    >>> setattr(C, 'x', sticky_property(3))
+    >>> c = C()
+    >>> c.x
+    3
+    >>> c.x = 2
+    >>> c.x
+    3
+    >>> del c.x
+    >>> c.x
+    3
+    """
+    return property(
+        fget=lambda self: val,
+        fset=lambda *args, **kwargs: None,
+        fdel=lambda *args, **kwargs: None,
+    )
+
+
 def get_running_loop() -> Optional[asyncio.AbstractEventLoop]:
     """Returns the asyncio event loop that is *running in this thread*, if any."""
     try:


Test patch:
diff --git a/tests/test_jsondb.py b/tests/test_jsondb.py
new file mode 100644
index 000000000000..16129e11493a
--- /dev/null
+++ b/tests/test_jsondb.py
@@ -0,0 +1,86 @@
+import contextlib
+import copy
+import traceback
+
+import jsonpatch
+from jsonpatch import JsonPatchException
+from jsonpointer import JsonPointerException
+
+from . import ElectrumTestCase
+
+
+class TestJsonpatch(ElectrumTestCase):
+
+    async def test_op_replace(self):
+        data1 = {'foo': 'bar', 'numbers': [1, 3, 4, 8], 'dictlevelA1': {'secret1': 2, 'secret2': 4, 'secret3': 6}}
+        patches = [{"op": "replace", "path": "/dictlevelA1/secret2", "value": 2222}]
+        jpatch = jsonpatch.JsonPatch(patches)
+        data2 = jpatch.apply(data1)
+        self.assertEqual(
+            {'foo': 'bar', 'numbers': [1, 3, 4, 8], 'dictlevelA1': {'secret1': 2, 'secret2': 2222, 'secret3': 6}},
+            data2
+        )
+
+    @contextlib.contextmanager
+    def _customAssertRaises(self, *args, **kwargs):
+        with self.assertRaises(*args, **kwargs) as ctx:
+            try:
+                yield ctx
+            except Exception as e:
+                # save original traceback now, as assertRaises will destroy most of it imminently:
+                ctx._customctx_original_tb = "".join(traceback.format_exception(e))
+                raise
+
+    async def test_patch_does_not_leak_privatekeys(self):
+        data1 = {
+            'dictlevelB1': 'secret77',
+            'dictlevelC1': [1, "secret99", 4, 8],
+            'dictlevelA1': {"dictlevelA2_aa": "secret11", "dictlevelA2_bb": "secret12", "dictlevelA2_cc": "secret13"}}
+        def fail_if_leaking_secret(ctx) -> None:
+            self.assertNotIn("secret", str(ctx.exception))
+            self.assertNotIn("secret", repr(ctx.exception))
+            self.assertNotIn("secret", ctx._customctx_original_tb)
+            self.assertNotIn("dictlevel", str(ctx.exception))
+            self.assertNotIn("dictlevel", repr(ctx.exception))
+            self.assertNotIn("dictlevel", ctx._customctx_original_tb)
+            self.assertIn("redacted", str(ctx.exception))  # injected by our monkeypatching
+            self.assertIn("redacted", repr(ctx.exception))  # injected by our monkeypatching
+        # op "replace"
+        with self.subTest(msg="replace_dict_inner_key_missing"):
+            patches = [{"op": "replace", "path": "/dictlevelA1/dictlevelX2", "value": "nakamoto_secret"}]
+            jpatch = jsonpatch.JsonPatch(patches)
+            with self._customAssertRaises(JsonPatchException) as ctx:
+                data2 = jpatch.apply(data1)
+            fail_if_leaking_secret(ctx)
+        with self.subTest(msg="replace_dict_outer_key_missing"):
+            patches = [{"op": "replace", "path": "/dictlevelX1/dictlevelX2", "value": "nakamoto_secret"}]
+            jpatch = jsonpatch.JsonPatch(patches)
+            with self._customAssertRaises(JsonPointerException) as ctx:
+                data2 = jpatch.apply(data1)
+            fail_if_leaking_secret(ctx)
+        # op "remove"
+        with self.subTest(msg="remove_dict_inner_key_missing"):
+            patches = [{"op": "remove", "path": "/dictlevelA1/dictlevelX2"}]
+            jpatch = jsonpatch.JsonPatch(patches)
+            with self._customAssertRaises(JsonPatchException) as ctx:
+                data2 = jpatch.apply(data1)
+            fail_if_leaking_secret(ctx)
+        with self.subTest(msg="remove_dict_outer_key_missing"):
+            patches = [{"op": "remove", "path": "/dictlevelX1/dictlevelX2"}]
+            jpatch = jsonpatch.JsonPatch(patches)
+            with self._customAssertRaises(JsonPointerException) as ctx:
+                data2 = jpatch.apply(data1)
+            fail_if_leaking_secret(ctx)
+        # op "add"
+        with self.subTest(msg="add_dict_inner_key_missing"):
+            patches = [{"op": "add", "path": "/dictlevelA1/dictlevelX2/dictlevelX3/dictlevelX4", "value": "nakamoto_secret"}]
+            jpatch = jsonpatch.JsonPatch(patches)
+            with self._customAssertRaises(JsonPointerException) as ctx:
+                data2 = jpatch.apply(data1)
+            fail_if_leaking_secret(ctx)
+        with self.subTest(msg="add_dict_outer_key_missing"):
+            patches = [{"op": "add", "path": "/dictlevelX1/dictlevelX2/dictlevelX3/dictlevelX4", "value": "nakamoto_secret"}]
+            jpatch = jsonpatch.JsonPatch(patches)
+            with self._customAssertRaises(JsonPointerException) as ctx:
+                data2 = jpatch.apply(data1)
+            fail_if_leaking_secret(ctx)