Repository: borgbackup/borg

Issue:
truncated json
```
_______________ RemoteArchiverTestCase.test_corrupted_repository _______________
[gw0] cygwin -- Python 3.6.1 /home/admin/w/borg/.tox/py36/bin/python3.6
self = <borg.testsuite.archiver.RemoteArchiverTestCase testMethod=test_corrupted_repository>

    def test_corrupted_repository(self):
        self.cmd('init', '--encryption=repokey', self.repository_location)
        self.create_src_archive('test')
        self.cmd('extract', '--dry-run', self.repository_location + '::test')
        output = self.cmd('check', '--show-version', self.repository_location)
        self.assert_in('borgbackup version', output)  # implied output even without --info given
        self.assert_not_in('Starting repository check', output)  # --info not given for root logger

        name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[1]
        with open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+b') as fd:
            fd.seek(100)
            fd.write(b'XXXX')
>       output = self.cmd('check', '--info', self.repository_location, exit_code=1)

.tox/py36/lib/python3.6/site-packages/borg/testsuite/archiver.py:1392:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.tox/py36/lib/python3.6/site-packages/borg/testsuite/archiver.py:280: in cmd
    ret, output = exec_cmd(*args, fork=fork, exe=self.EXE, archiver=self.archiver, **kw)
.tox/py36/lib/python3.6/site-packages/borg/testsuite/archiver.py:99: in exec_cmd
    ret = archiver.run(args)
.tox/py36/lib/python3.6/site-packages/borg/archiver.py:3951: in run
    return set_ec(func(args))
.tox/py36/lib/python3.6/site-packages/borg/archiver.py:131: in wrapper
    return method(self, args, repository=repository, **kwargs)
.tox/py36/lib/python3.6/site-packages/borg/archiver.py:270: in do_check
    if not repository.check(repair=args.repair, save_space=args.save_space):
.tox/py36/lib/python3.6/site-packages/borg/remote.py:480: in do_rpc
    return self.call(f.__name__, named, **extra)
.tox/py36/lib/python3.6/site-packages/borg/remote.py:720: in call
    for resp in self.call_many(cmd, [args], **kw):
.tox/py36/lib/python3.6/site-packages/borg/remote.py:850: in call_many
    handle_remote_line(line)
.tox/py36/lib/python3.6/site-packages/borg/remote.py:984: in handle_remote_line
    msg = json.loads(line)
/usr/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/usr/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <json.decoder.JSONDecoder object at 0x6ffff544518>
s = '{"type": "log_message", "time": 1496955191.276449, "message": "49934fdb70dd0194252f4649f659fe2f2a'
idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
            a JSON document) and return a 2-tuple of the Python
            representation and the index in ``s`` where the document ended.

            This can be used to decode a JSON document from a string that may
            have extraneous data at the end.

            """
        try:
>           obj, end = self.scan_once(s, idx)
E           json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 63 (char 62)

/usr/lib/python3.6/json/decoder.py:355: JSONDecodeError
```
When it happens, it is always in test_corrupted_repository.
But not every run of this test fails. Looks like some race condition.

Note: this log output could be from repository index check where it uses a line_format of:
'ID', 'rebuilt index', 'committed index'.



Discussion:
Is this on a Cygwin version with or without the pipe bug-fix?

If without, then this would seem a likely candidate — the os_write hack/workaround is not used for logging.
with fixed cygwin (2.8.0).

I did not check any fix being present in the src of 2.8.0, but the "remote" tests worked without our workaround and they did not work before without the workaround.
Adding this to the 1.1 milestone. If it can't be found/fixed until then, I guess we need to at least document that remote repos are broken.
Are they, though? There is one sometimes failing test, from past experience that doesn't necessarily mean that non-testing is actually affected.
IIRC the tests under cygwin always or mostly failed. The truncation was not always at the same offset though.
> When it happens, it is always in test_corrupted_repository.
> But not every run of this test fails. Looks like some race condition.
I did some small-scale practical tests (init, create, list, info, check, delete) and it seemed to work ok.
I also tried --json and --json lines and they also looked ok.

client: windows10+cygwin borg master branch
server: ubuntu linux 16.04 borg 1.1.0b6

From IRC, originally posted by ganja, happening with borg 1.1.0rc2 **on Linux**:
```
Remote: ory.compact_segments", "finished": false, "current": 102, "total": 131}
Local Exception
Traceback (most recent call last):
  File "borg/archiver.py", line 3968, in main
  File "borg/archiver.py", line 3896, in run
  File "borg/archiver.py", line 148, in wrapper
  File "borg/archiver.py", line 527, in do_create
  File "borg/archiver.py", line 492, in create_inner
  File "borg/archiver.py", line 613, in _process
  File "borg/archive.py", line 991, in process_file
  File "borg/archive.py", line 923, in chunk_file
  File "borg/archive.py", line 905, in write_part_file
  File "borg/archive.py", line 438, in write_checkpoint
  File "borg/archive.py", line 477, in save
  File "borg/remote.py", line 467, in do_rpc
  File "borg/remote.py", line 707, in call
  File "borg/remote.py", line 842, in call_many
  File "borg/remote.py", line 976, in handle_remote_line
  File "json/__init__.py", line 319, in loads
  File "json/decoder.py", line 339, in decode
  File "json/decoder.py", line 355, in raw_decode
json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 135 (char 134)

Platform: Linux pgc-db-01 3.16.0-4-amd64 #1 SMP Debian 3.16.36-1+deb8u1 (2016-09-03) x86_64 
Linux: debian 8.6 
Borg: 1.1.0rc2  Python: CPython 3.5.3
PID: 24210  CWD: /root
sys.argv: ['/usr/local/bin/borgbackup', 'create', '--lock-wait', '7200', '--read-special', '--compression', 'auto,lzma,3', 'borg@thor:/data/borg-WedFriSun/pgc-db-01.borg::full_{now}', '/etc', '/usr/local', '/root', '/home', '/usr/share/mysql/charsets/Index.xml', '/usr/share/mysql/charsets/Index.xml-10.1', '/usr/share/mysql/charsets/Index.xml-10.1.18', '/usr/share/mysql/charsets/Index.xml.140409', '/usr/share/mysql/charsets/Index.xml.140717', '/usr/share/mysql/charsets/Index.xml.original', '/usr/share/mysql/charsets/Index.xml.original131026', '/usr/share/mysql/charsets/Index.xml.original.mariadb', '/backup/full_mysqldump.sql']
SSH_ORIGINAL_COMMAND: None
```

Note: "finished", "total", "current" indicates this is partial progress indication output in json mode.

https://docs.python.org/3/library/sys.html#sys.stdin

"When interactive, standard streams are line-buffered. Otherwise, they are block-buffered like regular text files. You can override this value with the -u command-line option." (interactive = isatty() is True)

So borg has block buffering when it runs client / server over ssh.
I guess, due to that, the json progress and log output could fill up the block buffer (but the last json structure only fitting in there partially, so it would send a block with partial json at the end over the network).
The receiving side's code for stderr is not prepared for such partial json.

Just flushing with print(flush=True) would fix this, IF this is the (only) root cause.

Maybe this would also fix some issues people have reported when output was not visible when expected (because it was still stuck in an unflushed block buffer)?

progress:
- needs print(..., flush=True)

logging:
- logging.StreamHandler does flush when it emits:
https://github.com/python/cpython/blob/3.4/Lib/logging/__init__.py#L982
- but sys.stderr is still block buffered. shouldn't matter when flushed after writing.

@enkore what do you think?

Sounds plausible
The json mode progress fix in #3032 was merged into master (and soon will be also merged into 1.1-maint), so I am closing this for now.
Built some binaries with the current 1.1-maint code:

https://paste.thinkmo.de/MKKHNDzg#borg-20170922.txt

Should fix the truncated json issue for progress output, please test.

Also new: binaries based on python 3.5.4 (up from 3.5.3), osxfuse 3.7.1 (up from 3.6.3).

Still happens, new binary on both sides:
```
Remote: sitory.compact_segments", "time": 1506266906.1123605, "total": 206, "type": "progress_percent", "operation": 12, "message": "Compacting segments  81%", "info": null, "current": 168}
Local Exception
Traceback (most recent call last):
  File "borg/archiver.py", line 3983, in main
  File "borg/archiver.py", line 3911, in run
  File "borg/archiver.py", line 148, in wrapper
  File "borg/archiver.py", line 527, in do_create
  File "borg/archiver.py", line 492, in create_inner
  File "borg/archiver.py", line 613, in _process
  File "borg/archive.py", line 991, in process_file
  File "borg/archive.py", line 931, in chunk_file
  File "borg/archive.py", line 905, in write_part_file
  File "borg/archive.py", line 438, in write_checkpoint
  File "borg/archive.py", line 477, in save
  File "borg/remote.py", line 467, in do_rpc
  File "borg/remote.py", line 707, in call
  File "borg/remote.py", line 842, in call_many
  File "borg/remote.py", line 976, in handle_remote_line
  File "json/__init__.py", line 319, in loads
  File "json/decoder.py", line 339, in decode
  File "json/decoder.py", line 355, in raw_decode
json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 30 (char 29)

Platform: Linux pgc-db-01 3.16.0-4-amd64 #1 SMP Debian 3.16.36-1+deb8u1 (2016-09-03) x86_64 
Linux: debian 8.6 
Borg: 1.1.0rc4.dev3+g3f3aa24.d20170922  Python: CPython 3.5.4
PID: 16333  CWD: /root
sys.argv: ['/usr/local/bin/borgbackup-20170922', 'create', '--lock-wait', '7200', '--read-special', '--compression', 'auto,lzma,3', 'borg@thor:/data/borg-WedFriSun/pgc-db-01.borg::full_{now}', '/etc', '/usr/local', '/root', '/home', '/usr/share/mysql/charsets/Index.xml', '/usr/share/mysql/charsets/Index.xml-10.1', '/usr/share/mysql/charsets/Index.xml-10.1.18', '/usr/share/mysql/charsets/Index.xml.140409', '/usr/share/mysql/charsets/Index.xml.140717', '/usr/share/mysql/charsets/Index.xml.original', '/usr/share/mysql/charsets/Index.xml.original131026', '/usr/share/mysql/charsets/Index.xml.original.mariadb', '/backup/full_mysqldump.sql']
SSH_ORIGINAL_COMMAND: None
```
Hmm, how big is that buffer that is used for stderr when not connected to a tty?
BTW, if the flush() is not the fix for the root cause, it makes the 2 issues (logger output, progress output) logically consistent again (the logger always did a flush after each log output and also showed this problem).
Hmm, maybe its wrong thinking here.

The flush() only flushes borg serve's stderr buffer, so the data travels from borg over the local pipe to ssh. But there is another "network pipe" from sshd to ssh and another local pipe from the client ssh to the client borg.

Each of these pipes maybe does own buffering, so one json structure might end up in different packets due to that?
The client code may have to synchronize read data itself (on \n). In this case, it may also affect older versions and be independent of the message format.
@enkore yup, i have just implemented that. PR coming soon. also found another issue there, it could also fall over incomplete multi-byte utf-8 sequences for same reason.

Resolving patch:
diff --git a/src/borg/remote.py b/src/borg/remote.py
--- a/src/borg/remote.py
+++ b/src/borg/remote.py
@@ -520,6 +520,7 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock
         self.rx_bytes = 0
         self.tx_bytes = 0
         self.to_send = b''
+        self.stderr_received = b''  # incomplete stderr line bytes received (no \n yet)
         self.chunkid_to_msgids = {}
         self.ignore_responses = set()
         self.responses = {}
@@ -829,9 +830,16 @@ def handle_error(unpacked):
                     if not data:
                         raise ConnectionClosed()
                     self.rx_bytes += len(data)
-                    data = data.decode('utf-8')
-                    for line in data.splitlines(keepends=True):
-                        handle_remote_line(line)
+                    # deal with incomplete lines (may appear due to block buffering)
+                    if self.stderr_received:
+                        data = self.stderr_received + data
+                        self.stderr_received = b''
+                    lines = data.splitlines(keepends=True)
+                    if lines and not lines[-1].endswith((b'\r', b'\n')):
+                        self.stderr_received = lines.pop()
+                    # now we have complete lines in <lines> and any partial line in self.stderr_received.
+                    for line in lines:
+                        handle_remote_line(line.decode('utf-8'))  # decode late, avoid partial utf-8 sequences
             if w:
                 while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT:
                     if calls:
@@ -962,6 +970,7 @@ def handle_remote_line(line):
 
     This function is remarkably complex because it handles multiple wire formats.
     """
+    assert line.endswith(('\r', '\n'))
     if line.startswith('{'):
         # This format is used by Borg since 1.1.0b6 for new-protocol clients.
         # It is the same format that is exposed by --log-json.
diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py
--- a/src/borg/testsuite/repository.py
+++ b/src/borg/testsuite/repository.py
@@ -944,16 +944,22 @@ def tearDown(self):
         sys.stderr = self.old_stderr
 
     def test_stderr_messages(self):
-        handle_remote_line("unstructured stderr message")
+        handle_remote_line("unstructured stderr message\n")
         self.assert_equal(self.stream.getvalue(), '')
         # stderr messages don't get an implicit newline
-        self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message')
+        self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message\n')
+
+    def test_stderr_progress_messages(self):
+        handle_remote_line("unstructured stderr progress message\r")
+        self.assert_equal(self.stream.getvalue(), '')
+        # stderr messages don't get an implicit newline
+        self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr progress message\r')
 
     def test_pre11_format_messages(self):
         self.handler.setLevel(logging.DEBUG)
         logging.getLogger().setLevel(logging.DEBUG)
 
-        handle_remote_line("$LOG INFO Remote: borg < 1.1 format message")
+        handle_remote_line("$LOG INFO Remote: borg < 1.1 format message\n")
         self.assert_equal(self.stream.getvalue(), 'Remote: borg < 1.1 format message\n')
         self.assert_equal(self.stderr.getvalue(), '')
 
@@ -961,7 +967,7 @@ def test_post11_format_messages(self):
         self.handler.setLevel(logging.DEBUG)
         logging.getLogger().setLevel(logging.DEBUG)
 
-        handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message")
+        handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message\n")
         self.assert_equal(self.stream.getvalue(), 'Remote: borg >= 1.1 format message\n')
         self.assert_equal(self.stderr.getvalue(), '')
 
@@ -970,7 +976,7 @@ def test_remote_messages_screened(self):
         self.handler.setLevel(logging.WARNING)
         logging.getLogger().setLevel(logging.WARNING)
 
-        handle_remote_line("$LOG INFO borg.repository Remote: new format info message")
+        handle_remote_line("$LOG INFO borg.repository Remote: new format info message\n")
         self.assert_equal(self.stream.getvalue(), '')
         self.assert_equal(self.stderr.getvalue(), '')
 
@@ -990,7 +996,7 @@ def test_info_to_correct_local_child(self):
         foo_handler.setLevel(logging.INFO)
         logging.getLogger('borg.repository.foo').handlers[:] = [foo_handler]
 
-        handle_remote_line("$LOG INFO borg.repository Remote: new format child message")
+        handle_remote_line("$LOG INFO borg.repository Remote: new format child message\n")
         self.assert_equal(foo_stream.getvalue(), '')
         self.assert_equal(child_stream.getvalue(), 'Remote: new format child message\n')
         self.assert_equal(self.stream.getvalue(), '')


Test patch: