Repository: matrix-org/synapse

Issue:
error in background updates
### Description

My synapse logs show these error messages regularly, though synapse otherwise seems to keep working.

I noticed this after upgrading to 1.6.0rc1. Downgrading to 1.5.1 gives a different traceback, though also has an issue with the background update.

Traceback for 1.5.1:

```
2019-11-23 09:04:58,698 - synapse.storage.background_updates - 110 - ERROR - background_updates-0- Error doing update
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
StopIteration: 1.0

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/background_updates.py", line 107, in _run_background_updates
    self.BACKGROUND_UPDATE_DURATION_MS
KeyError: 'event_store_labels'
```

Traceback for 1.6.0rc1:
```
2019-11-23 09:19:59,010 - synapse.metrics - 462 - INFO - - Collecting gc 1
2019-11-23 09:19:59,012 - synapse.metrics - 462 - INFO - - Collecting gc 2
2019-11-23 09:19:59,247 - synapse.storage.background_updates - 210 - INFO - background_updates-0- Starting update batch on background update 'event_store_labels'
2019-11-23 09:19:59,278 - synapse.storage.background_updates - 113 - ERROR - background_updates-0- Error doing update
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/background_updates.py", line 110, in run_background_updates
    self.BACKGROUND_UPDATE_DURATION_MS
  File "/usr/local/lib/python3.7/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/background_updates.py", line 205, in do_next_background_update
    res = yield self._do_background_update(update_name, desired_duration_ms)
  File "/usr/local/lib/python3.7/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/background_updates.py", line 238, in _do_background_update
    items_updated = yield update_handler(progress, batch_size)
  File "/usr/local/lib/python3.7/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/data_stores/main/events_bg_updates.py", line 562, in _event_store_labels
    desc="event_store_labels", func=_event_store_labels_txn
  File "/usr/local/lib/python3.7/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/_base.py", line 504, in runInteraction
    **kwargs
  File "/usr/local/lib/python3.7/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/_base.py", line 552, in runWithConnection
    self._db_pool.runWithConnection(inner_func, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/threadpool.py", line 250, in inContext
    result = inContext.theWork()
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/threadpool.py", line 266, in <lambda>
    inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
  File "/usr/local/lib/python3.7/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection
    compat.reraise(excValue, excTraceback)
  File "/usr/local/lib/python3.7/dist-packages/twisted/python/compat.py", line 464, in reraise
    raise exception.with_traceback(traceback)
  File "/usr/local/lib/python3.7/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection
    result = func(conn, *args, **kw)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/_base.py", line 549, in inner_func
    return func(conn, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/_base.py", line 421, in _new_transaction
    r = func(txn, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/synapse/storage/data_stores/main/events_bg_updates.py", line 533, in _event_store_labels_txn
    event_json = json.loads(event_json_raw)
  File "/usr/local/lib/python3.7/dist-packages/simplejson/__init__.py", line 525, in loads
    return _default_decoder.decode(s)
  File "/usr/local/lib/python3.7/dist-packages/simplejson/decoder.py", line 370, in decode
    obj, end = self.raw_decode(s)
  File "/usr/local/lib/python3.7/dist-packages/simplejson/decoder.py", line 400, in raw_decode
    return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Invalid \X escape sequence '\\': line 1 column 636 (char 635)
```

### Version information

<!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->

<!-- Was this issue identified on matrix.org or another homeserver? -->
- **Homeserver**: matrix.vgorcum.com

<!--
 What version of Synapse is running?

You can find the Synapse version with this command:

$ curl http://localhost:8008/_synapse/admin/v1/server_version

(You may need to replace `localhost:8008` if Synapse is not configured to
listen on that port.)
-->
- **Version**: 1.5.1 and 1.6.0rc1

- **Install method**: [mvgorcum/docker-matrix](https://hub.docker.com/r/mvgorcum/docker-matrix/tags). Installed through `pip3 install --upgrade .[all]`, after `git clone --branch release-v1.6.0 --depth 1 https://github.com/matrix-org/synapse.git` 
<!-- examples: package manager/git clone/pip  -->

- **Platform**:
<!--
Tell us about the environment in which your homeserver is operating
distro, hardware, if it's running in a vm/container, etc.
-->
Debian buster, docker 19.03.5, psql (PostgreSQL) 11.5 (Debian 11.5-1+deb10u1)



Discussion:

Resolving patch:
diff --git a/synapse/storage/data_stores/main/events_bg_updates.py b/synapse/storage/data_stores/main/events_bg_updates.py
--- a/synapse/storage/data_stores/main/events_bg_updates.py
+++ b/synapse/storage/data_stores/main/events_bg_updates.py
@@ -530,24 +530,31 @@ def _event_store_labels_txn(txn):
             nbrows = 0
             last_row_event_id = ""
             for (event_id, event_json_raw) in results:
-                event_json = json.loads(event_json_raw)
-
-                self._simple_insert_many_txn(
-                    txn=txn,
-                    table="event_labels",
-                    values=[
-                        {
-                            "event_id": event_id,
-                            "label": label,
-                            "room_id": event_json["room_id"],
-                            "topological_ordering": event_json["depth"],
-                        }
-                        for label in event_json["content"].get(
-                            EventContentFields.LABELS, []
-                        )
-                        if isinstance(label, str)
-                    ],
-                )
+                try:
+                    event_json = json.loads(event_json_raw)
+
+                    self._simple_insert_many_txn(
+                        txn=txn,
+                        table="event_labels",
+                        values=[
+                            {
+                                "event_id": event_id,
+                                "label": label,
+                                "room_id": event_json["room_id"],
+                                "topological_ordering": event_json["depth"],
+                            }
+                            for label in event_json["content"].get(
+                                EventContentFields.LABELS, []
+                            )
+                            if isinstance(label, str)
+                        ],
+                    )
+                except Exception as e:
+                    logger.warning(
+                        "Unable to load event %s (no labels will be imported): %s",
+                        event_id,
+                        e,
+                    )
 
                 nbrows += 1
                 last_row_event_id = event_id


Test patch: