Repository: raiden-network/raiden (language: python)

Issue:
OSError: [Errno 24] Too many open files
```
raiden-v3.0.0rc0-linux-x86_64 version
{
  "raiden": "3.0.0rc0",
  "raiden_db_version": 27,
  "python_implementation": "CPython",
  "python_version": "3.7.11",
  "system": "Linux 64bit_ELF 5.13.10-arch1-1",
  "architecture": "x86_64",
  "distribution": "bundled"
}
```

## Problem Definition

While testing RC locally, I noticed it had shut down with `Errno 24`. Here are the application logs from the failure, starting from the earliest occurrence of the respective stack trace:
```
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"offer\", \"sdp\": \"v=0\\r\\no=- 3838870061 3838870061 IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\na=group:BUNDLE 0\\r\\na=msid-semantic:WMS *\\r\\nm=application 9 DTLS/SCTP 5000\\r\\nc=IN IP4 0.0.0.0\\r\\na=mid:0\\r\\na=sctpmap:5000 webrtc-datachannel 65535\\r\\na=max-message-size:65536\\r\\na=ice-ufrag:CBsi\\r\\na=ice-pwd:rylGhuOMAS6sDwPeqjAH6n\\r\\na=fingerprint:sha-256 24:FE:F9:B7:73:C0:7D:B7:59:38:C1:92:0E:77:C2:CB:09:E5:09:58:C9:EF:40:02:1E:19:57:8B:78:D2:BD:DC\\r\\na=setup:actpass\\r\\n\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881261.3405204\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:41.379900"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59037.237065})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:47:41.548419"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"hangup\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881231.508647\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:47:41.548952"}
Traceback (most recent call last):
  File "gevent/threadpool.py", line 167, in __run_task
OSError: [Errno 24] Too many open files
2021-08-25T08:47:41Z (<ThreadPoolWorker at 0x7f2667d86650 thread_ident=0x7f2661208640 threadpool-hub=<Hub at 0x7f266b763940 thread_ident=0x7f267419bb80>>, <built-in function gethostbyname>) failed with OSError

{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59037.280165})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:41.590947"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"offer\", \"sdp\": \"v=0\\r\\no=- 3838870061 3838870061 IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\na=group:BUNDLE 0\\r\\na=msid-semantic:WMS *\\r\\nm=application 9 DTLS/SCTP 5000\\r\\nc=IN IP4 0.0.0.0\\r\\na=mid:0\\r\\na=sctpmap:5000 webrtc-datachannel 65535\\r\\na=max-message-size:65536\\r\\na=ice-ufrag:1hfq\\r\\na=ice-pwd:Fhok9SzE6Alo3WrPB2FdPN\\r\\na=fingerprint:sha-256 B8:B8:C1:91:A3:6C:44:E2:22:25:CB:6A:85:F8:AF:E7:E9:48:87:FA:37:C7:2F:ED:07:0C:61:DF:E9:C1:4D:B2\\r\\na=setup:actpass\\r\\n\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881261.5497177\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:41.591280"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59037.369895})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:41.681227"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"candidates\", \"candidates\": [{\"candidate\": \"candidate:7d55b391b134bfe141365929d1c8313b 1 udp 2130706431 192.168.178.6 59771 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:a8b1877ef9990915b58aebbba364d5ae 1 udp 2130706431 2001:16b8:4673:8200:29aa:14ee:410c:cb92 46992 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:49c33e787d306b4426a76a34d1d432a6 1 udp 2130706431 fdf8:7fc9:a491::d9c 50522 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6230c530838cecb82504ec86edd992b7 1 udp 2130706431 fdf8:7fc9:a491::da2 52738 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6451508d0462784bc3174d5d7827fcbb 1 udp 2130706431 fdf8:7fc9:a491:0:fb9d:2a2d:cfbc:a2f0 56875 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:85ada7af61c4408bec84f419d62b2577 1 udp 2130706431 192.168.178.47 34645 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fe9f175acc8b9d4d80f75da4f9361f17 1 udp 2130706431 2001:16b8:4673:8200:d329:c3b6:3553:be87 48162 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:781197854a35c8d3c846272c3d5b4e91 1 udp 2130706431 fdf8:7fc9:a491::aba 58337 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:42497068085741913fe8e0d7964bbdac 1 udp 2130706431 fdf8:7fc9:a491:0:b1e5:de0e:361b:97d7 36863 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fdfb0a5ba6c06b3a5a99f34847055ace 1 udp 2130706431 10.104.7.11 35038 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:63cd8457f927b2d8778292d0a5769ff5 1 udp 1694498815 87.123.193.49 24041 typ srflx raddr 192.168.178.6 rport 59771\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:31fc10d8fb076fe57afaa5c86fcfd826 1 udp 1694498815 87.123.193.49 24042 typ srflx raddr 192.168.178.47 rport 34645\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}], \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881261.5497177\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:41.681782"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:43] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001653", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:43.140893"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:44] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.004252", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:44.143132"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:45] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002027", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:45.196191"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:45] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001365", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:45.198275"}
{"remaining_blocks_to_sync": 0, "blocks_per_second": 0.06465789692351209, "to_block": 5380667, "elapsed": 15.466014942969196, "event": "Synchronizing blockchain events", "logger": "raiden.raiden_service", "level": "info", "greenlet_name": "AlarmTask._run node:0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "timestamp": "2021-08-25 08:47:45.964798"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:46] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014577", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:46.209166"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:46] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.151681", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:46.361527"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59042.08903})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:46.399968"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"candidates\", \"candidates\": [{\"candidate\": \"candidate:7d55b391b134bfe141365929d1c8313b 1 udp 2130706431 192.168.178.6 39926 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:a8b1877ef9990915b58aebbba364d5ae 1 udp 2130706431 2001:16b8:4673:8200:29aa:14ee:410c:cb92 55650 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:49c33e787d306b4426a76a34d1d432a6 1 udp 2130706431 fdf8:7fc9:a491::d9c 50237 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6230c530838cecb82504ec86edd992b7 1 udp 2130706431 fdf8:7fc9:a491::da2 51561 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6451508d0462784bc3174d5d7827fcbb 1 udp 2130706431 fdf8:7fc9:a491:0:fb9d:2a2d:cfbc:a2f0 48082 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:85ada7af61c4408bec84f419d62b2577 1 udp 2130706431 192.168.178.47 33872 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fe9f175acc8b9d4d80f75da4f9361f17 1 udp 2130706431 2001:16b8:4673:8200:d329:c3b6:3553:be87 36278 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:781197854a35c8d3c846272c3d5b4e91 1 udp 2130706431 fdf8:7fc9:a491::aba 38855 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:42497068085741913fe8e0d7964bbdac 1 udp 2130706431 fdf8:7fc9:a491:0:b1e5:de0e:361b:97d7 34492 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fdfb0a5ba6c06b3a5a99f34847055ace 1 udp 2130706431 10.104.7.11 57316 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:63cd8457f927b2d8778292d0a5769ff5 1 udp 1694498815 87.123.193.49 24039 typ srflx raddr 192.168.178.6 rport 39926\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:31fc10d8fb076fe57afaa5c86fcfd826 1 udp 1694498815 87.123.193.49 23937 typ srflx raddr 192.168.178.47 rport 33872\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}], \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881261.3405204\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:47:46.400274"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:48] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001129", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:48.197714"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:49] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.007799", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:49.205282"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:50] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002029", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:50.251973"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:50] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001354", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:50.254044"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:51] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.008101", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:51.259953"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:51] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.137225", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:51.397583"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:53] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001172", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:53.252157"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:54] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.001922", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:54.252207"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:55] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001509", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:55.307866"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:55] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.000695", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:55.308944"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:56] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014760", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:56.320734"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:56] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.150918", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:56.472356"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:58] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001147", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:58.308744"}
{"event": "127.0.0.1 - - [2021-08-25 10:47:59] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.001914", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:47:59.308884"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:00] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001375", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:00.364047"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:00] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.000719", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:00.365148"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:01] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014577", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:01.376607"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:01] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.150434", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:01.527738"}
{"remaining_blocks_to_sync": 0, "blocks_per_second": 0.06404327503495538, "to_block": 5380668, "elapsed": 15.61444194498472, "event": "Synchronizing blockchain events", "logger": "raiden.raiden_service", "level": "info", "greenlet_name": "AlarmTask._run node:0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "timestamp": "2021-08-25 08:48:01.579639"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:03] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001438", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:03.365470"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:05] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002789", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:05.370992"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:06] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002012", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:06.371313"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:06] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001359", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:06.373400"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:07] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014861", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:07.385153"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:07] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.150128", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:07.535974"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:08] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001257", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:08.377469"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:10] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.001699", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:10.381153"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59067.070759})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:11.382460"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"hangup\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881261.3405204\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:11.383099"}
Traceback (most recent call last):
  File "gevent/threadpool.py", line 167, in __run_task
OSError: [Errno 24] Too many open files
2021-08-25T08:48:11Z (<ThreadPoolWorker at 0x7f2667d86650 thread_ident=0x7f2661208640 threadpool-hub=<Hub at 0x7f266b763940 thread_ident=0x7f267419bb80>>, <built-in function gethostbyname>) failed with OSError

{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59067.11496})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.425536"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"offer\", \"sdp\": \"v=0\\r\\no=- 3838870091 3838870091 IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\na=group:BUNDLE 0\\r\\na=msid-semantic:WMS *\\r\\nm=application 9 DTLS/SCTP 5000\\r\\nc=IN IP4 0.0.0.0\\r\\na=mid:0\\r\\na=sctpmap:5000 webrtc-datachannel 65535\\r\\na=max-message-size:65536\\r\\na=ice-ufrag:zwpq\\r\\na=ice-pwd:EnWsE5gZhZimBya7afa6NX\\r\\na=fingerprint:sha-256 CD:9C:D1:81:D4:AF:00:18:23:B1:95:12:1C:DE:2D:04:49:5A:3C:EA:CD:D4:8A:38:90:16:8E:FC:5D:0A:F1:3B\\r\\na=setup:actpass\\r\\n\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881291.3838077\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.425885"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:11] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001091", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:11.434490"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:11] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.000696", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:11.435564"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59067.27012})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.580776"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"candidates\", \"candidates\": [{\"candidate\": \"candidate:7d55b391b134bfe141365929d1c8313b 1 udp 2130706431 192.168.178.6 36866 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:a8b1877ef9990915b58aebbba364d5ae 1 udp 2130706431 2001:16b8:4673:8200:29aa:14ee:410c:cb92 56475 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:49c33e787d306b4426a76a34d1d432a6 1 udp 2130706431 fdf8:7fc9:a491::d9c 57923 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6230c530838cecb82504ec86edd992b7 1 udp 2130706431 fdf8:7fc9:a491::da2 58832 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6451508d0462784bc3174d5d7827fcbb 1 udp 2130706431 fdf8:7fc9:a491:0:fb9d:2a2d:cfbc:a2f0 42859 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:85ada7af61c4408bec84f419d62b2577 1 udp 2130706431 192.168.178.47 57007 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fe9f175acc8b9d4d80f75da4f9361f17 1 udp 2130706431 2001:16b8:4673:8200:d329:c3b6:3553:be87 51781 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:781197854a35c8d3c846272c3d5b4e91 1 udp 2130706431 fdf8:7fc9:a491::aba 58373 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:42497068085741913fe8e0d7964bbdac 1 udp 2130706431 fdf8:7fc9:a491:0:b1e5:de0e:361b:97d7 37548 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fdfb0a5ba6c06b3a5a99f34847055ace 1 udp 2130706431 10.104.7.11 39527 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:63cd8457f927b2d8778292d0a5769ff5 1 udp 1694498815 87.123.193.49 24049 typ srflx raddr 192.168.178.6 rport 36866\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:31fc10d8fb076fe57afaa5c86fcfd826 1 udp 1694498815 87.123.193.49 24048 typ srflx raddr 192.168.178.47 rport 57007\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}], \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881291.3838077\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.581100"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59067.415621})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:11.725454"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"hangup\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881261.5497177\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:11.725757"}
Traceback (most recent call last):
  File "gevent/threadpool.py", line 167, in __run_task
OSError: [Errno 24] Too many open files
2021-08-25T08:48:11Z (<ThreadPoolWorker at 0x7f2667d86b90 thread_ident=0x7f26537fe640 threadpool-hub=<Hub at 0x7f266b763940 thread_ident=0x7f267419bb80>>, <built-in function gethostbyname>) failed with OSError

{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59067.457003})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.767112"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"offer\", \"sdp\": \"v=0\\r\\no=- 3838870091 3838870091 IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\na=group:BUNDLE 0\\r\\na=msid-semantic:WMS *\\r\\nm=application 9 DTLS/SCTP 5000\\r\\nc=IN IP4 0.0.0.0\\r\\na=mid:0\\r\\na=sctpmap:5000 webrtc-datachannel 65535\\r\\na=max-message-size:65536\\r\\na=ice-ufrag:yu0G\\r\\na=ice-pwd:p0ThxDO1M6lcaEeIL1qXIo\\r\\na=fingerprint:sha-256 8B:15:49:2F:6D:63:A4:1E:6C:E5:2F:35:52:A1:E6:F5:05:55:27:9B:D1:E0:E1:91:DF:07:AB:01:91:39:51:8A\\r\\na=setup:actpass\\r\\n\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881291.7262049\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.767474"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59067.576259})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.886058"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"candidates\", \"candidates\": [{\"candidate\": \"candidate:7d55b391b134bfe141365929d1c8313b 1 udp 2130706431 192.168.178.6 41128 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:a8b1877ef9990915b58aebbba364d5ae 1 udp 2130706431 2001:16b8:4673:8200:29aa:14ee:410c:cb92 39415 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:49c33e787d306b4426a76a34d1d432a6 1 udp 2130706431 fdf8:7fc9:a491::d9c 45368 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6230c530838cecb82504ec86edd992b7 1 udp 2130706431 fdf8:7fc9:a491::da2 39977 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:6451508d0462784bc3174d5d7827fcbb 1 udp 2130706431 fdf8:7fc9:a491:0:fb9d:2a2d:cfbc:a2f0 46935 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:85ada7af61c4408bec84f419d62b2577 1 udp 2130706431 192.168.178.47 54159 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fe9f175acc8b9d4d80f75da4f9361f17 1 udp 2130706431 2001:16b8:4673:8200:d329:c3b6:3553:be87 57022 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:781197854a35c8d3c846272c3d5b4e91 1 udp 2130706431 fdf8:7fc9:a491::aba 56077 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:42497068085741913fe8e0d7964bbdac 1 udp 2130706431 fdf8:7fc9:a491:0:b1e5:de0e:361b:97d7 54112 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:fdfb0a5ba6c06b3a5a99f34847055ace 1 udp 2130706431 10.104.7.11 48708 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:63cd8457f927b2d8778292d0a5769ff5 1 udp 1694498815 87.123.193.49 24052 typ srflx raddr 192.168.178.6 rport 41128\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:31fc10d8fb076fe57afaa5c86fcfd826 1 udp 1694498815 87.123.193.49 24053 typ srflx raddr 192.168.178.47 rport 54159\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}], \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881291.7262049\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:11.886400"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:13] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014782", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:13.454915"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:13] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001289", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:13.456956"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:13] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.150911", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:13.608500"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:15] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.001829", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:15.389721"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:16] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001998", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:16.497124"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:16] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001388", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:16.499239"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:18] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.010726", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:18.507182"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:18] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001298", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:18.509217"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:18] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.150190", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:18.660045"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:20] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.001806", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:20.498320"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:21] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001305", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:21.555544"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:21] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.000717", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:21.556638"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:24] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.012614", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:24.385736"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:24] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.000690", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:24.386848"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:24] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.137066", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:24.524290"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:25] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002054", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:25.506875"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:26] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001369", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:26.609366"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:26] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001131", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:26.611072"}
{"remaining_blocks_to_sync": 0, "blocks_per_second": 0.0773191412145807, "to_block": 5380670, "elapsed": 25.86681601195596, "event": "Synchronizing blockchain events", "logger": "raiden.raiden_service", "level": "info", "greenlet_name": "AlarmTask._run node:0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "timestamp": "2021-08-25 08:48:27.446712"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:29] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014711", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:29.407452"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:29] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.149547", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:29.557714"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:30] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001344", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:30.393812"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:31] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.001720", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:31.393928"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:32] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002016", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:32.394968"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:32] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001354", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:32.397045"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:34] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.011679", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:34.426804"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:34] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.139362", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:34.566604"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:35] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001364", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:35.416274"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:36] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002564", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:36.415995"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:37] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.001949", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:37.447817"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:37] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001404", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:37.450023"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:39] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014759", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:39.461146"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:39] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.152828", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:39.614723"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:40] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.006674", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:40.453505"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59097.116963})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:41.427532"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"hangup\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881291.3838077\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:41.428196"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491::d9c - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.436885"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491::da2 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.437207"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491:0:fb9d:2a2d:cfbc:a2f0 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.437495"}
{"event": "Connection(200) Could not bind to 192.168.178.47 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.437783"}
{"event": "Connection(200) Could not bind to 2001:16b8:4673:8200:d329:c3b6:3553:be87 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438088"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491::aba - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438375"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491:0:b1e5:de0e:361b:97d7 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438656"}
{"event": "Connection(200) Could not bind to 10.104.7.11 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438937"}

Traceback (most recent call last):
  File "gevent/threadpool.py", line 167, in __run_task
OSError: [Errno 24] Too many open files
2021-08-25T08:48:41Z (<ThreadPoolWorker at 0x7f2667d86650 thread_ident=0x7f2661208640 threadpool-hub=<Hub at 0x7f266b763940 thread_ident=0x7f267419bb80>>, <built-in function gethostbyname>) failed with OSError

{"event": "127.0.0.1 - - [2021-08-25 10:48:41] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002587", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:41.449377"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59097.160814})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:41.470546"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"offer\", \"sdp\": \"v=0\\r\\no=- 3838870121 3838870121 IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\na=group:BUNDLE 0\\r\\na=msid-semantic:WMS *\\r\\nm=application 9 DTLS/SCTP 5000\\r\\nc=IN IP4 0.0.0.0\\r\\na=mid:0\\r\\na=sctpmap:5000 webrtc-datachannel 65535\\r\\na=max-message-size:65536\\r\\na=ice-ufrag:5NFO\\r\\na=ice-pwd:plhw1LfX2yoPNJsaGi5I3z\\r\\na=fingerprint:sha-256 42:DE:2C:74:4F:D2:21:D6:0D:3E:D2:01:E0:D3:9E:7D:BA:A4:4B:20:16:77:F9:C6:AF:14:69:63:16:7E:00:FA\\r\\na=setup:actpass\\r\\n\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881321.4288278\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:41.470963"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59097.168976})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:41.518839"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9", "send_medium": "toDevice", "data": "{\"type\": \"candidates\", \"candidates\": [{\"candidate\": \"candidate:7d55b391b134bfe141365929d1c8313b 1 udp 2130706431 192.168.178.6 46538 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}, {\"candidate\": \"candidate:a8b1877ef9990915b58aebbba364d5ae 1 udp 2130706431 2001:16b8:4673:8200:29aa:14ee:410c:cb92 53446 typ host\", \"sdpMid\": \"0\", \"sdpMLineIndex\": 0}], \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9|1629881321.4288278\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:41.519302"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59097.456919})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:41.767341"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"hangup\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881291.7262049\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "greenlet_name": "Greenlet<fn:_wrapped_initialize_web_rtc>", "timestamp": "2021-08-25 08:48:41.767909"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59097.500034})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:41.810219"}
{"current_user": "@0x0af26c3071b4a85191817ca3346883a11d3abfdf:transport.transport05.raiden.network", "node": "0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "transport_uuid": "80a1e4a7-ebfc-45b1-9ed2-4b5aae54bef3", "receiver": "0xaBb2a51FA215D5F866C370F224879e9A331c3a85", "send_medium": "toDevice", "data": "{\"type\": \"offer\", \"sdp\": \"v=0\\r\\no=- 3838870121 3838870121 IN IP4 0.0.0.0\\r\\ns=-\\r\\nt=0 0\\r\\na=group:BUNDLE 0\\r\\na=msid-semantic:WMS *\\r\\nm=application 9 DTLS/SCTP 5000\\r\\nc=IN IP4 0.0.0.0\\r\\na=mid:0\\r\\na=sctpmap:5000 webrtc-datachannel 65535\\r\\na=max-message-size:65536\\r\\na=ice-ufrag:3njK\\r\\na=ice-pwd:grosAyPVbnnNRaDgBsAMKs\\r\\na=fingerprint:sha-256 10:95:A2:74:2D:AA:9D:F2:4F:E9:53:C7:CE:1B:DC:66:51:B8:09:C7:AC:B1:47:05:30:F7:67:94:B0:81:87:18\\r\\na=setup:actpass\\r\\n\", \"call_id\": \"0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF|0xaBb2a51FA215D5F866C370F224879e9A331c3a85|1629881321.7687073\"}", "event": "Cannot send raw message without address metadata.", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:41.810566"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:42] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002063", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:42.502200"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:42] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001451", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:42.504386"}
{"remaining_blocks_to_sync": 0, "blocks_per_second": 0.0646652832543159, "to_block": 5380671, "elapsed": 15.464248352043796, "event": "Synchronizing blockchain events", "logger": "raiden.raiden_service", "level": "info", "greenlet_name": "AlarmTask._run node:0x0aF26c3071B4a85191817Ca3346883a11d3ABfDF", "timestamp": "2021-08-25 08:48:42.911216"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:44] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014725", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:44.514911"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:44] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.152097", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:44.667790"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:45] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.006644", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:45.506395"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:46] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002308", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:46.502630"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:47] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002021", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:47.557142"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:47] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001416", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:47.559268"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:49] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014695", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:49.570980"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:49] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.150372", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:49.722076"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:50] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.006768", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:50.564409"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:51] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002720", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:51.558406"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:52] \"GET /api/v1/tokens HTTP/1.1\" 200 228 0.002036", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:52.612218"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:52] \"GET /api/v1/connections HTTP/1.1\" 200 241 0.001369", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:52.614364"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:54] \"GET /api/v1/payments?offset=362 HTTP/1.1\" 200 137 0.014723", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:54.626677"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:54] \"GET /api/v1/payments HTTP/1.1\" 200 93099 0.156814", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:54.784191"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:55] \"GET /api/v1/pending_transfers HTTP/1.1\" 200 548 0.001681", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:55.611826"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:56] \"GET /api/v1/channels HTTP/1.1\" 200 965 0.002713", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:56.612773"}
Traceback (most recent call last):
  File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
  File "raiden/tasks.py", line 99, in check_gas_reserve
  File "raiden/utils/gas_reserve.py", line 136, in has_enough_gas_reserve
  File "raiden/utils/gas_reserve.py", line 114, in get_reserve_estimate
  File "raiden/utils/gas_reserve.py", line 103, in get_required_gas_estimate
  File "raiden_contracts/contract_manager.py", line 146, in gas_measurements
  File "pathlib.py", line 1208, in open
  File "pathlib.py", line 1063, in _opener
OSError: [Errno 24] Too many open files: '/tmp/_MEIWmPD2w/data_0.39.0/gas.json'
2021-08-25T08:48:56Z <Greenlet "check_gas_reserve" at 0x7f2661aa2ef0: check_gas_reserve(<RaidenService node:0x0aF26c3071B4a85191817Ca33468)> failed with OSError

{"errors": "Raiden API is shutting down", "status_code": 503, "event": "Error processing request", "logger": "raiden.api.rest_utils", "level": "error", "timestamp": "2021-08-25 08:48:57.664671"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:57] \"GET /api/v1/tokens HTTP/1.1\" 503 194 0.001189", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:57.665670"}
{"errors": "Raiden API is shutting down", "status_code": 503, "event": "Error processing request", "logger": "raiden.api.rest_utils", "level": "error", "timestamp": "2021-08-25 08:48:57.666428"}
{"event": "127.0.0.1 - - [2021-08-25 10:48:57] \"GET /api/v1/connections HTTP/1.1\" 503 194 0.000822", "logger": "raiden.api.rest.pywsgi", "level": "info", "timestamp": "2021-08-25 08:48:57.667198"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0x2ac648Bc29F9b9FCdd235433B8dACd6E42cCF1c9', 'seen_offline_since': 59113.501818})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:57.812325"}
{"event": "('There is no user found online for given address. (PFS error code: 2203)', {'address': '0xaBb2a51FA215D5F866C370F224879e9A331c3a85', 'seen_offline_since': 59113.54084})", "logger": "raiden.network.transport.matrix.transport", "level": "error", "timestamp": "2021-08-25 08:48:57.895009"}
{"event": "Task exception was never retrieved\nfuture: <Task finished coro=<_RTCConnection._set_local_description() done, defined at raiden/network/transport/matrix/rtc/web_rtc.py:214> exception=OSError(24, 'Too many open files')>", "logger": "asyncio", "level": "error", "timestamp": "2021-08-25 08:48:57.898999", "exception": "Traceback (most recent call last):\n  File \"raiden/network/transport/matrix/rtc/web_rtc.py\", line 216, in _set_local_description\n  File \"raiden/network/transport/matrix/rtc/web_rtc.py\", line 198, in _try_signaling\n  File \"aiortc/rtcpeerconnection.py\", line 771, in setLocalDescription\n  File \"aiortc/rtcpeerconnection.py\", line 1011, in __gather\n  File \"aiortc/rtcicetransport.py\", line 198, in gather\n  File \"aioice/ice.py\", line 410, in gather_candidates\n  File \"aioice/ice.py\", line 59, in get_host_addresses\nOSError: [Errno 24] Too many open files"}
Traceback (most recent call last):
  File "raiden/ui/cli.py", line 721, in _run
  File "raiden/ui/runners.py", line 105, in run_services
  File "src/gevent/greenlet.py", line 1057, in gevent._gevent_cgreenlet.joinall
  File "src/gevent/greenlet.py", line 1073, in gevent._gevent_cgreenlet.joinall
  File "src/gevent/greenlet.py", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception
  File "gevent/_compat.py", line 65, in reraise
  File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
  File "raiden/tasks.py", line 99, in check_gas_reserve
  File "raiden/utils/gas_reserve.py", line 136, in has_enough_gas_reserve
  File "raiden/utils/gas_reserve.py", line 114, in get_reserve_estimate
  File "raiden/utils/gas_reserve.py", line 103, in get_required_gas_estimate
  File "raiden_contracts/contract_manager.py", line 146, in gas_measurements
  File "pathlib.py", line 1208, in open
  File "pathlib.py", line 1063, in _opener
OSError: [Errno 24] Too many open files: '/tmp/_MEIWmPD2w/data_0.39.0/gas.json'
FATAL: An unexpected exception occurred. A traceback has been written to /tmp/raiden-exception-2021-08-25T08-489lkbqx8f.txt
[Errno 24] Too many open files: '/tmp/_MEIWmPD2w/data_0.39.0/gas.json'
ReturnCode.FATAL
```

```
cat raiden-exception-2021-08-25T08-489lkbqx8f.txt
Traceback (most recent call last):
  File "raiden/ui/cli.py", line 721, in _run
  File "raiden/ui/runners.py", line 105, in run_services
  File "src/gevent/greenlet.py", line 1057, in gevent._gevent_cgreenlet.joinall
  File "src/gevent/greenlet.py", line 1073, in gevent._gevent_cgreenlet.joinall
  File "src/gevent/greenlet.py", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception
  File "gevent/_compat.py", line 65, in reraise
  File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
  File "raiden/tasks.py", line 99, in check_gas_reserve
  File "raiden/utils/gas_reserve.py", line 136, in has_enough_gas_reserve
  File "raiden/utils/gas_reserve.py", line 114, in get_reserve_estimate
  File "raiden/utils/gas_reserve.py", line 103, in get_required_gas_estimate
  File "raiden_contracts/contract_manager.py", line 146, in gas_measurements
  File "pathlib.py", line 1208, in open
  File "pathlib.py", line 1063, in _opener
OSError: [Errno 24] Too many open files: '/tmp/_MEIWmPD2w/data_0.39.0/gas.json'
```


Discussion / hints:
Note the `aioice` log messages:
```
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491::d9c - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.436885"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491::da2 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.437207"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491:0:fb9d:2a2d:cfbc:a2f0 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.437495"}
{"event": "Connection(200) Could not bind to 192.168.178.47 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.437783"}
{"event": "Connection(200) Could not bind to 2001:16b8:4673:8200:d329:c3b6:3553:be87 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438088"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491::aba - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438375"}
{"event": "Connection(200) Could not bind to fdf8:7fc9:a491:0:b1e5:de0e:361b:97d7 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438656"}
{"event": "Connection(200) Could not bind to 10.104.7.11 - [Errno 24] Too many open files", "logger": "aioice.ice", "level": "info", "timestamp": "2021-08-25 08:48:41.438937"}
```
I'm monitoring the open file descriptors while running raiden. It seems the number is constantly growing. My machine is configured for a maximum of `1024` per process:
```
ulimit -n
1024
```
Monitoring from start of raiden in 2 second interval:
```
while true;
do
  lsof -p $(pidof raiden-v3.0.0rc0-linux-x86_64|cut -d' ' -f1)|wc -l
  sleep 2
done

0
116
137
136
147
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
173
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
195
217
217
217
217
217
217
217
217
217
217
217
217
217
217
217
[…]
415
415
415
415
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
437
459
```
The increase seems to coincide with log messages of
```
"Cannot send raw message without address metadata."
```
The bunch of these are `UDP`:
```
lsof -p $(pidof raiden-v3.0.0rc0-linux-x86_64|cut -d' ' -f1)|grep UDP|wc -l
528
```
```
lsof -p $(pidof raiden-v3.0.0rc0-linux-x86_64|cut -d' ' -f1)|grep -v UDP|wc -l
151
```
@konradkonrad thanks for the nice analysis!
Related: https://github.com/aiortc/aiortc/issues/535.
> The increase seems to coincide with log messages of
> 
> ```
> "Cannot send raw message without address metadata."
> ```

This points to the health checks.
@ulope found this: https://github.com/aiortc/aioice/blob/26abbb23e485aed9a338208f53b81727c3ca6206/src/aioice/ice.py#L842-L844.

`create_datagram_endpoint` returns a `(transport, protocol)` pair, and `protocol` is put into an internal list, which is only ever cleared when the connection's `close` method is called. `StunProtocol` seems to do the right thing in its own `close` method, i.e. it closes the `transport`.

One possibility for this to go wrong is if somehow the `transport` that is being returned from `create_datagram_endpoint` is not the same `transport` that is being given to `StunProtocol.connection_made`. That would mean the first `transport` and therefore the file descriptor would leak.

Fix patch:
diff --git a/raiden/__init__.py b/raiden/__init__.py
index e69de29bb2..894d771fee 100644
--- a/raiden/__init__.py
+++ b/raiden/__init__.py
@@ -0,0 +1,4 @@
+from . import _monkey
+
+_monkey.patch_all()
+del _monkey
diff --git a/raiden/_monkey.py b/raiden/_monkey.py
new file mode 100644
index 0000000000..c0bc371600
--- /dev/null
+++ b/raiden/_monkey.py
@@ -0,0 +1,33 @@
+import asyncio.selector_events
+
+import aioice.ice
+
+
+def _patch_aioice() -> None:
+    # local_candidate could be None so make sure we check for it before
+    # invoking data_received
+    def StunProtocol__connection_lost(self, exc: Exception) -> None:  # type: ignore
+        self._StunProtocol__log_debug("connection_lost(%s)", exc)
+        if self.local_candidate is not None:
+            self.receiver.data_received(None, self.local_candidate.component)
+        self._StunProtocol__closed.set_result(True)
+
+    aioice.ice.StunProtocol.connection_lost = StunProtocol__connection_lost  # type: ignore
+
+
+def _patch_asyncio() -> None:
+    # don't do anything if the transport is closing/closed
+    def wrap(orig):  # type: ignore
+        def sendto(self, data, addr=None):  # type: ignore
+            if not self._closing:
+                orig(self, data, addr)
+
+        return sendto
+
+    orig = asyncio.selector_events._SelectorDatagramTransport.sendto  # type: ignore
+    asyncio.selector_events._SelectorDatagramTransport.sendto = wrap(orig)  # type: ignore
+
+
+def patch_all() -> None:
+    _patch_aioice()
+    _patch_asyncio()
diff --git a/raiden/network/transport/matrix/rtc/web_rtc.py b/raiden/network/transport/matrix/rtc/web_rtc.py
index fd9c475056..33373faa6e 100644
--- a/raiden/network/transport/matrix/rtc/web_rtc.py
+++ b/raiden/network/transport/matrix/rtc/web_rtc.py
@@ -15,7 +15,7 @@
 from raiden.network.transport.matrix.utils import validate_and_parse_message
 from raiden.utils.formatting import to_checksum_address
 from raiden.utils.runnable import Runnable
-from raiden.utils.typing import Address, Any, Callable, Coroutine, Dict, List, Optional, Set, Union
+from raiden.utils.typing import Address, Any, Callable, Coroutine, Dict, List, Optional, Union
 
 log = structlog.get_logger(__name__)
 
@@ -54,7 +54,6 @@ class _ConnectionState(Enum):
     CLOSED = "closed"
 
 
-_WEB_RTC_CHANNEL_TIMEOUT = 30.0
 _SDP_MID_DEFAULT = "0"
 _SDP_MLINE_INDEX_DEFAULT = 0
 
@@ -142,6 +141,7 @@ def __init__(
         super().__init__()
         self.node_address = node_address
         self.partner_address = partner_address
+        self._closing_task: Optional[Task] = None
         self._call_id = self._make_call_id()
         self._signaling_send = signaling_send
         self._ice_connection_closed = ice_connection_closed
@@ -232,6 +232,9 @@ def _make_call_id(self) -> str:
     async def _initialize_signaling(self) -> Optional[RTCSessionDescription]:
         """Coroutine to create channel. Setting up channel in aiortc"""
 
+        if self._closing_task is not None:
+            return None
+
         self._channel = self.peer_connection.createDataChannel(self.call_id)
         self._set_channel_callbacks()
         offer = await self._try_signaling(self.peer_connection.createOffer())
@@ -251,6 +254,9 @@ async def _process_signaling(
         remote_description = RTCSessionDescription(description["sdp"], description["type"])
         sdp_type = description["type"]
 
+        if self._closing_task is not None:
+            return None
+
         # If we initiated the signaling process, wait until the local
         # description is set.
         if self._initiator_address == self.node_address:
@@ -323,7 +329,7 @@ async def _send_message(self, message: str) -> None:
                 await self.peer_connection.sctp._transmit()
             except ConnectionError:
                 self.log.debug("Connection error occurred while trying to send message")
-                await self._close()
+                self.close()
                 return
         else:
             self.log.debug(
@@ -338,16 +344,19 @@ def send_message(self, message: str) -> None:
 
     async def _close(self) -> None:
         self.log.debug("Closing peer connection")
-        await self.peer_connection.close()
+        await self.wait_for_tasks()
+        await wrap_greenlet(gevent.spawn(gevent.joinall, self._greenlets, raise_error=True))
         if self._channel is not None:
             self._channel.close()
             self._channel = None
-        await self.wait_for_tasks()
-        await wrap_greenlet(gevent.spawn(gevent.joinall, self._greenlets, raise_error=True))
+        await self.peer_connection.close()
+        self.peer_connection = None
         self._ice_connection_closed(self)
 
-    def close(self) -> None:
-        asyncio.create_task(self._close())
+    def close(self) -> Task:
+        if self._closing_task is None:
+            self._closing_task = asyncio.create_task(self._close())
+        return self._closing_task
 
     def _handle_candidates_callback(self, candidates: List[Dict[str, Union[int, str]]]) -> None:
         message = {
@@ -478,8 +487,6 @@ def __init__(
         self._process_messages = process_messages
         self._signaling_send = signaling_send
         self._stop_event = stop_event
-        # the addresses for which we're currently trying to initialize WebRTC channels
-        self._web_rtc_channel_inits: Set[Address] = set()
         self._address_to_connection: Dict[Address, _RTCConnection] = {}
         self.log = log.bind(node=to_checksum_address(node_address))
 
@@ -498,18 +505,17 @@ def _handle_ice_connection_closed(self, conn: _RTCConnection) -> None:
             self.health_check(conn.partner_address)
 
     def _wrapped_initialize_web_rtc(self, address: Address) -> None:
-        if address in self._web_rtc_channel_inits:
-            return
         attempt = 0
         while attempt < 3 and not self.has_ready_channel(address):
-            self._web_rtc_channel_inits.add(address)
-            try:
-                self._initialize_web_rtc(address)
-            finally:
-                self._web_rtc_channel_inits.remove(address)
+            self._initialize_web_rtc(address)
             attempt += 1
 
     def _initialize_web_rtc(self, partner_address: Address) -> None:
+        if partner_address in self._address_to_connection:
+            # don't do anything if signaling is already in progress or the
+            # connection has already been established
+            return
+
         if self._stop_event.is_set():
             return
 
@@ -528,8 +534,8 @@ def _initialize_web_rtc(self, partner_address: Address) -> None:
         self._add_connection(partner_address, conn)
         conn.initialize_signaling()
 
-        # wait for _WEB_RTC_CHANNEL_TIMEOUT seconds and check if connection was established
-        if self._stop_event.wait(timeout=_WEB_RTC_CHANNEL_TIMEOUT):
+        # wait for channel init timeout period and check if connection was established
+        if self._stop_event.wait(timeout=self.get_channel_init_timeout()):
             return
 
         if not self.has_ready_channel(partner_address):
@@ -540,6 +546,11 @@ def _initialize_web_rtc(self, partner_address: Address) -> None:
             conn.send_hangup_message()
             self.close_connection(partner_address)
 
+    def get_channel_init_timeout(self) -> float:
+        """Returns the number of seconds to wait for a channel to be established."""
+        # pylint: disable=no-self-use
+        return 30.0
+
     def _add_connection(self, partner_address: Address, conn: _RTCConnection) -> None:
         assert partner_address not in self._address_to_connection, "must not be there already"
         self._address_to_connection[partner_address] = conn
@@ -569,6 +580,10 @@ def _process_signaling_for_address(
                 else:
                     # drop the offer, it's older than what we have
                     return
+
+            if self._stop_event.is_set():
+                return
+
             conn = _RTCConnection.from_offer(
                 partner_address,
                 self.node_address,
@@ -593,11 +608,12 @@ def health_check(self, partner_address: Address) -> None:
         self._schedule_new_greenlet(self._wrapped_initialize_web_rtc, partner_address)
 
     def close_connection(self, partner_address: Address) -> None:
-        conn = self._address_to_connection.pop(partner_address, None)
+        conn = self._address_to_connection.get(partner_address)
         if conn is not None:
-            conn.close()
+            yield_future(conn.close())
+            conn.wait()
 
-    def process_signaling_message(
+    def _process_signaling_message(
         self, partner_address: Address, rtc_message_type: str, content: Dict[str, str]
     ) -> None:
         if (
@@ -616,6 +632,13 @@ def process_signaling_message(
                 type=rtc_message_type,
             )
 
+    def process_signaling_message(
+        self, partner_address: Address, rtc_message_type: str, content: Dict[str, str]
+    ) -> None:
+        self._schedule_new_greenlet(
+            self._process_signaling_message, partner_address, rtc_message_type, content
+        )
+
     def stop(self) -> None:
         self.log.debug("Closing WebRTC connections")
 
diff --git a/raiden/raiden_service.py b/raiden/raiden_service.py
index e4efea709d..52b445a7ff 100644
--- a/raiden/raiden_service.py
+++ b/raiden/raiden_service.py
@@ -506,7 +506,7 @@ def _run(self, *args: Any, **kwargs: Any) -> None:  # pylint: disable=method-hid
             self.stop_event.wait()
         except gevent.GreenletExit:  # killed without exception
             self.stop_event.set()
-            gevent.killall([self.alarm, self.transport])  # kill children
+            gevent.killall([self.alarm.greenlet, self.transport.greenlet])  # kill children
             raise  # re-raise to keep killed status
         except Exception:
             self.stop()


Test patch:
diff --git a/raiden/tests/unit/network/rtc/test_web_rtc.py b/raiden/tests/unit/network/rtc/test_web_rtc.py
index 4490bbc874..76f08f2f83 100644
--- a/raiden/tests/unit/network/rtc/test_web_rtc.py
+++ b/raiden/tests/unit/network/rtc/test_web_rtc.py
@@ -1,6 +1,8 @@
 import collections
 import json
-from typing import List
+import os
+import sys
+from typing import Any, List
 
 import gevent
 import pytest
@@ -66,3 +68,53 @@ def _send_to_2(*args, **kwargs) -> None:  # type: ignore
 
     while manager1.has_ready_channel(address2):
         gevent.sleep(0.1)
+
+    stop_event1.set()
+    manager1.stop()
+    stop_event2.set()
+    manager2.stop()
+
+
+def _get_file_descriptors() -> List:
+    pid = os.getpid()
+    return os.listdir(f"/proc/{pid}/fd/")
+
+
+@pytest.mark.skipif(sys.platform != "linux", reason="this is a Linux-only test")
+@pytest.mark.skipif(sys.version_info < (3, 8), reason="it seems we leak on Python < 3.8")
+def test_leak_file_descriptors(monkeypatch: Any) -> None:
+    address1 = make_signer().address
+    address2 = make_signer().address
+    stop_event1 = Event()
+    stop_event2 = Event()
+
+    def _noop_send(*args, **kwargs) -> None:  # type: ignore
+        pass
+
+    manager1 = WebRTCManager(address1, _ignore_web_rtc_messages, _noop_send, stop_event1)
+    manager2 = WebRTCManager(address2, _ignore_web_rtc_messages, _noop_send, stop_event2)
+
+    assert not manager1.has_ready_channel(address2), "no channel should exist yet"
+    assert not manager2.has_ready_channel(address1), "no channel should exist yet"
+
+    # reduce the channel init timeout so that the test does not take too long
+    monkeypatch.setattr(manager1, "get_channel_init_timeout", lambda *_: 3.0)
+
+    max_open_fds = 0
+    for x in range(20):
+        manager1.health_check(address2)
+        gevent.sleep(manager1.get_channel_init_timeout())
+        assert not manager1.has_ready_channel(address2), "must not be ready"
+        fds = _get_file_descriptors()
+        count = len(fds)
+        if x < 4:
+            # a few rounds should be enough to get a good upper bound on the
+            # number of open file descriptors
+            max_open_fds = max(count, max_open_fds)
+        else:
+            assert count <= max_open_fds, "possible leakage of file descriptors"
+
+    stop_event1.set()
+    manager1.stop()
+    stop_event2.set()
+    manager2.stop()