Repository: canonical/operator

Issue:
Better Error Handling if the pebble socket goes missing
On a Microk8s cloud there were some issues causing containers to get restarted.  Because containers were getting rescheduled, it ended up that during the time that a charm hook was firing, it ultimately failed to stay connected to pebble. The error message that you end up getting is "NoSuchFile" which is hard to interpret as "pebble went away".
The specific traceback was:
```
Traceback (most recent call last):
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/pebble.py", line 1331, in _request_raw
    response = self.opener.open(request, timeout=self.timeout)
  File "/usr/lib/python3.8/urllib/request.py", line 525, in open
    response = self._open(req, data)
  File "/usr/lib/python3.8/urllib/request.py", line 542, in _open
    result = self._call_chain(self.handle_open, protocol, protocol +
  File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain
    result = func(*args)
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/pebble.py", line 84, in http_open
    return self.do_open(_UnixSocketConnection, req, socket_path=self.socket_path)
  File "/usr/lib/python3.8/urllib/request.py", line 1357, in do_open
    raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 2] No such file or directory>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "./src/charm.py", line 777, in <module>
    main(PostgresqlOperatorCharm, use_juju_for_storage=True)
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/main.py", line 429, in main
    framework.reemit()
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/framework.py", line 794, in reemit
    self._reemit()
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/framework.py", line 857, in _reemit
    custom_handler(event)
  File "./src/charm.py", line 381, in _on_postgresql_pebble_ready
    current_layer = container.get_plan()
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/model.py", line 1367, in get_plan
    return self._pebble.get_plan()
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/pebble.py", line 1615, in get_plan
    resp = self._request('GET', '/v1/plan', {'format': 'yaml'})
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/pebble.py", line 1297, in _request
    response = self._request_raw(method, path, query, headers, data)
  File "/var/lib/juju/agents/unit-postgresql-k8s-0/charm/venv/ops/pebble.py", line 1344, in _request_raw
    raise ConnectionError(e.reason)
ops.pebble.ConnectionError: [Errno 2] No such file or directory
```

There is a 'pebble.ConnectionError' in there, but the feedback from others is that it really wasn't clear why this was failing.

It would be nice if we could more clearly articulate this as something like "could not connect to pebble, socket not found at /...". Or even better if we could tell that we *did* have a connection earlier in the process, and could report that we *lost* the connection to pebble.


Discussion:
tonyandrewmeyer: As well as the unit test, I manually tested this (by removing the socket while a long-running action was executing, after `can_connect()`).

<details>
<summary>Previous exception</summary>

```python
unit-uptime-0: 10:35:27 ERROR unit.uptime/0.juju-log Uncaught exception while in charm code:
Traceback (most recent call last):
  File "/usr/lib/python3.10/urllib/request.py", line 1348, in do_open
    h.request(req.get_method(), req.selector, req.data, headers,
  File "/usr/lib/python3.10/http/client.py", line 1283, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/usr/lib/python3.10/http/client.py", line 1329, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.10/http/client.py", line 1278, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/usr/lib/python3.10/http/client.py", line 1038, in _send_output
    self.send(msg)
  File "/usr/lib/python3.10/http/client.py", line 976, in send
    self.connect()
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/pebble.py", line 272, in connect
    self.sock.connect(self.socket_path)
FileNotFoundError: [Errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/pebble.py", line 1599, in _request_raw
    response = self.opener.open(request, timeout=self.timeout)
  File "/usr/lib/python3.10/urllib/request.py", line 519, in open
    response = self._open(req, data)
  File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
    result = self._call_chain(self.handle_open, protocol, protocol +
  File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
    result = func(*args)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/pebble.py", line 286, in http_open
    return self.do_open(_UnixSocketConnection, req,  # type:ignore
  File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
    raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 2] No such file or directory>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/var/lib/juju/agents/unit-uptime-0/charm/./src/charm.py", line 122, in <module>
    ops.main(UptimeCharm)  # type: ignore
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/main.py", line 456, in __call__
    return main(charm_class, use_juju_for_storage=use_juju_for_storage)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/main.py", line 441, in main
    _emit_charm_event(charm, dispatcher.event_name)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/main.py", line 149, in _emit_charm_event
    event_to_emit.emit(*args, **kwargs)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/framework.py", line 342, in emit
    framework._emit(event)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/framework.py", line 839, in _emit
    self._reemit(event_path)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/framework.py", line 928, in _reemit
    custom_handler(event)
  File "/var/lib/juju/agents/unit-uptime-0/charm/./src/charm.py", line 89, in _on_uptime_action
    container.pull("/proc/uptime", encoding="ascii").read().split(None, 1)[0]
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/model.py", line 2156, in pull
    return self._pebble.pull(str(path), encoding=encoding)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/pebble.py", line 1927, in pull
    response = self._request_raw('GET', '/v1/files', query, headers)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/pebble.py", line 1612, in _request_raw
    raise ConnectionError(e.reason)
ops.pebble.ConnectionError: [Errno 2] No such file or directory
```

</details>

The exception correctly showed up in debug-log as:

```python
unit-uptime-0: 10:42:25 ERROR unit.uptime/0.juju-log Uncaught exception while in charm code:
Traceback (most recent call last):
  File "/var/lib/juju/agents/unit-uptime-0/charm/./src/charm.py", line 164, in <module>
    ops.main(UptimeCharm)  # type: ignore
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/main.py", line 456, in __call__
    return main(charm_class, use_juju_for_storage=use_juju_for_storage)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/main.py", line 441, in main
    _emit_charm_event(charm, dispatcher.event_name)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/main.py", line 149, in _emit_charm_event
    event_to_emit.emit(*args, **kwargs)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/framework.py", line 342, in emit
    framework._emit(event)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/framework.py", line 839, in _emit
    self._reemit(event_path)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/framework.py", line 928, in _reemit
    custom_handler(event)
  File "/var/lib/juju/agents/unit-uptime-0/charm/./src/charm.py", line 131, in _on_uptime_action
    container.pull("/proc/uptime", encoding="ascii").read().split(None, 1)[0]
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/model.py", line 2156, in pull
    return self._pebble.pull(str(path), encoding=encoding)
  File "/var/lib/juju/agents/unit-uptime-0/charm/venv/ops/pebble.py", line 1927, in pull
    response = self._request_raw('GET', '/v1/files', query, headers)
  File "/var/lib/juju/agents/unit-uptime-0/charm/./src/charm.py", line 108, in _request_raw
    raise ConnectionError(
ConnectionError: Could not connect to Pebble: socket not found at '/charm/containers/deb2/pebble.socket' (container restarted?)
```

Patch:
diff --git a/ops/pebble.py b/ops/pebble.py
index 3654185..80ce8f6 100644
--- a/ops/pebble.py
+++ b/ops/pebble.py
@@ -1623,6 +1623,10 @@ class Client:
                 message = f'{type(e2).__name__} - {e2}'
             raise APIError(body, code, status, message)
         except urllib.error.URLError as e:
+            if e.args and isinstance(e.args[0], FileNotFoundError):
+                raise ConnectionError(
+                    f"Could not connect to Pebble: socket not found at {self.socket_path!r} "
+                    "(container restarted?)") from None
             raise ConnectionError(e.reason)
 
         return response


Test patch:
diff --git a/test/test_pebble.py b/test/test_pebble.py
index 47e7388..af4937d 100644
--- a/test/test_pebble.py
+++ b/test/test_pebble.py
@@ -2579,6 +2579,7 @@ class TestSocketClient(unittest.TestCase):
         with self.assertRaises(pebble.ConnectionError) as cm:
             client.get_system_info()
         self.assertIsInstance(cm.exception, pebble.Error)
+        self.assertIn("Could not connect to Pebble", str(cm.exception))
 
     def test_real_client(self):
         shutdown, socket_path = fake_pebble.start_server()