Repository: canonical/operator
Pull request #468 (2021-01-20 23:13:44)

Low-level Pebble API client
Basic support for talking to the Pebble API (HTTP over Unix socket). This PR adds the low-level API and unit tests for it (though no functional tests -- still thinking about the best way to do that).

Note that this doesn't integrate the API into the Operator Framework model. We'll want a slightly higher-level layer of abstraction for that, attached to the operator framework model. Still working on the design for that -- coming in a subsequent PR.

To test manually, you can run pebble locally (check out the `operator/pebble` repo, `go build`, and `pebble run`). Then you can run `python3 ops/pebble.py` as a command line client to exercise the Python API. For example:

``` bash
PEBBLE=/pebble/dir python3 ops/pebble.py --help
PEBBLE=/pebble/dir python3 ops/pebble.py changes
```


benhoyt: @facundobatista Thanks a lot for your feedback. I've addressed most of the comments, though there are a couple that I disagree with.

Patch:
diff --git a/ops/pebble.py b/ops/pebble.py
new file mode 100644
index 0000000..e550380
--- /dev/null
+++ b/ops/pebble.py
@@ -0,0 +1,538 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Client for the Pebble API (HTTP over Unix socket)."""
+
+from typing import Any, Dict, List, Optional
+import datetime
+import enum
+import http.client
+import json
+import os
+import re
+import socket
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+
+_not_provided = object()
+
+
+class _UnixSocketConnection(http.client.HTTPConnection):
+    """Implementation of HTTPConnection that connects to a named Unix socket."""
+
+    def __init__(self, host, timeout=_not_provided, socket_path=None):
+        if timeout is _not_provided:
+            super().__init__(host)
+        else:
+            super().__init__(host, timeout=timeout)
+        self.socket_path = socket_path
+
+    def connect(self):
+        """Override connect to use Unix socket (instead of TCP socket)."""
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(self.socket_path)
+        if self.timeout is not _not_provided:
+            self.sock.settimeout(self.timeout)
+
+
+class _UnixSocketHandler(urllib.request.AbstractHTTPHandler):
+    """Implementation of HTTPHandler that uses a named Unix socket."""
+
+    def __init__(self, socket_path):
+        super().__init__()
+        self.socket_path = socket_path
+
+    def http_open(self, req):
+        """Override http_open to use a Unix socket connection (instead of TCP)."""
+        return self.do_open(_UnixSocketConnection, req, socket_path=self.socket_path)
+
+
+# Matches yyyy-mm-ddTHH:MM:SS.sss[-+]zz(:)zz
+_TIMESTAMP_RE = re.compile(
+    r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d+)([-+])(\d{2}):?(\d{2})')
+
+
+def _parse_timestamp(s):
+    """Parse timestamp from Go-encoded JSON (which uses 9 decimal places for seconds)."""
+    match = _TIMESTAMP_RE.match(s)
+    if not match:
+        raise ValueError('invalid timestamp {!r}'.format(s))
+    y, m, d, hh, mm, ss, sub, plus_minus, z1, z2 = match.groups()
+    s = '{y}-{m}-{d}T{hh}:{mm}:{ss}.{sub}{plus_minus}{z1}{z2}'.format(
+        y=y, m=m, d=d, hh=hh, mm=mm, ss=ss, sub=sub[:6],
+        plus_minus=plus_minus, z1=z1, z2=z2,
+    )
+    return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%f%z')
+
+
+class ServiceError(Exception):
+    """Raised by API.wait_change when a service change is ready but has an error."""
+
+    def __init__(self, err, change):
+        super().__init__(err)
+        self.err = err
+        self.change = change
+
+
+class WarningState(enum.Enum):
+    """Enum of states for API.get_warnings() select parameter."""
+
+    ALL = 'all'
+    PENDING = 'pending'
+
+
+class ChangeState(enum.Enum):
+    """Enum of states for API.get_changes() select parameter."""
+
+    ALL = 'all'
+    IN_PROGRESS = 'in-progress'
+    READY = 'ready'
+
+
+class SystemInfo:
+    """System information object."""
+
+    def __init__(self, version: str):
+        self.version = version
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> 'SystemInfo':
+        """Create new object from dict parsed from JSON."""
+        return cls(version=d['version'])
+
+    def __repr__(self):
+        return 'SystemInfo(version={self.version!r})'.format(self=self)
+
+
+class Warning:
+    """Warning object."""
+
+    def __init__(
+        self,
+        message: str,
+        first_added: datetime.datetime,
+        last_added: datetime.datetime,
+        last_shown: Optional[datetime.datetime],
+        expire_after: str,
+        repeat_after: str,
+    ):
+        self.message = message
+        self.first_added = first_added
+        self.last_added = last_added
+        self.last_shown = last_shown
+        self.expire_after = expire_after
+        self.repeat_after = repeat_after
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> 'Warning':
+        """Create new object from dict parsed from JSON."""
+        return cls(
+            message=d['message'],
+            first_added=_parse_timestamp(d['first-added']),
+            last_added=_parse_timestamp(d['last-added']),
+            last_shown=_parse_timestamp(d['last-shown']) if d.get('last-shown') else None,
+            expire_after=d['expire-after'],
+            repeat_after=d['repeat-after'],
+        )
+
+    def __repr__(self):
+        return ('Warning('
+                'message={self.message!r}, '
+                'first_added={self.first_added!r}, '
+                'last_added={self.last_added!r}, '
+                'last_shown={self.last_shown!r}, '
+                'expire_after={self.expire_after!r}, '
+                'repeat_after={self.repeat_after!r})'
+                ).format(self=self)
+
+
+class TaskProgress:
+    """Task progress object."""
+
+    def __init__(
+        self,
+        label: str,
+        done: int,
+        total: int,
+    ):
+        self.label = label
+        self.done = done
+        self.total = total
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> 'TaskProgress':
+        """Create new object from dict parsed from JSON."""
+        return cls(
+            label=d['label'],
+            done=d['done'],
+            total=d['total'],
+        )
+
+    def __repr__(self):
+        return ('TaskProgress('
+                'label={self.label!r}, '
+                'done={self.done!r}, '
+                'total={self.total!r})'
+                ).format(self=self)
+
+
+class TaskID(str):
+    """Task ID (a more strongly-typed string)."""
+
+    def __repr__(self):
+        return 'TaskID({!r})'.format(str(self))
+
+
+class Task:
+    """Task object."""
+
+    def __init__(
+        self,
+        id: TaskID,
+        kind: str,
+        summary: str,
+        status: str,
+        log: List[str],
+        progress: TaskProgress,
+        spawn_time: datetime.datetime,
+        ready_time: Optional[datetime.datetime],
+    ):
+        self.id = id
+        self.kind = kind
+        self.summary = summary
+        self.status = status
+        self.log = log
+        self.progress = progress
+        self.spawn_time = spawn_time
+        self.ready_time = ready_time
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> 'Task':
+        """Create new object from dict parsed from JSON."""
+        return cls(
+            id=TaskID(d['id']),
+            kind=d['kind'],
+            summary=d['summary'],
+            status=d['status'],
+            log=d.get('log') or [],
+            progress=TaskProgress.from_dict(d['progress']),
+            spawn_time=_parse_timestamp(d['spawn-time']),
+            ready_time=_parse_timestamp(d['ready-time']) if d.get('ready-time') else None,
+        )
+
+    def __repr__(self):
+        return ('Task('
+                'id={self.id!r}, '
+                'kind={self.kind!r}, '
+                'summary={self.summary!r}, '
+                'status={self.status!r}, '
+                'log={self.log!r}, '
+                'progress={self.progress!r}, '
+                'spawn_time={self.spawn_time!r}, '
+                'ready_time={self.ready_time!r})'
+                ).format(self=self)
+
+
+class ChangeID(str):
+    """Change ID (a more strongly-typed string)."""
+
+    def __repr__(self):
+        return 'ChangeID({!r})'.format(str(self))
+
+
+class Change:
+    """Change object."""
+
+    def __init__(
+        self,
+        id: ChangeID,
+        kind: str,
+        summary: str,
+        status: str,
+        tasks: List[Task],
+        ready: bool,
+        err: Optional[str],
+        spawn_time: datetime.datetime,
+        ready_time: Optional[datetime.datetime],
+    ):
+        self.id = id
+        self.kind = kind
+        self.summary = summary
+        self.status = status
+        self.tasks = tasks
+        self.ready = ready
+        self.err = err
+        self.spawn_time = spawn_time
+        self.ready_time = ready_time
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> 'Change':
+        """Create new object from dict parsed from JSON."""
+        return cls(
+            id=ChangeID(d['id']),
+            kind=d['kind'],
+            summary=d['summary'],
+            status=d['status'],
+            tasks=[Task.from_dict(t) for t in d.get('tasks') or []],
+            ready=d['ready'],
+            err=d.get('err'),
+            spawn_time=_parse_timestamp(d['spawn-time']),
+            ready_time=_parse_timestamp(d['ready-time']) if d.get('ready-time') else None,
+        )
+
+    def __repr__(self):
+        return ('Change('
+                'id={self.id!r}, '
+                'kind={self.kind!r}, '
+                'summary={self.summary!r}, '
+                'status={self.status!r}, '
+                'tasks={self.tasks!r}, '
+                'ready={self.ready!r}, '
+                'err={self.err!r}, '
+                'spawn_time={self.spawn_time!r}, '
+                'ready_time={self.ready_time!r})'
+                ).format(self=self)
+
+
+class API:
+    """Pebble API client."""
+
+    def __init__(self, socket_path=None, opener=None, base_url='http://localhost', timeout=5.0):
+        """Initialize an instance. Defaults to Unix socket "$PEBBLE/.pebble.socket"."""
+        if opener is None:
+            opener = self._get_default_opener(socket_path)
+        self.opener = opener
+        self.base_url = base_url
+        self.timeout = timeout
+
+    @classmethod
+    def _get_default_opener(cls, socket_path):
+        """Build the default urllib opener to use for requests.
+
+        If socket_path is None, use "$PEBBLE/.pebble.socket".
+        """
+        if socket_path is None:
+            PEBBLE = os.getenv('PEBBLE')
+            if not PEBBLE:
+                raise ValueError('You must specify socket_path or set $PEBBLE')
+            socket_path = os.path.join(PEBBLE, '.pebble.socket')
+
+        opener = urllib.request.OpenerDirector()
+        opener.add_handler(_UnixSocketHandler(socket_path))
+        opener.add_handler(urllib.request.HTTPDefaultErrorHandler())
+        opener.add_handler(urllib.request.HTTPRedirectHandler())
+        opener.add_handler(urllib.request.HTTPErrorProcessor())
+        return opener
+
+    def _request(self, method: str, path: str, query: Dict = None, body: Dict = None) -> Dict:
+        """Make a request with the given HTTP method and path to the Pebble API.
+
+        If query dict is provided, it is encoded and appended as a query string
+        to the URL. If body dict is provided, it is serialied as JSON and used
+        as the HTTP body (with Content-Type "application/json").
+        """
+        url = self.base_url + path
+        if query:
+            url = url + '?' + urllib.parse.urlencode(query)
+
+        headers = {'Accept': 'application/json'}
+        data = None
+        if body is not None:
+            data = json.dumps(body).encode('utf-8')
+            headers['Content-Type'] = 'application/json'
+
+        request = urllib.request.Request(url, method=method, data=data, headers=headers)
+        response = self.opener.open(request, timeout=self.timeout)
+        result = json.load(response)
+        return result
+
+    def get_system_info(self) -> SystemInfo:
+        """Get system info."""
+        result = self._request('GET', '/v1/system-info')
+        return SystemInfo.from_dict(result['result'])
+
+    def get_warnings(self, select: WarningState = WarningState.PENDING) -> List[Warning]:
+        """Get list of warnings in given state (pending or all)."""
+        query = {'select': select.value}
+        result = self._request('GET', '/v1/warnings', query)
+        return [Warning.from_dict(w) for w in result['result']]
+
+    def ack_warnings(self, timestamp: datetime.datetime) -> int:
+        """Acknowledge warnings up to given timestamp, return number acknowledged."""
+        body = {'action': 'okay', 'timestamp': timestamp.isoformat()}
+        result = self._request('POST', '/v1/warnings', body=body)
+        return result['result']
+
+    def get_changes(
+        self, select: ChangeState = ChangeState.IN_PROGRESS, service: str = None,
+    ) -> List[Change]:
+        """Get list of changes in given state, filter by service name if given."""
+        query = {'select': select.value}
+        if service is not None:
+            query['for'] = service
+        result = self._request('GET', '/v1/changes', query)
+        return [Change.from_dict(c) for c in result['result']]
+
+    def get_change(self, change_id: ChangeID) -> Change:
+        """Get single change by ID."""
+        result = self._request('GET', '/v1/changes/{}'.format(change_id))
+        return Change.from_dict(result['result'])
+
+    def abort_change(self, change_id: ChangeID) -> Change:
+        """Abort change with given ID."""
+        body = {'action': 'abort'}
+        result = self._request('POST', '/v1/changes/{}'.format(change_id), body=body)
+        return Change.from_dict(result['result'])
+
+    def autostart_services(self, timeout: float = 30.0, delay: float = 0.1) -> ChangeID:
+        """Start the autostart services and wait (poll) for them to be started.
+
+        If timeout is 0, submit the action but don't wait.
+        """
+        return self._services_action('autostart', [], timeout, delay)
+
+    def start_services(
+        self, services: List[str], timeout: float = 30.0, delay: float = 0.1,
+    ) -> ChangeID:
+        """Start services by name and wait (poll) for them to be started.
+
+        If timeout is 0, submit the action but don't wait.
+        """
+        return self._services_action('start', services, timeout, delay)
+
+    def stop_services(
+        self, services: List[str], timeout: float = 30.0, delay: float = 0.1,
+    ) -> ChangeID:
+        """Stop services by name and wait (poll) for them to be started.
+
+        If timeout is 0, submit the action but don't wait.
+        """
+        return self._services_action('stop', services, timeout, delay)
+
+    def _services_action(
+        self, action: str, services: List[str], timeout: float, delay: float,
+    ) -> ChangeID:
+        body = {'action': action, 'services': services}
+        result = self._request('POST', '/v1/services', body=body)
+        change_id = ChangeID(result['change'])
+        if timeout:
+            self.wait_change(change_id, timeout=timeout, delay=delay)
+        return change_id
+
+    def wait_change(
+        self, change_id: ChangeID, timeout: float = 30.0, delay: float = 0.1,
+    ) -> Change:
+        """Poll change every delay seconds (up to timeout) for it to be ready."""
+        deadline = time.time() + timeout
+
+        while time.time() < deadline:
+            change = self.get_change(change_id)
+            if change.ready:
+                if change.err:
+                    raise ServiceError(change.err, change)
+                return change
+
+            time.sleep(delay)
+
+        raise TimeoutError(
+            'timed out waiting for change {} ({} seconds)'.format(change_id, timeout))
+
+
+# Make useable as a command line client for local testing
+if __name__ == '__main__':
+    import argparse
+    import sys
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--socket', help='pebble socket path, default $PEBBLE/.pebble.socket')
+    subparsers = parser.add_subparsers(dest='command', metavar='command')
+
+    p = subparsers.add_parser('abort', help='abort a change by ID')
+    p.add_argument('change_id', help='ID of change to abort')
+
+    p = subparsers.add_parser('ack', help='acknowledge warnings up to given time')
+    p.add_argument('--timestamp', help='time to acknowledge up to (YYYY-mm-ddTHH:MM:SS.f+ZZ:zz'
+                                       'format), default current time',
+                   type=_parse_timestamp)
+
+    p = subparsers.add_parser('autostart', help='autostart default service(s)')
+
+    p = subparsers.add_parser('change', help='show a single change by ID')
+    p.add_argument('change_id', help='ID of change to fetch')
+
+    p = subparsers.add_parser('changes', help='show (filtered) changes')
+    p.add_argument('--select', help='change state to filter on, default %(default)s',
+                   choices=[s.value for s in ChangeState], default='all')
+    p.add_argument('--service', help='optional service name to filter on')
+
+    p = subparsers.add_parser('start', help='start service(s)')
+    p.add_argument('service', help='name of service to start (can specify multiple)', nargs='+')
+
+    p = subparsers.add_parser('stop', help='stop service(s)')
+    p.add_argument('service', help='name of service to stop (can specify multiple)', nargs='+')
+
+    p = subparsers.add_parser('system-info', help='show Pebble system information')
+
+    p = subparsers.add_parser('warnings', help='show (filtered) warnings')
+    p.add_argument('--select', help='warning state to filter on, default %(default)s',
+                   choices=[s.value for s in WarningState], default='all')
+
+    args = parser.parse_args()
+
+    try:
+        api = API(socket_path=args.socket)
+    except ValueError as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        result = None  # type: Any
+        if args.command == 'abort':
+            result = api.abort_change(ChangeID(args.change_id))
+        elif args.command == 'ack':
+            timestamp = args.timestamp or datetime.datetime.now(tz=datetime.timezone.utc)
+            result = api.ack_warnings(timestamp)
+        elif args.command == 'autostart':
+            result = api.autostart_services()
+        elif args.command == 'change':
+            result = api.get_change(ChangeID(args.change_id))
+        elif args.command == 'changes':
+            result = api.get_changes(select=ChangeState(args.select), service=args.service)
+        elif args.command == 'start':
+            result = api.start_services(args.service)
+        elif args.command == 'stop':
+            result = api.stop_services(args.service)
+        elif args.command == 'system-info':
+            result = api.get_system_info()
+        elif args.command == 'warnings':
+            result = api.get_warnings(select=WarningState(args.select))
+        else:
+            parser.error('command required')
+    except urllib.error.HTTPError as e:
+        print(e, file=sys.stderr)
+        obj = json.load(e)
+        print(json.dumps(obj, sort_keys=True, indent=4), file=sys.stderr)
+        sys.exit(1)
+    except ServiceError as e:
+        print('ServiceError:', e.err, file=sys.stderr)
+        sys.exit(1)
+
+    if isinstance(result, list):
+        for x in result:
+            print(x)
+    else:
+        print(result)


Test patch:
diff --git a/test/test_pebble.py b/test/test_pebble.py
new file mode 100644
index 0000000..4722759
--- /dev/null
+++ b/test/test_pebble.py
@@ -0,0 +1,572 @@
+#!/usr/bin/python3
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import unittest
+import unittest.mock
+import unittest.util
+
+import ops.pebble as pebble
+
+
+# Ensure unittest diffs don't get truncated like "[17 chars]"
+unittest.util._MAX_LENGTH = 1000
+
+
+def datetime_utc(y, m, d, hour, min, sec, micro=0):
+    tz = datetime.timezone.utc
+    return datetime.datetime(y, m, d, hour, min, sec, micro, tzinfo=tz)
+
+
+def datetime_nzdt(y, m, d, hour, min, sec, micro=0):
+    tz = datetime.timezone(datetime.timedelta(hours=13))
+    return datetime.datetime(y, m, d, hour, min, sec, micro, tzinfo=tz)
+
+
+class TestHelpers(unittest.TestCase):
+    def test_parse_timestamp(self):
+        self.assertEqual(pebble._parse_timestamp('2020-12-25T13:45:50.123456789+13:00'),
+                         datetime_nzdt(2020, 12, 25, 13, 45, 50, 123456))
+
+        self.assertEqual(pebble._parse_timestamp('2020-12-25T13:45:50.123456789+00:00'),
+                         datetime_utc(2020, 12, 25, 13, 45, 50, 123456))
+
+        tzinfo = datetime.timezone(datetime.timedelta(hours=-11, minutes=-30))
+        self.assertEqual(pebble._parse_timestamp('2020-12-25T13:45:50.123456789-1130'),
+                         datetime.datetime(2020, 12, 25, 13, 45, 50, 123456, tzinfo=tzinfo))
+
+        tzinfo = datetime.timezone(datetime.timedelta(hours=4))
+        self.assertEqual(pebble._parse_timestamp('2000-01-02T03:04:05.006000+0400'),
+                         datetime.datetime(2000, 1, 2, 3, 4, 5, 6000, tzinfo=tzinfo))
+
+
+class TestTypes(unittest.TestCase):
+    maxDiff = None
+
+    def test_service_error(self):
+        change = pebble.Change(
+            id=pebble.ChangeID('1234'),
+            kind='start',
+            summary='Start service "foo"',
+            status='Done',
+            tasks=[],
+            ready=True,
+            err=None,
+            spawn_time=datetime.datetime.now(),
+            ready_time=datetime.datetime.now(),
+        )
+        error = pebble.ServiceError('Some error', change)
+        self.assertEqual(error.err, 'Some error')
+        self.assertEqual(error.change, change)
+        self.assertEqual(str(error), 'Some error')
+
+    def test_warning_state(self):
+        self.assertEqual(list(pebble.WarningState), [
+            pebble.WarningState.ALL,
+            pebble.WarningState.PENDING,
+        ])
+        self.assertEqual(pebble.WarningState.ALL.value, 'all')
+        self.assertEqual(pebble.WarningState.PENDING.value, 'pending')
+
+    def test_change_state(self):
+        self.assertEqual(list(pebble.ChangeState), [
+            pebble.ChangeState.ALL,
+            pebble.ChangeState.IN_PROGRESS,
+            pebble.ChangeState.READY,
+        ])
+        self.assertEqual(pebble.ChangeState.ALL.value, 'all')
+        self.assertEqual(pebble.ChangeState.IN_PROGRESS.value, 'in-progress')
+        self.assertEqual(pebble.ChangeState.READY.value, 'ready')
+
+    def test_system_info_init(self):
+        info = pebble.SystemInfo(version='1.2.3')
+        self.assertEqual(info.version, '1.2.3')
+
+    def test_system_info_from_dict(self):
+        info = pebble.SystemInfo.from_dict({'version': '3.2.1'})
+        self.assertEqual(info.version, '3.2.1')
+
+    def test_warning_init(self):
+        warning = pebble.Warning(
+            message='Beware!',
+            first_added=datetime_utc(2021, 1, 1, 1, 1, 1),
+            last_added=datetime_utc(2021, 1, 26, 2, 3, 4),
+            last_shown=None,
+            expire_after='1s',
+            repeat_after='2s',
+        )
+        self.assertEqual(warning.message, 'Beware!')
+        self.assertEqual(warning.first_added, datetime_utc(2021, 1, 1, 1, 1, 1))
+        self.assertEqual(warning.last_added, datetime_utc(2021, 1, 26, 2, 3, 4))
+        self.assertEqual(warning.last_shown, None)
+        self.assertEqual(warning.expire_after, '1s')
+        self.assertEqual(warning.repeat_after, '2s')
+
+    def test_warning_from_dict(self):
+        d = {
+            'message': 'Look out...',
+            'first-added': '2020-12-25T17:18:54.016273778+13:00',
+            'last-added': '2021-01-26T17:01:02.12345+13:00',
+            'expire-after': '1s',
+            'repeat-after': '2s',
+        }
+        warning = pebble.Warning.from_dict(d)
+        self.assertEqual(warning.message, 'Look out...')
+        self.assertEqual(warning.first_added, datetime_nzdt(2020, 12, 25, 17, 18, 54, 16273))
+        self.assertEqual(warning.last_added, datetime_nzdt(2021, 1, 26, 17, 1, 2, 123450))
+        self.assertEqual(warning.last_shown, None)
+        self.assertEqual(warning.expire_after, '1s')
+        self.assertEqual(warning.repeat_after, '2s')
+
+        d['last-shown'] = None
+        warning = pebble.Warning.from_dict(d)
+        self.assertEqual(warning.last_shown, None)
+
+        d['last-shown'] = '2021-08-04T03:02:01.000000000+13:00'
+        warning = pebble.Warning.from_dict(d)
+        self.assertEqual(warning.last_shown, datetime_nzdt(2021, 8, 4, 3, 2, 1))
+
+        d['first-added'] = '2020-02-03T02:00:40.000000+00:00'
+        d['last-added'] = '2021-03-04T03:01:41.100000+00:00'
+        d['last-shown'] = '2022-04-05T06:02:42.200000+00:00'
+        warning = pebble.Warning.from_dict(d)
+        self.assertEqual(warning.first_added, datetime_utc(2020, 2, 3, 2, 0, 40, 0))
+        self.assertEqual(warning.last_added, datetime_utc(2021, 3, 4, 3, 1, 41, 100000))
+        self.assertEqual(warning.last_shown, datetime_utc(2022, 4, 5, 6, 2, 42, 200000))
+
+    def test_task_progress_init(self):
+        tp = pebble.TaskProgress(label='foo', done=3, total=7)
+        self.assertEqual(tp.label, 'foo')
+        self.assertEqual(tp.done, 3)
+        self.assertEqual(tp.total, 7)
+
+    def test_task_progress_from_dict(self):
+        tp = pebble.TaskProgress.from_dict({
+            'label': 'foo',
+            'done': 3,
+            'total': 7,
+        })
+        self.assertEqual(tp.label, 'foo')
+        self.assertEqual(tp.done, 3)
+        self.assertEqual(tp.total, 7)
+
+    def test_task_id(self):
+        task_id = pebble.TaskID('1234')
+        self.assertEqual(task_id, '1234')
+
+    def test_task_init(self):
+        task = pebble.Task(
+            id=pebble.TaskID('42'),
+            kind='start',
+            summary='Start service "svc"',
+            status='Done',
+            log=[],
+            progress=pebble.TaskProgress(label='foo', done=3, total=7),
+            spawn_time=datetime_nzdt(2021, 1, 28, 14, 37, 3, 270218),
+            ready_time=datetime_nzdt(2021, 1, 28, 14, 37, 2, 247158),
+        )
+        self.assertEqual(task.id, '42')
+        self.assertEqual(task.kind, 'start')
+        self.assertEqual(task.summary, 'Start service "svc"')
+        self.assertEqual(task.status, 'Done')
+        self.assertEqual(task.log, [])
+        self.assertEqual(task.progress.label, 'foo')
+        self.assertEqual(task.progress.done, 3)
+        self.assertEqual(task.progress.total, 7)
+        self.assertEqual(task.spawn_time, datetime_nzdt(2021, 1, 28, 14, 37, 3, 270218))
+        self.assertEqual(task.ready_time, datetime_nzdt(2021, 1, 28, 14, 37, 2, 247158))
+
+    def test_task_from_dict(self):
+        d = {
+            "id": "78",
+            "kind": "start",
+            "progress": {
+                "done": 1,
+                "label": "",
+                "total": 1,
+            },
+            "ready-time": "2021-01-28T14:37:03.270218778+13:00",
+            "spawn-time": "2021-01-28T14:37:02.247158162+13:00",
+            "status": "Done",
+            "summary": 'Start service "svc"',
+        }
+        task = pebble.Task.from_dict(d)
+        self.assertEqual(task.id, '78')
+        self.assertEqual(task.kind, 'start')
+        self.assertEqual(task.summary, 'Start service "svc"')
+        self.assertEqual(task.status, 'Done')
+        self.assertEqual(task.log, [])
+        self.assertEqual(task.progress.label, '')
+        self.assertEqual(task.progress.done, 1)
+        self.assertEqual(task.progress.total, 1)
+        self.assertEqual(task.ready_time, datetime_nzdt(2021, 1, 28, 14, 37, 3, 270218))
+        self.assertEqual(task.spawn_time, datetime_nzdt(2021, 1, 28, 14, 37, 2, 247158))
+
+        d['ready-time'] = '2021-01-28T14:37:03.270218778+00:00'
+        d['spawn-time'] = '2021-01-28T14:37:02.247158162+00:00'
+        task = pebble.Task.from_dict(d)
+        self.assertEqual(task.ready_time, datetime_utc(2021, 1, 28, 14, 37, 3, 270218))
+        self.assertEqual(task.spawn_time, datetime_utc(2021, 1, 28, 14, 37, 2, 247158))
+
+    def test_change_id(self):
+        change_id = pebble.ChangeID('1234')
+        self.assertEqual(change_id, '1234')
+
+    def test_change_init(self):
+        change = pebble.Change(
+            id=pebble.ChangeID('70'),
+            kind='autostart',
+            err='SILLY',
+            ready=True,
+            ready_time=datetime_nzdt(2021, 1, 28, 14, 37, 4, 291517),
+            spawn_time=datetime_nzdt(2021, 1, 28, 14, 37, 2, 247202),
+            status='Done',
+            summary='Autostart service "svc"',
+            tasks=[],
+        )
+        self.assertEqual(change.id, '70')
+        self.assertEqual(change.kind, 'autostart')
+        self.assertEqual(change.err, 'SILLY')
+        self.assertEqual(change.ready, True)
+        self.assertEqual(change.ready_time, datetime_nzdt(2021, 1, 28, 14, 37, 4, 291517))
+        self.assertEqual(change.spawn_time, datetime_nzdt(2021, 1, 28, 14, 37, 2, 247202))
+        self.assertEqual(change.status, 'Done')
+        self.assertEqual(change.summary, 'Autostart service "svc"')
+        self.assertEqual(change.tasks, [])
+
+    def test_change_from_dict(self):
+        d = {
+            "id": "70",
+            "kind": "autostart",
+            "err": "SILLY",
+            "ready": True,
+            "ready-time": "2021-01-28T14:37:04.291517768+13:00",
+            "spawn-time": "2021-01-28T14:37:02.247202105+13:00",
+            "status": "Done",
+            "summary": 'Autostart service "svc"',
+            "tasks": [],
+        }
+        change = pebble.Change.from_dict(d)
+        self.assertEqual(change.id, '70')
+        self.assertEqual(change.kind, 'autostart')
+        self.assertEqual(change.err, 'SILLY')
+        self.assertEqual(change.ready, True)
+        self.assertEqual(change.ready_time, datetime_nzdt(2021, 1, 28, 14, 37, 4, 291517))
+        self.assertEqual(change.spawn_time, datetime_nzdt(2021, 1, 28, 14, 37, 2, 247202))
+        self.assertEqual(change.status, 'Done')
+        self.assertEqual(change.summary, 'Autostart service "svc"')
+        self.assertEqual(change.tasks, [])
+
+        d['ready-time'] = '2021-01-28T14:37:04.291517768+00:00'
+        d['spawn-time'] = '2021-01-28T14:37:02.247202105+00:00'
+        change = pebble.Change.from_dict(d)
+        self.assertEqual(change.ready_time, datetime_utc(2021, 1, 28, 14, 37, 4, 291517))
+        self.assertEqual(change.spawn_time, datetime_utc(2021, 1, 28, 14, 37, 2, 247202))
+
+
+class MockAPI(pebble.API):
+    """Mock Pebble client that simply records reqeusts and returns stored responses."""
+
+    def __init__(self):
+        self.requests = []
+        self.responses = []
+
+    def _request(self, method, path, query=None, body=None):
+        self.requests.append((method, path, query, body))
+        return self.responses.pop(0)
+
+
+class MockTime:
+    """Mocked versions of time.time() and time.sleep().
+
+    MockTime.sleep() advances the clock and MockTime.time() returns the current time.
+    """
+
+    def __init__(self):
+        self._time = 0
+
+    def time(self):
+        return self._time
+
+    def sleep(self, delay):
+        self._time += delay
+
+
+class TestAPI(unittest.TestCase):
+    maxDiff = None
+
+    def setUp(self):
+        self.api = MockAPI()
+
+    def test_get_system_info(self):
+        self.api.responses.append({
+            "result": {
+                "version": "1.2.3",
+                "extra-field": "foo",
+            },
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        info = self.api.get_system_info()
+        self.assertEqual(info.version, '1.2.3')
+        self.assertEqual(self.api.requests, [
+            ('GET', '/v1/system-info', None, None),
+        ])
+
+    def test_get_warnings(self):
+        empty = {
+            "result": [],
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        }
+        self.api.responses.append(empty)
+        warnings = self.api.get_warnings()
+        self.assertEqual(warnings, [])
+
+        self.api.responses.append(empty)
+        warnings = self.api.get_warnings(select=pebble.WarningState.ALL)
+        self.assertEqual(warnings, [])
+
+        self.assertEqual(self.api.requests, [
+            ('GET', '/v1/warnings', {'select': 'pending'}, None),
+            ('GET', '/v1/warnings', {'select': 'all'}, None),
+        ])
+
+    def test_ack_warnings(self):
+        self.api.responses.append({
+            "result": 0,
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        num = self.api.ack_warnings(datetime_nzdt(2021, 1, 28, 15, 11, 0))
+        self.assertEqual(num, 0)
+        self.assertEqual(self.api.requests, [
+            ('POST', '/v1/warnings', None, {
+                'action': 'okay',
+                'timestamp': '2021-01-28T15:11:00+13:00',
+            }),
+        ])
+
+    def build_mock_change_dict(self):
+        return {
+            "id": "70",
+            "kind": "autostart",
+            "ready": True,
+            "ready-time": "2021-01-28T14:37:04.291517768+13:00",
+            "spawn-time": "2021-01-28T14:37:02.247202105+13:00",
+            "status": "Done",
+            "summary": 'Autostart service "svc"',
+            "tasks": [
+                {
+                    "id": "78",
+                    "kind": "start",
+                    "progress": {
+                        "done": 1,
+                        "label": "",
+                        "total": 1,
+                        "extra-field": "foo",
+                    },
+                    "ready-time": "2021-01-28T14:37:03.270218778+13:00",
+                    "spawn-time": "2021-01-28T14:37:02.247158162+13:00",
+                    "status": "Done",
+                    "summary": 'Start service "svc"',
+                    "extra-field": "foo",
+                },
+            ],
+            "extra-field": "foo",
+        }
+
+    def assert_mock_change(self, change):
+        self.assertEqual(change.id, '70')
+        self.assertEqual(change.kind, 'autostart')
+        self.assertEqual(change.summary, 'Autostart service "svc"')
+        self.assertEqual(change.status, 'Done')
+        self.assertEqual(len(change.tasks), 1)
+        self.assertEqual(change.tasks[0].id, '78')
+        self.assertEqual(change.tasks[0].kind, 'start')
+        self.assertEqual(change.tasks[0].summary, 'Start service "svc"')
+        self.assertEqual(change.tasks[0].status, 'Done')
+        self.assertEqual(change.tasks[0].log, [])
+        self.assertEqual(change.tasks[0].progress.done, 1)
+        self.assertEqual(change.tasks[0].progress.label, '')
+        self.assertEqual(change.tasks[0].progress.total, 1)
+        self.assertEqual(change.tasks[0].ready_time,
+                         datetime_nzdt(2021, 1, 28, 14, 37, 3, 270218))
+        self.assertEqual(change.tasks[0].spawn_time,
+                         datetime_nzdt(2021, 1, 28, 14, 37, 2, 247158))
+        self.assertEqual(change.ready, True)
+        self.assertEqual(change.err, None)
+        self.assertEqual(change.ready_time, datetime_nzdt(2021, 1, 28, 14, 37, 4, 291517))
+        self.assertEqual(change.spawn_time, datetime_nzdt(2021, 1, 28, 14, 37, 2, 247202))
+
+    def test_get_changes(self):
+        empty = {
+            "result": [],
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        }
+        self.api.responses.append(empty)
+        changes = self.api.get_changes()
+        self.assertEqual(changes, [])
+
+        self.api.responses.append(empty)
+        changes = self.api.get_changes(select=pebble.ChangeState.ALL)
+        self.assertEqual(changes, [])
+
+        self.api.responses.append(empty)
+        changes = self.api.get_changes(select=pebble.ChangeState.ALL, service='foo')
+        self.assertEqual(changes, [])
+
+        self.api.responses.append({
+            "result": [
+                self.build_mock_change_dict(),
+            ],
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        changes = self.api.get_changes()
+        self.assertEqual(len(changes), 1)
+        self.assert_mock_change(changes[0])
+
+        self.assertEqual(self.api.requests, [
+            ('GET', '/v1/changes', {'select': 'in-progress'}, None),
+            ('GET', '/v1/changes', {'select': 'all'}, None),
+            ('GET', '/v1/changes', {'select': 'all', 'for': 'foo'}, None),
+            ('GET', '/v1/changes', {'select': 'in-progress'}, None),
+        ])
+
+    def test_get_change(self):
+        self.api.responses.append({
+            "result": self.build_mock_change_dict(),
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        change = self.api.get_change('70')
+        self.assert_mock_change(change)
+        self.assertEqual(self.api.requests, [
+            ('GET', '/v1/changes/70', None, None),
+        ])
+
+    def test_abort_change(self):
+        self.api.responses.append({
+            "result": self.build_mock_change_dict(),
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        change = self.api.abort_change('70')
+        self.assert_mock_change(change)
+        self.assertEqual(self.api.requests, [
+            ('POST', '/v1/changes/70', None, {'action': 'abort'}),
+        ])
+
+    def _services_action_helper(self, action, api_func, services):
+        self.api.responses.append({
+            "change": "70",
+            "result": None,
+            "status": "Accepted",
+            "status-code": 202,
+            "type": "async"
+        })
+        change = self.build_mock_change_dict()
+        change['ready'] = False
+        self.api.responses.append({
+            "result": change,
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        change = self.build_mock_change_dict()
+        change['ready'] = True
+        self.api.responses.append({
+            "result": change,
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        change_id = api_func()
+        self.assertEqual(change_id, '70')
+        self.assertEqual(self.api.requests, [
+            ('POST', '/v1/services', None, {'action': action, 'services': services}),
+            ('GET', '/v1/changes/70', None, None),
+            ('GET', '/v1/changes/70', None, None),
+        ])
+
+    def _services_action_async_helper(self, action, api_func, services):
+        self.api.responses.append({
+            "change": "70",
+            "result": None,
+            "status": "Accepted",
+            "status-code": 202,
+            "type": "async"
+        })
+        change_id = api_func(timeout=0)
+        self.assertEqual(change_id, '70')
+        self.assertEqual(self.api.requests, [
+            ('POST', '/v1/services', None, {'action': action, 'services': services}),
+        ])
+
+    def test_autostart_services(self):
+        self._services_action_helper('autostart', self.api.autostart_services, [])
+
+    def test_autostart_services_async(self):
+        self._services_action_async_helper('autostart', self.api.autostart_services, [])
+
+    def test_start_services(self):
+        def api_func():
+            return self.api.start_services(['svc'])
+        self._services_action_helper('start', api_func, ['svc'])
+
+    def test_start_services_async(self):
+        def api_func(timeout=30):
+            return self.api.start_services(['svc'], timeout=timeout)
+        self._services_action_async_helper('start', api_func, ['svc'])
+
+    def test_stop_services(self):
+        def api_func():
+            return self.api.stop_services(['svc'])
+        self._services_action_helper('stop', api_func, ['svc'])
+
+    def test_stop_services_async(self):
+        def api_func(timeout=30):
+            return self.api.stop_services(['svc'], timeout=timeout)
+        self._services_action_async_helper('stop', api_func, ['svc'])
+
+    def test_wait_change_timeout(self):
+        with unittest.mock.patch('ops.pebble.time', MockTime()):
+            change = self.build_mock_change_dict()
+            change['ready'] = False
+            for _ in range(3):
+                self.api.responses.append({
+                    "result": change,
+                    "status": "OK",
+                    "status-code": 200,
+                    "type": "sync"
+                })
+
+            with self.assertRaises(TimeoutError):
+                self.api.wait_change('70', timeout=3, delay=1)
+
+            self.assertEqual(self.api.requests, [
+                ('GET', '/v1/changes/70', None, None),
+                ('GET', '/v1/changes/70', None, None),
+                ('GET', '/v1/changes/70', None, None),
+            ])