Repository: canonical/operator
Pull request #668 (2021-12-01 03:54:55)

Add checks config fields and get_checks API
Add support for the new backoff and checks-related fields in the plan configuration types, as well as support for the new `/v1/checks` API (`Client.get_checks` and `Container.get_checks` and `Container.get_check`).

See also the brief docs I'm adding in https://github.com/benhoyt/juju-docs/pull/3 (in addition to the API reference docs in this PR).

rwcarlsen: I assume we're waiting on this: https://github.com/canonical/pebble/pull/86?
benhoyt: @rwcarlsen Yep, that's right. I'll note that in the PR description.
benhoyt: @rwcarlsen Yep, here they are: https://github.com/benhoyt/juju-docs/pull/1 ... the plan is to add them (and Dora will review/edit) once https://github.com/canonical/pebble/pull/86 and then this PR is merged.
jnsgruk: @benhoyt Looking at activity today, looks like we're pretty darn close to getting this merged? :)
rwcarlsen: @benhoyt - have things settled enough here for another review?
benhoyt: @rwcarlsen Yeah, definitely. The Pebble changes are now merged - go ahead and review. Also I think we're waiting on @niemeyer's review here, as this is an API change (though that probably won't be before next week, given the sprint).

Patch:
diff --git a/ops/model.py b/ops/model.py
index 849a590..583c55b 100644
--- a/ops/model.py
+++ b/ops/model.py
@@ -1204,7 +1204,7 @@ class Container:
     def get_service(self, service_name: str) -> 'pebble.ServiceInfo':
         """Get status information for a single named service.
 
-        Raises model error if service_name is not found.
+        Raises :class:`ModelError` if service_name is not found.
         """
         services = self.get_services(service_name)
         if not services:
@@ -1213,6 +1213,33 @@ class Container:
             raise RuntimeError('expected 1 service, got {}'.format(len(services)))
         return services[service_name]
 
+    def get_checks(
+            self,
+            *check_names: str,
+            level: 'pebble.CheckLevel' = None) -> 'CheckInfoMapping':
+        """Fetch and return a mapping of check information indexed by check name.
+
+        Args:
+            check_names: Optional check names to query for. If no check names
+                are specified, return checks with any name.
+            level: Optional check level to query for. If not specified, fetch
+                checks with any level.
+        """
+        checks = self._pebble.get_checks(names=check_names or None, level=level)
+        return CheckInfoMapping(checks)
+
+    def get_check(self, check_name: str) -> 'pebble.CheckInfo':
+        """Get check information for a single named check.
+
+        Raises :class:`ModelError` if check_name is not found.
+        """
+        checks = self.get_checks(check_name)
+        if not checks:
+            raise ModelError('check {!r} not found'.format(check_name))
+        if len(checks) > 1:
+            raise RuntimeError('expected 1 check, got {}'.format(len(checks)))
+        return checks[check_name]
+
     def pull(self, path: str, *, encoding: str = 'utf-8') -> typing.Union[typing.BinaryIO,
                                                                           typing.TextIO]:
         """Read a file's content from the remote system.
@@ -1402,7 +1429,7 @@ class ContainerMapping(Mapping):
 
 
 class ServiceInfoMapping(Mapping):
-    """Map of service names to pebble.ServiceInfo objects.
+    """Map of service names to :class:`ops.pebble.ServiceInfo` objects.
 
     This is done as a mapping object rather than a plain dictionary so that we
     can extend it later, and so it's not mutable.
@@ -1424,6 +1451,29 @@ class ServiceInfoMapping(Mapping):
         return repr(self._services)
 
 
+class CheckInfoMapping(Mapping):
+    """Map of check names to :class:`ops.pebble.CheckInfo` objects.
+
+    This is done as a mapping object rather than a plain dictionary so that we
+    can extend it later, and so it's not mutable.
+    """
+
+    def __init__(self, checks: typing.Iterable['pebble.CheckInfo']):
+        self._checks = {c.name: c for c in checks}
+
+    def __getitem__(self, key: str):
+        return self._checks[key]
+
+    def __iter__(self):
+        return iter(self._checks)
+
+    def __len__(self):
+        return len(self._checks)
+
+    def __repr__(self):
+        return repr(self._checks)
+
+
 class ModelError(Exception):
     """Base class for exceptions raised when interacting with the Model."""
     pass
diff --git a/ops/pebble.py b/ops/pebble.py
index bd4273f..24437d9 100644
--- a/ops/pebble.py
+++ b/ops/pebble.py
@@ -19,6 +19,7 @@ For a command-line interface for local testing, see test/pebble_cli.py.
 
 import binascii
 import cgi
+import copy
 import datetime
 import email.parser
 import enum
@@ -512,13 +513,19 @@ class Change:
 
 
 class Plan:
-    """Represents the effective Pebble configuration."""
+    """Represents the effective Pebble configuration.
+
+    A plan is the combined layer configuration. The layer configuration is
+    documented at https://github.com/canonical/pebble/#layer-specification.
+    """
 
     def __init__(self, raw: str):
         d = yaml.safe_load(raw) or {}
         self._raw = raw
         self._services = {name: Service(name, service)
                           for name, service in d.get('services', {}).items()}
+        self._checks = {name: Check(name, check)
+                        for name, check in d.get('checks', {}).items()}
 
     @property
     def services(self):
@@ -528,14 +535,21 @@ class Plan:
         """
         return self._services
 
+    @property
+    def checks(self):
+        """This plan's checks mapping (maps check name to :class:`Check`).
+
+        This property is currently read-only.
+        """
+        return self._checks
+
     def to_dict(self) -> typing.Dict[str, typing.Any]:
         """Convert this plan to its dict representation."""
-        as_dicts = {name: service.to_dict() for name, service in self._services.items()}
-        if not as_dicts:
-            return {}
-        return {
-            'services': as_dicts,
-        }
+        fields = [
+            ('services', {name: service.to_dict() for name, service in self._services.items()}),
+            ('checks', {name: check.to_dict() for name, check in self._checks.items()}),
+        ]
+        return {name: value for name, value in fields if value}
 
     def to_yaml(self) -> str:
         """Return this plan's YAML representation."""
@@ -547,13 +561,14 @@ class Plan:
 class Layer:
     """Represents a Pebble configuration layer.
 
-    The format of this is not documented, but is captured in code here:
-    https://github.com/canonical/pebble/blob/master/internal/plan/plan.go
+    The format of this is documented at
+    https://github.com/canonical/pebble/#layer-specification.
 
     Attributes:
         summary: A summary of the purpose of this layer
         description: A long form description of this layer
-        services: A mapping of name: :class:`Service` defined by this layer
+        services: A mapping of name to :class:`Service` defined by this layer
+        checks: A mapping of check to :class:`Check` defined by this layer
     """
 
     # This is how you do type annotations, but it is not supported by Python 3.5
@@ -570,6 +585,8 @@ class Layer:
         self.description = d.get('description', '')
         self.services = {name: Service(name, service)
                          for name, service in d.get('services', {}).items()}
+        self.checks = {name: Check(name, check)
+                       for name, check in d.get('checks', {}).items()}
 
     def to_yaml(self) -> str:
         """Convert this layer to its YAML representation."""
@@ -580,7 +597,8 @@ class Layer:
         fields = [
             ('summary', self.summary),
             ('description', self.description),
-            ('services', {name: service.to_dict() for name, service in self.services.items()})
+            ('services', {name: service.to_dict() for name, service in self.services.items()}),
+            ('checks', {name: check.to_dict() for name, check in self.checks.items()}),
         ]
         return {name: value for name, value in fields if value}
 
@@ -609,8 +627,14 @@ class Service:
         self.user_id = raw.get('user-id')
         self.group = raw.get('group', '')
         self.group_id = raw.get('group-id')
+        self.on_success = raw.get('on-success', '')
+        self.on_failure = raw.get('on-failure', '')
+        self.on_check_failure = dict(raw.get('on-check-failure', {}))
+        self.backoff_delay = raw.get('backoff-delay', '')
+        self.backoff_factor = raw.get('backoff-factor')
+        self.backoff_limit = raw.get('backoff-limit', '')
 
-    def to_dict(self) -> typing.Dict:
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
         """Convert this service object to its dict representation."""
         fields = [
             ('summary', self.summary),
@@ -626,6 +650,12 @@ class Service:
             ('user-id', self.user_id),
             ('group', self.group),
             ('group-id', self.group_id),
+            ('on-success', self.on_success),
+            ('on-failure', self.on_failure),
+            ('on-check-failure', self.on_check_failure),
+            ('backoff-delay', self.backoff_delay),
+            ('backoff-factor', self.backoff_factor),
+            ('backoff-limit', self.backoff_limit),
         ]
         return {name: value for name, value in fields if value}
 
@@ -701,6 +731,78 @@ class ServiceInfo:
                 ).format(self=self)
 
 
+class Check:
+    """Represents a check in a Pebble configuration layer."""
+
+    def __init__(self, name: str, raw: typing.Dict = None):
+        self.name = name
+        raw = raw or {}
+        self.override = raw.get('override', '')
+        try:
+            self.level = CheckLevel(raw.get('level', ''))
+        except ValueError:
+            self.level = raw.get('level')
+        self.period = raw.get('period', '')
+        self.timeout = raw.get('timeout', '')
+        self.threshold = raw.get('threshold')
+
+        http = raw.get('http')
+        if http is not None:
+            http = copy.deepcopy(http)
+        self.http = http
+
+        tcp = raw.get('tcp')
+        if tcp is not None:
+            tcp = copy.deepcopy(tcp)
+        self.tcp = tcp
+
+        exec_ = raw.get('exec')
+        if exec_ is not None:
+            exec_ = copy.deepcopy(exec_)
+        self.exec = exec_
+
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        """Convert this check object to its dict representation."""
+        fields = [
+            ('override', self.override),
+            ('level', self.level.value),
+            ('period', self.period),
+            ('timeout', self.timeout),
+            ('threshold', self.threshold),
+            ('http', self.http),
+            ('tcp', self.tcp),
+            ('exec', self.exec),
+        ]
+        return {name: value for name, value in fields if value}
+
+    def __repr__(self) -> str:
+        return 'Check({!r})'.format(self.to_dict())
+
+    def __eq__(self, other: typing.Union[typing.Dict, 'Check']) -> bool:
+        """Compare this check configuration to another."""
+        if isinstance(other, dict):
+            return self.to_dict() == other
+        elif isinstance(other, Check):
+            return self.to_dict() == other.to_dict()
+        else:
+            raise ValueError("Cannot compare pebble.Check to {}".format(type(other)))
+
+
+class CheckLevel(enum.Enum):
+    """Enum of check levels."""
+
+    UNSET = ''
+    ALIVE = 'alive'
+    READY = 'ready'
+
+
+class CheckStatus(enum.Enum):
+    """Enum of check statuses."""
+
+    UP = 'up'
+    DOWN = 'down'
+
+
 class FileType(enum.Enum):
     """Enum of file types."""
 
@@ -775,6 +877,68 @@ class FileInfo:
                 ).format(self=self)
 
 
+class CheckInfo:
+    """Check status information.
+
+    A list of these objects is returned from :meth:`Client.get_checks`.
+
+    Attributes:
+        name: The name of the check.
+        level: The check level: :attr:`CheckLevel.ALIVE`,
+            :attr:`CheckLevel.READY`, or None (level not set).
+        status: The status of the check: :attr:`CheckStatus.UP` means the
+            check is healthy (the number of failures is less than the
+            threshold), :attr:`CheckStatus.DOWN` means the check is unhealthy
+            (the number of failures has reached the threshold).
+        failures: The number of failures since the check last succeeded (reset
+            to zero if the check succeeds).
+        threshold: The failure threshold, that is, how many consecutive
+            failures for the check to be considered "down".
+    """
+
+    def __init__(
+        self,
+        name: str,
+        level: typing.Optional[typing.Union[CheckLevel, str]],
+        status: typing.Union[CheckStatus, str],
+        failures: int = 0,
+        threshold: int = 0,
+    ):
+        self.name = name
+        self.level = level
+        self.status = status
+        self.failures = failures
+        self.threshold = threshold
+
+    @classmethod
+    def from_dict(cls, d: typing.Dict) -> 'CheckInfo':
+        """Create new :class:`CheckInfo` object from dict parsed from JSON."""
+        try:
+            level = CheckLevel(d.get('level', ''))
+        except ValueError:
+            level = d.get('level')
+        try:
+            status = CheckStatus(d['status'])
+        except ValueError:
+            status = d['status']
+        return cls(
+            name=d['name'],
+            level=level,
+            status=status,
+            failures=d.get('failures', 0),
+            threshold=d['threshold'],
+        )
+
+    def __repr__(self):
+        return ('CheckInfo('
+                'name={self.name!r}, '
+                'level={self.level!r}, '
+                'status={self.status}, '
+                'failures={self.failures}, '
+                'threshold={self.threshold!r})'
+                ).format(self=self)
+
+
 class ExecProcess:
     """Represents a process started by :meth:`Client.exec`.
 
@@ -1135,7 +1299,7 @@ class Client:
         """Make a request to the Pebble server; return the raw HTTPResponse object."""
         url = self.base_url + path
         if query:
-            url = url + '?' + urllib.parse.urlencode(query)
+            url = url + '?' + urllib.parse.urlencode(query, doseq=True)
 
         # python 3.5 urllib requests require their data to be a bytes object -
         # generators won't work.
@@ -1430,7 +1594,7 @@ class Client:
         self._request('POST', '/v1/layers', body=body)
 
     def get_plan(self) -> Plan:
-        """Get the Pebble plan (currently contains only combined services)."""
+        """Get the Pebble plan (contains combined layer configuration)."""
         resp = self._request('GET', '/v1/plan', {'format': 'yaml'})
         return Plan(resp['result'])
 
@@ -1976,3 +2140,27 @@ class Client:
             'services': services,
         }
         self._request('POST', '/v1/signals', body=body)
+
+    def get_checks(
+        self,
+        level: CheckLevel = None,
+        names: typing.List[str] = None
+    ) -> typing.List[CheckInfo]:
+        """Get the check status for the configured checks.
+
+        Args:
+            level: Optional check level to query for (default is to fetch
+                checks with any level).
+            names: Optional list of check names to query for (default is to
+                fetch checks with any name).
+
+        Returns:
+            List of :class:`CheckInfo` objects.
+        """
+        query = {}
+        if level is not None:
+            query['level'] = level.value
+        if names:
+            query['names'] = names
+        resp = self._request('GET', '/v1/checks', query)
+        return [CheckInfo.from_dict(info) for info in resp['result']]


Test patch:
diff --git a/.github/workflows/framework-tests.yaml b/.github/workflows/framework-tests.yaml
index 1c25ec6..d529f24 100644
--- a/.github/workflows/framework-tests.yaml
+++ b/.github/workflows/framework-tests.yaml
@@ -67,7 +67,7 @@ jobs:
       - name: Start Pebble
         run: |
           umask 0
-          $HOME/go/bin/pebble run --create-dirs &
+          $HOME/go/bin/pebble run --create-dirs --http=:4000 &
         env:
           PEBBLE: /tmp/pebble
 
diff --git a/ops/testing.py b/ops/testing.py
index 019c1b2..df4f7d1 100755
--- a/ops/testing.py
+++ b/ops/testing.py
@@ -1628,6 +1628,9 @@ ChangeError: cannot perform the following tasks:
                 status='Internal Server Error',
                 message=message)
 
+    def get_checks(self, level=None, names=None):
+        raise NotImplementedError(self.get_checks)
+
 
 class NonAbsolutePathError(Exception):
     """Error raised by _MockFilesystem.
diff --git a/test/pebble_cli.py b/test/pebble_cli.py
index 46add0f..07dd222 100644
--- a/test/pebble_cli.py
+++ b/test/pebble_cli.py
@@ -57,6 +57,11 @@ def main():
                    choices=[s.value for s in pebble.ChangeState], default='all')
     p.add_argument('--service', help='optional service name to filter on')
 
+    p = subparsers.add_parser('checks', help='show (filtered) checks')
+    p.add_argument('--level', help='check level to filter on, default all levels',
+                   choices=[c.value for c in pebble.CheckLevel], default='')
+    p.add_argument('name', help='check name(s) to filter on', nargs='*')
+
     p = subparsers.add_parser('exec', help='execute a command')
     p.add_argument('--env', help='environment variables to set', action='append',
                    metavar='KEY=VALUE')
@@ -157,6 +162,8 @@ def main():
         elif args.command == 'changes':
             result = client.get_changes(select=pebble.ChangeState(args.select),
                                         service=args.service)
+        elif args.command == 'checks':
+            result = client.get_checks(level=pebble.CheckLevel(args.level), names=args.name)
         elif args.command == 'exec':
             environment = {}
             for env in args.env or []:
diff --git a/test/test_model.py b/test/test_model.py
index 5004d88..1b79bd8 100755
--- a/test/test_model.py
+++ b/test/test_model.py
@@ -1106,6 +1106,94 @@ containers:
         with self.assertRaises(RuntimeError):
             self.container.get_service('s1')
 
+    def test_get_checks(self):
+        response_checks = [
+            ops.pebble.CheckInfo.from_dict({
+                'name': 'c1',
+                'status': 'up',
+                'failures': 0,
+                'threshold': 3,
+            }),
+            ops.pebble.CheckInfo.from_dict({
+                'name': 'c2',
+                'level': 'alive',
+                'status': 'down',
+                'failures': 2,
+                'threshold': 2,
+            }),
+        ]
+
+        self.pebble.responses.append(response_checks)
+        checks = self.container.get_checks()
+        self.assertEqual(len(checks), 2)
+        self.assertEqual(checks['c1'].name, 'c1')
+        self.assertEqual(checks['c1'].level, ops.pebble.CheckLevel.UNSET)
+        self.assertEqual(checks['c1'].status, ops.pebble.CheckStatus.UP)
+        self.assertEqual(checks['c1'].failures, 0)
+        self.assertEqual(checks['c1'].threshold, 3)
+        self.assertEqual(checks['c2'].name, 'c2')
+        self.assertEqual(checks['c2'].level, ops.pebble.CheckLevel.ALIVE)
+        self.assertEqual(checks['c2'].status, ops.pebble.CheckStatus.DOWN)
+        self.assertEqual(checks['c2'].failures, 2)
+        self.assertEqual(checks['c2'].threshold, 2)
+
+        self.pebble.responses.append(response_checks[1:2])
+        checks = self.container.get_checks('c1', 'c2', level=ops.pebble.CheckLevel.ALIVE)
+        self.assertEqual(len(checks), 1)
+        self.assertEqual(checks['c2'].name, 'c2')
+        self.assertEqual(checks['c2'].level, ops.pebble.CheckLevel.ALIVE)
+        self.assertEqual(checks['c2'].status, ops.pebble.CheckStatus.DOWN)
+        self.assertEqual(checks['c2'].failures, 2)
+        self.assertEqual(checks['c2'].threshold, 2)
+
+        self.assertEqual(self.pebble.requests, [
+            ('get_checks', None, None),
+            ('get_checks', ops.pebble.CheckLevel.ALIVE, ('c1', 'c2')),
+        ])
+
+    def test_get_check(self):
+        # Single check returned successfully
+        self.pebble.responses.append([
+            ops.pebble.CheckInfo.from_dict({
+                'name': 'c1',
+                'status': 'up',
+                'failures': 0,
+                'threshold': 3,
+            })
+        ])
+        c = self.container.get_check('c1')
+        self.assertEqual(self.pebble.requests, [('get_checks', None, ('c1', ))])
+        self.assertEqual(c.name, 'c1')
+        self.assertEqual(c.level, ops.pebble.CheckLevel.UNSET)
+        self.assertEqual(c.status, ops.pebble.CheckStatus.UP)
+        self.assertEqual(c.failures, 0)
+        self.assertEqual(c.threshold, 3)
+
+        # If Pebble returns no checks, should be a ModelError
+        self.pebble.responses.append([])
+        with self.assertRaises(ops.model.ModelError) as cm:
+            self.container.get_check('c2')
+        self.assertEqual(str(cm.exception), "check 'c2' not found")
+
+        # If Pebble returns more than one check, RuntimeError is raised
+        self.pebble.responses.append([
+            ops.pebble.CheckInfo.from_dict({
+                'name': 'c1',
+                'status': 'up',
+                'failures': 0,
+                'threshold': 3,
+            }),
+            ops.pebble.CheckInfo.from_dict({
+                'name': 'c2',
+                'level': 'alive',
+                'status': 'down',
+                'failures': 2,
+                'threshold': 2,
+            }),
+        ])
+        with self.assertRaises(RuntimeError):
+            self.container.get_check('c1')
+
     def test_pull(self):
         self.pebble.responses.append('dummy1')
         got = self.container.pull('/path/1')
@@ -1277,6 +1365,10 @@ class MockPebbleClient:
         self.requests.append(('get_services', names))
         return self.responses.pop(0)
 
+    def get_checks(self, level=None, names=None):
+        self.requests.append(('get_checks', level, names))
+        return self.responses.pop(0)
+
     def pull(self, path, *, encoding='utf-8'):
         self.requests.append(('pull', path, encoding))
         return self.responses.pop(0)
diff --git a/test/test_pebble.py b/test/test_pebble.py
index db21281..9f7365a 100644
--- a/test/test_pebble.py
+++ b/test/test_pebble.py
@@ -28,6 +28,8 @@ import time
 import unittest
 import unittest.mock
 import unittest.util
+import urllib.error
+import urllib.request
 
 import pytest
 
@@ -509,6 +511,22 @@ class TestPlan(unittest.TestCase):
         with self.assertRaises(AttributeError):
             plan.services = {}
 
+    def test_checks(self):
+        plan = pebble.Plan('')
+        self.assertEqual(plan.checks, {})
+
+        plan = pebble.Plan(
+            'checks:\n bar:\n  override: replace\n  http:\n   url: https://example.com/')
+
+        self.assertEqual(len(plan.checks), 1)
+        self.assertEqual(plan.checks['bar'].name, 'bar')
+        self.assertEqual(plan.checks['bar'].override, 'replace')
+        self.assertEqual(plan.checks['bar'].http, {'url': 'https://example.com/'})
+
+        # Should be read-only ("can't set attribute")
+        with self.assertRaises(AttributeError):
+            plan.checks = {}
+
     def test_yaml(self):
         # Starting with nothing, we get the empty result
         plan = pebble.Plan('')
@@ -521,6 +539,11 @@ services:
  foo:
   override: replace
   command: echo foo
+
+checks:
+ bar:
+  http:
+   https://example.com/
 '''
         plan = pebble.Plan(raw)
         reformed = yaml.safe_dump(yaml.safe_load(raw))
@@ -528,9 +551,6 @@ services:
         self.assertEqual(str(plan), reformed)
 
     def test_service_equality(self):
-        plan = pebble.Plan('')
-        self.assertEqual(plan.services, {})
-
         plan = pebble.Plan('services:\n foo:\n  override: replace\n  command: echo foo')
 
         old_service = pebble.Service(name="foo",
@@ -592,7 +612,11 @@ class TestLayer(unittest.TestCase):
         s = pebble.Layer('')
         self._assert_empty(s)
 
-        yaml = """description: The quick brown fox!
+        yaml = """checks:
+  chk:
+    http:
+      url: https://example.com/
+description: The quick brown fox!
 services:
   bar:
     command: echo bar
@@ -625,6 +649,9 @@ summary: Sum Mary
         self.assertEqual(s.services['bar'].group, 'staff')
         self.assertEqual(s.services['bar'].group_id, 2000)
 
+        self.assertEqual(s.checks['chk'].name, 'chk')
+        self.assertEqual(s.checks['chk'].http, {'url': 'https://example.com/'})
+
         self.assertEqual(s.to_yaml(), yaml)
         self.assertEqual(str(s), yaml)
 
@@ -668,6 +695,12 @@ class TestService(unittest.TestCase):
         self.assertIs(service.user_id, None)
         self.assertEqual(service.group, '')
         self.assertIs(service.group_id, None)
+        self.assertEqual(service.on_success, '')
+        self.assertEqual(service.on_failure, '')
+        self.assertEqual(service.on_check_failure, {})
+        self.assertEqual(service.backoff_delay, '')
+        self.assertIs(service.backoff_factor, None)
+        self.assertEqual(service.backoff_limit, '')
         self.assertEqual(service.to_dict(), {})
 
     def test_name_only(self):
@@ -692,6 +725,12 @@ class TestService(unittest.TestCase):
             'user-id': 1000,
             'group': 'staff',
             'group-id': 2000,
+            'on-success': 'restart',
+            'on-failure': 'ignore',
+            'on-check-failure': {'chk1': 'halt'},
+            'backoff-delay': '1s',
+            'backoff-factor': 4,
+            'backoff-limit': '10s',
         }
         s = pebble.Service('Name 2', d)
         self.assertEqual(s.name, 'Name 2')
@@ -707,6 +746,12 @@ class TestService(unittest.TestCase):
         self.assertEqual(s.user_id, 1000)
         self.assertEqual(s.group, 'staff')
         self.assertEqual(s.group_id, 2000)
+        self.assertEqual(s.on_success, 'restart')
+        self.assertEqual(s.on_failure, 'ignore')
+        self.assertEqual(s.on_check_failure, {'chk1': 'halt'})
+        self.assertEqual(s.backoff_delay, '1s')
+        self.assertEqual(s.backoff_factor, 4)
+        self.assertEqual(s.backoff_limit, '10s')
 
         self.assertEqual(s.to_dict(), d)
 
@@ -715,6 +760,7 @@ class TestService(unittest.TestCase):
         s.before.append('b3')
         s.requires.append('r3')
         s.environment['k3'] = 'v3'
+        s.on_check_failure['chk2'] = 'ignore'
         self.assertEqual(s.after, ['a1', 'a2', 'a3'])
         self.assertEqual(s.before, ['b1', 'b2', 'b3'])
         self.assertEqual(s.requires, ['r1', 'r2', 'r3'])
@@ -723,6 +769,7 @@ class TestService(unittest.TestCase):
         self.assertEqual(d['before'], ['b1', 'b2'])
         self.assertEqual(d['requires'], ['r1', 'r2'])
         self.assertEqual(d['environment'], {'k1': 'v1', 'k2': 'v2'})
+        self.assertEqual(d['on-check-failure'], {'chk1': 'halt'})
 
     def test_equality(self):
         d = {
@@ -761,6 +808,82 @@ class TestService(unittest.TestCase):
         }
         self.assertEqual(one, as_dict)
 
+        with self.assertRaises(ValueError):
+            self.assertEqual(one, 5)
+
+
+class TestCheck(unittest.TestCase):
+    def _assert_empty(self, check, name):
+        self.assertEqual(check.name, name)
+        self.assertEqual(check.override, '')
+        self.assertEqual(check.level, pebble.CheckLevel.UNSET)
+        self.assertEqual(check.period, '')
+        self.assertEqual(check.timeout, '')
+        self.assertIs(check.threshold, None)
+        self.assertIs(check.http, None)
+        self.assertIs(check.tcp, None)
+        self.assertIs(check.exec, None)
+
+    def test_name_only(self):
+        check = pebble.Check('chk')
+        self._assert_empty(check, 'chk')
+
+    def test_dict(self):
+        d = {
+            'override': 'replace',
+            'level': 'alive',
+            'period': '10s',
+            'timeout': '3s',
+            'threshold': 5,
+            # Not valid for Pebble to have more than one of http, tcp, and exec,
+            # but it makes things simpler for the unit tests.
+            'http': {'url': 'https://example.com/'},
+            'tcp': {'port': 80},
+            'exec': {'command': 'echo foo'},
+        }
+        check = pebble.Check('chk-http', d)
+        self.assertEqual(check.name, 'chk-http')
+        self.assertEqual(check.override, 'replace')
+        self.assertEqual(check.level, pebble.CheckLevel.ALIVE)
+        self.assertEqual(check.period, '10s')
+        self.assertEqual(check.timeout, '3s')
+        self.assertEqual(check.threshold, 5)
+        self.assertEqual(check.http, {'url': 'https://example.com/'})
+        self.assertEqual(check.tcp, {'port': 80})
+        self.assertEqual(check.exec, {'command': 'echo foo'})
+
+        self.assertEqual(check.to_dict(), d)
+
+        # Ensure pebble.Check has made copies of mutable objects
+        check.http['url'] = 'https://www.google.com'
+        self.assertEqual(d['http'], {'url': 'https://example.com/'})
+        check.tcp['port'] = 81
+        self.assertEqual(d['tcp'], {'port': 80})
+        check.exec['command'] = 'foo'
+        self.assertEqual(d['exec'], {'command': 'echo foo'})
+
+    def test_equality(self):
+        d = {
+            'override': 'replace',
+            'level': 'alive',
+            'period': '10s',
+            'timeout': '3s',
+            'threshold': 5,
+            'http': {'url': 'https://example.com/'},
+        }
+        one = pebble.Check('one', d)
+        two = pebble.Check('two', d)
+        self.assertEqual(one, two)
+        self.assertEqual(one, d)
+        self.assertEqual(two, d)
+        self.assertEqual(one, one.to_dict())
+        self.assertEqual(two, two.to_dict())
+        d['level'] = 'ready'
+        self.assertNotEqual(one, d)
+
+        with self.assertRaises(ValueError):
+            self.assertEqual(one, 5)
+
 
 class TestServiceInfo(unittest.TestCase):
     def test_service_startup(self):
@@ -787,6 +910,14 @@ class TestServiceInfo(unittest.TestCase):
         self.assertEqual(s.startup, pebble.ServiceStartup.ENABLED)
         self.assertEqual(s.current, pebble.ServiceStatus.ACTIVE)
 
+        s = pebble.ServiceInfo(
+            'svc1',
+            pebble.ServiceStartup.ENABLED,
+            pebble.ServiceStatus.ACTIVE)
+        self.assertEqual(s.name, 'svc1')
+        self.assertEqual(s.startup, pebble.ServiceStartup.ENABLED)
+        self.assertEqual(s.current, pebble.ServiceStatus.ACTIVE)
+
         s = pebble.ServiceInfo.from_dict({
             'name': 'svc2',
             'startup': 'disabled',
@@ -813,6 +944,76 @@ class TestServiceInfo(unittest.TestCase):
             self.assertFalse(s.is_running())
 
 
+class TestCheckInfo(unittest.TestCase):
+    def test_check_level(self):
+        self.assertEqual(list(pebble.CheckLevel), [
+            pebble.CheckLevel.UNSET,
+            pebble.CheckLevel.ALIVE,
+            pebble.CheckLevel.READY,
+        ])
+        self.assertEqual(pebble.CheckLevel.UNSET.value, '')
+        self.assertEqual(pebble.CheckLevel.ALIVE.value, 'alive')
+        self.assertEqual(pebble.CheckLevel.READY.value, 'ready')
+
+    def test_check_status(self):
+        self.assertEqual(list(pebble.CheckStatus), [
+            pebble.CheckStatus.UP,
+            pebble.CheckStatus.DOWN,
+        ])
+        self.assertEqual(pebble.CheckStatus.UP.value, 'up')
+        self.assertEqual(pebble.CheckStatus.DOWN.value, 'down')
+
+    def test_check_info(self):
+        check = pebble.CheckInfo(
+            name='chk1',
+            level=pebble.CheckLevel.READY,
+            status=pebble.CheckStatus.UP,
+            threshold=3,
+        )
+        self.assertEqual(check.name, 'chk1')
+        self.assertEqual(check.level, pebble.CheckLevel.READY)
+        self.assertEqual(check.status, pebble.CheckStatus.UP)
+        self.assertEqual(check.failures, 0)
+        self.assertEqual(check.threshold, 3)
+
+        check = pebble.CheckInfo(
+            name='chk2',
+            level=pebble.CheckLevel.ALIVE,
+            status=pebble.CheckStatus.DOWN,
+            failures=5,
+            threshold=3,
+        )
+        self.assertEqual(check.name, 'chk2')
+        self.assertEqual(check.level, pebble.CheckLevel.ALIVE)
+        self.assertEqual(check.status, pebble.CheckStatus.DOWN)
+        self.assertEqual(check.failures, 5)
+        self.assertEqual(check.threshold, 3)
+
+        check = pebble.CheckInfo.from_dict({
+            'name': 'chk3',
+            'status': 'up',
+            'threshold': 3,
+        })
+        self.assertEqual(check.name, 'chk3')
+        self.assertEqual(check.level, pebble.CheckLevel.UNSET)
+        self.assertEqual(check.status, pebble.CheckStatus.UP)
+        self.assertEqual(check.failures, 0)
+        self.assertEqual(check.threshold, 3)
+
+        check = pebble.CheckInfo.from_dict({
+            'name': 'chk4',
+            'level': pebble.CheckLevel.UNSET,
+            'status': pebble.CheckStatus.DOWN,
+            'failures': 3,
+            'threshold': 3,
+        })
+        self.assertEqual(check.name, 'chk4')
+        self.assertEqual(check.level, pebble.CheckLevel.UNSET)
+        self.assertEqual(check.status, pebble.CheckStatus.DOWN)
+        self.assertEqual(check.failures, 3)
+        self.assertEqual(check.threshold, 3)
+
+
 class MockClient(pebble.Client):
     """Mock Pebble client that simply records requests and returns stored responses."""
 
@@ -2048,6 +2249,69 @@ bad path
         with self.assertRaises(TypeError):
             self.client.send_signal('SIGHUP', [1, 2])
 
+    def test_get_checks_all(self):
+        self.client.responses.append({
+            "result": [
+                {
+                    "name": "chk1",
+                    "status": "up",
+                    "threshold": 2,
+                },
+                {
+                    "name": "chk2",
+                    "level": "alive",
+                    "status": "down",
+                    "failures": 5,
+                    "threshold": 3,
+                }
+            ],
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        checks = self.client.get_checks()
+        self.assertEqual(len(checks), 2)
+        self.assertEqual(checks[0].name, 'chk1')
+        self.assertEqual(checks[0].level, pebble.CheckLevel.UNSET)
+        self.assertEqual(checks[0].status, pebble.CheckStatus.UP)
+        self.assertEqual(checks[0].failures, 0)
+        self.assertEqual(checks[0].threshold, 2)
+        self.assertEqual(checks[1].name, 'chk2')
+        self.assertEqual(checks[1].level, pebble.CheckLevel.ALIVE)
+        self.assertEqual(checks[1].status, pebble.CheckStatus.DOWN)
+        self.assertEqual(checks[1].failures, 5)
+        self.assertEqual(checks[1].threshold, 3)
+
+        self.assertEqual(self.client.requests, [
+            ('GET', '/v1/checks', {}, None),
+        ])
+
+    def test_get_checks_filters(self):
+        self.client.responses.append({
+            "result": [
+                {
+                    "name": "chk2",
+                    "level": "ready",
+                    "status": "up",
+                    "threshold": 3,
+                },
+            ],
+            "status": "OK",
+            "status-code": 200,
+            "type": "sync"
+        })
+        checks = self.client.get_checks(level=pebble.CheckLevel.READY, names=['chk2'])
+        self.assertEqual(len(checks), 1)
+        self.assertEqual(checks[0].name, 'chk2')
+        self.assertEqual(checks[0].level, pebble.CheckLevel.READY)
+        self.assertEqual(checks[0].status, pebble.CheckStatus.UP)
+        self.assertEqual(checks[0].failures, 0)
+        self.assertEqual(checks[0].threshold, 3)
+
+        self.assertEqual(self.client.requests, [
+            ('GET', '/v1/checks', {'level': 'ready', 'names': ['chk2']}, None),
+        ])
+
 
 @unittest.skipIf(sys.platform == 'win32', "Unix sockets don't work on Windows")
 class TestSocketClient(unittest.TestCase):
@@ -2717,7 +2981,7 @@ class TestExec(unittest.TestCase):
 # Set the RUN_REAL_PEBBLE_TESTS environment variable to run these tests
 # against a real Pebble server. For example, in one terminal, run Pebble:
 #
-# $ PEBBLE=~/pebble pebble run
+# $ PEBBLE=~/pebble pebble run --http=:4000
 # 2021-09-20T04:10:34.934Z [pebble] Started daemon
 #
 # In another terminal, run the tests:
@@ -2736,6 +3000,96 @@ class TestRealPebble(unittest.TestCase):
 
         self.client = pebble.Client(socket_path=socket_path)
 
+    def test_checks_and_health(self):
+        self.client.add_layer('layer', {
+            'checks': {
+                'bad': {
+                    'override': 'replace',
+                    'level': 'ready',
+                    'period': '50ms',
+                    'threshold': 2,
+                    'exec': {
+                        'command': 'sleep x',
+                    },
+                },
+                'good': {
+                    'override': 'replace',
+                    'level': 'alive',
+                    'period': '50ms',
+                    'exec': {
+                        'command': 'echo foo',
+                    },
+                },
+                'other': {
+                    'override': 'replace',
+                    'exec': {
+                        'command': 'echo bar',
+                    },
+                },
+            },
+        }, combine=True)
+
+        # Checks should all be "up" initially
+        checks = self.client.get_checks()
+        self.assertEqual(len(checks), 3)
+        self.assertEqual(checks[0].name, 'bad')
+        self.assertEqual(checks[0].level, pebble.CheckLevel.READY)
+        self.assertEqual(checks[0].status, pebble.CheckStatus.UP)
+        self.assertEqual(checks[1].name, 'good')
+        self.assertEqual(checks[1].level, pebble.CheckLevel.ALIVE)
+        self.assertEqual(checks[1].status, pebble.CheckStatus.UP)
+        self.assertEqual(checks[2].name, 'other')
+        self.assertEqual(checks[2].level, pebble.CheckLevel.UNSET)
+        self.assertEqual(checks[2].status, pebble.CheckStatus.UP)
+
+        # And /v1/health should return "healthy"
+        health = self._get_health()
+        self.assertEqual(health, {
+            'result': {'healthy': True},
+            'status': 'OK',
+            'status-code': 200,
+            'type': 'sync',
+        })
+
+        # After two retries the "bad" check should go down
+        for i in range(5):
+            checks = self.client.get_checks()
+            bad_check = [c for c in checks if c.name == 'bad'][0]
+            if bad_check.status == pebble.CheckStatus.DOWN:
+                break
+            time.sleep(0.06)
+        else:
+            assert False, 'timed out waiting for "bad" check to go down'
+        self.assertEqual(bad_check.failures, 2)
+        self.assertEqual(bad_check.threshold, 2)
+        good_check = [c for c in checks if c.name == 'good'][0]
+        self.assertEqual(good_check.status, pebble.CheckStatus.UP)
+
+        # And /v1/health should return "unhealthy" (with status HTTP 502)
+        with self.assertRaises(urllib.error.HTTPError) as cm:
+            self._get_health()
+        self.assertEqual(cm.exception.code, 502)
+        health = pebble._json_loads(cm.exception.read())
+        self.assertEqual(health, {
+            'result': {'healthy': False},
+            'status': 'Bad Gateway',
+            'status-code': 502,
+            'type': 'sync',
+        })
+
+        # Then test filtering by check level and by name
+        checks = self.client.get_checks(level=pebble.CheckLevel.ALIVE)
+        self.assertEqual(len(checks), 1)
+        self.assertEqual(checks[0].name, 'good')
+        checks = self.client.get_checks(names=['good', 'bad'])
+        self.assertEqual(len(checks), 2)
+        self.assertEqual(checks[0].name, 'bad')
+        self.assertEqual(checks[1].name, 'good')
+
+    def _get_health(self):
+        f = urllib.request.urlopen('http://localhost:4000/v1/health')
+        return pebble._json_loads(f.read())
+
     def test_exec_wait(self):
         process = self.client.exec(['true'])
         process.wait()