Repository: zalando-stups/pierone-cli

Issue:
Display Clair security information.
PierOne now supports Clair for vulnerability scanning. Pierone now exposes the following three information for each tag:

- clair_id
- severity_fix_available
- severity_no_fix_available

The PierOne CLI should now also enhance the `tags` subcommand with these information and provide a new `cves` subcommand to display full CVE reports.

    $ pierone tags foo bar

Team | Artifact | Tag | Created | By | Fixable CVE Severity | Unfixable CVE Severity |
--------|-----------|-------|------------|-----|-----------------------|--------------------------
foo | bar | 1.0 | 5d ago | example | **Critical** | Medium |
foo | bar | 1.1 | 2d ago | example | **Critical** | Medium |
foo | bar | 2.0 | 1d ago | example | None | Medium |

`High` and `Critical` severities should be highlighted.


    $ pierone cves foo bar 1.0

CVE | Severity | Affected Feature | Fixing Feature | Link
-------|------------|------------------------|---------------------|-------
CVE-2014-9471 | Low | coreutils:8.23-4 | coreutils:9.23-5 | https://security-tracker.debian.org/tracker/CVE-2014-9471

Again, `High` and `Critical` needs to be highlighted and the whole table should be sorted by severity. PierOne source contains an ordered list of possible values.

The information for this output can be retrieved via the [Clair API](https://github.com/coreos/clair/blob/master/api/v1/README.md#get-layersname) using the PierOne provided Clair ID. For this, the PierOne CLI will need to learn about the Clair API's endpoint.

Comments:

Patch:
diff --git a/.gitignore b/.gitignore
index 1e365e8..e60d986 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ htmlcov/
 virtualenv
 *.sw*
 .cache/
+.tox/
diff --git a/pierone/cli.py b/pierone/cli.py
index 467ff32..1af5790 100644
--- a/pierone/cli.py
+++ b/pierone/cli.py
@@ -1,20 +1,18 @@
 import datetime
 import os
 import re
-
-import click
-
-import requests
 import tarfile
 import tempfile
 import time
-import zign.api
-from clickclick import error, AliasedGroup, print_table, OutputFormat, UrlType
 
-from .api import docker_login, request, get_latest_tag, DockerImage
+import click
 import pierone
+import requests
 import stups_cli.config
+import zign.api
+from clickclick import AliasedGroup, OutputFormat, UrlType, error, print_table
 
+from .api import DockerImage, docker_login, get_latest_tag, request
 
 KEYRING_KEY = 'pierone'
 
@@ -24,6 +22,48 @@ output_option = click.option('-o', '--output', type=click.Choice(['text', 'json'
                              help='Use alternative output format')
 
 url_option = click.option('--url', help='Pier One URL', metavar='URI')
+clair_url_option = click.option('--clair-url', help='Clair URL', metavar='CLAIR_URI')
+
+CVE_STYLES = {
+    'TOO_OLD': {
+        'bold': True,
+        'fg': 'red'
+    },
+    'NOT_PROCESSED_YET': {
+        'bold': True,
+        'fg': 'red'
+    },
+    'COULDNT_FIGURE_OUT': {
+        'bold': True,
+        'fg': 'red'
+    },
+    'CRITICAL': {
+        'bold': True,
+        'fg': 'red'
+    },
+    'HIGH': {
+        'bold': True,
+        'fg': 'red'
+    },
+    'MEDIUM': {
+        'fg': 'yellow'
+    },
+    'LOW': {
+        'fg': 'yellow'
+    },
+    'NEGLIGIBLE': {
+        'fg': 'yellow'
+    },
+    'UNKNOWN': {
+        'fg': 'yellow'
+    },
+    'PENDING': {
+        'fg': 'yellow'
+    },
+    'NO_CVES_FOUND': {
+        'fg': 'green'
+    }
+}
 
 TEAM_PATTERN_STR = r'[a-z][a-z0-9-]+'
 TEAM_PATTERN = re.compile(r'^{}$'.format(TEAM_PATTERN_STR))
@@ -54,6 +94,19 @@ def parse_time(s: str) -> float:
         return None
 
 
+def parse_severity(value, clair_id_exists):
+    '''Parse severity values to displayable values'''
+    if value is None and clair_id_exists:
+        return 'NOT_PROCESSED_YET'
+    elif value is None:
+        return 'TOO_OLD'
+
+    value = re.sub('^clair:', '', value)
+    value = re.sub('(?P<upper_letter>(?<=[a-z])[A-Z])', '_\g<upper_letter>', value)
+
+    return value.upper()
+
+
 def print_version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
@@ -82,6 +135,28 @@ def set_pierone_url(config: dict, url: str) -> None:
     return url
 
 
+def set_clair_url(config: dict, url: str) -> None:
+    '''Read Clair URL from cli, from config file or from stdin.'''
+    url = url or config.get('clair_url')
+
+    while not url:
+        url = click.prompt('Please enter the Clair URL', type=UrlType())
+
+        try:
+            requests.get(url, timeout=5)
+        except:
+            error('Could not reach {}'.format(url))
+            url = None
+
+    if '://' not in url:
+        # issue 63: gracefully handle URLs without scheme
+        url = 'https://{}'.format(url)
+
+    config['clair_url'] = url
+    stups_cli.config.store_config(config, 'pierone')
+    return url
+
+
 @click.group(cls=AliasedGroup, context_settings=CONTEXT_SETTINGS)
 @click.option('-V', '--version', is_flag=True, callback=print_version, expose_value=False, is_eager=True,
               help='Print the current version number and exit.')
@@ -147,6 +222,19 @@ def get_tags(url, team, art, access_token):
     return r.json()
 
 
+def get_clair_features(url, layer_id, access_token):
+    if layer_id is None:
+        return []
+
+    r = request(url, '/v1/layers/{}?vulnerabilities&features'.format(layer_id), access_token)
+    if r.status_code == 404:
+        # empty list of tags (layer does not exist)
+        return []
+    else:
+        r.raise_for_status()
+    return r.json()['Layer']['Features']
+
+
 @cli.command()
 @click.argument('team', callback=validate_team)
 @url_option
@@ -184,14 +272,69 @@ def tags(config, team: str, artifact, url, output):
                       'artifact': art,
                       'tag': row['name'],
                       'created_by': row['created_by'],
-                      'created_time': parse_time(row['created'])}
+                      'created_time': parse_time(row['created']),
+                      'severity_fix_available': parse_severity(
+                          row.get('severity_fix_available'), row.get('clair_id', False)),
+                      'severity_no_fix_available': parse_severity(
+                          row.get('severity_no_fix_available'), row.get('clair_id', False))}
                      for row in r])
 
     # sorts are guaranteed to be stable, i.e. tags will be sorted by time (as returned from REST service)
     rows.sort(key=lambda row: (row['team'], row['artifact']))
     with OutputFormat(output):
-        print_table(['team', 'artifact', 'tag', 'created_time', 'created_by'], rows,
-                    titles={'created_time': 'Created', 'created_by': 'By'})
+        titles = {
+            'created_time': 'Created',
+            'created_by': 'By',
+            'severity_fix_available': 'Fixable CVE Severity',
+            'severity_no_fix_available': 'Unfixable CVE Severity'
+        }
+        print_table(['team', 'artifact', 'tag', 'created_time', 'created_by',
+                     'severity_fix_available', 'severity_no_fix_available'],
+                    rows, titles=titles, styles=CVE_STYLES)
+
+
+@cli.command()
+@click.argument('team', callback=validate_team)
+@click.argument('artifact')
+@click.argument('tag')
+@url_option
+@clair_url_option
+@output_option
+@click.pass_obj
+def cves(config, team, artifact, tag, url, clair_url, output):
+    '''List all CVE's found by Clair service for a specific artifact tag'''
+    set_pierone_url(config, url)
+    set_clair_url(config, clair_url)
+
+    rows = []
+    token = get_token()
+    for artifact_tag in get_tags(config.get('url'), team, artifact, token):
+        if artifact_tag['name'] == tag:
+            installed_software = get_clair_features(config.get('clair_url'), artifact_tag.get('clair_id'), token)
+            for software_pkg in installed_software:
+                for cve in software_pkg.get('Vulnerabilities', []):
+                    rows.append({
+                        'cve': cve['Name'],
+                        'severity': cve['Severity'].upper(),
+                        'affected_feature': '{}:{}'.format(software_pkg['Name'],
+                                                           software_pkg['Version']),
+                        'fixing_feature': cve.get(
+                            'FixedBy') and '{}:{}'.format(software_pkg['Name'],
+                                                          cve['FixedBy']),
+                        'link': cve['Link'],
+                    })
+    severity_rating = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'NEGLIGIBLE', 'UNKNOWN', 'PENDING']
+    rows.sort(key=lambda row: severity_rating.index(row['severity']))
+    with OutputFormat(output):
+        titles = {
+            'cve': 'CVE',
+            'severity': 'Severity',
+            'affected_feature': 'Affected Feature',
+            'fixing_feature': 'Fixing Feature',
+            'link': 'Link'
+        }
+        print_table(['cve', 'severity', 'affected_feature', 'fixing_feature', 'link'],
+                    rows, titles=titles, styles=CVE_STYLES)
 
 
 @cli.command()
diff --git a/tox.ini b/tox.ini
index aa079ec..4644fe1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,2 +1,8 @@
 [flake8]
 max-line-length=120
+
+[tox]
+envlist=py34,py35
+
+[testenv]
+commands=python setup.py test


Test patch:
diff --git a/setup.py b/setup.py
old mode 100644
new mode 100755
diff --git a/tests/fixtures/clair_response.json b/tests/fixtures/clair_response.json
new file mode 100644
index 0000000..2638daa
--- /dev/null
+++ b/tests/fixtures/clair_response.json
@@ -0,0 +1,70 @@
+{
+  "Layer": {
+    "Name": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
+    "NamespaceName": "ubuntu:16.04",
+    "ParentName": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
+    "IndexedByVersion": 2,
+    "Features": [
+      {
+        "Name": "python3.5",
+        "NamespaceName": "ubuntu:16.04",
+        "Version": "3.5.1-10",
+        "AddedBy": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+      },
+      {
+        "Name": "python-pip",
+        "NamespaceName": "ubuntu:16.04",
+        "Version": "8.1.1-2",
+        "Vulnerabilities": [
+          {
+            "Name": "CVE-2013-5123",
+            "NamespaceName": "ubuntu:16.04",
+            "Description": "The mirroring support (-M, --use-mirrors) was implemented without any sort of authenticity checks and is downloaded over plaintext HTTP. Further more by default it will dynamically discover the list of available mirrors by querying a DNS entry and extrapolating from that data. It does not attempt to use any sort of method of securing this querying of the DNS like DNSSEC. Software packages are downloaded over these insecure links, unpacked, and then typically the setup.py python file inside of them is executed.",
+            "Link": "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2013-5123",
+            "Severity": "Medium"
+          },
+          {
+            "Name": "CVE-2014-8991",
+            "NamespaceName": "ubuntu:16.04",
+            "Description": "pip 1.3 through 1.5.6 allows local users to cause a denial of service (prevention of package installation) by creating a /tmp/pip-build-* file for another user.",
+            "Link": "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2014-8991",
+            "Severity": "Low",
+            "Metadata": {
+              "NVD": {
+                "CVSSv2": {
+                  "Score": 2.1,
+                  "Vectors": "AV:L/AC:L/Au:N/C:N/I:N"
+                }
+              }
+            }
+          }
+        ],
+        "AddedBy": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+      },
+      {
+        "Name": "openssl",
+        "NamespaceName": "ubuntu:16.04",
+        "Version": "1.0.2g-1ubuntu4",
+        "Vulnerabilities": [
+          {
+            "Name": "CVE-2016-2108",
+            "NamespaceName": "ubuntu:16.04",
+            "Description": "The ASN.1 implementation in OpenSSL before 1.0.1o and 1.0.2 before 1.0.2c allows remote attackers to execute arbitrary code or cause a denial of service (buffer underflow and memory corruption) via an ANY field in crafted serialized data, aka the \"negative zero\" issue.",
+            "Link": "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2016-2108",
+            "Severity": "High",
+            "Metadata": {
+              "NVD": {
+                "CVSSv2": {
+                  "Score": 10,
+                  "Vectors": "AV:N/AC:L/Au:N/C:C/I:C"
+                }
+              }
+            },
+            "FixedBy": "1.0.2g-1ubuntu4.1"
+          }
+        ],
+        "AddedBy": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
+      }
+    ]
+  }
+}
diff --git a/tests/test_api.py b/tests/test_api.py
index 5cb2fc7..3548e01 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1,9 +1,11 @@
 import json
 import os
 from unittest.mock import MagicMock
-import yaml
-from pierone.api import docker_login, DockerImage, get_latest_tag, Unauthorized, image_exists
+
 import pytest
+import yaml
+from pierone.api import (DockerImage, Unauthorized, docker_login,
+                         get_latest_tag, image_exists)
 
 
 def test_docker_login(monkeypatch, tmpdir):
@@ -12,22 +14,22 @@ def test_docker_login(monkeypatch, tmpdir):
     response.status_code = 200
     response.json.return_value = {'access_token': '12377'}
     monkeypatch.setattr('requests.get', MagicMock(return_value=response))
-    token = docker_login('https://pierone.example.org', 'services', 'mytok',
-                         'myuser', 'mypass', 'https://token.example.org', use_keyring=False)
+    docker_login('https://pierone.example.org', 'services', 'mytok',
+                 'myuser', 'mypass', 'https://token.example.org', use_keyring=False)
     path = os.path.expanduser('~/.docker/config.json')
     with open(path) as fd:
         data = yaml.safe_load(fd)
-    assert {'auth': 'b2F1dGgyOjEyMzc3', 'email': 'no-mail-required@example.org'} == data.get('auths').get('https://pierone.example.org')
+        assert {'auth': 'b2F1dGgyOjEyMzc3', 'email': 'no-mail-required@example.org'} == data.get('auths').get('https://pierone.example.org')
 
 
 def test_docker_login_service_token(monkeypatch, tmpdir):
     monkeypatch.setattr('os.path.expanduser', lambda x: x.replace('~', str(tmpdir)))
     monkeypatch.setattr('tokens.get', lambda x: '12377')
-    token = docker_login('https://pierone.example.org', None, 'mytok', 'myuser', 'mypass', 'https://token.example.org')
+    docker_login('https://pierone.example.org', None, 'mytok', 'myuser', 'mypass', 'https://token.example.org')
     path = os.path.expanduser('~/.docker/config.json')
     with open(path) as fd:
         data = yaml.safe_load(fd)
-    assert {'auth': 'b2F1dGgyOjEyMzc3', 'email': 'no-mail-required@example.org'} == data.get('auths').get('https://pierone.example.org')
+        assert {'auth': 'b2F1dGgyOjEyMzc3', 'email': 'no-mail-required@example.org'} == data.get('auths').get('https://pierone.example.org')
 
 
 def test_keep_dockercfg_entries(monkeypatch, tmpdir):
@@ -49,12 +51,12 @@ def test_keep_dockercfg_entries(monkeypatch, tmpdir):
     with open(path, 'w') as fd:
         json.dump(existing_data, fd)
 
-    token = docker_login('https://pierone.example.org', 'services', 'mytok',
-                         'myuser', 'mypass', 'https://token.example.org', use_keyring=False)
+    docker_login('https://pierone.example.org', 'services', 'mytok',
+                 'myuser', 'mypass', 'https://token.example.org', use_keyring=False)
     with open(path) as fd:
         data = yaml.safe_load(fd)
-    assert {'auth': 'b2F1dGgyOjEyMzc3', 'email': 'no-mail-required@example.org'} == data.get('auths', {}).get('https://pierone.example.org')
-    assert existing_data.get(key) == data.get(key)
+        assert {'auth': 'b2F1dGgyOjEyMzc3', 'email': 'no-mail-required@example.org'} == data.get('auths', {}).get('https://pierone.example.org')
+        assert existing_data.get(key) == data.get(key)
 
 
 def test_get_latest_tag(monkeypatch):
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 6f58d15..6282253 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,9 +1,9 @@
 import json
 import os
-from click.testing import CliRunner
+import re
 from unittest.mock import MagicMock
-import yaml
-import zign.api
+
+from click.testing import CliRunner
 from pierone.cli import cli
 
 
@@ -40,6 +40,7 @@ def test_login_given_url_option(monkeypatch, tmpdir):
     runner = CliRunner()
 
     config = {}
+
     def store(data, section):
         config.update(**data)
 
@@ -50,9 +51,9 @@ def test_login_given_url_option(monkeypatch, tmpdir):
     monkeypatch.setattr('requests.get', lambda x, timeout: response)
 
     with runner.isolated_filesystem():
-        result = runner.invoke(cli, ['login'], catch_exceptions=False, input='pieroneurl\n')
+        runner.invoke(cli, ['login'], catch_exceptions=False, input='pieroneurl\n')
         assert config == {'url': 'https://pieroneurl'}
-        result = runner.invoke(cli, ['login', '--url', 'someotherregistry'], catch_exceptions=False)
+        runner.invoke(cli, ['login', '--url', 'someotherregistry'], catch_exceptions=False)
         with open(os.path.join(str(tmpdir), '.docker/config.json')) as fd:
             data = json.load(fd)
         assert data['auths']['https://pieroneurl']['auth'] == 'b2F1dGgyOnRvazEyMw=='
@@ -65,7 +66,7 @@ def test_scm_source(monkeypatch, tmpdir):
     response.json.return_value = {'url': 'git:somerepo', 'revision': 'myrev123'}
 
     runner = CliRunner()
-    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url':'foobar'})
+    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url': 'foobar'})
     monkeypatch.setattr('zign.api.get_token', MagicMock(return_value='tok123'))
     monkeypatch.setattr('pierone.cli.get_tags', MagicMock(return_value={}))
     monkeypatch.setattr('os.path.expanduser', lambda x: x.replace('~', str(tmpdir)))
@@ -75,12 +76,13 @@ def test_scm_source(monkeypatch, tmpdir):
         assert 'myrev123' in result.output
         assert 'git:somerepo' in result.output
 
+
 def test_image(monkeypatch, tmpdir):
     response = MagicMock()
     response.json.return_value = [{'name': '1.0', 'team': 'stups', 'artifact': 'kio'}]
 
     runner = CliRunner()
-    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url':'foobar'})
+    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url': 'foobar'})
     monkeypatch.setattr('zign.api.get_token', MagicMock(return_value='tok123'))
     monkeypatch.setattr('os.path.expanduser', lambda x: x.replace('~', str(tmpdir)))
     monkeypatch.setattr('pierone.api.session.get', MagicMock(return_value=response))
@@ -93,16 +95,130 @@ def test_image(monkeypatch, tmpdir):
 
 def test_tags(monkeypatch, tmpdir):
     response = MagicMock()
-    response.json.return_value = [{'name': '1.0', 'created_by': 'myuser', 'created': '2015-08-20T08:14:59.432Z'}]
+    response.json.return_value = [
+        # Former pierone payload
+        {
+            'name': '1.0',
+            'created_by': 'myuser',
+            'created': '2015-08-20T08:14:59.432Z'
+        },
+        # New pierone payload with clair but no information about CVEs -- old images
+        {
+            "name": "1.1",
+            "created": "2016-05-19T15:23:41.065Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": None,
+            "severity_fix_available": None,
+            "severity_no_fix_available": None
+        },
+        # New pierone payload with clair but no information about CVEs -- still processing
+        {
+            "name": "1.1",
+            "created": "2016-05-19T15:23:41.065Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": "sha256:here",
+            "severity_fix_available": None,
+            "severity_no_fix_available": None
+        },
+        # New pierone payload with clair but could not figure out
+        {
+            "name": "1.1",
+            "created": "2016-05-19T15:23:41.065Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": "sha256:here",
+            "severity_fix_available": "clair:CouldntFigureOut",
+            "severity_no_fix_available": "clair:CouldntFigureOut"
+        },
+        # New pierone payload with clair with no CVEs found
+        {
+            "name": "1.1",
+            "created": "2016-05-19T15:23:41.065Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": "sha256:here",
+            "severity_fix_available": "clair:NoCVEsFound",
+            "severity_no_fix_available": "clair:NoCVEsFound"
+        },
+        # New pierone payload with clair input and info about CVEs
+        {
+            "name": "1.2",
+            "created": "2016-05-23T13:29:17.753Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": "sha256:here",
+            "severity_fix_available": "High",
+            "severity_no_fix_available": "Medium"
+        }
+    ]
 
     runner = CliRunner()
-    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url':'foobar'})
+    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url': 'foobar'})
     monkeypatch.setattr('zign.api.get_token', MagicMock(return_value='tok123'))
     monkeypatch.setattr('os.path.expanduser', lambda x: x.replace('~', str(tmpdir)))
     monkeypatch.setattr('pierone.api.session.get', MagicMock(return_value=response))
     with runner.isolated_filesystem():
         result = runner.invoke(cli, ['tags', 'myteam', 'myart'], catch_exceptions=False)
         assert '1.0' in result.output
+        assert 'Fixable CVE Severity' in result.output
+        assert 'Unfixable CVE Severity' in result.output
+        assert 'TOO_OLD' in result.output
+        assert 'NOT_PROCESSED_YET' in result.output
+        assert 'NO_CVES_FOUND' in result.output
+        assert re.search('HIGH\s+MEDIUM', result.output), 'Should how information about CVEs'
+
+
+def test_cves(monkeypatch, tmpdir):
+    pierone_service_payload = [
+        # Former pierone payload
+        {
+            'name': '1.0',
+            'created_by': 'myuser',
+            'created': '2015-08-20T08:14:59.432Z'
+        },
+        # New pierone payload with clair but no information about CVEs
+        {
+            "name": "1.1",
+            "created": "2016-05-19T15:23:41.065Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": None,
+            "severity_fix_available": None,
+            "severity_no_fix_available": None
+        },
+        # New pierone payload with clair input and info about CVEs
+        {
+            "name": "1.2",
+            "created": "2016-05-23T13:29:17.753Z",
+            "created_by": "myuser",
+            "image": "sha256:here",
+            "clair_id": "sha256:here",
+            "severity_fix_available": "High",
+            "severity_no_fix_available": "Medium"
+        }
+    ]
+
+    with open(os.path.join(os.path.dirname(__file__),
+                           'fixtures', 'clair_response.json'), 'r') as fixture:
+        clair_service_payload = json.loads(fixture.read())
+
+    response = MagicMock()
+    response.json.side_effect = [
+        pierone_service_payload,
+        clair_service_payload
+    ]
+
+    runner = CliRunner()
+    monkeypatch.setattr('stups_cli.config.load_config', lambda x: {'url': 'foobar', 'clair_url': 'barfoo'})
+    monkeypatch.setattr('zign.api.get_token', MagicMock(return_value='tok123'))
+    monkeypatch.setattr('os.path.expanduser', lambda x: x.replace('~', str(tmpdir)))
+    monkeypatch.setattr('pierone.api.session.get', MagicMock(return_value=response))
+    with runner.isolated_filesystem():
+        result = runner.invoke(cli, ['cves', 'myteam', 'myart', '1.2'], catch_exceptions=False)
+        assert 'CVE-2013-5123' in result.output
+        assert re.match('[^\n]+\n[^\n]+HIGH', result.output), 'Results should be ordered by highest priority'
 
 
 def test_latest(monkeypatch, tmpdir):