Repository: oasis-open/cti-taxii-client

Issue:
Verify correct handling of all optional fields.
For example: If the server's API root is without description, the client will throw an exception. 
But the description is optional according to the spec, I expected there is no error here.

Discussion:

Patch:
diff --git a/taxii2client/__init__.py b/taxii2client/__init__.py
index 85a7b7f..1fddedb 100644
--- a/taxii2client/__init__.py
+++ b/taxii2client/__init__.py
@@ -220,23 +220,50 @@ class Status(_TAXIIEndpoint):
             self.refresh()
             elapsed = time.time() - start_time
 
-    def _populate_fields(self, id, status, total_count, success_count,
-                         failure_count, pending_count, request_timestamp=None,
+    def _populate_fields(self, id=None, status=None, total_count=None,
+                         success_count=None, failure_count=None,
+                         pending_count=None, request_timestamp=None,
                          successes=None, failures=None, pendings=None):
-        self.id = id
-        self.status = status
-        self.request_timestamp = request_timestamp
-        self.total_count = total_count
-        self.success_count = success_count
-        self.failure_count = failure_count
-        self.pending_count = pending_count
-        self.successes = successes or []
-        self.failures = failures or []
-        self.pendings = pendings or []
+        self.id = id  # required
+        self.status = status  # required
+        self.request_timestamp = request_timestamp  # optional
+        self.total_count = total_count  # required
+        self.success_count = success_count  # required
+        self.failure_count = failure_count  # required
+        self.pending_count = pending_count  # required
+        self.successes = successes or []  # optional
+        self.failures = failures or []  # optional
+        self.pendings = pendings or []  # optional
 
         self._validate_status()
 
     def _validate_status(self):
+        """Validates Status information. Raises errors for required
+        properties."""
+        if not self.id:
+            msg = "No 'id' in Status for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if not self.status:
+            msg = "No 'status' in Status for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self.total_count is None:
+            msg = "No 'total_count' in Status for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self.success_count is None:
+            msg = "No 'success_count' in Status for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self.failure_count is None:
+            msg = "No 'failure_count' in Status for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self.pending_count is None:
+            msg = "No 'pending_count' in Status for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
         if len(self.successes) != self.success_count:
             msg = "Found successes={}, but success_count={} in status '{}'"
             raise ValidationError(msg.format(self.successes,
@@ -356,18 +383,34 @@ class Collection(_TAXIIEndpoint):
 
     def _populate_fields(self, id=None, title=None, description=None,
                          can_read=None, can_write=None, media_types=None):
-        if media_types is None:
-            media_types = []
-        self._id = id
-        self._title = title
-        self._description = description
-        self._can_read = can_read
-        self._can_write = can_write
-        self._media_types = media_types
+        self._id = id  # required
+        self._title = title  # required
+        self._description = description  # optional
+        self._can_read = can_read  # required
+        self._can_write = can_write  # required
+        self._media_types = media_types or []  # optional
 
         self._validate_collection()
 
     def _validate_collection(self):
+        """Validates Collection information. Raises errors for required
+        properties."""
+        if not self._id:
+            msg = "No 'id' in Collection for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if not self._title:
+            msg = "No 'title' in Collection for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self._can_read is None:
+            msg = "No 'can_read' in Collection for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self._can_write is None:
+            msg = "No 'can_write' in Collection for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
         if self._id not in self.url:
             msg = "The collection '{}' does not match the url for queries '{}'"
             raise ValidationError(msg.format(self._id, self.url))
@@ -545,6 +588,21 @@ class ApiRoot(_TAXIIEndpoint):
         if not self._loaded_information:
             self.refresh_information()
 
+    def _validate_api_root(self):
+        """Validates API Root information. Raises errors for required
+        properties."""
+        if not self._title:
+            msg = "No 'title' in API Root for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if not self._versions:
+            msg = "No 'versions' in API Root for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
+        if self._max_content_length is None:
+            msg = "No 'max_content_length' in API Root for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
     def refresh(self):
         """Update the API Root's information and list of Collections"""
         self.refresh_information()
@@ -557,11 +615,12 @@ class ApiRoot(_TAXIIEndpoint):
         """
         response = self._conn.get(self.url, accept=MEDIA_TYPE_TAXII_V20)
 
-        self._title = response["title"]
-        self._description = response["description"]
-        self._versions = response["versions"]
-        self._max_content_length = response["max_content_length"]
+        self._title = response.get("title")  # required
+        self._description = response.get("description")  # optional
+        self._versions = response.get("versions", [])  # required
+        self._max_content_length = response.get("max_content_length")  # required
 
+        self._validate_api_root()
         self._loaded_information = True
 
     def refresh_collections(self):
@@ -573,7 +632,7 @@ class ApiRoot(_TAXIIEndpoint):
         response = self._conn.get(url, accept=MEDIA_TYPE_TAXII_V20)
 
         self._collections = []
-        for item in response["collections"]:
+        for item in response.get("collections", []):  # optional
             collection_url = url + item["id"] + "/"
             collection = Collection(collection_url, conn=self._conn, **item)
             self._collections.append(collection)
@@ -582,8 +641,8 @@ class ApiRoot(_TAXIIEndpoint):
 
     def get_status(self, status_id):
         status_url = self.url + "status/" + status_id + "/"
-        info = self._conn.get(status_url, accept=MEDIA_TYPE_TAXII_V20)
-        return Status(status_url, conn=self._conn, **info)
+        response = self._conn.get(status_url, accept=MEDIA_TYPE_TAXII_V20)
+        return Status(status_url, conn=self._conn, **response)
 
 
 class Server(_TAXIIEndpoint):
@@ -649,13 +708,20 @@ class Server(_TAXIIEndpoint):
         if not self._loaded:
             self.refresh()
 
+    def _validate_server(self):
+        """Validates server information. Raises errors for required properties.
+        """
+        if not self._title:
+            msg = "No 'title' in Server Discovery for request '{}'"
+            raise ValidationError(msg.format(self.url))
+
     def refresh(self):
         response = self._conn.get(self.url, accept=MEDIA_TYPE_TAXII_V20)
 
-        self._title = response.get("title")
-        self._description = response.get("description")
-        self._contact = response.get("contact")
-        roots = response.get("api_roots", [])
+        self._title = response.get("title")  # required
+        self._description = response.get("description")  # optional
+        self._contact = response.get("contact")  # optional
+        roots = response.get("api_roots", [])  # optional
         self._api_roots = [ApiRoot(url,
                                    user=self._user,
                                    password=self._password,
@@ -665,7 +731,8 @@ class Server(_TAXIIEndpoint):
         # rather than creating a duplicate. The TAXII 2.0 spec says that the
         # `default` API Root MUST be an item in `api_roots`.
         root_dict = dict(zip(roots, self._api_roots))
-        self._default = root_dict.get(response.get("default"))
+        self._default = root_dict.get(response.get("default"))  # optional
+        self._validate_server()
 
         self._loaded = True
 


Test patch:
diff --git a/taxii2client/test/test_client.py b/taxii2client/test/test_client.py
index de2e303..afdb0fe 100644
--- a/taxii2client/test/test_client.py
+++ b/taxii2client/test/test_client.py
@@ -7,7 +7,7 @@ import six
 
 from taxii2client import (
     MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20, AccessError, ApiRoot,
-    Collection, InvalidArgumentsError, Server, TAXIIServiceException,
+    Collection, InvalidArgumentsError, Server, Status, TAXIIServiceException,
     ValidationError, _filter_kwargs_to_query_params, _HTTPConnection,
     _TAXIIEndpoint, get_collection_by_id
 )
@@ -187,6 +187,46 @@ STATUS_RESPONSE = """{
 }"""
 
 
+@pytest.fixture
+def status_dict():
+    return {
+        "id": "2d086da7-4bdc-4f91-900e-d77486753710",
+        "status": "pending",
+        "request_timestamp": "2016-11-02T12:34:34.12345Z",
+        "total_count": 4,
+        "success_count": 1,
+        "successes": [
+            "indicator--c410e480-e42b-47d1-9476-85307c12bcbf"
+        ],
+        "failure_count": 1,
+        "failures": [
+            {
+              "id": "malware--664fa29d-bf65-4f28-a667-bdb76f29ec98",
+              "message": "Unable to process object"
+            }
+        ],
+        "pending_count": 2,
+        "pendings": [
+            "indicator--252c7c11-daf2-42bd-843b-be65edca9f61",
+            "relationship--045585ad-a22f-4333-af33-bfd503a683b5"
+        ]
+    }
+
+
+@pytest.fixture
+def collection_dict():
+    return {
+        "id": "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb",
+        "title": "Writable Collection",
+        "description": "This collection is a dropbox for submitting indicators",
+        "can_read": False,
+        "can_write": True,
+        "media_types": [
+            "application/vnd.oasis.stix+json; version=2.0"
+        ]
+    }
+
+
 @pytest.fixture
 def server():
     """Default server object for example.com"""
@@ -222,6 +262,11 @@ def bad_writable_collection():
     return Collection(COLLECTION_URL)
 
 
+def set_api_root_response(response):
+    responses.add(responses.GET, API_ROOT_URL, body=response,
+                  status=200, content_type=MEDIA_TYPE_TAXII_V20)
+
+
 def set_discovery_response(response):
     responses.add(responses.GET, DISCOVERY_URL, body=response, status=200,
                   content_type=MEDIA_TYPE_TAXII_V20)
@@ -282,6 +327,66 @@ def test_discovery_with_no_default(server):
     assert server.default is None
 
 
+@responses.activate
+def test_discovery_with_no_title(server):
+    response = """{
+      "description": "This TAXII Server contains a listing of...",
+      "contact": "string containing contact information",
+      "api_roots": [
+        "https://example.com/api1/",
+        "https://example.com/api2/",
+        "https://example.net/trustgroup1/"
+      ]
+    }"""
+    set_discovery_response(response)
+    with pytest.raises(ValidationError) as excinfo:
+        server.refresh()
+
+    assert "No 'title' in Server Discovery for request 'https://example.com/taxii/'" == str(excinfo.value)
+
+
+@responses.activate
+def test_api_root_no_title(api_root):
+    set_api_root_response("""{
+      "description": "A trust group setup for malware researchers",
+      "versions": ["taxii-2.0"],
+      "max_content_length": 9765625
+    }""")
+    with pytest.raises(ValidationError) as excinfo:
+        assert api_root._loaded_information is False
+        api_root.refresh_information()
+
+    assert "No 'title' in API Root for request 'https://example.com/api1/'" == str(excinfo.value)
+
+
+@responses.activate
+def test_api_root_no_versions(api_root):
+    set_api_root_response("""{
+      "title": "Malware Research Group",
+      "description": "A trust group setup for malware researchers",
+      "max_content_length": 9765625
+    }""")
+    with pytest.raises(ValidationError) as excinfo:
+        assert api_root._loaded_information is False
+        api_root.refresh_information()
+
+    assert "No 'versions' in API Root for request 'https://example.com/api1/'" == str(excinfo.value)
+
+
+@responses.activate
+def test_api_root_no_max_content_length(api_root):
+    set_api_root_response("""{
+      "title": "Malware Research Group",
+      "description": "A trust group setup for malware researchers",
+      "versions": ["taxii-2.0"]
+    }""")
+    with pytest.raises(ValidationError) as excinfo:
+        assert api_root._loaded_information is False
+        api_root.refresh_information()
+
+    assert "No 'max_content_length' in API Root for request 'https://example.com/api1/'" == str(excinfo.value)
+
+
 @responses.activate
 def test_api_root(api_root):
     responses.add(responses.GET, API_ROOT_URL, API_ROOT_RESPONSE,
@@ -563,3 +668,93 @@ def test_taxii_endpoint_raises_exception():
         _TAXIIEndpoint("https://example.com/api1/collections/", conn, "other", "test")
 
     assert "A connection and user/password may not both be provided." in str(excinfo.value)
+
+
+def test_status_missing_id_property(status_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        status_dict.pop("id")
+        Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/",
+               user="foo", password="bar", verify=False, **status_dict)
+
+    assert "No 'id' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value)
+
+
+def test_status_missing_status_property(status_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        status_dict.pop("status")
+        Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/",
+               user="foo", password="bar", verify=False, **status_dict)
+
+    assert "No 'status' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value)
+
+
+def test_status_missing_total_count_property(status_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        status_dict.pop("total_count")
+        Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/",
+               user="foo", password="bar", verify=False, **status_dict)
+
+    assert "No 'total_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value)
+
+
+def test_status_missing_success_count_property(status_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        status_dict.pop("success_count")
+        Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/",
+               user="foo", password="bar", verify=False, **status_dict)
+
+    assert "No 'success_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value)
+
+
+def test_status_missing_failure_count_property(status_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        status_dict.pop("failure_count")
+        Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/",
+               user="foo", password="bar", verify=False, **status_dict)
+
+    assert "No 'failure_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value)
+
+
+def test_status_missing_pending_count_property(status_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        status_dict.pop("pending_count")
+        Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/",
+               user="foo", password="bar", verify=False, **status_dict)
+
+    assert "No 'pending_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value)
+
+
+def test_collection_missing_id_property(collection_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        collection_dict.pop("id")
+        Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/",
+                   user="foo", password="bar", verify=False, **collection_dict)
+
+    assert "No 'id' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value)
+
+
+def test_collection_missing_title_property(collection_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        collection_dict.pop("title")
+        Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/",
+                   user="foo", password="bar", verify=False, **collection_dict)
+
+    assert "No 'title' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value)
+
+
+def test_collection_missing_can_read_property(collection_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        collection_dict.pop("can_read")
+        Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/",
+                   user="foo", password="bar", verify=False, **collection_dict)
+
+    assert "No 'can_read' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value)
+
+
+def test_collection_missing_can_write_property(collection_dict):
+    with pytest.raises(ValidationError) as excinfo:
+        collection_dict.pop("can_write")
+        Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/",
+                   user="foo", password="bar", verify=False, **collection_dict)
+
+    assert "No 'can_write' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value)