Repository: RockefellerArchiveCenter/rac_aspace

Issue:
Add unit tests for data helpers that do not require data from ArchivesSpace
**Is your feature request related to a problem? Please describe.**
In order to know that our code is working (and remains working as we update it) we should have unit tests in place which assert expected outputs given prescribed inputs.

**Describe the solution you'd like**
Unit tests for all of the functions in `data_helpers.py`

Discussion:

Patch:
diff --git a/fixtures/archival_object.json b/fixtures/archival_object.json
new file mode 100644
index 0000000..26f2d42
--- /dev/null
+++ b/fixtures/archival_object.json
@@ -0,0 +1,145 @@
+{
+    "lock_version": 3,
+    "title": "kmtests",
+    "publish": true,
+    "restrictions": false,
+    "created_by": "k.martin",
+    "last_modified_by": "admin",
+    "create_time": "2020-03-09T16:25:14Z",
+    "system_mtime": "2020-03-19T13:07:13Z",
+    "user_mtime": "2020-03-19T13:07:13Z",
+    "suppressed": false,
+    "is_slug_auto": false,
+    "id_0": "1",
+    "id_1": "2",
+    "id_2": "3",
+    "id_3": "4",
+    "language": "ang",
+    "level": "collection",
+    "jsonmodel_type": "resource",
+    "external_ids": [],
+    "subjects": [],
+    "linked_events": [],
+    "extents": [
+        {
+            "lock_version": 0,
+            "number": "1",
+            "created_by": "admin",
+            "last_modified_by": "admin",
+            "create_time": "2020-03-19T13:07:40Z",
+            "system_mtime": "2020-03-19T13:07:40Z",
+            "user_mtime": "2020-03-19T13:07:40Z",
+            "portion": "whole",
+            "extent_type": "Cubic Feet",
+            "jsonmodel_type": "extent"
+        }
+    ],
+    "dates": [
+        {
+            "lock_version": 0,
+            "begin": "1990",
+            "end": "2000",
+            "created_by": "admin",
+            "last_modified_by": "admin",
+            "create_time": "2020-03-19T13:07:40Z",
+            "system_mtime": "2020-03-19T13:07:40Z",
+            "user_mtime": "2020-03-19T13:07:40Z",
+            "date_type": "bulk",
+            "label": "creation",
+            "era": "ce",
+            "calendar": "gregorian",
+            "jsonmodel_type": "date"
+        }
+    ],
+    "external_documents": [],
+    "rights_statements": [
+        {
+            "lock_version": 0,
+            "identifier": "e5dc6d09b9fe8640c05fe3de2926868c",
+            "created_by": "admin",
+            "last_modified_by": "admin",
+            "create_time": "2020-03-19T13:07:41Z",
+            "system_mtime": "2020-03-19T13:07:41Z",
+            "user_mtime": "2020-03-19T13:07:41Z",
+            "start_date": "2020-02-01",
+            "end_date": "2090-05-06",
+            "rights_type": "copyright",
+            "status": "copyrighted",
+            "jurisdiction": "US",
+            "jsonmodel_type": "rights_statement",
+            "external_documents": [],
+            "acts": [
+                {
+                    "start_date": "2020-01-01",
+                    "end_date": "2090-12-31",
+                    "created_by": "admin",
+                    "last_modified_by": "admin",
+                    "create_time": "2020-03-19T13:07:41Z",
+                    "system_mtime": "2020-03-19T13:07:41Z",
+                    "user_mtime": "2020-03-19T13:07:41Z",
+                    "act_type": "use",
+                    "restriction": "disallow",
+                    "jsonmodel_type": "rights_statement_act",
+                    "notes": []
+                }
+            ],
+            "linked_agents": [],
+            "notes": []
+        }
+    ],
+    "linked_agents": [],
+    "revision_statements": [],
+    "instances": [
+        {
+            "lock_version": 0,
+            "created_by": "admin",
+            "last_modified_by": "admin",
+            "create_time": "2020-03-19T13:07:41Z",
+            "system_mtime": "2020-03-19T13:07:41Z",
+            "user_mtime": "2020-03-19T13:07:41Z",
+            "instance_type": "mixed materials",
+            "jsonmodel_type": "instance",
+            "is_representative": false,
+            "sub_container": {
+                "lock_version": 0,
+                "created_by": "admin",
+                "last_modified_by": "admin",
+                "create_time": "2020-03-19T13:07:41Z",
+                "system_mtime": "2020-03-19T13:07:41Z",
+                "user_mtime": "2020-03-19T13:07:41Z",
+                "jsonmodel_type": "sub_container",
+                "top_container": {
+                    "ref": "/repositories/2/top_containers/191155"
+                }
+            }
+        }
+    ],
+    "deaccessions": [],
+    "related_accessions": [],
+    "classifications": [],
+    "notes": [
+        {
+            "jsonmodel_type": "note_multipart",
+            "persistent_id": "7e39d839488e068c65f822e52f5ffa01",
+            "type": "accessrestrict",
+            "rights_restriction": {
+                "local_access_restriction_type": []
+            },
+            "subnotes": [
+                {
+                    "jsonmodel_type": "note_text",
+                    "content": "materials are restricted",
+                    "publish": true
+                }
+            ],
+            "publish": true
+        }
+    ],
+    "uri": "/repositories/2/resources/13064",
+    "repository": {
+        "ref": "/repositories/2"
+    },
+    "tree": {
+        "ref": "/repositories/2/resources/13064/tree"
+    }
+}
diff --git a/fixtures/date_expression.json b/fixtures/date_expression.json
new file mode 100644
index 0000000..66e7b43
--- /dev/null
+++ b/fixtures/date_expression.json
@@ -0,0 +1,14 @@
+{
+  "lock_version": 0,
+  "expression": "1905-1980",
+  "begin": "1905",
+  "end": "1980",
+  "created_by": "greg",
+  "last_modified_by": "greg",
+  "create_time": "2017-08-08T21:11:46Z",
+  "system_mtime": "2017-08-08T21:11:46Z",
+  "user_mtime": "2017-08-08T21:11:46Z",
+  "date_type": "inclusive",
+  "label": "creation",
+  "jsonmodel_type": "date"
+}
diff --git a/fixtures/date_no_expression.json b/fixtures/date_no_expression.json
new file mode 100644
index 0000000..a2db572
--- /dev/null
+++ b/fixtures/date_no_expression.json
@@ -0,0 +1,13 @@
+{
+  "lock_version": 0,
+  "begin": "1905",
+  "end": "1980",
+  "created_by": "greg",
+  "last_modified_by": "greg",
+  "create_time": "2017-08-08T21:11:46Z",
+  "system_mtime": "2017-08-08T21:11:46Z",
+  "user_mtime": "2017-08-08T21:11:46Z",
+  "date_type": "inclusive",
+  "label": "creation",
+  "jsonmodel_type": "date"
+}
diff --git a/fixtures/note_multi.json b/fixtures/note_multi.json
new file mode 100644
index 0000000..4488f58
--- /dev/null
+++ b/fixtures/note_multi.json
@@ -0,0 +1,17 @@
+{
+  "jsonmodel_type": "note_multipart",
+  "persistent_id": "d08e4b20fceb371af437d3e2a23dda6f",
+  "label": "Access",
+  "type": "accessrestrict",
+  "rights_restriction": {
+    "local_access_restriction_type": []
+  },
+  "subnotes": [
+    {
+      "jsonmodel_type": "note_text",
+      "content": "materials are restricted",
+      "publish": true
+    }
+  ],
+  "publish": true
+}
diff --git a/fixtures/note_single.json b/fixtures/note_single.json
new file mode 100644
index 0000000..406ab14
--- /dev/null
+++ b/fixtures/note_single.json
@@ -0,0 +1,9 @@
+{
+  "jsonmodel_type": "note_singlepart",
+  "persistent_id": "ec9e58a73212a0b4782e512339a3fa83",
+  "type": "abstract",
+  "content": [
+    "New York Mets"
+  ],
+  "publish": true
+}
diff --git a/fixtures/top_container.json b/fixtures/top_container.json
new file mode 100644
index 0000000..b64a04f
--- /dev/null
+++ b/fixtures/top_container.json
@@ -0,0 +1,39 @@
+{
+    "lock_version": 2,
+    "indicator": "1",
+    "created_by": "admin",
+    "last_modified_by": "admin",
+    "create_time": "2020-03-19T13:07:07Z",
+    "system_mtime": "2020-03-19T13:14:45Z",
+    "user_mtime": "2020-03-19T13:14:44Z",
+    "created_for_collection": "/repositories/2/resources/13064",
+    "type": "box",
+    "jsonmodel_type": "top_container",
+    "active_restrictions": [],
+    "container_locations": [
+        {
+            "jsonmodel_type": "container_location",
+            "status": "current",
+            "start_date": "2020-03-19",
+            "system_mtime": "2020-03-19 13:14:44 UTC",
+            "user_mtime": "2020-03-19 13:14:44 UTC",
+            "ref": "/locations/7799"
+        }
+    ],
+    "series": [],
+    "collection": [
+        {
+            "ref": "/repositories/2/resources/13064",
+            "identifier": "1",
+            "display_string": "kmtests"
+        }
+    ],
+    "uri": "/repositories/2/top_containers/191155",
+    "repository": {
+        "ref": "/repositories/2"
+    },
+    "restricted": false,
+    "is_linked_to_published_record": true,
+    "display_string": "Box 1",
+    "long_display_string": "1, Box 1"
+}
diff --git a/rac_aspace/data_helpers.py b/rac_aspace/data_helpers.py
index e53a24a..c062b01 100644
--- a/rac_aspace/data_helpers.py
+++ b/rac_aspace/data_helpers.py
@@ -43,11 +43,13 @@ def get_note_text(note):
         return content
 
     if note.jsonmodel_type == "note_singlepart":
-        content = note.source.content.strip("]['").split(', ')
+        content = note.content
     elif note.jsonmodel_type == "note_index":
-        content = note.source.items.strip("]['").split(', ')
+        content = note.items
     else:
-        content = (parse_subnote(sn) for sn in note.subnotes)
+        subnote_content_list = list(parse_subnote(sn) for sn in note.subnotes)
+        content = [
+            c for subnote_content in subnote_content_list for c in subnote_content]
     return content
 
 
@@ -65,7 +67,8 @@ def text_in_note(note, query_string):
     CONFIDENCE_RATIO = 97
     """int: Minimum confidence ratio to match against."""
     note_content = get_note_text(note)
-    ratio = fuzz.token_sort_ratio(note_content.lower(), query_string.lower())
+    ratio = fuzz.token_sort_ratio(
+        " ".join([n.lower() for n in note_content]), query_string.lower())
     return (True if ratio > CONFIDENCE_RATIO else False)
 
 
@@ -110,7 +113,7 @@ def format_container(top_container):
     Returns:
         str: a concatenation of top container type and indicator.
     """
-    return "{0} {1}".format(top_container.container_type,
+    return "{0} {1}".format(top_container.type,
                             top_container.indicator)
 
 
@@ -126,7 +129,7 @@ def format_resource_id(resource, separator=":"):
         str: a concatenated four-part ID for the resource record.
     """
     resource_id = []
-    for x in range(3):
+    for x in range(4):
         try:
             resource_id.append(getattr(resource, "id_{0}".format(x)))
         except AttributeError:
@@ -175,17 +178,20 @@ def get_expression(date):
     Concatenates start and end dates if no date expression exists.
 
     Args:
-        date (dict): an ArchivesSpace date object.
+        date (obj): an ArchivesSpace date object
 
     Returns:
         str: a date expression for the date object.
     """
-    if date.expression:
-        return date.expression
-    if date.date_end:
-        return "{0} - {1}".format(date.date_start, date.date_end)
-    else:
-        return date.date_start
+    date_json = date.json()
+    try:
+        expression = date_json["expression"]
+    except KeyError:
+        if date_json.get("end"):
+            expression = "{0}-{1}".format(date_json["begin"], date_json["end"])
+        else:
+            expression = date_json["begin"]
+    return expression
 
 
 def associated_objects(top_container):
@@ -235,10 +241,10 @@ def is_restricted(archival_object):
     """
     query_string = "materials are restricted"
     for note in archival_object.notes:
-        if note.jsonmodel_type == 'accessrestrict':
+        if note.type == 'accessrestrict':
             if text_in_note(note, query_string):
                 return True
-    for rights_statement in archival_object.rights_statement:
+    for rights_statement in archival_object.rights_statements:
         if indicates_restriction(rights_statement):
             return True
     return False


Test patch:
diff --git a/tests/test_data_helpers.py b/tests/test_data_helpers.py
index 8ea489c..0caa5a2 100644
--- a/tests/test_data_helpers.py
+++ b/tests/test_data_helpers.py
@@ -1,16 +1,170 @@
 """
 Unit tests for Data Helpers
 """
+import json
+import os
 import unittest
 
+from asnake.jsonmodel import wrap_json_object
 from rac_aspace import data_helpers
 
 
 class TestDataHelpers(unittest.TestCase):
+    """
+    Tests the data helper functions.
+    """
 
-    def test_helpers(self):
+    def test_get_note_text(self):
+        """
+        Checks whether the returned note text matches the selected query string.
+
+        Args:
+            note (dict): an ArchivesSpace note object
+
+        Returns:
+            bool: True if the content matches the supplied text.
+        """
+        with open(os.path.join("fixtures", "note_multi.json"), "r") as json_file:
+            data = json.load(json_file)
+            note = wrap_json_object(data)
+            result = data_helpers.get_note_text(note)
+            self.assertTrue(result, list)
+            self.assertEqual(result, ["materials are restricted"])
+
+    def test_text_in_note(self):
+        """
+        Checks whether the query string and note content are close to a match.
+
+        Args:
+            note (dict): an ArchivesSpace note object
+
+            query_string (str): a string object used to check against note content
+
+        Returns:
+            bool: True if the sort ratio is greater than or equal to 97.
+        """
+        query_string = "New York Mets"
+        with open(os.path.join("fixtures", "note_single.json"), "r") as json_file:
+            data = json.load(json_file)
+            note = wrap_json_object(data)
+            result = data_helpers.text_in_note(note, query_string)
+            self.assertTrue(result)
+
+    # def test_get_locations(self):
+        """
+        Data helper as written won't work without AS calls. Container and Location
+        information are in different objects, not just Archival Objects.
+        """
+        # """
+        # Checks whether the function returns a list and if it is empty.
+        # Need to write a way to get the archival_object information from AS.
+
+        # Args:
+        # archival_object (dict): an ArchivesSpace archival object
+
+        # Returns:
+        # bool: Boolean. True if locations exists and is not empty.
+        # bool: Boolean. True if locations is a list. False if any other data type.
+        # """
+        # with open(os.path.join("fixtures", "archival_object.json"), "r") as json_file:
+        # data = json.load(json_file)
+        # archival_object = wrap_json_object(data)
+        # locations = get_locations(archival_object)
+        # self.assertNotEqual(
+        # locations, False,
+        # "Get locatins returned an error: {}".format(locations)
+        # )
+        # self.assertIsInstance(locations, list)
+
+    def test_format_location(self):
         pass
 
+    def test_format_container(self):
+        """
+        Checks whether the function returns a string as expected.
+
+        Args:
+            top_container (dict): an archivespace top container
+
+        Returns:
+            bool: Boolean. True if the top_container string matches expected output and type.
+        """
+        with open(os.path.join("fixtures", "top_container.json"), "r") as json_file:
+            data = json.load(json_file)
+            top_container = wrap_json_object(data)
+            result = data_helpers.format_container(top_container)
+            self.assertEqual(result, 'box 1')
+            self.assertIsInstance(result, str)
+
+    def test_format_resource_id(self):
+        """
+        Checks whether the function returns a concatenated string as expected.
+
+        Args:
+            resource (dict): an ArchivesSpace object.
+
+            separator (str): a string separator that will be added in between each section.
+
+        Returns:
+            bool: Boolean. True if the top_container string matches expected output and type.
+        """
+        separator = ":"
+        with open(os.path.join("fixtures", "archival_object.json"), "r") as json_file:
+            data = json.load(json_file)
+            resource = wrap_json_object(data)
+            result = data_helpers.format_resource_id(resource, separator)
+            self.assertIsInstance(result, str)
+            self.assertEqual(result, '1:2:3:4')
+
+    def test_closest_value(self):
+        pass
+
+    def test_get_orphans(self):
+        pass
+
+    def test_get_expression(self):
+        """
+        Tests whether the date expression function works as intended.
+
+        Args:
+            date (dict): an ArchivesSpace date object
+
+        Returns:
+            bool: Returns true if the function creates the expected string output.
+        """
+        files = ['date_expression.json', 'date_no_expression.json']
+        for f in files:
+            with open(os.path.join("fixtures", f), "r") as json_file:
+                data = json.load(json_file)
+                date = wrap_json_object(data)
+                result = data_helpers.get_expression(date)
+                self.assertTrue(result, "1905-1980")
+
+    def test_associated_objects(self):
+        pass
+
+    def test_indicates_restriction(self):
+        pass
+
+    def test_is_restricted(self):
+        """
+        Tests whether the function can find restrictions in an AS archival object.
+
+        Args:
+            archival_object (dict): and ArchivesSpace archival object
+
+        Returns:
+            bool: Returns true when finding a restriction in a note or rights statement.
+        """
+        with open(os.path.join("fixtures", "archival_object.json"), "r") as json_file:
+            data = json.load(json_file)
+            archival_object = wrap_json_object(data)
+            result = data_helpers.is_restricted(archival_object)
+            self.assertEqual(result, True)
+            """
+            Cannot currently test the restrictions portion because data helper is unwritten
+            """
+
     def test_strip_html_tags(self):
         input = "<h1>Title</h1><p>This is <i>some</i> text! It is wrapped in a variety of html tags, which should <strong>all</strong> be stripped &amp; not returned.</p>"
         expected = "TitleThis is some text! It is wrapped in a variety of html tags, which should all be stripped &amp; not returned."
diff --git a/tests/test_serializers.py b/tests/test_serializers.py
index 90c7b97..4a831fa 100644
--- a/tests/test_serializers.py
+++ b/tests/test_serializers.py
@@ -34,7 +34,8 @@ class TestSerializers(unittest.TestCase):
                     reader = csv.reader(
                         out_file, delimiter=serializer.delimiter)
                     out_data = [r for r in reader]
-                    self.assertEqual(out_data[0], ["column1", "column2"])
+                    self.assertEqual(set(out_data[0]), set(
+                        ["column1", "column2"]))
                     self.assertEqual(len(out_data), len(data) + 1)
                 remove(expected_filepath)