Repository: openaddresses/machine
Pull request #633 (2017-05-08 00:24:20)

"chain" conform function
Hey all -  finally found a minute to implement this.

In some cases it would be useful to have the ability to apply more than a single standard conform function to a given field. One example is the city of Wichita, KS in https://github.com/openaddresses/openaddresses/pull/2423, where there's a "Prop_Addr" field which contains number, street, and sometimes unit, and also a separate unit field which needs to be removed from the street name. Ideally we could run a "postfixed_street" to separate the street (+unit) from the initial house number, and then remove the unit suffix with a "remove_postix". However, the conform only allows running a single function per field. It might be possible to devise a (nasty) regex to handle all of the cases, but said regex might be brittle if e.g. the data source is updated with a pattern that wasn't initially handled.

This PR allows the chaining of multiple primitive functions from the OpenAddresses conforms. Here's an example for the Wichita case:

```json
{
    "coverage": {
        "geometry": {
            "type": "Point",
            "coordinates": [
                -97.34,
                37.68
            ]
        },
        "US Census": {
            "geoid": "2079000",
            "name": "City of Witchita",
            "state": "Kansas"
        },
        "country": "us",
        "state": "ks",
        "city": "Witchita"
    },
    "data": "http://gismaps.wichita.gov/agsweb/rest/services/COWGIS/Property_and_Location/MapServer/4",
    "type": "ESRI",
    "conform": {
        "type": "geojson",
        "number": {
            "function": "prefixed_number",
            "field": "Prop_Addr"
        },
        "street": {
            "function": "chain",
            "variable": "street_wip",
            "functions": [
                {
                    "function": "postfixed_street",
                    "field": "Prop_Addr"
                },
                {
                    "function": "remove_postfix",
                    "field": "street_wip",
                    "field_to_remove": "Prop_Addr_Unit"
                }
            ]
        },
        "postcode": "Prop_zip",
        "city": "Prop_City"
    }
}
```

Notice that the field we operate on in the second chained function is the user-defined variable "street_wip". This field is added to the current row and stores the computed result between function calls. As far as the inner steps in the chain are concerned, they're simply storing their output in the "street_wip" field instead of "OA:street". Each step in a conform chain works exactly the way it did before, but in the context of a chain, there's this extra variable that can be referenced along with source fields. Note that chains can be nested, so one could, for example, run various regexes on different source fields and then combine their output in a specific format. I include a very simple example of a nested config in the tests for this new function.

The only requirement for the user-specified variable names is that they should not conflict with the source fields or the standard field names like "street", "number", etc.

Note that there was some refactoring of the current row functions required to make this work. Namely, each row function needs to be passed its local config dictionary instead of just the top-level config and a key (which assumes that the function dictionaries can be referenced shallowly with a single string key).

Also related: https://github.com/openaddresses/openaddresses/pull/2909


Patch:
diff --git a/openaddr/conform.py b/openaddr/conform.py
index e21537c..f0c4e86 100644
--- a/openaddr/conform.py
+++ b/openaddr/conform.py
@@ -60,6 +60,8 @@ attrib_types = {
     'id':       'OA:id'
 }
 
+var_types = attrib_types.copy()
+
 UNZIPPED_DIRNAME = 'unzipped'
 
 geometry_types = {
@@ -886,6 +888,30 @@ def row_extract_and_reproject(source_definition, source_row):
     out_row[Y_FIELDNAME] = out_y
     return out_row
 
+
+def row_function(sd, row, key, fxn):
+    function = fxn["function"]
+    if function == "join":
+        row = row_fxn_join(sd, row, key, fxn)
+    elif function == "regexp":
+        row = row_fxn_regexp(sd, row, key, fxn)
+    elif function == "format":
+        row = row_fxn_format(sd, row, key, fxn)
+    elif function == "prefixed_number":
+        row = row_fxn_prefixed_number(sd, row, key, fxn)
+    elif function == "postfixed_street":
+        row = row_fxn_postfixed_street(sd, row, key, fxn)
+    elif function == "remove_prefix":
+        row = row_fxn_remove_prefix(sd, row, key, fxn)
+    elif function == "remove_postfix":
+        row = row_fxn_remove_postfix(sd, row, key, fxn)
+    elif function == "chain":
+        row = row_fxn_chain(sd, row, key, fxn)
+
+    return row
+
+
+
 ### Row-level conform code. Inputs and outputs are individual rows in a CSV file.
 ### The input row may or may not be modified in place. The output row is always returned.
 
@@ -896,28 +922,15 @@ def row_transform_and_convert(sd, row):
     row = row_smash_case(sd, row)
 
     c = sd["conform"]
-    
+
     "Attribute tags can utilize processing fxns"
     for k, v in c.items():
-        if k in attrib_types and type(c[k]) is list:
+        if k in attrib_types and type(v) is list:
             "Lists are a concat shortcut to concat fields with spaces"
             row = row_merge(sd, row, k)
-        if k in attrib_types and type(c[k]) is dict:
+        if k in attrib_types and type(v) is dict:
             "Dicts are custom processing functions"
-            if c[k]["function"] == "join":
-                row = row_fxn_join(sd, row, k) 
-            elif c[k]["function"] == "regexp":
-                row = row_fxn_regexp(sd, row, k)
-            elif c[k]["function"] == "format":
-                row = row_fxn_format(sd, row, k)
-            elif c[k]["function"] == "prefixed_number":
-                row = row_fxn_prefixed_number(sd, row, k)
-            elif c[k]["function"] == "postfixed_street":
-                row = row_fxn_postfixed_street(sd, row, k)
-            elif c[k]["function"] == "remove_prefix":
-                row = row_fxn_remove_prefix(sd, row, k)
-            elif c[k]["function"] == "remove_postfix":
-                row = row_fxn_remove_postfix(sd, row, k)
+            row = row_function(sd, row, k, v)
 
     if "advanced_merge" in c:
         raise ValueError('Found unsupported "advanced_merge" option in conform')
@@ -964,77 +977,67 @@ def row_smash_case(sd, input):
 def row_merge(sd, row, key):
     "Merge multiple columns like 'Maple','St' to 'Maple St'"
     merge_data = [row[field] for field in sd["conform"][key]]
-    row[attrib_types[key]] = ' '.join(merge_data)
+    row[var_types[key]] = ' '.join(merge_data)
     return row
 
-def row_fxn_join(sd, row, key):
+def row_fxn_join(sd, row, key, fxn):
     "Create new columns by merging arbitrary other columns with a separator"
-    fxn = sd["conform"][key]
     separator = fxn.get("separator", " ")
     try:
         fields = [(row[n] or u'').strip() for n in fxn["fields"]]
-        row[attrib_types[key]] = separator.join([f for f in fields if f])
+        row[var_types[key]] = separator.join([f for f in fields if f])
     except Exception as e:
         _L.debug("Failure to merge row %r %s", e, row)
     return row
 
-def row_fxn_regexp(sd, row, key):
+def row_fxn_regexp(sd, row, key, fxn):
     "Split addresses like '123 Maple St' into '123' and 'Maple St'"
-    fxn = sd["conform"][key]
     pattern = re.compile(fxn.get("pattern", False))
     replace = fxn.get('replace', False)
     if replace:
         match = re.sub(pattern, convert_regexp_replace(replace), row[fxn["field"]])
-        row[attrib_types[key]] = match;
+        row[var_types[key]] = match;
     else:
         match = pattern.search(row[fxn["field"]])
-        row[attrib_types[key]] = ''.join(match.groups()) if match else '';
+        row[var_types[key]] = ''.join(match.groups()) if match else '';
     return row
 
-def row_fxn_prefixed_number(sd, row, key):
+def row_fxn_prefixed_number(sd, row, key, fxn):
     "Extract '123' from '123 Maple St'"
-    fxn = sd["conform"][key]
 
     match = prefixed_number_pattern.search(row[fxn["field"]])
-    row[attrib_types[key]] = ''.join(match.groups()) if match else '';
+    row[var_types[key]] = ''.join(match.groups()) if match else '';
 
     return row
 
-def row_fxn_postfixed_street(sd, row, key):
+def row_fxn_postfixed_street(sd, row, key, fxn):
     "Extract 'Maple St' from '123 Maple St'"
-    fxn = sd["conform"][key]
 
     match = postfixed_street_pattern.search(row[fxn["field"]])
-    row[attrib_types[key]] = ''.join(match.groups()) if match else '';
+    row[var_types[key]] = ''.join(match.groups()) if match else '';
 
     return row
 
-def row_fxn_remove_prefix(sd, row, key):
+def row_fxn_remove_prefix(sd, row, key, fxn):
     "Remove a 'field_to_remove' from the beginning of 'field' if it is a prefix"
-    fxn = sd["conform"][key]
-
     if row[fxn["field"]].startswith(row[fxn["field_to_remove"]]):
-        row[attrib_types[key]] = row[fxn["field"]][len(row[fxn["field_to_remove"]]):].lstrip(' ')
+        row[var_types[key]] = row[fxn["field"]][len(row[fxn["field_to_remove"]]):].lstrip(' ')
     else:
-        row[attrib_types[key]] = row[fxn["field"]]
+        row[var_types[key]] = row[fxn["field"]]
 
     return row
 
-def row_fxn_remove_postfix(sd, row, key):
+def row_fxn_remove_postfix(sd, row, key, fxn):
     "Remove a 'field_to_remove' from the end of 'field' if it is a postfix"
-    fxn = sd["conform"][key]
-
     if row[fxn["field_to_remove"]] != "" and row[fxn["field"]].endswith(row[fxn["field_to_remove"]]):
-        row[attrib_types[key]] = row[fxn["field"]][0:len(row[fxn["field_to_remove"]])*-1].rstrip(' ')
+        row[var_types[key]] = row[fxn["field"]][0:len(row[fxn["field_to_remove"]])*-1].rstrip(' ')
     else:
-        row[attrib_types[key]] = row[fxn["field"]]
+        row[var_types[key]] = row[fxn["field"]]
 
     return row
 
-def row_fxn_format(sd, row, key):
+def row_fxn_format(sd, row, key, fxn):
     "Format multiple fields using a user-specified format string"
-    fxn = sd["conform"][key]
-
     format_var_pattern = re.compile('\$([0-9]+)')
 
     fields = [(row[n] or u'').strip() for n in fxn["fields"]]
@@ -1063,9 +1066,29 @@ def row_fxn_format(sd, row, key):
 
     if num_fields_added > 0:
         parts.append(format_str[idx:])
-        row[attrib_types[key]] = u''.join(parts)
+        row[var_types[key]] = u''.join(parts)
+    else:
+        row[var_types[key]] = u''
+
+    return row
+
+def row_fxn_chain(sd, row, key, fxn):
+    functions = fxn["functions"]
+    var = fxn.get("variable")
+
+    original_key = key
+
+    if var and var not in attrib_types and var.lstrip('OA:') not in attrib_types and var not in row:
+        var_types[var] = var
+        row[var_types[var]] = u''
+        key = var
     else:
-        row[attrib_types[key]] = u''
+        var = None
+
+    for func in functions:
+        row = row_function(sd, row, key, func)
+
+    row[var_types[original_key]] = row[var_types[key]]
 
     return row
 


Test patch:
diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py
index d903450..08433bd 100644
--- a/openaddr/tests/conform.py
+++ b/openaddr/tests/conform.py
@@ -18,7 +18,7 @@ from ..conform import (
     row_fxn_regexp, row_smash_case, row_round_lat_lon, row_merge,
     row_extract_and_reproject, row_convert_to_out, row_fxn_join, row_fxn_format,
     row_fxn_prefixed_number, row_fxn_postfixed_street,
-    row_fxn_remove_prefix, row_fxn_remove_postfix,
+    row_fxn_remove_prefix, row_fxn_remove_postfix, row_fxn_chain,
     row_canonicalize_unit_and_number, conform_smash_case, conform_cli,
     convert_regexp_replace, conform_license,
     conform_attribution, conform_sharealike, normalize_ogr_filename_case,
@@ -75,14 +75,14 @@ class TestConformTransforms (unittest.TestCase):
         d = { "a1": "va1", "b1": "vb1", "b2": "vb2" }
         e = copy.deepcopy(d)
         e.update({ "OA:number": "va1", "OA:street": "vb1-vb2" })
-        d = row_fxn_join(c, d, "number")
-        d = row_fxn_join(c, d, "street")
+        d = row_fxn_join(c, d, "number", c["conform"]["number"])
+        d = row_fxn_join(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
         d = { "a1": "va1", "b1": "vb1", "b2": None}
         e = copy.deepcopy(d)
         e.update({ "OA:number": "va1", "OA:street": "vb1" })
-        d = row_fxn_join(c, d, "number")
-        d = row_fxn_join(c, d, "street")
+        d = row_fxn_join(c, d, "number", c["conform"]["number"])
+        d = row_fxn_join(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
     def test_row_fxn_format(self):
@@ -101,19 +101,90 @@ class TestConformTransforms (unittest.TestCase):
 
         d = {"a1": "12", "a2": "34", "a3": "56", "b1": "1", "b2": "B", "b3": "3"}
         e = copy.deepcopy(d)
-        d = row_fxn_format(c, d, "number")
-        d = row_fxn_format(c, d, "street")
+        d = row_fxn_format(c, d, "number", c["conform"]["number"])
+        d = row_fxn_format(c, d, "street", c["conform"]["street"])
         self.assertEqual(d.get("OA:number", ""), "12-34-56")
         self.assertEqual(d.get("OA:street", ""), "foo 1B-3 bar")
 
         d = copy.deepcopy(e)
         d["a2"] = None
         d["b3"] = None
-        d = row_fxn_format(c, d, "number")
-        d = row_fxn_format(c, d, "street")
+        d = row_fxn_format(c, d, "number", c["conform"]["number"])
+        d = row_fxn_format(c, d, "street", c["conform"]["street"])
         self.assertEqual(d.get("OA:number", ""), "12-56")
         self.assertEqual(d.get("OA:street", ""), "foo 1B bar")
 
+    def test_row_fxn_chain(self):
+        c = { "conform": {
+            "number": {
+                "function": "chain",
+                "functions": [
+                    {
+                        "function": "format",
+                        "fields": ["a1", "a2", "a3"],
+                        "format": "$1-$2-$3"
+                    },
+                    {
+                        "function": "remove_postfix",
+                        "field": "OA:number",
+                        "field_to_remove": "b1"
+                    }
+                ]
+            }
+        } }
+
+        d = {"a1": "12", "a2": "34", "a3": "56 UNIT 5", "b1": "UNIT 5"}
+        e = copy.deepcopy(d)
+        d = row_fxn_chain(c, d, "number", c["conform"]["number"])
+        self.assertEqual(d.get("OA:number", ""), "12-34-56")
+
+        d = copy.deepcopy(e)
+        d["a2"] = None
+        d = row_fxn_chain(c, d, "number", c["conform"]["number"])
+        self.assertEqual(d.get("OA:number", ""), "12-56")
+
+
+    def test_row_fxn_chain_nested(self):
+        c = { "conform": {
+            "number": {
+                "function": "chain",
+                "variable": "foo",
+                "functions": [
+                    {
+                        "function": "format",
+                        "fields": ["a1", "a2"],
+                        "format": "$1-$2"
+                    },
+                    {
+                        "function": "chain",
+                        "variable": "bar",
+                        "functions": [
+                            {
+                                "function": "format",
+                                "fields": ["foo", "a3"],
+                                "format": "$1-$2"
+                            },
+                            {
+                                "function": "remove_postfix",
+                                "field": "bar",
+                                "field_to_remove": "b1"
+                            }
+                        ]
+                    }
+                ]
+            }
+        } }
+
+        d = {"a1": "12", "a2": "34", "a3": "56 UNIT 5", "b1": "UNIT 5"}
+        e = copy.deepcopy(d)
+        d = row_fxn_chain(c, d, "number", c["conform"]["number"])
+        self.assertEqual(d.get("OA:number", ""), "12-34-56")
+
+        d = copy.deepcopy(e)
+        d["a2"] = None
+        d = row_fxn_chain(c, d, "number", c["conform"]["number"])
+        self.assertEqual(d.get("OA:number", ""), "12-56")
+
     def test_row_fxn_regexp(self):
         "Regex split - replace"
         c = { "conform": {
@@ -134,8 +205,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123", "OA:street": "MAPLE ST" })
         
-        d = row_fxn_regexp(c, d, "number")
-        d = row_fxn_regexp(c, d, "street")
+        d = row_fxn_regexp(c, d, "number", c["conform"]["number"])
+        d = row_fxn_regexp(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
         
         "Regex split - no replace - good match"
@@ -155,8 +226,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123", "OA:street": "MAPLE ST" })
         
-        d = row_fxn_regexp(c, d, "number")
-        d = row_fxn_regexp(c, d, "street")
+        d = row_fxn_regexp(c, d, "number", c["conform"]["number"])
+        d = row_fxn_regexp(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "regex split - no replace - bad match"
@@ -176,8 +247,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123", "OA:street": "" })
         
-        d = row_fxn_regexp(c, d, "number")
-        d = row_fxn_regexp(c, d, "street")
+        d = row_fxn_regexp(c, d, "number", c["conform"]["number"])
+        d = row_fxn_regexp(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
         
     def test_transform_and_convert(self):
@@ -293,8 +364,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123", "OA:street": "MAPLE ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_street - no number"
@@ -312,8 +383,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "", "OA:street": "MAPLE ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_street - empty input"
@@ -331,8 +402,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "", "OA:street": "" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_street - no spaces after number"
@@ -350,8 +421,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "", "OA:street": "123MAPLE ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_street - excess whitespace"
@@ -369,8 +440,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123", "OA:street": "MAPLE ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
     
         "Regex prefixed_number and postfixed_number - ordinal street w/house number"
@@ -388,8 +459,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "12", "OA:street": "3RD ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_number - ordinal street w/o house number"
@@ -407,8 +478,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "", "OA:street": "3RD ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_number - combined house number and suffix"
@@ -426,8 +497,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123A", "OA:street": "3RD ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_number - hyphenated house number and suffix"
@@ -445,8 +516,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123-A", "OA:street": "3RD ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_number - queens-style house number"
@@ -464,8 +535,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123-45", "OA:street": "3RD ST" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "Regex prefixed_number and postfixed_number - should be case-insenstive"
@@ -483,8 +554,8 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:number": "123-a", "OA:street": "3rD St" })
 
-        d = row_fxn_prefixed_number(c, d, "number")
-        d = row_fxn_postfixed_street(c, d, "street")
+        d = row_fxn_prefixed_number(c, d, "number", c["conform"]["number"])
+        d = row_fxn_postfixed_street(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
     def test_row_fxn_remove_prefix(self):
@@ -500,7 +571,7 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:street": "MAPLE ST" })
 
-        d = row_fxn_remove_prefix(c, d, "street")
+        d = row_fxn_remove_prefix(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "remove_prefix - field_to_remove is not a prefix"
@@ -515,7 +586,7 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:street": "123 MAPLE ST" })
 
-        d = row_fxn_remove_prefix(c, d, "street")
+        d = row_fxn_remove_prefix(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "remove_prefix - field_to_remove value is empty string"
@@ -530,7 +601,7 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:street": "123 MAPLE ST" })
 
-        d = row_fxn_remove_prefix(c, d, "street")
+        d = row_fxn_remove_prefix(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
     def test_row_fxn_remove_postfix(self):
@@ -546,7 +617,7 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:street": "MAPLE ST" })
 
-        d = row_fxn_remove_postfix(c, d, "street")
+        d = row_fxn_remove_postfix(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "remove_postfix - field_to_remove is not a postfix"
@@ -561,7 +632,7 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:street": "123 MAPLE ST" })
 
-        d = row_fxn_remove_postfix(c, d, "street")
+        d = row_fxn_remove_postfix(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
         "remove_postfix - field_to_remove value is empty string"
@@ -576,7 +647,7 @@ class TestConformTransforms (unittest.TestCase):
         e = copy.deepcopy(d)
         e.update({ "OA:street": "123 MAPLE ST" })
 
-        d = row_fxn_remove_postfix(c, d, "street")
+        d = row_fxn_remove_postfix(c, d, "street", c["conform"]["street"])
         self.assertEqual(e, d)
 
 class TestConformCli (unittest.TestCase):