Repository: openaddresses/machine
Pull request #464 (2016-11-28 21:17:27)

added `prefix_number` and `prefix_street` conform functions
See openaddresses/openaddresses#2075

These 2 functions abstract the common `regexp` patterns in use for US (and other country) sources.  By internalizing these 2 commons patterns, modifications can be implemented without having to modify the many sources that use these patterns.

Its current implementation only covers a few basic scenarios:

- `number` is only defined as optional whitespace + 1 or more digits + at least one whitespace
- `street` is defined as anything after `number`

2 functions were used instead of a single function that writes both `number` and `street` fields in the event that the source already contains a separate `number` or `street` field.

*This is my first foray into python, my apologies if it's terrible*


Patch:
diff --git a/openaddr/conform.py b/openaddr/conform.py
index 192f55c..358d4c4 100644
--- a/openaddr/conform.py
+++ b/openaddr/conform.py
@@ -82,6 +82,9 @@ geometry_types = {
     ogr.wkbUnknown: 'Unknown'
     }
 
+prefix_number_pattern = re.compile("^\s*([0-9]+)\s+", False)
+prefix_street_pattern = re.compile("^(?:\s*[0-9]+\s+)?(.*)", False)
+
 def mkdirsp(path):
     try:
         os.makedirs(path)
@@ -875,6 +878,10 @@ def row_transform_and_convert(sd, row):
                 row = row_fxn_regexp(sd, row, k)
             elif c[k]["function"] == "format":
                 row = row_fxn_format(sd, row, k)
+            elif c[k]["function"] == "prefix_number":
+                row = row_fxn_prefix_number(sd, row, k)
+            elif c[k]["function"] == "prefix_street":
+                row = row_fxn_prefix_street(sd, row, k)
 
     if "advanced_merge" in c:
         raise ValueError('Found unsupported "advanced_merge" option in conform')
@@ -944,6 +951,24 @@ def row_fxn_regexp(sd, row, key):
         row[attrib_types[key]] = ''.join(match.groups()) if match else '';
     return row
 
+def row_fxn_prefix_number(sd, row, key):
+    "Extract '123' from '123 Maple St'"
+    fxn = sd["conform"][key]
+
+    match = prefix_number_pattern.search(row[fxn["field"]])
+    row[attrib_types[key]] = ''.join(match.groups()) if match else '';
+
+    return row
+
+def row_fxn_prefix_street(sd, row, key):
+    "Extract 'Maple St' from '123 Maple St'"
+    fxn = sd["conform"][key]
+
+    match = prefix_street_pattern.search(row[fxn["field"]])
+    row[attrib_types[key]] = ''.join(match.groups()) if match else '';
+
+    return row
+
 def row_fxn_format(sd, row, key):
     "Format multiple fields using a user-specified format string"
     fxn = sd["conform"][key]


Test patch:
diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py
index 8e32d0a..1a536a6 100644
--- a/openaddr/tests/conform.py
+++ b/openaddr/tests/conform.py
@@ -16,6 +16,7 @@ from ..conform import (
     csv_source_to_csv, find_source_path, row_transform_and_convert,
     row_fxn_regexp, row_smash_case, row_round_lat_lon, row_merge,
     row_extract_and_reproject, row_convert_to_out, row_fxn_join, row_fxn_format,
+    row_fxn_prefix_number, row_fxn_prefix_street,
     row_canonicalize_unit_and_number, conform_smash_case, conform_cli,
     csvopen, csvDictReader, convert_regexp_replace, conform_license,
     conform_attribution, conform_sharealike, normalize_ogr_filename_case,
@@ -274,6 +275,102 @@ class TestConformTransforms (unittest.TestCase):
         r = row_extract_and_reproject(d, {"LONG_WGS84": "-21,77", "LAT_WGS84": "64,11"})
         self.assertEqual({Y_FIELDNAME: "64.11", X_FIELDNAME: "-21.77"}, r)
 
+    def test_row_fxn_prefix_number_and_street(self):
+        "Regex prefix_number and prefix_street - both fields present"
+        c = { "conform": {
+            "number": {
+                "function": "prefix_number",
+                "field": "ADDRESS"
+            },
+            "street": {
+                "function": "prefix_street",
+                "field": "ADDRESS"
+            }
+        } }
+        d = { "ADDRESS": "123 MAPLE ST" }
+        e = copy.deepcopy(d)
+        e.update({ "OA:number": "123", "OA:street": "MAPLE ST" })
+
+        d = row_fxn_prefix_number(c, d, "number")
+        d = row_fxn_prefix_street(c, d, "street")
+        self.assertEqual(e, d)
+
+        "Regex prefix_number and prefix_street - no number"
+        c = { "conform": {
+            "number": {
+                "function": "prefix_number",
+                "field": "ADDRESS"
+            },
+            "street": {
+                "function": "prefix_street",
+                "field": "ADDRESS"
+            }
+        } }
+        d = { "ADDRESS": "MAPLE ST" }
+        e = copy.deepcopy(d)
+        e.update({ "OA:number": "", "OA:street": "MAPLE ST" })
+
+        d = row_fxn_prefix_number(c, d, "number")
+        d = row_fxn_prefix_street(c, d, "street")
+        self.assertEqual(e, d)
+
+        "Regex prefix_number and prefix_street - empty input"
+        c = { "conform": {
+            "number": {
+                "function": "prefix_number",
+                "field": "ADDRESS"
+            },
+            "street": {
+                "function": "prefix_street",
+                "field": "ADDRESS"
+            }
+        } }
+        d = { "ADDRESS": "" }
+        e = copy.deepcopy(d)
+        e.update({ "OA:number": "", "OA:street": "" })
+
+        d = row_fxn_prefix_number(c, d, "number")
+        d = row_fxn_prefix_street(c, d, "street")
+        self.assertEqual(e, d)
+
+        "Regex prefix_number and prefix_street - no spaces after number"
+        c = { "conform": {
+            "number": {
+                "function": "prefix_number",
+                "field": "ADDRESS"
+            },
+            "street": {
+                "function": "prefix_street",
+                "field": "ADDRESS"
+            }
+        } }
+        d = { "ADDRESS": "123MAPLE ST" }
+        e = copy.deepcopy(d)
+        e.update({ "OA:number": "", "OA:street": "123MAPLE ST" })
+
+        d = row_fxn_prefix_number(c, d, "number")
+        d = row_fxn_prefix_street(c, d, "street")
+        self.assertEqual(e, d)
+
+        "Regex prefix_number and prefix_street - excess whitespace"
+        c = { "conform": {
+            "number": {
+                "function": "prefix_number",
+                "field": "ADDRESS"
+            },
+            "street": {
+                "function": "prefix_street",
+                "field": "ADDRESS"
+            }
+        } }
+        d = { "ADDRESS": " \t 123 \t MAPLE ST" }
+        e = copy.deepcopy(d)
+        e.update({ "OA:number": "123", "OA:street": "MAPLE ST" })
+
+        d = row_fxn_prefix_number(c, d, "number")
+        d = row_fxn_prefix_street(c, d, "street")
+        self.assertEqual(e, d)
+
 class TestConformCli (unittest.TestCase):
     "Test the command line interface creates valid output files from test input"
     def setUp(self):