Repository: openaddresses/machine
Pull request #775 (2020-08-25 02:50:01)

Basic support for v2 sources in machine
Machine already supported V2 sources since #691, but it only made assumptions about the V2 `layer` and `layersource` it should use when the sources were still V1. When we switched sources to V2 schema in https://github.com/openaddresses/openaddresses/pull/5165, `process-one` stopped making the assumption for layer and layersource and started failing instead.

This PR changes the logic for the CI worker to default to layer `addresses` and layersource `primary`.


Patch:
diff --git a/openaddr/ci/objects.py b/openaddr/ci/objects.py
index efee9aa..e6a1693 100644
--- a/openaddr/ci/objects.py
+++ b/openaddr/ci/objects.py
@@ -518,13 +518,10 @@ def read_completed_runs_to_date(db, starting_set_id):
         return None
 
     # Get IDs for latest successful source runs of any run in the requested set.
-    db.execute('''SELECT MAX(id), source_path FROM runs
-                  WHERE source_path IN (
-                      -- Get all source paths for successful runs in this set.
-                      SELECT source_path FROM runs
-                      WHERE set_id = %s
-                    )
-                    -- Get only successful, merged runs.
+    db.execute('''SELECT MAX(id), source_path
+                  FROM runs
+                  WHERE
+                    set_id = %s
                     AND status = true
                     AND (is_merged = true OR is_merged IS NULL)
                   GROUP BY source_path''',
@@ -533,13 +530,10 @@ def read_completed_runs_to_date(db, starting_set_id):
     run_path_ids = {path: run_id for (run_id, path) in db.fetchall()}
 
     # Get IDs for latest unsuccessful source runs of any run in the requested set.
-    db.execute('''SELECT MAX(id), source_path FROM runs
-                  WHERE source_path IN (
-                      -- Get all source paths for failed runs in this set.
-                      SELECT source_path FROM runs
-                      WHERE set_id = %s
-                    )
-                    -- Get only unsuccessful, merged runs.
+    db.execute('''SELECT MAX(id), source_path
+                  FROM runs
+                  WHERE
+                    set_id = %s
                     AND status = false
                     AND (is_merged = true OR is_merged IS NULL)
                   GROUP BY source_path''',
diff --git a/openaddr/ci/work.py b/openaddr/ci/work.py
index ae4af66..9607237 100644
--- a/openaddr/ci/work.py
+++ b/openaddr/ci/work.py
@@ -106,6 +106,10 @@ def do_work(s3, run_id, source_name, job_contents_b64, render_preview, output_di
     else:
         cmd += ('--skip-preview', )
 
+    # For now, assume 'addresses' layer for V2 sources
+    cmd += ('--layer', 'addresses')
+    cmd += ('--layersource', 'primary')
+
     try:
         known_error, cmd_status = False, 0
         timeout_seconds = JOB_TIMEOUT.seconds + JOB_TIMEOUT.days * 86400
diff --git a/openaddr/process_one.py b/openaddr/process_one.py
index a8b9586..5ede4b1 100644
--- a/openaddr/process_one.py
+++ b/openaddr/process_one.py
@@ -88,15 +88,10 @@ def process(source, destination, layer, layersource, do_preview, mapbox_key=None
                     layer = 'addresses'
                     layersource = 'primary'
 
-                if type(layer) is not str:
-                    layer = ''
-                if type(layersource) is not str:
-                    layersource = ''
-
-                if len(layer) == 0:
+                if not layer:
                     _L.error('explicit --layer arg is required for v2 sources')
                     raise ValueError('explicit --layer arg is required for v2 sources')
-                elif len(layersource) == 0:
+                if not layersource:
                     _L.error('explicit --layersource arg is required for v2 sources')
                     raise ValueError('explicit --layersource arg is required for v2 sources')
 


Test patch:
diff --git a/openaddr/tests/ci.py b/openaddr/tests/ci.py
index 16dfc10..d33ba8a 100644
--- a/openaddr/tests/ci.py
+++ b/openaddr/tests/ci.py
@@ -739,25 +739,19 @@ class TestObjects (unittest.TestCase):
             = exec1[1][:], exec2[1][:], exec3[1][:]
 
         self.assertEqual(e1_query,
-               '''SELECT MAX(id), source_path FROM runs
-                  WHERE source_path IN (
-                      -- Get all source paths for successful runs in this set.
-                      SELECT source_path FROM runs
-                      WHERE set_id = %s
-                    )
-                    -- Get only successful, merged runs.
+               '''SELECT MAX(id), source_path
+                  FROM runs
+                  WHERE
+                    set_id = %s
                     AND status = true
                     AND (is_merged = true OR is_merged IS NULL)
                   GROUP BY source_path''')
 
         self.assertEqual(e2_query,
-               '''SELECT MAX(id), source_path FROM runs
-                  WHERE source_path IN (
-                      -- Get all source paths for failed runs in this set.
-                      SELECT source_path FROM runs
-                      WHERE set_id = %s
-                    )
-                    -- Get only unsuccessful, merged runs.
+               '''SELECT MAX(id), source_path
+                  FROM runs
+                  WHERE
+                    set_id = %s
                     AND status = false
                     AND (is_merged = true OR is_merged IS NULL)
                   GROUP BY source_path''')
@@ -3078,7 +3072,9 @@ class TestWorker (unittest.TestCase):
             os.path.join(self.output_dir, u'work/so--exalté.txt'),
             os.path.join(self.output_dir, 'work/out'),
             '--render-preview',
-            '--mapbox-key', 'mapbox-XXXX'
+            '--mapbox-key', 'mapbox-XXXX',
+            '--layer', 'addresses',
+            '--layersource', 'primary',
             ))
 
         self.assertEqual(check_output.mock_calls[-1][2]['timeout'],
@@ -3136,7 +3132,9 @@ class TestWorker (unittest.TestCase):
             os.path.join(self.output_dir, 'work/logfile.txt'),
             os.path.join(self.output_dir, 'work/angry.txt'),
             os.path.join(self.output_dir, 'work/out'),
-            '--skip-preview'
+            '--skip-preview',
+            '--layer', 'addresses',
+            '--layersource', 'primary',
             ))
 
         self.assertEqual(check_output.mock_calls[-1][2]['timeout'],
@@ -3182,7 +3180,9 @@ class TestWorker (unittest.TestCase):
             os.path.join(self.output_dir, u'work/so--exalté.txt'),
             os.path.join(self.output_dir, 'work/out'),
             '--render-preview',
-            '--mapbox-key', 'mapbox-XXXX'
+            '--mapbox-key', 'mapbox-XXXX',
+            '--layer', 'addresses',
+            '--layersource', 'primary',
             ))
 
         self.assertEqual(check_output.mock_calls[-1][2]['timeout'],