Repository: MITLibraries/slingshot

Issue:
Set script failure conditions
The script should not completely fail for one bad layer, but rather log it and continue. It should fail immediately in certain cases, such as, 404 and 401. It should probably fail after some number of layers have failed consecutively, since that would likely indicate something more fundamental is wrong.

Discussion:

Patch:
diff --git a/slingshot/cli.py b/slingshot/cli.py
index 047d98e..75ad0c9 100644
--- a/slingshot/cli.py
+++ b/slingshot/cli.py
@@ -28,7 +28,9 @@ def main():
 @click.option('--username', help="Username for kepler submission.")
 @click.option('--password',
               help="Password for kepler submission. Omit for prompt.")
-def run(layers, store, url, namespace, username, password):
+@click.option('--fail-after', default=5,
+              help="Stop after number of consecutive failures. Default is 5.")
+def run(layers, store, url, namespace, username, password, fail_after):
     """Create and upload bags to the specified endpoint.
 
     This script will create bags from all the layers in the LAYERS
@@ -50,6 +52,7 @@ def run(layers, store, url, namespace, username, password):
     auth = username, password
     if not all(auth):
         auth = None
+    failures = 0
     for data_layer in uploadable(layers, store):
         bag = prep_bag(os.path.join(layers, data_layer), store)
         try:
@@ -57,10 +60,15 @@ def run(layers, store, url, namespace, username, password):
             bag_name = make_uuid(os.path.basename(bag), namespace)
             with temp_archive(bag, bag_name) as zf:
                 submit(zf, url, auth)
+            click.echo("%sZ: %s uploaded" % (datetime.utcnow().isoformat(),
+                                             data_layer))
+            failures = 0
         except Exception as e:
             shutil.rmtree(bag, ignore_errors=True)
+            failures += 1
             click.echo("%sZ: %s failed with %r" %
                        (datetime.utcnow().isoformat(), data_layer, e))
-            raise e
-        click.echo("%sZ: %s uploaded" % (datetime.utcnow().isoformat(),
-                                         data_layer))
+            if failures >= fail_after:
+                click.echo("%sZ: Maximum number of consecutive failures (%d)" %
+                           (datetime.utcnow().isoformat(), failures))
+                raise e


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 61eca94..e32903d 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -77,3 +77,12 @@ def test_run_logs_failed_layers_to_stdout(runner, layers_dir):
         res = runner.invoke(main, ['run', layers_dir, store,
                                    'http://localhost'])
         assert 'SDE_DATA_BD_A8GNS_2003.zip failed' in res.output
+
+
+def test_run_fails_after_consecutive_failures(runner, layers_dir):
+    with requests_mock.Mocker() as m:
+        store = tempfile.mkdtemp()
+        m.post('http://localhost', status_code=500)
+        res = runner.invoke(main, ['run', layers_dir, store,
+                                   'http://localhost', '--fail-after', 1])
+        assert 'Maximum number of consecutive failures' in res.output