Repository: google/fuzzbench

Issue:
Experiment is not ended and dispatcher stays alive even at end of experiment.
Looks like measurer is not finished, scheduler did finish.


Discussion:
What experiment was this?
aarya-test-6, one i did on friday (only 1 hour test one).
Do you know why it didn't finish? I think under the current setup, the measurer is pretty fragile and won't finish if any trials are missing snapshots.
Nothing was missing (simple 2 trial experiment - http://commondatastorage.googleapis.com/fuzzbench-reports/aarya-test-6/index.html), but i will try debugging more today with a new one. looks like a regression.
Something is up with
all_trials_ended. measure_all_trials is complete, it returns False. but all_trials_ended is not causing the break to happen.

                all_trials_ended = scheduler.all_trials_ended(experiment)

                if not measure_all_trials(experiment, max_total_time, pool, q):
                    # We didn't measure any trials.
                    if all_trials_ended:
                        # There are no trials producing snapshots to measure.
                        # Given that we couldn't measure any snapshots, we won't
                        # be able to measure any the future, so break now.
                        break

Resolving patch:
diff --git a/common/gcloud.py b/common/gcloud.py
--- a/common/gcloud.py
+++ b/common/gcloud.py
@@ -120,7 +120,8 @@ def create_instance(instance_name: str,
 
 
 def delete_instances(instance_names: List[str], zone: str, **kwargs) -> bool:
-    """Delete gcloud instance |instance_names|."""
+    """Delete gcloud instance |instance_names|. Returns true if the operation
+    succeeded or false otherwise."""
     error_occurred = False
     # Delete instances in batches, otherwise we run into rate limit errors.
     for idx in range(0, len(instance_names), INSTANCE_BATCH_SIZE):
@@ -131,7 +132,7 @@ def delete_instances(instance_names: List[str], zone: str, **kwargs) -> bool:
         result = new_process.execute(command, expect_zero=False, **kwargs)
         error_occurred = error_occurred or result.retcode != 0
 
-    return error_occurred
+    return not error_occurred
 
 
 def list_instances() -> List[str]:
diff --git a/experiment/scheduler.py b/experiment/scheduler.py
--- a/experiment/scheduler.py
+++ b/experiment/scheduler.py
@@ -115,6 +115,7 @@ def end_expired_trials(experiment_config: dict):
         # If we failed to delete some instances, then don't update the status
         # of expired trials in database as we don't know which instances were
         # successfully deleted. Wait for next iteration of end_expired_trials.
+        logger.error('Failed to delete instances after trial expiry.')
         return
 
     db_utils.bulk_save(trials_past_expiry)


Test patch:
diff --git a/common/test_gcloud.py b/common/test_gcloud.py
--- a/common/test_gcloud.py
+++ b/common/test_gcloud.py
@@ -156,11 +156,11 @@ def test_delete_instances_less_than_batch_size(mocked_execute):
     than batch size."""
     instances = ['instance-%d' % i for i in range(5)]
     mocked_execute.return_value = new_process.ProcessResult(0, '', False)
-    # -q is needed otherwise gcloud will prompt "Y/N?".
     zone = 'us-central1-a'
     expected_command = (['gcloud', 'compute', 'instances', 'delete', '-q'] +
                         instances + ['--zone', zone])
-    gcloud.delete_instances(instances, zone)
+    result = gcloud.delete_instances(instances, zone)
+    assert result
     mocked_execute.assert_called_with(expected_command, expect_zero=False)
 
 
@@ -170,9 +170,9 @@ def test_delete_instances_greater_than_batch_size(mocked_execute):
   than batch size."""
     instances = ['instance-%d' % i for i in range(103)]
     mocked_execute.return_value = new_process.ProcessResult(0, '', False)
-    # -q is needed otherwise gcloud will prompt "Y/N?".
     zone = 'us-central1-a'
-    gcloud.delete_instances(instances, zone)
+    result = gcloud.delete_instances(instances, zone)
+    assert result
     expected_command_1 = (['gcloud', 'compute', 'instances', 'delete', '-q'] +
                           ['instance-%d' % i for i in range(100)] +
                           ['--zone', zone])
@@ -183,3 +183,16 @@ def test_delete_instances_greater_than_batch_size(mocked_execute):
         mock.call(expected_command_1, expect_zero=False),
         mock.call(expected_command_2, expect_zero=False)
     ])
+
+
+@mock.patch('common.new_process.execute')
+def test_delete_instances_fail(mocked_execute):
+    """Test that delete_instances returns False when instance deletion fails."""
+    instances = ['instance-%d' % i for i in range(5)]
+    mocked_execute.return_value = new_process.ProcessResult(1, 'Error', False)
+    zone = 'us-central1-a'
+    expected_command = (['gcloud', 'compute', 'instances', 'delete', '-q'] +
+                        instances + ['--zone', zone])
+    result = gcloud.delete_instances(instances, zone)
+    assert not result
+    mocked_execute.assert_called_with(expected_command, expect_zero=False)