Repository: mozilla/telemetry-analysis-service

Issue:
AttributeError: 'SparkJobProvisioner' object has no attribute 'stop'
While testing locally in a branch I bumped into this. I didn't test it on master but looking at the code this seems like it may also be a problem.

The `SparkJob.terminate` calls `self.provisioner.stop(self.current_run_jobflow_id)`, where `self.provisioner` is a `SparkJobProvisioner`. But currently the `SparkJobProvisioner` has no `stop` method and neither does the parent class, it is only defined on the `ClusterProvisioner`. In my branch I changed the `terminate` method to call `self.cluster_provisioner.stop` instead and it fixed my issue locally.



Discussion:

Resolving patch:
diff --git a/atmo/jobs/models.py b/atmo/jobs/models.py
--- a/atmo/jobs/models.py
+++ b/atmo/jobs/models.py
@@ -210,8 +210,8 @@ def run(self):
 
     def terminate(self):
         """Stop the currently running scheduled Spark job."""
-        if self.current_run_jobflow_id:
-            self.provisioner.stop(self.current_run_jobflow_id)
+        if self.is_expired and self.current_run_jobflow_id:
+            self.cluster_provisioner.stop(self.current_run_jobflow_id)
 
     def cleanup(self):
         """Remove the Spark job notebook file from S3"""


Test patch:
diff --git a/tests/test_jobs.py b/tests/test_jobs.py
--- a/tests/test_jobs.py
+++ b/tests/test_jobs.py
@@ -14,6 +14,16 @@
 from atmo.jobs import models
 
 
+@pytest.fixture
+def cluster_provisioner_mocks(mocker):
+    return {
+        'stop': mocker.patch(
+            'atmo.clusters.provisioners.ClusterProvisioner.stop',
+            return_value=None,
+        ),
+    }
+
+
 @pytest.fixture
 def sparkjob_provisioner_mocks(mocker):
     return {
@@ -393,6 +403,8 @@ def test_spark_job_second_run_should_run(now, test_user):
 
 
 def test_spark_job_is_expired(now, test_user):
+    # Test that a spark job `is_expired` if it has run for longer than
+    # its timeout.
     spark_job = models.SparkJob.objects.create(
         identifier='test-spark-job-6',
         description='description',
@@ -405,9 +417,6 @@ def test_spark_job_is_expired(now, test_user):
         created_by=test_user,
         current_run_jobflow_id='my-jobflow-id',
     )
-    # A spark job expires if:
-    # or has run before and finished (or not)
-    # it hasn't run for longer than its timeout
 
     timeout_run_date = now - timedelta(hours=12)
     running_status = Cluster.STATUS_RUNNING
@@ -438,6 +447,37 @@ def test_spark_job_is_expired(now, test_user):
     assert spark_job.is_expired
 
 
+def test_spark_job_terminates(now, test_user, cluster_provisioner_mocks):
+    # Test that a spark job's `terminate` tells the EMR to terminate correctly.
+    spark_job = models.SparkJob.objects.create(
+        identifier='test-spark-job-7',
+        description='description',
+        notebook_s3_key=u'jobs/test-spark-job/test-notebook.ipynb',
+        result_visibility='private',
+        size=5,
+        interval_in_hours=1,
+        job_timeout=12,
+        start_date=now - timedelta(days=1),
+        created_by=test_user,
+        current_run_jobflow_id='jobflow-id',
+    )
+
+    timeout_run_date = now - timedelta(hours=12)
+    running_status = Cluster.STATUS_RUNNING
+
+    # Test job does not terminate if not expired.
+    spark_job.last_run_date = timeout_run_date + timedelta(seconds=1)
+    spark_job.most_recent_status = running_status
+    spark_job.terminate()
+    cluster_provisioner_mocks['stop'].assert_not_called()
+
+    # Test job terminates when expired.
+    spark_job.last_run_date = timeout_run_date
+    spark_job.most_recent_status = running_status
+    spark_job.terminate()
+    cluster_provisioner_mocks['stop'].assert_called_with(u'jobflow-id')
+
+
 def test_check_identifier_taken(client, test_user):
     # create a test job to edit later
     identifier = 'test-spark-job'