internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mozilla/telemetry-analysis-service

Issue:
Record cluster size when running jobs for historical log



Discussion:

Resolving patch:
diff --git a/atmo/jobs/migrations/0026_sparkjobrun_size.py b/atmo/jobs/migrations/0026_sparkjobrun_size.py
new file mode 100644
--- /dev/null
+++ b/atmo/jobs/migrations/0026_sparkjobrun_size.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.1 on 2017-05-19 18:26
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0025_populate_job_schedule'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sparkjobrun',
+            name='size',
+            field=models.IntegerField(blank=True, help_text='Number of computers used to run the job.', null=True),
+        ),
+    ]
diff --git a/atmo/jobs/models.py b/atmo/jobs/models.py
--- a/atmo/jobs/models.py
+++ b/atmo/jobs/models.py
@@ -239,6 +239,7 @@ def run(self):
             jobflow_id=jobflow_id,
             scheduled_date=timezone.now(),
             emr_release_version=self.emr_release.version,
+            size=self.size,
         )
         # Remove the cached latest run to this objects will requery it.
         try:
@@ -314,6 +315,11 @@ class SparkJobRun(EditedAtModel):
         blank=True,
         null=True,
     )
+    size = models.IntegerField(
+        help_text="Number of computers used to run the job.",
+        blank=True,
+        null=True,
+    )
     status = models.CharField(
         max_length=50,
         blank=True,
@@ -346,7 +352,7 @@ def spark_job_identifier(self):
         return self.spark_job.identifier
 
     __repr__ = autorepr(
-        ['jobflow_id', 'spark_job_identifier'],
+        ['jobflow_id', 'spark_job_identifier', 'emr_release_version', 'size'],
         spark_job_identifier=spark_job_identifier,
     )
 


Test patch:
Repository: mozilla/telemetry-analysis-service

Issue:
EMR release form shows inactive records
The EMR release model has a column for `is_active`, but it's not being considered when querying the list of EMR releases in the form. 


Discussion:

Resolving patch:
diff --git a/atmo/clusters/forms.py b/atmo/clusters/forms.py
--- a/atmo/clusters/forms.py
+++ b/atmo/clusters/forms.py
@@ -4,6 +4,7 @@
 from django import forms
 from django.conf import settings
 from django.core.urlresolvers import reverse
+from django.utils.safestring import mark_safe
 
 from . import models
 from ..forms.mixins import AutoClassFormMixin, CreatedByModelFormMixin
@@ -14,7 +15,7 @@ class EMRReleaseChoiceField(forms.ModelChoiceField):
     def __init__(self, *args, **kwargs):
         super().__init__(
             label='EMR release',
-            queryset=models.EMRRelease.objects.all(),
+            queryset=models.EMRRelease.objects.active(),
             required=True,
             empty_label=None,
             widget=forms.RadioSelect(attrs={
@@ -28,11 +29,11 @@ def label_from_instance(self, obj):
         label = obj.version
         extra = []
         if obj.is_experimental:
-            extra.append('experimental')
+            extra.append('<span class="label label-info">experimental</span>')
         elif obj.is_deprecated:
-            extra.append('deprecated')
+            extra.append('<span class="label label-warning">deprecated</span>')
         if extra:
-            label = '%s (%s)' % (label, ', '.join(extra))
+            label = mark_safe('%s %s' % (label, ''.join(extra)))
         return label
 
 
diff --git a/atmo/clusters/queries.py b/atmo/clusters/queries.py
--- a/atmo/clusters/queries.py
+++ b/atmo/clusters/queries.py
@@ -6,6 +6,11 @@
 
 class EMRReleaseQuerySet(models.QuerySet):
 
+    def active(self):
+        return self.filter(
+            is_active=True,
+        )
+
     def stable(self):
         return self.filter(
             is_experimental=False,


Test patch:
diff --git a/tests/clusters/test_forms.py b/tests/clusters/test_forms.py
--- a/tests/clusters/test_forms.py
+++ b/tests/clusters/test_forms.py
@@ -5,12 +5,22 @@
 
 
 def test_emr_release_choice_field(emr_release_factory):
+
+    def make_label(label, text):
+        return '<span class="label label-%s">%s</span>' % (label, text)
+
     regular = emr_release_factory()
+    inactive = emr_release_factory(is_active=False)
     deprecated = emr_release_factory(is_deprecated=True)
     experimental = emr_release_factory(is_experimental=True)
 
     choice_field = EMRReleaseChoiceField()
     result = choice_field.widget.render('test', regular.pk)
+    assert inactive.version not in result
     assert '%s</label>' % regular.version in result
-    assert '%s (experimental)</label>' % experimental.version in result
-    assert '%s (deprecated)</label>' % deprecated.version in result
+    assert ('%s %s</label>' % (deprecated.version,
+                               make_label('warning', 'deprecated'))
+            in result)
+    assert ('%s %s</label>' % (experimental.version,
+                               make_label('info', 'experimental'))
+            in result)
Repository: mozilla/telemetry-analysis-service

Issue:
Separate EMR bootstrap deploy for staging and production
This will allow us to make changes to the bootstrap scripts and test them on ATMO staging easily before pushing to production.

This involves updating the ansible playbook to deploy the bootstrap scripts to either a staging or production location on S3. And setting up a configuration variable to point ATMO to the staging or production buckets.




Discussion:

Resolving patch:
diff --git a/atmo/settings.py b/atmo/settings.py
--- a/atmo/settings.py
+++ b/atmo/settings.py
@@ -216,7 +216,9 @@ class AWS:
         ),
         'SPARK_INSTANCE_PROFILE': 'telemetry-spark-cloudformation-'
                                   'TelemetrySparkInstanceProfile-1SATUBVEXG7E3',
-        'SPARK_EMR_BUCKET': 'telemetry-spark-emr-2',
+        'SPARK_EMR_BUCKET': values.Value(default='telemetry-spark-emr-2',
+                                         environ_prefix=None,
+                                         environ_name='AWS_SPARK_EMR_BUCKET'),
         'INSTANCE_APP_TAG': 'telemetry-analysis-worker-instance',
         'EMAIL_SOURCE': 'telemetry-alerts@mozilla.com',
         'MAX_CLUSTER_SIZE': 30,


Test patch:
Repository: mozilla/telemetry-analysis-service

Issue:
Separate EMR bootstrap deploy for staging and production
This will allow us to make changes to the bootstrap scripts and test them on ATMO staging easily before pushing to production.

This involves updating the ansible playbook to deploy the bootstrap scripts to either a staging or production location on S3. And setting up a configuration variable to point ATMO to the staging or production buckets.




Discussion:

Resolving patch:
diff --git a/atmo/settings.py b/atmo/settings.py
--- a/atmo/settings.py
+++ b/atmo/settings.py
@@ -209,11 +209,16 @@ class AWS:
         'MASTER_INSTANCE_TYPE': 'c3.4xlarge',
         'WORKER_INSTANCE_TYPE': 'c3.4xlarge',
 
-        'SPARK_INSTANCE_PROFILE': 'telemetry-spark-cloudformation-'
-                                  'TelemetrySparkInstanceProfile-1SATUBVEXG7E3',
-        'SPARK_EMR_BUCKET': values.Value(default='telemetry-spark-emr-2',
-                                         environ_prefix=None,
-                                         environ_name='AWS_SPARK_EMR_BUCKET'),
+        'SPARK_INSTANCE_PROFILE': values.Value(
+            default='telemetry-spark-cloudformation-'
+                    'TelemetrySparkInstanceProfile-1SATUBVEXG7E3',
+            environ_prefix=None,
+            environ_name='AWS_SPARK_INSTANCE_PROFILE'
+        ),
+        'SPARK_EMR_BUCKET': values.Value(
+            default='telemetry-spark-emr-2',
+            environ_prefix=None,
+            environ_name='AWS_SPARK_EMR_BUCKET'),
         'INSTANCE_APP_TAG': 'telemetry-analysis-worker-instance',
         'EMAIL_SOURCE': 'telemetry-alerts@mozilla.com',
         'MAX_CLUSTER_SIZE': 30,


Test patch:
Repository: mozilla/telemetry-analysis-service

Issue:
Better naming for scheduled jobs
In #459 identifiers for adhoc clusters and scheduled jobs were changed to be randomly generated. This is fine for a single-use cluster, but makes notifications for scheduled jobs meaningless. 

For example, I received a notification today that job "fervent-snyder-2799" has failed. It is owned by someone else, but emails are cc'd to telemetry-alerts as well (general audience to monitor for important failures).

I would prefer that the name for a scheduled job be reflective of what is being done in the job. 

Alternatively, we could ensure that the "description" field has been filled in and include that information in the notification emails.


Discussion:

Resolving patch:
diff --git a/atmo/jobs/views.py b/atmo/jobs/views.py
--- a/atmo/jobs/views.py
+++ b/atmo/jobs/views.py
@@ -14,12 +14,10 @@
 from django.utils.safestring import mark_safe
 from django.utils.text import get_valid_filename
 
-from .. import names
 from ..clusters.models import EMRRelease
 from ..decorators import (change_permission_required,
                           delete_permission_required, modified_date,
                           view_permission_required)
-from ..models import next_field_value
 from .forms import EditSparkJobForm, NewSparkJobForm, SparkJobAvailableForm
 from .models import SparkJob
 
@@ -48,10 +46,8 @@ def new_spark_job(request):
     """
     View to schedule a new Spark job to run on AWS EMR.
     """
-    identifier = names.random_scientist()
-    next_identifier = next_field_value(SparkJob, 'identifier', identifier)
     initial = {
-        'identifier': next_identifier,
+        'identifier': '',
         'size': 1,
         'interval_in_hours': SparkJob.INTERVAL_WEEKLY,
         'job_timeout': 24,


Test patch:
diff --git a/tests/jobs/test_views.py b/tests/jobs/test_views.py
--- a/tests/jobs/test_views.py
+++ b/tests/jobs/test_views.py
@@ -17,6 +17,10 @@ def test_new_spark_job(client):
     response = client.get(reverse('jobs-new'))
     assert response.status_code == 200
     assert 'form' in response.context
+    f = response.context['form']
+    assert f.initial['identifier'] == ''
+    assert f.initial['size'] == 1
+    assert f.initial['job_timeout'] == 24
 
 
 @pytest.mark.usefixtures('transactional_db')
Repository: mozilla/telemetry-analysis-service

Issue:
When a job terminates early a metrics recording error is thrown
There might be more than one issue here...

Recently a few jobs terminated early b/c of EBS volume limits. When that happened the status failed to update correctly and an error was reported to sentry when we try to calculate one of the metrics which is missing the `ready_at` timestamp.

If a job terminates with errors I think it best to not attempt to record the usual metrics.


Discussion:
Decided it best to continue to record the metrics if we have the data for them. So this issue requires adding more data checking. The code assumes incorrectly that the `ready_at` datetime will exist if the `finished_at` datetime is set -- this is true when the job completes successfully but not true if the job terminates with errors during bootstrapping.

Resolving patch:
diff --git a/atmo/jobs/models.py b/atmo/jobs/models.py
--- a/atmo/jobs/models.py
+++ b/atmo/jobs/models.py
@@ -461,8 +461,10 @@ def sync(self, info=None):
                         }
                     )
 
+                if self.finished_at and self.ready_at:
                     # When job is finished, record time in seconds it took the
-                    # scheduled job to run.
+                    # scheduled job to run. Sometimes `ready_at` won't be
+                    # available if the cluster terminated with errors.
                     run_time = (self.finished_at - self.ready_at).seconds
                     Metric.record(
                         'sparkjob-run-time', run_time,


Test patch:
diff --git a/tests/jobs/test_models.py b/tests/jobs/test_models.py
--- a/tests/jobs/test_models.py
+++ b/tests/jobs/test_models.py
@@ -227,9 +227,9 @@ def test_sync_terminated_with_errors(request, mocker,
     mocker.patch(
         'atmo.clusters.provisioners.ClusterProvisioner.info',
         return_value={
-            'creation_datetime': now,
+            'creation_datetime': one_hour_ago,
             'ready_datetime': None,
-            'end_datetime': None,
+            'end_datetime': now,
             'state': Cluster.STATUS_TERMINATED_WITH_ERRORS,
             'state_change_reason_code': Cluster.STATE_CHANGE_REASON_BOOTSTRAP_FAILURE,
             'state_change_reason_message': 'Bootstrapping steps failed.',
@@ -243,6 +243,19 @@ def test_sync_terminated_with_errors(request, mocker,
     assert alert.mail_sent_date is None
     assert spark_job.latest_run.status == Cluster.STATUS_TERMINATED_WITH_ERRORS
 
+    metrics = Metric.objects.filter(key='sparkjob-normalized-instance-hours')
+    assert metrics.count() == 1
+    metric = metrics.first()
+    assert metric.value == 5
+    assert metric.data == {
+        'identifier': spark_job.identifier,
+        'size': spark_job.size,
+        'jobflow_id': spark_job.latest_run.jobflow_id,
+    }
+
+    assert Metric.objects.filter(key='sparkjob-time-to-ready').count() == 0
+    assert Metric.objects.filter(key='sparkjob-run-time').count() == 0
+
 
 def test_first_run_without_run(mocker, spark_job):
     apply_async = mocker.patch('atmo.jobs.tasks.run_job.apply_async')
Repository: mozilla/telemetry-analysis-service

Issue:
Logout doesn't work as GET request
With the new mozilla-django-oidc implementation the logout view requires using POST to affect the logout (which is a common pattern, although not necessarily a good one). We need to update the logout link to submit a form against the logout view.


Discussion:

Resolving patch:
diff --git a/atmo/settings.py b/atmo/settings.py
--- a/atmo/settings.py
+++ b/atmo/settings.py
@@ -17,7 +17,7 @@
 from datetime import timedelta
 
 from celery.schedules import crontab
-from configurations import Configuration, values
+from configurations import Configuration, pristinemethod, values
 from django.contrib.messages import constants as messages
 from django.core.urlresolvers import reverse_lazy
 from dockerflow.version import get_version
@@ -376,9 +376,21 @@ def DJANGO_AMAZON_SES_REGION(self):
     LOGIN_URL = reverse_lazy('users-login')
     LOGOUT_URL = reverse_lazy('oidc_logout')
     LOGIN_REDIRECT_URL = reverse_lazy('dashboard')
-    LOGIN_REDIRECT_URL_FAILURE = '/'
+    LOGOUT_REDIRECT_URL = reverse_lazy('dashboard')
+    LOGIN_REDIRECT_URL_FAILURE = reverse_lazy('dashboard')
     OIDC_STORE_ACCESS_TOKEN = True
-    OIDC_USERNAME_ALGO = 'atmo.users.utils.generate_username_from_email'
+
+    @pristinemethod
+    def OIDC_USERNAME_ALGO(self, email):
+        """
+        Use the unique part of the email as the username for mozilla.com
+        and the full email address for all other users.
+        """
+        if '@' in email and email.endswith('@mozilla.com'):
+            return email.split('@')[0]
+        else:
+            return email
+
     OIDC_EXEMPT_URLS = [
         'users-login',
     ]
diff --git a/atmo/urls.py b/atmo/urls.py
--- a/atmo/urls.py
+++ b/atmo/urls.py
@@ -23,7 +23,7 @@
     url(r'keys/', include('atmo.keys.urls')),
     url(r'news/', include('atmo.news.urls')),
     url(r'users/', include('atmo.users.urls')),
-    url(r'^oidc/', include('mozilla_django_oidc.urls')),
+    url(r'oidc/', include('mozilla_django_oidc.urls')),
 
     # contribute.json url
     url(r'^(?P<path>contribute\.json)$', static.serve, {'document_root': settings.BASE_DIR}),
diff --git a/atmo/users/utils.py b/atmo/users/utils.py
deleted file mode 100644
--- a/atmo/users/utils.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, you can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-def generate_username_from_email(email):
-    """
-    Use the unique part of the email as the username for mozilla.com
-    and the full email address for all other users.
-    """
-    if '@' in email and email.endswith('@mozilla.com'):
-        return email.split('@')[0]
-    else:
-        return email


Test patch:
Repository: mozilla/telemetry-analysis-service

Issue:
Don't overwrite creator when saving jobs
Since we can provide view and editing permissions to other users (e.g. admins) with row-level permissions we should stop overwriting the creator on every save of a scheduled Spark job since it would prevent an effective trail of ownership and has in the past led to inconsistencies when updating jobs by an admin.


Discussion:

Resolving patch:
diff --git a/atmo/forms/mixins.py b/atmo/forms/mixins.py
--- a/atmo/forms/mixins.py
+++ b/atmo/forms/mixins.py
@@ -5,6 +5,7 @@
 from collections import OrderedDict
 
 from django import forms
+from django.contrib.auth.models import User
 
 from .cache import CachedFileCache
 from .fields import CachedFileField
@@ -52,7 +53,10 @@ def save(self, commit=True):
         obj = super().save(commit=False)
 
         # set the field to the user that created the object
-        obj.created_by = self.created_by
+        try:
+            obj.created_by
+        except User.DoesNotExist:
+            obj.created_by = self.created_by
 
         if commit:
             # actually start the real object, and return the model object


Test patch:
Repository: mozilla/telemetry-analysis-service

Issue:
Add OIDC claim verification for LDAP groups



Discussion:

Resolving patch:
diff --git a/atmo/settings.py b/atmo/settings.py
--- a/atmo/settings.py
+++ b/atmo/settings.py
@@ -369,7 +369,7 @@ def AWS_DEFAULT_REGION(self):
 
     AUTHENTICATION_BACKENDS = (
         'django.contrib.auth.backends.ModelBackend',
-        'mozilla_django_oidc.auth.OIDCAuthenticationBackend',
+        'atmo.users.backends.AtmoOIDCAuthenticationBackend',
         'guardian.backends.ObjectPermissionBackend',
     )
 
@@ -383,6 +383,11 @@ def AWS_DEFAULT_REGION(self):
     OIDC_EXEMPT_URLS = [
         'users-login',
     ]
+    # When enabled this will match the remote groups provided via the OIDC
+    # claims with configured list of allowed user groups using UNIX shell-style
+    # wildcards such as * and ?.
+    REMOTE_GROUPS_ENABLED = values.BooleanValue(default=False)
+    REMOTE_GROUPS_ALLOWED = values.SetValue(set(), separator=',')
 
     MESSAGE_TAGS = {
         messages.ERROR: 'danger'
diff --git a/atmo/users/backends.py b/atmo/users/backends.py
new file mode 100644
--- /dev/null
+++ b/atmo/users/backends.py
@@ -0,0 +1,22 @@
+from django.conf import settings
+
+from mozilla_django_oidc.auth import OIDCAuthenticationBackend
+
+
+class AtmoOIDCAuthenticationBackend(OIDCAuthenticationBackend):
+
+    def verify_claims(self, claims):
+        """
+        See if the claims contain a list of user groups (in various forms)
+        and then check it against a configured list of allowed groups.
+        """
+        # shortcut in case remote groups aren't enabled
+        if not settings.REMOTE_GROUPS_ENABLED:
+            return True
+        remote_groups = set(
+            claims.get('groups') or
+            claims.get('https://sso.mozilla.com/claim/groups') or
+            []
+        )
+        allowed_groups = settings.REMOTE_GROUPS_ALLOWED
+        return bool(allowed_groups.intersection(remote_groups))


Test patch:
Repository: mozilla/telemetry-analysis-service

Issue:
Sort EMR versions naturally
The EMR versions are currently not sorted naturally but by alphabet, which breaks for example with EMR 5.11.0.


Discussion:

Resolving patch:
diff --git a/atmo/clusters/forms.py b/atmo/clusters/forms.py
--- a/atmo/clusters/forms.py
+++ b/atmo/clusters/forms.py
@@ -21,7 +21,7 @@ class EMRReleaseChoiceField(forms.ModelChoiceField):
     def __init__(self, *args, **kwargs):
         super().__init__(
             label='EMR release',
-            queryset=models.EMRRelease.objects.active(),
+            queryset=models.EMRRelease.objects.active().natural_sort_by_version(),
             required=True,
             empty_label=None,
             widget=forms.RadioSelect(attrs={
diff --git a/atmo/clusters/queries.py b/atmo/clusters/queries.py
--- a/atmo/clusters/queries.py
+++ b/atmo/clusters/queries.py
@@ -8,6 +8,15 @@ class EMRReleaseQuerySet(models.QuerySet):
     """
     A Django queryset for the :class:`~atmo.clusters.models.EMRRelease` model.
     """
+    def natural_sort_by_version(self):
+        """
+        Sorts this queryset by the EMR version naturally (human-readable).
+        """
+        return self.extra(
+            select={
+                'natural_version': "string_to_array(version, '.')::int[]",
+            },
+        ).order_by('-natural_version')
 
     def active(self):
         return self.filter(


Test patch:
diff --git a/tests/clusters/test_models.py b/tests/clusters/test_models.py
--- a/tests/clusters/test_models.py
+++ b/tests/clusters/test_models.py
@@ -89,3 +89,25 @@ def test_metric_records_emr_version(cluster_provisioner_mocks, cluster_factory):
 
     assert (Metric.objects.get(key='cluster-emr-version').data ==
             {'version': str(cluster.emr_release.version)})
+
+
+def test_natural_sorting(emr_release_factory):
+    # create EMR releases out of order to force PKs to be not consecutive
+    third = emr_release_factory(version='5.9.0')
+    second = emr_release_factory(version='5.8.0')
+    first = emr_release_factory(version='5.2.1')
+    fourth = emr_release_factory(version='5.11.0')
+
+    expected = [fourth.version, third.version, second.version, first.version]
+
+    versions = models.EMRRelease.objects.all().values_list(
+        'version', flat=True
+    )
+
+    assert list(versions) != expected
+
+    versions = models.EMRRelease.objects.natural_sort_by_version().values_list(
+        'version', flat=True
+    )
+    assert versions.ordered
+    assert list(versions) == expected