internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mozmeao/snippets-service

Issue:
Add Product Field in ASRSnippets
- [x] Populate list of products
- [x] Add permissions to Snippet Creators to edit products
- [ ] Remove `null=True` from ForeignKey relation


Discussion:

Resolving patch:
diff --git a/snippets/base/migrations/0032_auto_20200221_1312.py b/snippets/base/migrations/0032_auto_20200221_1312.py
new file mode 100644
--- /dev/null
+++ b/snippets/base/migrations/0032_auto_20200221_1312.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.2.6 on 2020-02-21 13:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0031_auto_20200219_0949'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='asrsnippet',
+            name='product',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='snippets', to='base.Product'),
+        ),
+    ]
diff --git a/snippets/base/models.py b/snippets/base/models.py
--- a/snippets/base/models.py
+++ b/snippets/base/models.py
@@ -2052,8 +2052,7 @@ class ASRSnippet(models.Model):
     name = models.CharField(max_length=255, unique=True)
 
     category = models.ForeignKey(Category, on_delete=models.PROTECT, related_name='snippets')
-    product = models.ForeignKey(Product, null=True, on_delete=models.PROTECT,
-                                related_name='snippets')
+    product = models.ForeignKey(Product, on_delete=models.PROTECT, related_name='snippets')
 
     tags = TaggableManager(blank=True)
 


Test patch:
Repository: mozmeao/snippets-service

Issue:
Stop pulling data from RedShift 
Starting in version 72 (Jan 2020), Firefox Telemetry uses BigQuery instead of RedShift. 

We currently pull data from both data sources for frequency capping and performance reports. 

In about a year from now the usage of pre-72 versions will be limited and we will be able to remove the RedShift queries from the codebase.

- [x] Stop pulling for Freq Capped Jobs
- [x] Stop pulling Daily Data


Discussion:
@glogiotatidis set a reminder for **Jan 8th 2021**
Apparently the Redshift pipeline was shut down on 2020-02-05. We can go ahead and remove this code

Resolving patch:
diff --git a/snippets/base/etl.py b/snippets/base/etl.py
--- a/snippets/base/etl.py
+++ b/snippets/base/etl.py
@@ -11,10 +11,12 @@
 
 
 REDASH_QUERY_IDS = {
-    'redshift-job': 68135,
     'bq-job': 68136,
-    'redshift-impressions': 68345,
     'bq-impressions': 68341,
+
+    # Not currently used but kept here for reference.
+    'redshift-job': 68135,
+    'redshift-impressions': 68345,
 }
 
 redash = RedashDynamicQuery(
@@ -145,10 +147,7 @@ def prosses_rows(rows, key='message_id'):
 
 
 def update_job_metrics(date):
-    rows = []
-    for query in ['redshift-job', 'bq-job']:
-        rows += redash_rows(query, date)
-
+    rows = redash_rows('bq-job', date)
     processed = prosses_rows(rows, key='message_id')
     with atomic():
         JobDailyPerformance.objects.filter(date=date).delete()
@@ -162,11 +161,7 @@ def update_job_metrics(date):
 
 
 def update_impressions(date):
-    rows = []
-
-    for query in ['redshift-impressions', 'bq-impressions']:
-        rows += redash_rows(query, date)
-
+    rows = redash_rows('bq-impressions', date)
     details = []
     for row in rows:
         # Normalize channel name, based on what kind of snippets they get.


Test patch:
diff --git a/snippets/base/tests/test_etl.py b/snippets/base/tests/test_etl.py
--- a/snippets/base/tests/test_etl.py
+++ b/snippets/base/tests/test_etl.py
@@ -84,9 +84,7 @@ def test_base(self):
                     'channel': 'demo',
                     'country_code': 'GR',
                     'counts': 5,
-                }
-            ],
-            [
+                },
                 {
                     'message_id': '1000',
                     'event_context': '',
@@ -225,8 +223,6 @@ def test_base(self):
                         'counts': 10,
                         'duration': '5',
                     },
-                ],
-                [
                     {
                         'channel': 'release',
                         'counts': 2,
Repository: mozmeao/snippets-service

Issue:
Add CSV export admin action to Jobs admin view



Discussion:

Resolving patch:
diff --git a/snippets/base/admin/actions.py b/snippets/base/admin/actions.py
--- a/snippets/base/admin/actions.py
+++ b/snippets/base/admin/actions.py
@@ -1,4 +1,8 @@
+import csv
+from datetime import datetime
+
 from django.db import transaction
+from django.http import HttpResponse
 
 
 @transaction.atomic
@@ -6,3 +10,21 @@ def duplicate_snippets_action(modeladmin, request, queryset):
     for snippet in queryset:
         snippet.duplicate(request.user)
 duplicate_snippets_action.short_description = 'Duplicate selected snippets'  # noqa
+
+
+def export_as_csv(modeladmin, request, queryset):
+    """Adapted from https://books.agiliq.com/projects/django-admin-cookbook/en/latest/export.html"""
+    meta = modeladmin.model._meta
+    field_names = [field.name for field in meta.fields]
+    filename = f'{meta}-{datetime.today().strftime("%Y-%m-%d-%H-%M")}'
+
+    response = HttpResponse(content_type='text/csv')
+    response['Content-Disposition'] = 'attachment; filename={}.csv'.format(filename)
+    writer = csv.writer(response)
+
+    writer.writerow(field_names)
+    for obj in queryset:
+        writer.writerow([getattr(obj, field) for field in field_names])
+
+    return response
+export_as_csv.short_description = 'Export Selected to CSV'  # noqa
diff --git a/snippets/base/admin/adminmodels.py b/snippets/base/admin/adminmodels.py
--- a/snippets/base/admin/adminmodels.py
+++ b/snippets/base/admin/adminmodels.py
@@ -1562,6 +1562,9 @@ class JobDailyPerformanceAdmin(admin.ModelAdmin):
             ),
         }),
     ]
+    actions = [
+        actions.export_as_csv,
+    ]
 
     def has_add_permission(self, request):
         return False
@@ -1585,6 +1588,9 @@ class DailyImpressionsAdmin(admin.ModelAdmin):
             ),
         }),
     ]
+    actions = [
+        actions.export_as_csv,
+    ]
 
     def has_add_permission(self, request):
         return False


Test patch:
Repository: mozmeao/snippets-service

Issue:
Fix documentation links for UITour Highlights and Menus



Discussion:

Resolving patch:
diff --git a/snippets/base/admin/adminmodels.py b/snippets/base/admin/adminmodels.py
--- a/snippets/base/admin/adminmodels.py
+++ b/snippets/base/admin/adminmodels.py
@@ -596,8 +596,8 @@ class ASRSnippetAdmin(admin.ModelAdmin):
                   <li><code>special:monitor</code> links User to Firefox Monitor and directly authenticates them. Works only in buttons. Works only after Firefox 69.</li>
                   <li><code>special:about:ABOUT_PAGE</code> links to an About page. Get a list of About pages by typing <code>about:about</code> in your awesome bar. Example: <code>special:about:protections</code>.
                   <li><code>special:preferences</code> opens the Firefox Preferences tab. Example: <code>special:preferences</code>.
-                  <li><code>special:highlight:HIGHLIGHT</code> highlights a button in the browser chrome. Get a list of <a href="https://bedrock.readthedocs.io/en/latest/uitour.html#showhighlight-target-effect">available highlights</a>. Example: <code>special:highlight:logins</code>. Works only after Firefox 71.
-                  <li><code>special:menu:MENU</code> opens a targeted menu in the browser chrome. Get a list of <a href="https://bedrock.readthedocs.io/en/latest/uitour.html#showmenu-target-callback">available menus</a>. Example: <code>special:menu:appMenu</code>.
+                  <li><code>special:highlight:HIGHLIGHT</code> highlights a button in the browser chrome. Get a list of <a href="https://firefox-source-docs.mozilla.org/browser/components/uitour/docs/UITour-lib.html#Mozilla.UITour.Target">available highlights</a>. Example: <code>special:highlight:logins</code>. Works only after Firefox 71.
+                  <li><code>special:menu:MENU</code> opens a targeted menu in the browser chrome. Get a list of <a href="https://firefox-source-docs.mozilla.org/browser/components/uitour/docs/UITour-lib.html#Mozilla.UITour.MenuName">available menus</a>. Example: <code>special:menu:appMenu</code>.
                 </ol><br/>
                 <strong>Content Variables:</strong><br/>
                 You can use <code>[[snippet_id]]</code> in any field and it


Test patch:
Repository: mozmeao/snippets-service

Issue:
Exception: ZeroDivisionError: division by zero
Exception

```python
KeyError: 'click_rate'
  File "django/db/models/options.py", line 565, in get_field
    return self.fields_map[field_name]

FieldDoesNotExist: JobDailyPerformance has no field named 'click_rate'
  File "django/contrib/admin/utils.py", line 262, in lookup_field
    f = _get_non_gfk_field(opts, name)
  File "django/contrib/admin/utils.py", line 293, in _get_non_gfk_field
    field = opts.get_field(name)
  File "django/db/models/options.py", line 567, in get_field
    raise FieldDoesNotExist("%s has no field named '%s'" % (self.object_name, field_name))

ZeroDivisionError: division by zero
(32 additional frame(s) were not displayed)
...
  File "django/contrib/admin/templatetags/admin_list.py", line 315, in results
    yield ResultList(None, items_for_result(cl, res, None))
  File "django/contrib/admin/templatetags/admin_list.py", line 306, in __init__
    super().__init__(*items)
  File "django/contrib/admin/templatetags/admin_list.py", line 229, in items_for_result
    f, attr, value = lookup_field(field_name, result, cl.model_admin)
  File "django/contrib/admin/utils.py", line 273, in lookup_field
    attr = getattr(obj, name)
  File "snippets/base/models.py", line 2301, in click_rate
    return float(f'{(self.click / self.impression) * 100:.4f}')

```


Discussion:

Resolving patch:
diff --git a/snippets/base/models.py b/snippets/base/models.py
--- a/snippets/base/models.py
+++ b/snippets/base/models.py
@@ -2290,14 +2290,20 @@ def calculate_adj_impression(self):
 
     @property
     def adj_block_rate(self):
+        if self.adj_impression == 0:
+            return 'N/A'
         return float(f'{(self.block / self.adj_impression) * 100:.4f}')
 
     @property
     def adj_click_rate(self):
+        if self.adj_impression == 0:
+            return 'N/A'
         return float(f'{(self.click / self.adj_impression) * 100:.4f}')
 
     @property
     def click_rate(self):
+        if self.impression == 0:
+            return 'N/A'
         return float(f'{(self.click / self.impression) * 100:.4f}')
 
 


Test patch:
Repository: mozmeao/snippets-service

Issue:
Re-generate all bundles with Distribution changes after Timestamp



Discussion:

Resolving patch:
diff --git a/snippets/base/management/commands/generate_bundles.py b/snippets/base/management/commands/generate_bundles.py
--- a/snippets/base/management/commands/generate_bundles.py
+++ b/snippets/base/management/commands/generate_bundles.py
@@ -34,7 +34,10 @@ def handle(self, *args, **options):
             self.stdout.write(
                 'Generating bundles with Jobs modified on or after {}'.format(options['timestamp'])
             )
-            total_jobs = Job.objects.filter(snippet__modified__gte=options['timestamp'])
+            total_jobs = Job.objects.filter(
+                Q(snippet__modified__gte=options['timestamp']) |
+                Q(distribution__distributionbundle__modified__gte=options['timestamp'])
+            ).distinct()
 
         if not total_jobs:
             self.stdout.write('Nothing to do…')


Test patch:
diff --git a/snippets/base/tests/test_commands.py b/snippets/base/tests/test_commands.py
--- a/snippets/base/tests/test_commands.py
+++ b/snippets/base/tests/test_commands.py
@@ -5,6 +5,7 @@
 from django.conf import settings
 from django.core.management import call_command
 from django.core.management.base import CommandError
+from django.db.models import Q
 from django.test.utils import override_settings
 
 from snippets.base import models
@@ -291,17 +292,20 @@ def setUp(self):
 
     def test_generate_all(self):
         with patch('snippets.base.management.commands.generate_bundles.Job') as job_mock:
-            job_mock.objects.all.return_value = []
+            job_mock.objects.all.return_value = models.Job.objects.none()
             call_command('generate_bundles', stdout=Mock())
         job_mock.objects.all.assert_called()
         job_mock.objects.filter.assert_not_called()
 
     def test_generate_after_timestamp(self):
         with patch('snippets.base.management.commands.generate_bundles.Job') as job_mock:
-            job_mock.objects.filter.return_value = []
+            job_mock.objects.filter.return_value = models.Job.objects.none()
             call_command('generate_bundles', timestamp='2019-01-01', stdout=Mock())
         job_mock.objects.all.assert_not_called()
-        job_mock.objects.filter.assert_called_with(snippet__modified__gte='2019-01-01')
+        job_mock.objects.filter.assert_called_with(
+            Q(snippet__modified__gte='2019-01-01') |
+            Q(distribution__distributionbundle__modified__gte='2019-01-01')
+        )
 
     @override_settings(MEDIA_BUNDLES_PREGEN_ROOT='pregen')
     def test_generation(self):
Repository: mozmeao/snippets-service

Issue:
Add Firefox Relay to Firefox Services Targeting Field



Discussion:

Resolving patch:
diff --git a/snippets/base/admin/fields.py b/snippets/base/admin/fields.py
--- a/snippets/base/admin/fields.py
+++ b/snippets/base/admin/fields.py
@@ -198,6 +198,7 @@ def __init__(self, **kwargs):
             ('802d56ef2a9af9fa', 'Firefox Monitor'),
             ('1f30e32975ae5112|20f7931c9054d833', 'Firefox Send'),
             ('a8c528140153d1c6|565585c1745a144d|e6eb0d1e856335fc', 'Firefox Private Network'),
+            ('9ebfe2c2f9ea3c58', 'Firefox Private Relay'),
             ('7ad9917f6c55fb77', 'Firefox Reality'),
             ('7377719276ad44ee|749818d3f2e7857f', 'Pocket'),
 


Test patch:
Repository: mozmeao/snippets-service

Issue:
Add Firefox VPN target
It seems the Firefox VPN uses `e6eb0d1e856335fc`, previously named `Firefox FPN`


Discussion:

Resolving patch:
diff --git a/snippets/base/admin/fields.py b/snippets/base/admin/fields.py
--- a/snippets/base/admin/fields.py
+++ b/snippets/base/admin/fields.py
@@ -197,7 +197,8 @@ def __init__(self, **kwargs):
             ('e7ce535d93522896|98adfa37698f255b', 'Firefox Lockwise'),
             ('802d56ef2a9af9fa', 'Firefox Monitor'),
             ('1f30e32975ae5112|20f7931c9054d833', 'Firefox Send'),
-            ('a8c528140153d1c6|565585c1745a144d|e6eb0d1e856335fc', 'Firefox Private Network'),
+            ('a8c528140153d1c6|565585c1745a144d', 'Firefox Private Network'),
+            ('e6eb0d1e856335fc', 'Firefox VPN'),
             ('9ebfe2c2f9ea3c58', 'Firefox Private Relay'),
             ('7ad9917f6c55fb77', 'Firefox Reality'),
             ('7377719276ad44ee|749818d3f2e7857f', 'Pocket'),


Test patch:
Repository: mathesar-foundation/mathesar

Issue:
Configure flake8 & GitHub Action correctly
Our flake8 setup has a couple of issues:
- Failures on the GitHub Action don't actually block merge.
- We need to set up our style guide for flake8.


Discussion:

Resolving patch:
diff --git a/mathesar/forms/forms.py b/mathesar/forms/forms.py
--- a/mathesar/forms/forms.py
+++ b/mathesar/forms/forms.py
@@ -4,10 +4,12 @@
 from mathesar.database.schemas import get_all_schemas
 from mathesar.forms.widgets import DataListInput
 
+
 def validate_csv(value):
     if not value.name.lower().endswith(".csv"):
         raise ValidationError(f"{value.name} is not a CSV file")
 
+
 class UploadFileForm(forms.Form):
     collection_name = forms.CharField(min_length=1, label="Collection Name")
 
diff --git a/mathesar/forms/widgets.py b/mathesar/forms/widgets.py
--- a/mathesar/forms/widgets.py
+++ b/mathesar/forms/widgets.py
@@ -1,5 +1,6 @@
 from django.forms.widgets import TextInput
 
+
 class DataListInput(TextInput):
     """
     Widget that adds a <data_list> element to the standard text input widget.
@@ -37,4 +38,3 @@ def render(self, name, value, attrs=None, renderer=None):
         attrs = {} if attrs is None else attrs
         attrs["list"] = self.data_list_id
         return super().render(name, value, attrs, renderer)
-


Test patch:
Repository: mathesar-foundation/mathesar

Issue:
Schema getter should return public, but not mathesar_types
**Describe the bug**
<!-- A clear and concise description of what the bug is. -->

If a user wants to create a table the `public` schema, they can't currently, because the logic in the `db.schemas.get_all_schemas` function ignores it.  This means when they try, an error is thrown.  This is especially a problem when they've imported a DB, since most tables are in the `public` schema in most installations of PostgreSQL in the wild.

**Expected behavior**
<!-- A clear and concise description of what you expected to happen. -->
The public schema should be available for holding mathesar tables.

**To Reproduce**
Please try to provide a [Minimal, Complete, and Verifiable](http://stackoverflow.com/help/mcve) example.

Start the webapp using the README.  Try to upload a CSV to the `public` schema.  See the error.

**Have a nice day!**



Discussion:

Resolving patch:
diff --git a/db/schemas.py b/db/schemas.py
--- a/db/schemas.py
+++ b/db/schemas.py
@@ -1,24 +1,28 @@
 from sqlalchemy.schema import CreateSchema
 from sqlalchemy import inspect
 
+from db import types
 
-def get_all_schemas(engine):
-    inspector = inspect(engine)
+TYPES_SCHEMA = types.base.SCHEMA
+
+
+def get_mathesar_schemas(engine):
     return [
         schema
-        for schema in inspector.get_schema_names()
-        if schema not in ["public", "information_schema"]
+        for schema in get_all_schemas(engine)
+        if schema not in [TYPES_SCHEMA, "information_schema"]
     ]
 
 
-def schema_exists(schema, engine):
-    return schema in get_all_schemas(engine)
+def get_all_schemas(engine):
+    inspector = inspect(engine)
+    return inspector.get_schema_names()
 
 
 def create_schema(schema, engine):
     """
     This method creates a Postgres schema.
     """
-    if not schema_exists(schema, engine):
+    if schema not in get_all_schemas(engine):
         with engine.begin() as connection:
             connection.execute(CreateSchema(schema))


Test patch:
diff --git a/db/tests/test_schemas.py b/db/tests/test_schemas.py
new file mode 100644
--- /dev/null
+++ b/db/tests/test_schemas.py
@@ -0,0 +1,27 @@
+from sqlalchemy import create_engine
+from db import schemas
+from db import types
+
+
+def test_get_mathesar_schemas_gets_public(monkeypatch):
+    engine = create_engine("postgresql://")
+    monkeypatch.setattr(
+        schemas,
+        "get_all_schemas",
+        lambda x: ["public"]
+    )
+    actual_schemas = schemas.get_mathesar_schemas(engine)
+    expect_schemas = ["public"]
+    assert actual_schemas == expect_schemas
+
+
+def test_get_mathesar_schemas_avoids_utility_schemata(monkeypatch):
+    engine = create_engine("postgresql://")
+    monkeypatch.setattr(
+        schemas,
+        "get_all_schemas",
+        lambda x: ["public", "information_schema", types.base.SCHEMA]
+    )
+    actual_schemas = schemas.get_mathesar_schemas(engine)
+    expect_schemas = ["public"]
+    assert actual_schemas == expect_schemas
Repository: mathesar-foundation/mathesar

Issue:
API tests using wrong tables database
**Describe the bug**
The django API tests are running on the `test_mathesar_db_test_database`, which differs from the `mathesar_db_test_database` tables database we should be using. As a result, we don't have a proper reference to the database being used by the API functions, which prevents us from certain operations like installing types for a test. 

**Expected behavior**
We should ensure `pytest-django` doesn't build a separate tables database.

**Additional context**
Currently blocking #276 



Discussion:

Resolving patch:
diff --git a/config/settings.py b/config/settings.py
--- a/config/settings.py
+++ b/config/settings.py
@@ -74,6 +74,13 @@
     decouple_config('MATHESAR_DATABASE_KEY'): decouple_config('MATHESAR_DATABASE_URL', cast=db_url)
 }
 
+# pytest-django will create a new database named 'test_{DATABASES[table_db]['NAME']}'
+# and use it for our API tests if we don't specify DATABASES[table_db]['TEST']['NAME']
+if decouple_config('TEST', default=False, cast=bool):
+    DATABASES[decouple_config('MATHESAR_DATABASE_KEY')]['TEST'] = {
+        'NAME': DATABASES[decouple_config('MATHESAR_DATABASE_KEY')]['NAME']
+    }
+
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/


Test patch:
Repository: mathesar-foundation/mathesar

Issue:
User should be able to configure multiple databases in settings
**Problem**
<!-- Please provide a clear and concise description of the problem that this feature request is designed to solve.-->
Currently, the user can only configure one Mathesar database in the settings. They should be able to configure as many databases to connect to Mathesar as they want.

**Proposed solution**
<!-- A clear and concise description of your proposed solution or feature. -->
The user should be able to configure multiple databases in the `.env` file.

**Additional context**
<!-- Add any other context or screenshots about the feature request here.-->
We might want to use `python-decouple`'s [built in CSV helper](https://github.com/henriquebastos/python-decouple/#built-in-csv-helper) for this.

Ideally, the user would be able to associate the database key with the connection information directly using a tuple or something like that.


Discussion:

Resolving patch:
diff --git a/config/settings.py b/config/settings.py
--- a/config/settings.py
+++ b/config/settings.py
@@ -16,6 +16,16 @@
 from decouple import Csv, config as decouple_config
 from dj_database_url import parse as db_url
 
+
+# We use a 'tuple' with pipes as delimiters as decople naively splits the global
+# variables on commas when casting to Csv()
+def pipe_delim(pipe_string):
+    # Remove opening and closing brackets
+    pipe_string = pipe_string[1:-1]
+    # Split on pipe delim
+    return pipe_string.split("|")
+
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -69,17 +79,20 @@
 # https://docs.djangoproject.com/en/3.1/ref/settings/#databases
 
 # TODO: Add to documentation that database keys should not be than 128 characters.
+
+# MATHESAR_DATABASES should be of the form '({db_name}|{db_url}), ({db_name}|{db_url})'
+# See pipe_delim above for why we use pipes as delimiters
 DATABASES = {
-    decouple_config('DJANGO_DATABASE_KEY'): decouple_config('DJANGO_DATABASE_URL', cast=db_url),
-    decouple_config('MATHESAR_DATABASE_KEY'): decouple_config('MATHESAR_DATABASE_URL', cast=db_url)
+    db_key: db_url(url_string)
+    for db_key, url_string in decouple_config('MATHESAR_DATABASES', cast=Csv(pipe_delim))
 }
+DATABASES[decouple_config('DJANGO_DATABASE_KEY')] = decouple_config('DJANGO_DATABASE_URL', cast=db_url)
 
 # pytest-django will create a new database named 'test_{DATABASES[table_db]['NAME']}'
 # and use it for our API tests if we don't specify DATABASES[table_db]['TEST']['NAME']
 if decouple_config('TEST', default=False, cast=bool):
-    DATABASES[decouple_config('MATHESAR_DATABASE_KEY')]['TEST'] = {
-        'NAME': DATABASES[decouple_config('MATHESAR_DATABASE_KEY')]['NAME']
-    }
+    for db_key, _ in decouple_config('MATHESAR_DATABASES', cast=Csv(pipe_delim)):
+        DATABASES[db_key]['TEST'] = {'NAME': DATABASES[db_key]['NAME']}
 
 
 # Quick-start development settings - unsuitable for production


Test patch: