Repository: mozmeao/basket

Issue:
Upgrade to latest Celery
4.2 as of this writing.


Discussion:

Resolving patch:
diff --git a/basket/news/celery.py b/basket/news/celery.py
--- a/basket/news/celery.py
+++ b/basket/news/celery.py
@@ -6,14 +6,12 @@
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'basket.settings')
 
-from django.conf import settings
-
-import celery
+from celery import Celery as CeleryBase
 from raven.contrib.celery import register_signal, register_logger_signal
 from raven.contrib.django.raven_compat.models import client
 
 
-class Celery(celery.Celery):
+class Celery(CeleryBase):
     def on_configure(self):
         # register a custom filter to filter out duplicate logs
         register_logger_signal(client)
@@ -24,10 +22,13 @@ def on_configure(self):
 
 app = Celery('basket')
 
-# Using a string here means the worker will not have to
-# pickle the object when using Windows.
-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
 
 
 @app.task(bind=True)
diff --git a/basket/settings.py b/basket/settings.py
--- a/basket/settings.py
+++ b/basket/settings.py
@@ -41,7 +41,7 @@ def path(*args):
 if REDIS_URL:
     REDIS_URL = REDIS_URL.rstrip('/0')
     # use redis for celery and cache
-    os.environ['BROKER_URL'] = REDIS_URL + '/' + config('REDIS_CELERY_DB', '0')
+    os.environ['CELERY_BROKER_URL'] = REDIS_URL + '/' + config('REDIS_CELERY_DB', '0')
     os.environ['CACHE_URL'] = REDIS_URL + '/' + config('REDIS_CACHE_DB', '1')
 
 # Production uses MySQL, but Sqlite should be sufficient for local development.
@@ -237,23 +237,23 @@ def path(*args):
 # view rate limiting
 RATELIMIT_VIEW = 'basket.news.views.ratelimited'
 
-CELERY_ALWAYS_EAGER = config('CELERY_ALWAYS_EAGER', DEBUG, cast=bool)
-BROKER_URL = config('BROKER_URL', None)
+CELERY_TASK_ALWAYS_EAGER = config('CELERY_TASK_ALWAYS_EAGER', DEBUG, cast=bool)
+CELERY_BROKER_URL = config('CELERY_BROKER_URL', None)
 CELERY_REDIS_MAX_CONNECTIONS = config('CELERY_REDIS_MAX_CONNECTIONS', 2, cast=int)
-CELERY_DISABLE_RATE_LIMITS = True
-CELERY_IGNORE_RESULT = True
-CELERYD_PREFETCH_MULTIPLIER = config('CELERYD_PREFETCH_MULTIPLIER', 1, cast=int)
-CELERY_MESSAGE_COMPRESSION = 'gzip'
-CELERY_ROUTES = {
+CELERY_WORKER_DISABLE_RATE_LIMITS = True
+CELERY_TASK_IGNORE_RESULT = True
+CELERY_WORKER_PREFETCH_MULTIPLIER = config('CELERY_WORKER_PREFETCH_MULTIPLIER', 1, cast=int)
+CELERY_TASK_COMPRESSION = 'gzip'
+CELERY_TASK_ROUTES = {
     'basket.news.tasks.snitch': {'queue': 'snitch'},
 }
 
 SNITCH_ID = config('SNITCH_ID', None)
 
-CELERYBEAT_SCHEDULE = {}
+CELERY_BEAT_SCHEDULE = {}
 
 if SNITCH_ID:
-    CELERYBEAT_SCHEDULE['snitch'] = {
+    CELERY_BEAT_SCHEDULE['snitch'] = {
         'task': 'basket.news.tasks.snitch',
         'schedule': timedelta(minutes=5),
     }
@@ -368,7 +368,7 @@ def get_default_gateway_linux():
 
 FXA_ACCESS_KEY_ID = config('FXA_ACCESS_KEY_ID', default='')
 FXA_SECRET_ACCESS_KEY = config('FXA_SECRET_ACCESS_KEY', default='')
-FXA_S3_BUCKET = config('FXA_S3_BUCKET', default='')
+FXA_CELERY_S3_BUCKET = config('FXA_CELERY_S3_BUCKET', default='')
 FXA_SFMC_DE = config('FXA_SFMC_DE', default='FXA_Logins')
 FXA_SNITCH_URL = config('FXA_SNITCH_URL', default='')
 # stable, stage, or production
@@ -407,7 +407,7 @@ def get_default_gateway_linux():
 
 if sys.argv[0].endswith('py.test') or (len(sys.argv) > 1 and sys.argv[1] == 'test'):
     # stuff that's absolutely required for a test run
-    CELERY_ALWAYS_EAGER = True
+    CELERY_TASK_ALWAYS_EAGER = True
     SFDC_SETTINGS.pop('username', None)
     SFDC_SETTINGS.pop('password', None)
     SFMC_SETTINGS.pop('clientid', None)


Test patch:
diff --git a/basket/news/tests/test_confirm.py b/basket/news/tests/test_confirm.py
--- a/basket/news/tests/test_confirm.py
+++ b/basket/news/tests/test_confirm.py
@@ -1,5 +1,6 @@
 from django.test import TestCase
 
+from celery.exceptions import Retry
 from mock import patch, Mock
 
 from basket.news.backends.common import NewsletterException
@@ -15,7 +16,7 @@ def test_error(self, get_user_data, sfdc_mock):
         an exception so our task logic will retry
         """
         get_user_data.side_effect = NewsletterException('Stuffs broke yo.')
-        with self.assertRaises(NewsletterException):
+        with self.assertRaises(Retry):
             confirm_user('token')
         self.assertFalse(sfdc_mock.update.called)
 
diff --git a/basket/news/tests/test_tasks.py b/basket/news/tests/test_tasks.py
--- a/basket/news/tests/test_tasks.py
+++ b/basket/news/tests/test_tasks.py
@@ -1,5 +1,6 @@
 import json
 from copy import deepcopy
+from datetime import datetime
 from urllib2 import URLError
 
 from django.conf import settings
@@ -7,9 +8,8 @@
 from django.test import TestCase
 from django.test.utils import override_settings
 
-from datetime import datetime
-
 import simple_salesforce as sfapi
+from celery.exceptions import Retry
 from mock import ANY, Mock, patch
 
 from basket.news.celery import app as celery_app
@@ -201,7 +201,7 @@ def test_signature_with_new_user_retry(self, gt_mock, sfdc_mock, gud_mock, uu_mo
         contact_data['email'] = data['form']['email']
         contact_data['record_type'] = settings.DONATE_CONTACT_RECORD_TYPE
         gud_mock.return_value = None
-        with self.assertRaises(RetryTask):
+        with self.assertRaises(Retry):
             process_petition_signature(data)
 
         sfdc_mock.update.assert_not_called()
@@ -313,7 +313,7 @@ def test_name_splitting(self, sfdc_mock, gud_mock):
         gud_mock.return_value = None
         del data['first_name']
         data['last_name'] = 'Theodore Donald Kerabatsos'
-        with self.assertRaises(RetryTask):
+        with self.assertRaises(Retry):
             # raises retry b/c the 2nd call to get_user_data returns None
             process_donation(data)
         sfdc_mock.add.assert_called_with({
@@ -334,7 +334,7 @@ def test_name_empty(self, sfdc_mock, gud_mock):
         gud_mock.return_value = None
         del data['first_name']
         data['last_name'] = '  '
-        with self.assertRaises(RetryTask):
+        with self.assertRaises(Retry):
             # raises retry b/c the 2nd call to get_user_data returns None
             process_donation(data)
         sfdc_mock.add.assert_called_with({
@@ -353,7 +353,7 @@ def test_name_none(self, sfdc_mock, gud_mock):
         gud_mock.return_value = None
         del data['first_name']
         data['last_name'] = None
-        with self.assertRaises(RetryTask):
+        with self.assertRaises(Retry):
             # raises retry b/c the 2nd call to get_user_data returns None
             process_donation(data)
         sfdc_mock.add.assert_called_with({
@@ -447,7 +447,7 @@ def test_donation_normal_failure_not_dupe(self, sfdc_mock, gud_mock):
         }]
         exc = sfapi.SalesforceMalformedRequest('url', 400, 'opportunity', error_content)
         sfdc_mock.opportunity.create.side_effect = exc
-        with self.assertRaises(sfapi.SalesforceMalformedRequest):
+        with self.assertRaises(Retry):
             process_donation(data)
 
 
@@ -540,7 +540,7 @@ def test_et_error(self, mock_look_for_user, mock_send):
         """Error talking to Basket. I'm shocked, SHOCKED!"""
         mock_look_for_user.side_effect = NewsletterException('ET has failed to achieve.')
 
-        with self.assertRaises(NewsletterException):
+        with self.assertRaises(Retry):
             send_recovery_message_task(self.email)
 
         self.assertFalse(mock_send.called)
diff --git a/docker/envfiles/test.env b/docker/envfiles/test.env
--- a/docker/envfiles/test.env
+++ b/docker/envfiles/test.env
@@ -3,5 +3,5 @@ DEV=False
 ALLOWED_HOSTS=*
 SECRET_KEY=ssssssssshhhhhhhhhh
 ADMINS=["thedude@example.com"]
-CELERY_ALWAYS_EAGER=True
+CELERY_TASK_ALWAYS_EAGER=True
 DATABASE_URL=sqlite:///basket.db