Repository: celery/celery

Issue:
autoretry_for and retry_kwargs for class-based tasks
python==3.6.4
celery==4.1.0

## Steps to reproduce
```
from project.celery import app


class test_task(celery.Task):
    autoretry_for = (Exception,)
    retry_kwargs = {
        'max_retries': 5,
        'countdown': 10,
    }

    def run(self, *args, **kwargs):
        raise Exception('test')


test_task = app.register_task(test_task())

test_task.delay()
```

## Expected behavior
5 retries of this task with countdown of 10 seconds each.

## Actual behavior
No retries at all.


So the main question - is there any way for specifying  autoretry_for and retry_kwargs for class-based tasks? Just to mention - function-based tasks works properly with shared_task decorator and this arguments for it.


Discussion:
Exactly the same issue for me. It looks like the `autoretry_for` attribute is only touched in `_task_from_fun`, which isn't used for a class based task. At least it looks like this. Is there a special reason or was this just forgotten?
@CompadreP maybe as a little workaround a class similar to this could help you:
```python
class AutoRetryTask(Task):
    default_retry_delay = 3
    max_retries = 5
    autoretry_for = None
    retry_kwargs = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if self.autoretry_for and not hasattr(self, '_orig_run'):
            @wraps(self.run)
            def run(*args, **kwargs):
                try:
                    return self._orig_run(*args, **kwargs)
                except self.autoretry_for as exc:
                    if 'countdown' not in self.retry_kwargs:
                        countdown = int(random.uniform(2, 4) ** self.request.retries)
                        retry_kwargs = self.retry_kwargs.copy()
                        retry_kwargs.update({'countdown': countdown})
                    else:
                        retry_kwargs = self.retry_kwargs
                    raise self.retry(exc=exc, **retry_kwargs)

            self._orig_run, self.run = self.run, run
```
I think this would be a very neat feature (unless we want to actively discourage the use of class based tasks altogether) - if one of the maintainers could mention their opinion on this (and the `Is there a special reason or was this just forgotten?` question), I could take a stab at implementing it.
Any update on this?
I was able to fix that easily by overriding Celery task decorator:
```
class CustomCelery(Celery):

    def task(self, *args, **opts):

        # Adds autoretry kwargs to @celery.task() decorators
        if opts.get('base') == BaseTask:
            opts['autoretry_for'] = (requests.ConnectionError, ConnectionError)
            opts['retry_kwargs'] = {'max_retries': 5}
            opts['retry_backoff'] = True

        return super().task(*args, **opts)
```
@amir-hadi Thanks for the idea. With small modifications I've got working retries with exponential backoff.

Celery 4.3.0
Python 3.7.4

```python
from celery import Task
from celery.utils.time import get_exponential_backoff_interval

class AutoRetryTask(Task):
    retry_kwargs = {
        'max_retries': 5,
    }
    retry_backoff = True
    retry_backoff_max = 600
    retry_jitter = True
    
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        if self.autoretry_for and not hasattr(self, '_orig_run'):
            def run(*args, **kwargs):
                try:
                    return self._orig_run(*args, **kwargs)
                except self.autoretry_for as exc:
                    if 'countdown' not in self.retry_kwargs:
                        countdown = get_exponential_backoff_interval(
                            factor=self.retry_backoff,
                            retries=self.request.retries,
                            maximum=self.retry_backoff_max,
                            full_jitter=self.retry_jitter,
                        )

                        retry_kwargs = self.retry_kwargs.copy()
                        retry_kwargs.update({'countdown': countdown})
                    else:
                        retry_kwargs = self.retry_kwargs

                    retry_kwargs.update({'exc': exc})
                    raise self.retry(**retry_kwargs)

            self._orig_run, self.run = self.run, run
```

Further improvements:
* Make it a base task
* Define `retry_*` properties from some config object/file

Resolving patch:
diff --git a/celery/app/base.py b/celery/app/base.py
--- a/celery/app/base.py
+++ b/celery/app/base.py
@@ -460,11 +460,24 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options):
             self._tasks[task.name] = task
             task.bind(self)  # connects task to this app
 
-            autoretry_for = tuple(options.get('autoretry_for', ()))
-            retry_kwargs = options.get('retry_kwargs', {})
-            retry_backoff = int(options.get('retry_backoff', False))
-            retry_backoff_max = int(options.get('retry_backoff_max', 600))
-            retry_jitter = options.get('retry_jitter', True)
+            autoretry_for = tuple(
+                options.get('autoretry_for',
+                            getattr(task, 'autoretry_for', ()))
+            )
+            retry_kwargs = options.get(
+                'retry_kwargs', getattr(task, 'retry_kwargs', {})
+            )
+            retry_backoff = int(
+                options.get('retry_backoff',
+                            getattr(task, 'retry_backoff', False))
+            )
+            retry_backoff_max = int(
+                options.get('retry_backoff_max',
+                            getattr(task, 'retry_backoff_max', 600))
+            )
+            retry_jitter = options.get(
+                'retry_jitter', getattr(task, 'retry_jitter', True)
+            )
 
             if autoretry_for and not hasattr(task, '_orig_run'):
 


Test patch:
diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py
--- a/t/unit/tasks/test_tasks.py
+++ b/t/unit/tasks/test_tasks.py
@@ -43,6 +43,14 @@ class TaskWithPriority(Task):
     priority = 10
 
 
+class TaskWithRetry(Task):
+    autoretry_for = (TypeError,)
+    retry_kwargs = {'max_retries': 5}
+    retry_backoff = True
+    retry_backoff_max = 700
+    retry_jitter = False
+
+
 class TasksCase:
 
     def setup(self):
@@ -152,6 +160,81 @@ def autoretry_backoff_jitter_task(self, url):
 
         self.autoretry_backoff_jitter_task = autoretry_backoff_jitter_task
 
+        @self.app.task(bind=True, base=TaskWithRetry, shared=False)
+        def autoretry_for_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.autoretry_for_from_base_task = autoretry_for_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry,
+                       autoretry_for=(ZeroDivisionError,), shared=False)
+        def override_autoretry_for_from_base_task(self, a, b):
+            self.iterations += 1
+            return a / b
+
+        self.override_autoretry_for = override_autoretry_for_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry, shared=False)
+        def retry_kwargs_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.retry_kwargs_from_base_task = retry_kwargs_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry,
+                       retry_kwargs={'max_retries': 2}, shared=False)
+        def override_retry_kwargs_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.override_retry_kwargs = override_retry_kwargs_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry, shared=False)
+        def retry_backoff_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.retry_backoff_from_base_task = retry_backoff_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry,
+                       retry_backoff=False, shared=False)
+        def override_retry_backoff_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.override_retry_backoff = override_retry_backoff_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry, shared=False)
+        def retry_backoff_max_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.retry_backoff_max_from_base_task = retry_backoff_max_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry,
+                       retry_backoff_max=16, shared=False)
+        def override_retry_backoff_max_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.override_backoff_max = override_retry_backoff_max_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry, shared=False)
+        def retry_backoff_jitter_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.retry_backoff_jitter_from_base = retry_backoff_jitter_from_base_task
+
+        @self.app.task(bind=True, base=TaskWithRetry,
+                       retry_jitter=True, shared=False)
+        def override_backoff_jitter_from_base_task(self, a, b):
+            self.iterations += 1
+            return a + b
+
+        self.override_backoff_jitter = override_backoff_jitter_from_base_task
+
         @self.app.task(bind=True)
         def task_check_request_context(self):
             assert self.request.hostname == socket.gethostname()
@@ -373,6 +456,94 @@ def test_autoretry_backoff_jitter(self, randrange):
         ]
         assert retry_call_countdowns == [0, 1, 3, 7]
 
+    def test_autoretry_for_from_base(self):
+        self.autoretry_for_from_base_task.iterations = 0
+        self.autoretry_for_from_base_task.apply((1, "a"))
+        assert self.autoretry_for_from_base_task.iterations == 6
+
+    def test_override_autoretry_for_from_base(self):
+        self.override_autoretry_for.iterations = 0
+        self.override_autoretry_for.apply((1, 0))
+        assert self.override_autoretry_for.iterations == 6
+
+    def test_retry_kwargs_from_base(self):
+        self.retry_kwargs_from_base_task.iterations = 0
+        self.retry_kwargs_from_base_task.apply((1, "a"))
+        assert self.retry_kwargs_from_base_task.iterations == 6
+
+    def test_override_retry_kwargs_from_base(self):
+        self.override_retry_kwargs.iterations = 0
+        self.override_retry_kwargs.apply((1, "a"))
+        assert self.override_retry_kwargs.iterations == 3
+
+    def test_retry_backoff_from_base(self):
+        task = self.retry_backoff_from_base_task
+        task.iterations = 0
+        with patch.object(task, 'retry', wraps=task.retry) as fake_retry:
+            task.apply((1, "a"))
+
+        assert task.iterations == 6
+        retry_call_countdowns = [
+            call[1]['countdown'] for call in fake_retry.call_args_list
+        ]
+        assert retry_call_countdowns == [1, 2, 4, 8, 16, 32]
+
+    @patch('celery.app.base.get_exponential_backoff_interval')
+    def test_override_retry_backoff_from_base(self, backoff):
+        self.override_retry_backoff.iterations = 0
+        self.override_retry_backoff.apply((1, "a"))
+        assert self.override_retry_backoff.iterations == 6
+        assert backoff.call_count == 0
+
+    def test_retry_backoff_max_from_base(self):
+        task = self.retry_backoff_max_from_base_task
+        task.iterations = 0
+        with patch.object(task, 'retry', wraps=task.retry) as fake_retry:
+            task.apply((1, "a"))
+
+        assert task.iterations == 6
+        retry_call_countdowns = [
+            call[1]['countdown'] for call in fake_retry.call_args_list
+        ]
+        assert retry_call_countdowns == [1, 2, 4, 8, 16, 32]
+
+    def test_override_retry_backoff_max_from_base(self):
+        task = self.override_backoff_max
+        task.iterations = 0
+        with patch.object(task, 'retry', wraps=task.retry) as fake_retry:
+            task.apply((1, "a"))
+
+        assert task.iterations == 6
+        retry_call_countdowns = [
+            call[1]['countdown'] for call in fake_retry.call_args_list
+        ]
+        assert retry_call_countdowns == [1, 2, 4, 8, 16, 16]
+
+    def test_retry_backoff_jitter_from_base(self):
+        task = self.retry_backoff_jitter_from_base
+        task.iterations = 0
+        with patch.object(task, 'retry', wraps=task.retry) as fake_retry:
+            task.apply((1, "a"))
+
+        assert task.iterations == 6
+        retry_call_countdowns = [
+            call[1]['countdown'] for call in fake_retry.call_args_list
+        ]
+        assert retry_call_countdowns == [1, 2, 4, 8, 16, 32]
+
+    @patch('random.randrange', side_effect=lambda i: i - 2)
+    def test_override_backoff_jitter_from_base(self, randrange):
+        task = self.override_backoff_jitter
+        task.iterations = 0
+        with patch.object(task, 'retry', wraps=task.retry) as fake_retry:
+            task.apply((1, "a"))
+
+        assert task.iterations == 6
+        retry_call_countdowns = [
+            call[1]['countdown'] for call in fake_retry.call_args_list
+        ]
+        assert retry_call_countdowns == [0, 1, 3, 7, 15, 31]
+
     def test_retry_wrong_eta_when_not_enable_utc(self):
         """Issue #3753"""
         self.app.conf.enable_utc = False