Repository: readthedocs/readthedocs.org

Issue:
Task for update_docs can timeout apparently
Stack trace:

```
Jul 13 18:43:51 build02 readthedocs/celery.worker.job[11229]: ERROR Task update_docs[12609cdb-ee39-45f2-b3a6-616a1a920c3c] raised unexpected: TimeLimitExceeded(3600,) [celery.worker.job:282]
Traceback (most recent call last):
  File "/home/docs/local/lib/python2.7/site-packages/billiard/pool.py", line 641, in on_hard_timeout
    raise TimeLimitExceeded(job._timeout)
TimeLimitExceeded: TimeLimitExceeded(3600,)
Jul 13 18:43:51 build02 readthedocs/celery.worker.job[11229]: ERROR Hard time limit (3600s) exceeded for update_docs[12609cdb-ee39-45f2-b3a6-616a1a920c3c] [celery.worker.job:368]
```

This should instead report a failure on the build, not just die. When the celery process is killed, the lock stays around, user is not notified of failure, and the build stays in "Building" state.



Discussion:
This is a celery setting that we have set. 

We can remove it or raise it if we don't think this is an issue now that we have the Docker time-limiting in place. 

Yeah, the reason for termination is clear. We'll want to do this as an argument to calling the task, as the value of Docker time limit changes with projects.


Resolving patch:
diff --git a/readthedocs/core/utils/__init__.py b/readthedocs/core/utils/__init__.py
--- a/readthedocs/core/utils/__init__.py
+++ b/readthedocs/core/utils/__init__.py
@@ -7,6 +7,7 @@
 from django.conf import settings
 
 from readthedocs.builds.constants import LATEST
+from readthedocs.doc_builder.constants import DOCKER_LIMITS
 from ..tasks import send_email_task
 
 
@@ -102,6 +103,18 @@ def trigger_build(project, version=None, record=True, force=False, basic=False):
     if project.build_queue:
         options['queue'] = project.build_queue
 
+    # Set per-task time limit
+    time_limit = DOCKER_LIMITS['time']
+    try:
+        if project.container_time_limit:
+            time_limit = int(project.container_time_limit)
+    except ValueError:
+        pass
+    # Add 20% overhead to task, to ensure the build can timeout and the task
+    # will cleanly finish.
+    options['soft_time_limit'] = time_limit
+    options['time_limit'] = int(time_limit * 1.2)
+
     update_docs.apply_async(kwargs=kwargs, **options)
 
     return build


Test patch:
diff --git a/readthedocs/rtd_tests/tests/test_core_utils.py b/readthedocs/rtd_tests/tests/test_core_utils.py
new file mode 100644
--- /dev/null
+++ b/readthedocs/rtd_tests/tests/test_core_utils.py
@@ -0,0 +1,80 @@
+"""Test core util functions"""
+
+import mock
+
+from django_dynamic_fixture import get
+from django.test import TestCase
+from django.test.utils import override_settings
+
+from readthedocs.projects.models import Project
+from readthedocs.builds.models import Version
+from readthedocs.core.utils import trigger_build
+
+
+class CoreUtilTests(TestCase):
+
+    def setUp(self):
+        self.project = get(Project, container_time_limit=None)
+        self.version = get(Version, project=self.project)
+
+    @mock.patch('readthedocs.projects.tasks.update_docs')
+    def test_trigger_build_time_limit(self, update_docs):
+        """Pass of time limit"""
+        trigger_build(project=self.project, version=self.version)
+        update_docs.assert_has_calls([
+            mock.call.apply_async(
+                time_limit=720,
+                soft_time_limit=600,
+                queue=mock.ANY,
+                kwargs={
+                    'pk': self.project.id,
+                    'force': False,
+                    'basic': False,
+                    'record': True,
+                    'build_pk': mock.ANY,
+                    'version_pk': self.version.id
+                }
+            )
+        ])
+
+    @mock.patch('readthedocs.projects.tasks.update_docs')
+    def test_trigger_build_invalid_time_limit(self, update_docs):
+        """Time limit as string"""
+        self.project.container_time_limit = '200s'
+        trigger_build(project=self.project, version=self.version)
+        update_docs.assert_has_calls([
+            mock.call.apply_async(
+                time_limit=720,
+                soft_time_limit=600,
+                queue=mock.ANY,
+                kwargs={
+                    'pk': self.project.id,
+                    'force': False,
+                    'basic': False,
+                    'record': True,
+                    'build_pk': mock.ANY,
+                    'version_pk': self.version.id
+                }
+            )
+        ])
+
+    @mock.patch('readthedocs.projects.tasks.update_docs')
+    def test_trigger_build_rounded_time_limit(self, update_docs):
+        """Time limit should round down"""
+        self.project.container_time_limit = 3
+        trigger_build(project=self.project, version=self.version)
+        update_docs.assert_has_calls([
+            mock.call.apply_async(
+                time_limit=3,
+                soft_time_limit=3,
+                queue=mock.ANY,
+                kwargs={
+                    'pk': self.project.id,
+                    'force': False,
+                    'basic': False,
+                    'record': True,
+                    'build_pk': mock.ANY,
+                    'version_pk': self.version.id
+                }
+            )
+        ])