Repository: readthedocs/readthedocs.org

Issue:
Build: cancel a build when a new one for the same branch/PR is triggered
Now that we implemented a way to revoke Celery tasks we are able to cancel a running build. This is useful for people to manually cancel a build they are not interested in finishing for different reasons and free up the concurrency limit they have.

There are some scenarios where we can handle this automatically because we already know that the running build is useless and can be canceled:

- a new commit was pushed to a branch
- a build for the same branch was triggered (do not confuse with "Duplicated Build")
- ...

Related #8850


Discussion:
that would be a really nice feature :)
This would help us over at https://github.com/xgcm/xgcm quite a bit. +1 👍
I think the SunPy and Astropy folks would love having this too :) @nabobalis @pllim
> Astropy folks

I cannot speak for all of Astropy but personally I would love auto-cancel. Otherwise, sometimes when we get commit spam, RTD gives us a long queue.

Thanks for the ping, @astrojuanlu !
Yeah from the SunPy side, we waste so much of your CI time (with how we commit) and implementing this would help. 

I try to cancel builds manually but I don't keep on top of this. 
Also chiming in that this would be a welcome addition for our PRs at https://github.com/Ouranosinc/xclim

Resolving patch:
diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py
--- a/readthedocs/builds/views.py
+++ b/readthedocs/builds/views.py
@@ -1,6 +1,5 @@
 """Views for builds app."""
 
-import signal
 import textwrap
 from urllib.parse import urlparse
 
@@ -23,10 +22,9 @@
 from readthedocs.builds.filters import BuildListFilter
 from readthedocs.builds.models import Build, Version
 from readthedocs.core.permissions import AdminPermission
-from readthedocs.core.utils import trigger_build
-from readthedocs.doc_builder.exceptions import BuildAppError, BuildCancelled
+from readthedocs.core.utils import cancel_build, trigger_build
+from readthedocs.doc_builder.exceptions import BuildAppError
 from readthedocs.projects.models import Project
-from readthedocs.worker import app
 
 log = structlog.get_logger(__name__)
 
@@ -167,33 +165,7 @@ def post(self, request, project_slug, build_pk):
         if not AdminPermission.is_admin(request.user, project):
             return HttpResponseForbidden()
 
-        # NOTE: `terminate=True` is required for the child to attend our call
-        # immediately when it's running the build. Otherwise, it finishes the
-        # task. However, to revoke a task that has not started yet, we don't
-        # need it.
-        if build.state == BUILD_STATE_TRIGGERED:
-            # Since the task won't be executed at all, we need to update the
-            # Build object here.
-            terminate = False
-            build.state = BUILD_STATE_CANCELLED
-            build.success = False
-            build.error = BuildCancelled.message
-            build.length = 0
-            build.save()
-        else:
-            # In this case, we left the update of the Build object to the task
-            # itself to be executed in the `on_failure` handler.
-            terminate = True
-
-        log.warning(
-            'Canceling build.',
-            project_slug=project.slug,
-            version_slug=build.version.slug,
-            build_id=build.pk,
-            build_task_id=build.task_id,
-            terminate=terminate,
-        )
-        app.control.revoke(build.task_id, signal=signal.SIGINT, terminate=terminate)
+        cancel_build(build)
 
         return HttpResponseRedirect(
             reverse('builds_detail', args=[project.slug, build.pk]),
diff --git a/readthedocs/core/utils/__init__.py b/readthedocs/core/utils/__init__.py
--- a/readthedocs/core/utils/__init__.py
+++ b/readthedocs/core/utils/__init__.py
@@ -2,6 +2,7 @@
 
 import datetime
 import re
+import signal
 
 import structlog
 from django.conf import settings
@@ -19,9 +20,11 @@
     EXTERNAL,
 )
 from readthedocs.doc_builder.exceptions import (
+    BuildCancelled,
     BuildMaxConcurrencyError,
     DuplicatedBuildError,
 )
+from readthedocs.worker import app
 
 log = structlog.get_logger(__name__)
 
@@ -52,10 +55,11 @@ def prepare_build(
     from readthedocs.projects.tasks.builds import update_docs_task
     from readthedocs.projects.tasks.utils import send_external_build_status
 
+    log.bind(project_slug=project.slug)
+
     if not Project.objects.is_active(project):
         log.warning(
             'Build not triggered because project is not active.',
-            project_slug=project.slug,
         )
         return (None, None)
 
@@ -72,7 +76,10 @@ def prepare_build(
         commit=commit
     )
 
-    log.bind(build_id=build.id)
+    log.bind(
+        build_id=build.id,
+        version_slug=version.slug,
+    )
 
     options = {}
     if project.build_queue:
@@ -90,7 +97,7 @@ def prepare_build(
         if project.container_time_limit:
             time_limit = int(project.container_time_limit)
     except ValueError:
-        log.warning('Invalid time_limit for project.', project_slug=project.slug)
+        log.warning("Invalid time_limit for project.")
 
     # Add 20% overhead to task, to ensure the build can timeout and the task
     # will cleanly finish.
@@ -98,6 +105,8 @@ def prepare_build(
     options['time_limit'] = int(time_limit * 1.2)
 
     if commit:
+        log.bind(commit=commit)
+
         # Send pending Build Status using Git Status API for External Builds.
         send_external_build_status(
             version_type=version.type,
@@ -115,56 +124,86 @@ def prepare_build(
         )
 
     skip_build = False
-    if commit:
-        skip_build = (
+    # Reduce overhead when doing multiple push on the same version.
+    if project.has_feature(Feature.CANCEL_OLD_BUILDS):
+        running_builds = (
             Build.objects
             .filter(
                 project=project,
                 version=version,
-                commit=commit,
             ).exclude(
                 state__in=BUILD_FINAL_STATES,
             ).exclude(
                 pk=build.pk,
-            ).exists()
+            )
         )
+        if running_builds.count() > 0:
+            log.warning(
+                "Canceling running builds automatically due a new one arrived.",
+                running_builds=running_builds.count(),
+            )
+
+        # If there are builds triggered/running for this particular project and version,
+        # we cancel all of them and trigger a new one for the latest commit received.
+        for running_build in running_builds:
+            cancel_build(running_build)
     else:
-        skip_build = Build.objects.filter(
-            project=project,
-            version=version,
-            state=BUILD_STATE_TRIGGERED,
-            # By filtering for builds triggered in the previous 5 minutes we
-            # avoid false positives for builds that failed for any reason and
-            # didn't update their state, ending up on blocked builds for that
-            # version (all the builds are marked as DUPLICATED in that case).
-            # Adding this date condition, we reduce the risk of hitting this
-            # problem to 5 minutes only.
-            date__gte=timezone.now() - datetime.timedelta(minutes=5),
-        ).count() > 1
-
-    if not project.has_feature(Feature.DEDUPLICATE_BUILDS):
-        log.debug(
-            'Skipping deduplication of builds. Feature not enabled.',
-            project_slug=project.slug,
-        )
-        skip_build = False
+        # NOTE: de-duplicate builds won't be required if we enable `CANCEL_OLD_BUILDS`,
+        # since canceling a build is more effective.
+        # However, we are keepting `DEDUPLICATE_BUILDS` code around while we test
+        # `CANCEL_OLD_BUILDS` with a few projects and we are happy with the results.
+        # After that, we can remove `DEDUPLICATE_BUILDS` code
+        # and make `CANCEL_OLD_BUILDS` the default behavior.
+        if commit:
+            skip_build = (
+                Build.objects.filter(
+                    project=project,
+                    version=version,
+                    commit=commit,
+                )
+                .exclude(
+                    state__in=BUILD_FINAL_STATES,
+                )
+                .exclude(
+                    pk=build.pk,
+                )
+                .exists()
+            )
+        else:
+            skip_build = (
+                Build.objects.filter(
+                    project=project,
+                    version=version,
+                    state=BUILD_STATE_TRIGGERED,
+                    # By filtering for builds triggered in the previous 5 minutes we
+                    # avoid false positives for builds that failed for any reason and
+                    # didn't update their state, ending up on blocked builds for that
+                    # version (all the builds are marked as DUPLICATED in that case).
+                    # Adding this date condition, we reduce the risk of hitting this
+                    # problem to 5 minutes only.
+                    date__gte=timezone.now() - datetime.timedelta(minutes=5),
+                ).count()
+                > 1
+            )
 
-    if skip_build:
-        # TODO: we could mark the old build as duplicated, however we reset our
-        # position in the queue and go back to the end of it --penalization
-        log.warning(
-            'Marking build to be skipped by builder.',
-            project_slug=project.slug,
-            version_slug=version.slug,
-            build_id=build.pk,
-            commit=commit,
-        )
-        build.error = DuplicatedBuildError.message
-        build.status = DuplicatedBuildError.status
-        build.exit_code = DuplicatedBuildError.exit_code
-        build.success = False
-        build.state = BUILD_STATE_CANCELLED
-        build.save()
+        if not project.has_feature(Feature.DEDUPLICATE_BUILDS):
+            log.debug(
+                "Skipping deduplication of builds. Feature not enabled.",
+            )
+            skip_build = False
+
+        if skip_build:
+            # TODO: we could mark the old build as duplicated, however we reset our
+            # position in the queue and go back to the end of it --penalization
+            log.warning(
+                "Marking build to be skipped by builder.",
+            )
+            build.error = DuplicatedBuildError.message
+            build.status = DuplicatedBuildError.status
+            build.exit_code = DuplicatedBuildError.exit_code
+            build.success = False
+            build.state = BUILD_STATE_CANCELLED
+            build.save()
 
     # Start the build in X minutes and mark it as limited
     if not skip_build and project.has_feature(Feature.LIMIT_CONCURRENT_BUILDS):
@@ -172,8 +211,6 @@ def prepare_build(
         if limit_reached:
             log.warning(
                 'Delaying tasks at trigger step due to concurrency limit.',
-                project_slug=project.slug,
-                version_slug=version.slug,
             )
             # Delay the start of the build for the build retry delay.
             # We're still triggering the task, but it won't run immediately,
@@ -245,6 +282,48 @@ def trigger_build(project, version=None, commit=None):
     return task, build
 
 
+def cancel_build(build):
+    """
+    Cancel a triggered/running build.
+
+    Depending on the current state of the build, it takes one approach or the other:
+
+    - Triggered:
+        Update the build status and tells Celery to revoke this task.
+        Workers will know about this and will discard it.
+    - Running:
+        Communicate Celery to force the termination of the current build
+        and rely on the worker to update the build's status.
+    """
+    # NOTE: `terminate=True` is required for the child to attend our call
+    # immediately when it's running the build. Otherwise, it finishes the
+    # task. However, to revoke a task that has not started yet, we don't
+    # need it.
+    if build.state == BUILD_STATE_TRIGGERED:
+        # Since the task won't be executed at all, we need to update the
+        # Build object here.
+        terminate = False
+        build.state = BUILD_STATE_CANCELLED
+        build.success = False
+        build.error = BuildCancelled.message
+        build.length = 0
+        build.save()
+    else:
+        # In this case, we left the update of the Build object to the task
+        # itself to be executed in the `on_failure` handler.
+        terminate = True
+
+    log.warning(
+        "Canceling build.",
+        project_slug=build.project.slug,
+        version_slug=build.version.slug,
+        build_id=build.pk,
+        build_task_id=build.task_id,
+        terminate=terminate,
+    )
+    app.control.revoke(build.task_id, signal=signal.SIGINT, terminate=terminate)
+
+
 def send_email(
         recipient, subject, template, template_html, context=None, request=None,
         from_email=None, **kwargs
diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py
--- a/readthedocs/projects/models.py
+++ b/readthedocs/projects/models.py
@@ -1859,12 +1859,13 @@ def add_features(sender, **kwargs):
     DEFAULT_TO_FUZZY_SEARCH = 'default_to_fuzzy_search'
     INDEX_FROM_HTML_FILES = 'index_from_html_files'
 
-    LIST_PACKAGES_INSTALLED_ENV = 'list_packages_installed_env'
-    VCS_REMOTE_LISTING = 'vcs_remote_listing'
-    SPHINX_PARALLEL = 'sphinx_parallel'
-    USE_SPHINX_BUILDERS = 'use_sphinx_builders'
-    DEDUPLICATE_BUILDS = 'deduplicate_builds'
-    DONT_CREATE_INDEX = 'dont_create_index'
+    LIST_PACKAGES_INSTALLED_ENV = "list_packages_installed_env"
+    VCS_REMOTE_LISTING = "vcs_remote_listing"
+    SPHINX_PARALLEL = "sphinx_parallel"
+    USE_SPHINX_BUILDERS = "use_sphinx_builders"
+    DEDUPLICATE_BUILDS = "deduplicate_builds"
+    CANCEL_OLD_BUILDS = "cancel_old_builds"
+    DONT_CREATE_INDEX = "dont_create_index"
 
     FEATURES = (
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
@@ -2015,6 +2016,12 @@ def add_features(sender, **kwargs):
             DEDUPLICATE_BUILDS,
             _('Mark duplicated builds as NOOP to be skipped by builders'),
         ),
+        (
+            CANCEL_OLD_BUILDS,
+            _(
+                "Cancel triggered/running builds when a new one with same project/version arrives"
+            ),
+        ),
         (
             DONT_CREATE_INDEX,
             _('Do not create index.md or README.rst if the project does not have one.'),


Test patch:
diff --git a/readthedocs/builds/tests/test_trigger_build.py b/readthedocs/builds/tests/test_trigger_build.py
new file mode 100644
--- /dev/null
+++ b/readthedocs/builds/tests/test_trigger_build.py
@@ -0,0 +1,95 @@
+from unittest import mock
+
+import django_dynamic_fixture as fixture
+import pytest
+
+from readthedocs.builds.constants import (
+    BUILD_FINAL_STATES,
+    BUILD_STATE_BUILDING,
+    BUILD_STATE_CANCELLED,
+    BUILD_STATE_CLONING,
+    BUILD_STATE_FINISHED,
+    BUILD_STATE_INSTALLING,
+    BUILD_STATE_TRIGGERED,
+    BUILD_STATE_UPLOADING,
+)
+from readthedocs.builds.models import Build, Version
+from readthedocs.core.utils import trigger_build
+from readthedocs.projects.models import Feature, Project
+
+
+@pytest.mark.django_db
+class TestCancelOldBuilds:
+    @pytest.fixture(autouse=True)
+    def setup(self):
+        self.project = fixture.get(Project)
+        self.version = fixture.get(Version, project=self.project)
+        self.feature = fixture.get(Feature, feature_id=Feature.CANCEL_OLD_BUILDS)
+        self.feature.projects.add(self.project)
+
+    @pytest.mark.parametrize(
+        "state",
+        [
+            BUILD_STATE_TRIGGERED,
+            BUILD_STATE_CLONING,
+            BUILD_STATE_INSTALLING,
+            BUILD_STATE_BUILDING,
+            BUILD_STATE_UPLOADING,
+        ],
+    )
+    @mock.patch("readthedocs.core.utils.cancel_build")
+    @mock.patch("readthedocs.projects.tasks.builds.update_docs_task")
+    def test_cancel_old_running_build(self, update_docs_task, cancel_build, state):
+        # Create a running build
+        build = fixture.get(
+            Build, project=self.project, version=self.version, state=state
+        )
+
+        builds_count_before = Build.objects.count()
+
+        # Trigger a new build for the same project/version
+        result = trigger_build(
+            project=self.project,
+            version=self.version,
+        )
+
+        triggered_build = Build.objects.first()
+        builds_count_after = Build.objects.count()
+
+        cancel_build.assert_called_once_with(build)
+        assert result == (mock.ANY, triggered_build)
+        assert builds_count_before == builds_count_after - 1
+        assert update_docs_task.signature.called
+        assert update_docs_task.signature().apply_async.called
+
+    @pytest.mark.parametrize(
+        "state",
+        [
+            BUILD_STATE_CANCELLED,
+            BUILD_STATE_FINISHED,
+        ],
+    )
+    @mock.patch("readthedocs.core.utils.cancel_build")
+    @mock.patch("readthedocs.projects.tasks.builds.update_docs_task")
+    def test_not_cancel_old_finished_build(self, update_docs_task, cancel_build, state):
+        # Create a running build
+        build = fixture.get(
+            Build, project=self.project, version=self.version, state=state
+        )
+
+        builds_count_before = Build.objects.count()
+
+        # Trigger a new build for the same project/version
+        result = trigger_build(
+            project=self.project,
+            version=self.version,
+        )
+
+        triggered_build = Build.objects.first()
+        builds_count_after = Build.objects.count()
+
+        cancel_build.assert_not_called()
+        assert result == (mock.ANY, triggered_build)
+        assert builds_count_before == builds_count_after - 1
+        assert update_docs_task.signature.called
+        assert update_docs_task.signature().apply_async.called
diff --git a/readthedocs/builds/tests/test_views.py b/readthedocs/builds/tests/test_views.py
--- a/readthedocs/builds/tests/test_views.py
+++ b/readthedocs/builds/tests/test_views.py
@@ -16,7 +16,7 @@
 from readthedocs.projects.models import Project
 
 
-@mock.patch("readthedocs.builds.views.app")
+@mock.patch("readthedocs.core.utils.app")
 @override_settings(RTD_ALLOW_ORGANIZATIONS=False)
 class CancelBuildViewTests(TestCase):
     def setUp(self):