diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml
index de8542359..c207f5515 100644
--- a/.github/workflows/build-docker-images.yml
+++ b/.github/workflows/build-docker-images.yml
@@ -2,6 +2,7 @@ name: Build Docker images (scheduled)
 
 on:
   workflow_dispatch:
+  workflow_call:
   schedule:
     - cron: "0 1 * * *"
 
diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml
new file mode 100644
index 000000000..65afdd6c9
--- /dev/null
+++ b/.github/workflows/check_dependencies.yml
@@ -0,0 +1,39 @@
+name: Check for dependency modification
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  check-for-setup:
+    runs-on: ubuntu-latest
+    name: Check if setup was changed
+    outputs:
+      changed: ${{ steps.was_changed.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with: 
+          fetch-depth: "2"
+      
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v22.2
+      
+      - name: Was setup changed 
+        id: was_changed
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            if [ `basename "${file}"` = "setup.py" ]; then
+              echo ::set-output name=changed::"1"
+            fi
+          done
+          
+  build-docker-containers:
+    needs: check-for-setup
+    if: (github.event_name == 'push') && (needs.check-for-setup.outputs.changed == '1')
+    uses: ./.github/workflows/build-docker-images.yml
\ No newline at end of file
diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml
index 4d603bf8e..f8b9ccddb 100644
--- a/.github/workflows/on-merge.yml
+++ b/.github/workflows/on-merge.yml
@@ -1,10 +1,11 @@
 name: Self-hosted runner (push to "main")
 
 on:
+  workflow_run:
+    workflows: ["Check for dependency modification"]
+    branches: ["main"]
+    types: [completed]
   workflow_dispatch:
-  push:
-    branches:
-      - "main"
 
 env:
   TESTING_MOCKED_DATALOADERS: "1"


Review comments: body: This is not set anywhere, so I don't think this will work (also pashing the github token on a PR is not a trivial thing as you don't get secrets on pull requests. I'd just remove this whole writing comments thingy.
diff_hunk: @@ -0,0 +1,97 @@
+# Copyright 2022 The HuggingFace Team, the AllenNLP library authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Script to check if the setup.py was modified during a Pull Request,
+and if so will write a comment stating that Docker Images will be 
+rebuilt upon a merge. 
+
+Environment variables available:
+- PR_NUMBER: The number of the currently open pull request
+- REPO: The full name of the repository (such as 'huggingface/accelerate')
+"""
+
+import os
+import re
+from github import Github, PullRequest
+
+def get_setup_diff(pull_request:PullRequest):
+    """Checks whether `setup.py` was changed during this pull request, and will 
+    return the diff if so
+
+    Args:
+        pull_request (`PullRequest`):
+            A pull request returned from `Github.repo.get_pull_request`
+    """
+    files = pull_request.get_files()
+    for f in files:
+        if f.filename == "setup.py":
+            return f'''```diff\n{f.patch}\n```'''
+    return None
+
+def does_comment_exist(pull_request:PullRequest):
+    """Checks whether the bot has already commented on this pull request
+
+    Args:
+        pull_request (`PullRequest`):
+            A pull request returned from `Github.repo.get_pull_request`
+    """
+    comments = pull_request.get_issue_comments()
+    for c in comments:
+        if c.user.login == "github-actions[bot]" and 'This PR modifies `setup.py`.' in c.body:
+            return True
+    return False
+
+def write_comment(pull_request:PullRequest, diff:str):
+    """Writes a comment stating that the pr modified setup.py, and that new Docker images will be built
+
+    Args:
+        pull_request (`PullRequest`):
+            A pull request returned from `Github.repo.get_pull_request`
+        diff (`str`):
+            The diff of the modified setup.py
+    """
+    s = f'This PR modifies `setup.py`. New latest Docker images will be built and deployed before running tests once this has been merged:\n\n{diff}'
+    pull_request.create_issue_comment(s)
+
+def update_diff(pull_request:PullRequest, diff:str):
+    """Updates the diff of the setup.py in the existing comment
+
+    Args:
+        pull_request (`PullRequest`):
+            A pull request returned from `Github.repo.get_pull_request`
+        diff (`str`):
+            The diff of the modified setup.py
+    """
+    comments = pull_request.get_issue_comments()
+    for c in comments:
+        if c.user.login == "github-actions[bot]" and 'This PR modifies `setup.py`.' in c.body:
+            comment = c
+            break
+    original_diff = re.search(r'```([^`]*)```', comment.body).group(0)
+    new_body = comment.body.replace(original_diff, diff)
+    comment.edit(new_body)
+
+def main():
+    g = Github(os.environ["GITHUB_TOKEN"])
body: I'd remove this job personally.
diff_hunk: @@ -0,0 +1,68 @@
+name: Check for dependency modification
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  check-for-setup:
+    runs-on: ubuntu-latest
+    name: Check if setup was changed
+    outputs:
+      changed: ${{ steps.was_changed.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with: 
+          fetch-depth: "2"
+      
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v22.2
+      
+      - name: Was setup changed 
+        id: was_changed
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            if [ `basename "${file}"` = "setup.py" ]; then
+              echo ::set-output name=changed::"1"
+            fi
+          done
+
+  write-message:
body: Is there any reason that we should run this workflow `on pull_request`?
diff_hunk: @@ -0,0 +1,41 @@
+name: Check for dependency modification
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
body: Very nice - we should build the image only for the push event (to `main`).
diff_hunk: @@ -0,0 +1,41 @@
+name: Check for dependency modification
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  check-for-setup:
+    runs-on: ubuntu-latest
+    name: Check if setup was changed
+    outputs:
+      changed: ${{ steps.was_changed.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with: 
+          fetch-depth: "2"
+      
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v22.2
+      
+      - name: Was setup changed 
+        id: was_changed
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            if [ `basename "${file}"` = "setup.py" ]; then
+              echo ::set-output name=changed::"1"
+            fi
+          done
+          
+  build-docker-containers:
+    needs: check-for-setup
+    if: (github.event_name == 'push') && (needs.check-for-setup.outputs.changed == '1')
body: I was thinking that `completed` means the CI will be run only if the workflow `Check for dependency modification` is successful.

After reading this [doc](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-a-workflow-based-on-the-conclusion-of-another-workflow), it is not the case. And this is good, as we want to run the push CI anyway (even if something wrong in the setup check).


diff_hunk: @@ -1,10 +1,11 @@
 name: Self-hosted runner (push to "main")
 
 on:
+  workflow_run:
+    workflows: ["Check for dependency modification"]
+    branches: ["main"]
+    types: [completed]
body: Yup exactly
diff_hunk: @@ -1,10 +1,11 @@
 name: Self-hosted runner (push to "main")
 
 on:
+  workflow_run:
+    workflows: ["Check for dependency modification"]
+    branches: ["main"]
+    types: [completed]
body: This was due to the overengineering part. We definitely could just trigger it on a push to main. 
diff_hunk: @@ -0,0 +1,41 @@
+name: Check for dependency modification
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
body: Yeah, I meanly mean `push` in this remark, and you are doing it as what I would expect. (`to main` is to make it more clear only 😄 )
diff_hunk: @@ -0,0 +1,41 @@
+name: Check for dependency modification
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  check-for-setup:
+    runs-on: ubuntu-latest
+    name: Check if setup was changed
+    outputs:
+      changed: ${{ steps.was_changed.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with: 
+          fetch-depth: "2"
+      
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v22.2
+      
+      - name: Was setup changed 
+        id: was_changed
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            if [ `basename "${file}"` = "setup.py" ]; then
+              echo ::set-output name=changed::"1"
+            fi
+          done
+          
+  build-docker-containers:
+    needs: check-for-setup
+    if: (github.event_name == 'push') && (needs.check-for-setup.outputs.changed == '1')

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: @ydshieh you can see my almost 250 debug logs here: https://github.com/muellerzr/accelerate/actions

**Very** *very* comfortable with this working