diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index cb0a7ee42..277a81df9 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -7,6 +7,7 @@ on:
 
 env:
   RUN_SLOW: "yes"
+  IS_GITHUB_CI: "1"
 
 jobs:
   run_all_tests_single_gpu:
@@ -27,6 +28,7 @@ jobs:
           git config --global --add safe.directory '*'
           git fetch && git checkout ${{ github.sha }} 
           pip install -e . --no-deps
+          pip install pytest-reportlog
 
       - name: Run test on GPUs
         run: |
@@ -37,6 +39,11 @@ jobs:
           source activate accelerate
           pip uninstall comet_ml -y
           make test_examples
+          
+      - name: Generate Report
+        if: always()
+        run: |
+          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
 
   run_all_tests_multi_gpu:
     runs-on: [self-hosted, docker-gpu, multi-gpu]
@@ -56,6 +63,7 @@ jobs:
           git config --global --add safe.directory '*'
           git fetch && git checkout ${{ github.sha }}
           pip install -e . --no-deps
+          pip install pytest-reportlog
 
       - name: Run core and big modeling tests on GPUs
         run: |
@@ -72,4 +80,9 @@ jobs:
         run: |
           source activate accelerate
           pip uninstall comet_ml -y
-          make test_examples
\ No newline at end of file
+          make test_examples
+
+      - name: Generate Report
+        if: always()
+        run: |
+          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml
index 028b99d9a..a794cd72b 100644
--- a/.github/workflows/on-merge.yml
+++ b/.github/workflows/on-merge.yml
@@ -6,6 +6,7 @@ on:
 
 env:
   TESTING_MOCKED_DATALOADERS: "1"
+  IS_GITHUB_CI: "1"
 
 jobs:
   run_all_tests_single_gpu:
@@ -26,6 +27,7 @@ jobs:
           git config --global --add safe.directory '*'
           git fetch && git checkout ${{ github.sha }}
           pip install -e .[testing,test_trackers]
+          pip install pytest-reportlog
 
       - name: Run test on GPUs
         run: |
@@ -37,6 +39,11 @@ jobs:
           pip uninstall comet_ml -y
           make test_examples
 
+      - name: Generate Report
+        if: always()
+        run: |
+          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
+
   run_all_tests_multi_gpu:
     runs-on: [self-hosted, docker-gpu, multi-gpu]
     container:
@@ -53,6 +60,7 @@ jobs:
           git config --global --add safe.directory '*'
           git fetch && git checkout ${{ github.sha }}
           pip install -e .[testing,test_trackers]
+          pip install pytest-reportlog
 
       - name: Run test on GPUs
         run: |
@@ -63,4 +71,9 @@ jobs:
         run: |
           source activate accelerate
           pip uninstall comet_ml -y
-          make test_examples
\ No newline at end of file
+          make test_examples
+
+      - name: Generate Report
+        if: always()
+        run: |
+          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2cf8e34a7..d39d36951 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -5,7 +5,7 @@ on: [pull_request]
 env:
   HF_HOME: ~/hf_cache
   TESTING_MOCKED_DATALOADERS: "1"
-  CI: 1
+  IS_GITHUB_CI: "1"
 
 jobs:
   run-tests:
diff --git a/Makefile b/Makefile
index 0c9ad0ca9..52c594ae7 100644
--- a/Makefile
+++ b/Makefile
@@ -25,40 +25,40 @@ style:
 	
 # Run tests for the library
 test:
-	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py $(if $(CI),--report-log 'all.log',)
+	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py $(if $(IS_GITHUB_CI),--report-log 'all.log',)
 
 test_big_modeling:
-	python -m pytest -s -v ./tests/test_big_modeling.py $(if $(CI),--report-log 'big_modeling.log',)
+	python -m pytest -s -v ./tests/test_big_modeling.py $(if $(IS_GITHUB_CI),--report-log 'big_modeling.log',)
 
 test_core:
 	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py --ignore=./tests/deepspeed --ignore=./tests/test_big_modeling.py \
-	--ignore=./tests/fsdp $(if $(CI),--report-log 'core.log',)
+	--ignore=./tests/fsdp $(if $(IS_GITHUB_CI),--report-log 'core.log',)
 
 test_deepspeed:
-	python -m pytest -s -v ./tests/deepspeed $(if $(CI),--report-log 'deepspeed.log',)
+	python -m pytest -s -v ./tests/deepspeed $(if $(IS_GITHUB_CI),--report-log 'deepspeed.log',)
 
 test_fsdp:
-	python -m pytest -s -v ./tests/fsdp $(if $(CI),--report-log 'fsdp.log',)
+	python -m pytest -s -v ./tests/fsdp $(if $(IS_GITHUB_CI),--report-log 'fsdp.log',)
 
 test_examples:
-	python -m pytest -s -v ./tests/test_examples.py $(if $(CI),--report-log 'examples.log',)
+	python -m pytest -s -v ./tests/test_examples.py $(if $(IS_GITHUB_CI),--report-log 'examples.log',)
 
 # Broken down example tests for the CI runners
 test_integrations:
-	python -m pytest -s -v ./tests/deepspeed ./tests/fsdp $(if $(CI),--report-log 'integrations.log',)
+	python -m pytest -s -v ./tests/deepspeed ./tests/fsdp $(if $(IS_GITHUB_CI),--report-log 'integrations.log',)
 
 test_example_differences:
-	python -m pytest -s -v ./tests/test_examples.py::ExampleDifferenceTests $(if $(CI),--report-log 'example_diff.log',)
+	python -m pytest -s -v ./tests/test_examples.py::ExampleDifferenceTests $(if $(IS_GITHUB_CI),--report-log 'example_diff.log',)
 
 test_checkpoint_epoch:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_epoch" $(if $(CI),--report-log 'checkpoint_epoch.log',)
+	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_epoch" $(if $(IS_GITHUB_CI),--report-log 'checkpoint_epoch.log',)
 
 test_checkpoint_step:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_step" $(if $(CI),--report-log 'checkpoint_step.log',)
+	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_step" $(if $(IS_GITHUB_CI),--report-log 'checkpoint_step.log',)
 
 # Same as test but used to install only the base dependencies
 test_prod:
 	$(MAKE) test_core
 
 test_rest:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "not by_step and not by_epoch" $(if $(CI),--report-log 'rest.log',)
+	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "not by_step and not by_epoch" $(if $(IS_GITHUB_CI),--report-log 'rest.log',)



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._