diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3fec513d2..2cf8e34a7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -5,11 +5,13 @@ on: [pull_request]
 env:
   HF_HOME: ~/hf_cache
   TESTING_MOCKED_DATALOADERS: "1"
+  CI: 1
 
 jobs:
   run-tests:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         pytorch-version: [
           latest,
@@ -48,7 +50,13 @@ jobs:
         if [[ ${{ matrix.test-kind }} != test_prod ]]; then pip install -e .[testing,test_trackers]; fi
         if [[ ${{ matrix.test-kind }} = test_rest ]]; then pip uninstall comet_ml -y; fi
         if [[ ${{ matrix.pytorch-version }} = minimum ]]; then pip install torch==1.6.0; fi
+        pip install pytest-reportlog
     
     - name: Run Tests
       run: |
-        make ${{ matrix.test-kind }}
\ No newline at end of file
+        make ${{ matrix.test-kind }}
+
+    - name: Generate Report
+      if: always()
+      run: |
+        python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/Makefile b/Makefile
index c1c79d6e8..0c9ad0ca9 100644
--- a/Makefile
+++ b/Makefile
@@ -25,39 +25,40 @@ style:
 	
 # Run tests for the library
 test:
-	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py
+	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py $(if $(CI),--report-log 'all.log',)
 
 test_big_modeling:
-	python -m pytest -s -v ./tests/test_big_modeling.py
+	python -m pytest -s -v ./tests/test_big_modeling.py $(if $(CI),--report-log 'big_modeling.log',)
 
 test_core:
 	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py --ignore=./tests/deepspeed --ignore=./tests/test_big_modeling.py \
-	--ignore=./tests/fsdp
+	--ignore=./tests/fsdp $(if $(CI),--report-log 'core.log',)
 
 test_deepspeed:
-	python -m pytest -s -v ./tests/deepspeed
+	python -m pytest -s -v ./tests/deepspeed $(if $(CI),--report-log 'deepspeed.log',)
 
 test_fsdp:
-	python -m pytest -s -v ./tests/fsdp
+	python -m pytest -s -v ./tests/fsdp $(if $(CI),--report-log 'fsdp.log',)
 
 test_examples:
-	python -m pytest -s -v ./tests/test_examples.py
+	python -m pytest -s -v ./tests/test_examples.py $(if $(CI),--report-log 'examples.log',)
 
 # Broken down example tests for the CI runners
 test_integrations:
-	python -m pytest -s -v ./tests/deepspeed ./tests/fsdp
+	python -m pytest -s -v ./tests/deepspeed ./tests/fsdp $(if $(CI),--report-log 'integrations.log',)
+
 test_example_differences:
-	python -m pytest -s -v ./tests/test_examples.py::ExampleDifferenceTests
+	python -m pytest -s -v ./tests/test_examples.py::ExampleDifferenceTests $(if $(CI),--report-log 'example_diff.log',)
 
 test_checkpoint_epoch:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_epoch"
+	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_epoch" $(if $(CI),--report-log 'checkpoint_epoch.log',)
 
 test_checkpoint_step:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_step"
+	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_step" $(if $(CI),--report-log 'checkpoint_step.log',)
 
 # Same as test but used to install only the base dependencies
 test_prod:
 	$(MAKE) test_core
 
 test_rest:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "not by_step and not by_epoch"
+	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "not by_step and not by_epoch" $(if $(CI),--report-log 'rest.log',)
diff --git a/utils/log_reports.py b/utils/log_reports.py
new file mode 100644
index 000000000..f701f08c2
--- /dev/null
+++ b/utils/log_reports.py
@@ -0,0 +1,34 @@
+import json
+from pathlib import Path 
+import subprocess
+
+failed = []
+passed = []
+
+group_info = []
+
+total_num_failed = 0
+for log in Path().glob("*.log"):
+    section_num_failed = 0
+    with open(log, "r") as f:
+        for line in f:
+            line = json.loads(line)
+            if line.get("nodeid", "") != "":
+                test = line["nodeid"]
+                if line.get("duration", None) is not None:
+                    duration = f'{line["duration"]:.4f}'
+                    if line.get("outcome", "") == "failed":
+                        section_num_failed += 1
+                        failed.append([test, duration])
+                    else:
+                        passed.append([test, duration])
+    group_info.append([str(log), section_num_failed])
+
+if len(failed) > 0:
+    result = "## Failed Tests:\n"
+    failed_table = '| Test Location | Test Class | Test Name |\n|---|---|---|\n| '
+    for test in failed:
+        failed_table += ' | '.join(test[0].split("::"))
+    failed_table += " |"
+    result += failed_table
+    print(result)
\ No newline at end of file



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._