Repository: fitbenchmarking/fitbenchmarking

Issue:
CSV output
**Is your feature request related to a problem? Please describe.**
The current text format is human readable, but for further processing, it would make more sense as a csv. 

**Describe the solution you'd like**
The text file should have the same structure but with a comma somewhere in each column (preferably down a column)


Discussion:
github-actions[bot]: ## Unit & System Test Results
    4 files  ±0      4 suites  ±0   7m 30s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") +23s
360 tests ±0  348 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests")  - 4    8 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  4 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") +4 
688 runs  ±0  633 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests")  - 5  50 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  5 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") +5 

For more details on these failures, see [this check](https://github.com/fitbenchmarking/fitbenchmarking/runs/9746128668).

Results for commit ba81044a. ± Comparison against base commit 3fa74abb.

[test-results]:data:application/gzip;base64,H4sIAAO/hGMC/02MwQ7CIBAFf6Xh7AFdWLf+jAGEhNgWQ+HU+O9CWqjHmfcyG3N+sit7DOIysDX71OGVo0o+LBUlL6JMqY6AnZ5rNqYqQad6+09Rp3DKT0dzFzbGEIupmZiX2kSiA1oSAbrZi7L9W1E2/g+aMM8+FWBa0ZULoRxYqe+aOI5IgKAcd8oQ8psYnRbAvj9JWLd5BgEAAA==


Patch:
diff --git a/fitbenchmarking/results_processing/base_table.py b/fitbenchmarking/results_processing/base_table.py
index 35c4a6e3..b2f49742 100644
--- a/fitbenchmarking/results_processing/base_table.py
+++ b/fitbenchmarking/results_processing/base_table.py
@@ -160,7 +160,7 @@ class Table:
         """
         Generate a dictionary of results lists with rows and columns as the key
         and list elements respectively.
-        This is used to create HTML and txt tables.
+        This is used to create HTML and csv tables.
         This is stored in self.sorted_results
         """
         self.sorted_results = {k: [r for cat in row.values() for r in cat]
@@ -363,7 +363,7 @@ class Table:
 
         return table_style.to_html(table_uuid='table')
 
-    def to_txt(self):
+    def to_csv_file(self):
         """
         Generate a plain text version of the table
 
@@ -371,7 +371,7 @@ class Table:
         :rtype: str
         """
         table = self.create_pandas_data_frame(html=False)
-        return table.to_string()
+        return table.to_csv()
 
     def get_description(self, html_description):
         """
diff --git a/fitbenchmarking/results_processing/tables.py b/fitbenchmarking/results_processing/tables.py
index 5ccf2d54..f3c2ac0e 100644
--- a/fitbenchmarking/results_processing/tables.py
+++ b/fitbenchmarking/results_processing/tables.py
@@ -31,7 +31,7 @@ SORTED_TABLE_NAMES = ["compare", "acc", "runtime", "local_min"]
 def create_results_tables(options, results, best_results, group_dir, fig_dir,
                           pp_locations, failed_problems, unselected_minimzers):
     """
-    Saves the results of the fitting to html/txt tables.
+    Saves the results of the fitting to html/csv tables.
 
     :param options: The options used in the fitting problem and plotting
     :type options: fitbenchmarking.utils.options.Options
@@ -68,7 +68,7 @@ def create_results_tables(options, results, best_results, group_dir, fig_dir,
             table_names[suffix] = f'{suffix}_table.'
 
             try:
-                table, html, txt_table, cbar = \
+                table, html, csv_table, cbar = \
                     generate_table(results=results,
                                    best_results=best_results,
                                    options=options,
@@ -96,10 +96,10 @@ def create_results_tables(options, results, best_results, group_dir, fig_dir,
             env = Environment(loader=FileSystemLoader(template_dir))
             template = env.get_template("table_template.html")
             html_output_file = file_path + 'html'
-            txt_output_file = file_path + 'txt'
+            csv_output_file = file_path + 'csv'
 
-            with open(txt_output_file, "w") as f:
-                f.write(txt_table)
+            with open(csv_output_file, "w") as f:
+                f.write(csv_table)
             failed_minimzers = sum(list(unselected_minimzers.values()), [])
             report_failed_min = failed_minimzers != []
 
@@ -159,7 +159,7 @@ def load_table(table):
 def generate_table(results, best_results, options, group_dir, fig_dir,
                    pp_locations, table_name, suffix):
     """
-    Generate html/txt tables.
+    Generate html/csv tables.
 
     :param results: Results grouped by row and category (for colouring)
     :type results: dict[str, dict[str, list[utils.fitbm_result.FittingResult]]]
@@ -189,7 +189,7 @@ def generate_table(results, best_results, options, group_dir, fig_dir,
                          pp_locations, table_name)
 
     html_table = table.to_html()
-    txt_table = table.to_txt()
+    csv_table = table.to_csv_file()
     cbar = table.save_colourbar(fig_dir)
 
     problem_dropdown_html = table.problem_dropdown_html()
@@ -201,4 +201,4 @@ def generate_table(results, best_results, options, group_dir, fig_dir,
         'minim_dropdown': minimizer_dropdown_html
     }
 
-    return table, html_dict, txt_table, cbar
+    return table, html_dict, csv_table, cbar


Test patch:
diff --git a/fitbenchmarking/results_processing/tests/expected_results/acc.csv b/fitbenchmarking/results_processing/tests/expected_results/acc.csv
new file mode 100644
index 00000000..27c91b16
--- /dev/null
+++ b/fitbenchmarking/results_processing/tests/expected_results/acc.csv
@@ -0,0 +1,4 @@
+,scipy_ls,scipy_ls,scipy_ls
+,dogbox,lm-scipy,trf
+prob_0,2 (2)[2],1 (1)[1],5 (5)
+prob_1,5 (1.667)[3],7 (2.333),3 (1)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/acc.txt b/fitbenchmarking/results_processing/tests/expected_results/acc.txt
deleted file mode 100644
index e49247c4..00000000
--- a/fitbenchmarking/results_processing/tests/expected_results/acc.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-            scipy_ls                     
-              dogbox   lm-scipy       trf
-prob_0      2 (2)[2]   1 (1)[1]     5 (5)
-prob_1  5 (1.667)[3]  7 (2.333)  3 (1)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/compare.csv b/fitbenchmarking/results_processing/tests/expected_results/compare.csv
new file mode 100644
index 00000000..38196e22
--- /dev/null
+++ b/fitbenchmarking/results_processing/tests/expected_results/compare.csv
@@ -0,0 +1,4 @@
+,scipy_ls,scipy_ls,scipy_ls
+,dogbox,lm-scipy,trf
+prob_0,2 (2)<br>4.2e-05 (1)[2],1 (1)<br>0.01 (238.1)[1],5 (5)<br>0.0022 (52.38)
+prob_1,5 (1.667)<br>1e-07 (2e+06)[3],7 (2.333)<br>3e-10 (6000),3 (1)<br>5e-14 (1)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/compare.txt b/fitbenchmarking/results_processing/tests/expected_results/compare.txt
deleted file mode 100644
index 2f6c49b4..00000000
--- a/fitbenchmarking/results_processing/tests/expected_results/compare.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-                             scipy_ls                                                    
-                               dogbox                   lm-scipy                      trf
-prob_0        2 (2)<br>4.2e-05 (1)[2]   1 (1)<br>0.01 (238.1)[1]  5 (5)<br>0.0022 (52.38)
-prob_1  5 (1.667)<br>1e-07 (2e+06)[3]  7 (2.333)<br>3e-10 (6000)    3 (1)<br>5e-14 (1)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/local_min.csv b/fitbenchmarking/results_processing/tests/expected_results/local_min.csv
new file mode 100644
index 00000000..9d619a5c
--- /dev/null
+++ b/fitbenchmarking/results_processing/tests/expected_results/local_min.csv
@@ -0,0 +1,4 @@
+,scipy_ls,scipy_ls,scipy_ls
+,dogbox,lm-scipy,trf
+prob_0,False (2.941)[2],False (0.5174)[1],False (1.48)
+prob_1,False (15.01)[3],False (6.708),False (4.5)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/local_min.txt b/fitbenchmarking/results_processing/tests/expected_results/local_min.txt
deleted file mode 100644
index ed18d730..00000000
--- a/fitbenchmarking/results_processing/tests/expected_results/local_min.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-                scipy_ls                                   
-                  dogbox           lm-scipy             trf
-prob_0  False (2.941)[2]  False (0.5174)[1]    False (1.48)
-prob_1  False (15.01)[3]      False (6.708)  False (4.5)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/runtime.csv b/fitbenchmarking/results_processing/tests/expected_results/runtime.csv
new file mode 100644
index 00000000..38469a52
--- /dev/null
+++ b/fitbenchmarking/results_processing/tests/expected_results/runtime.csv
@@ -0,0 +1,4 @@
+,scipy_ls,scipy_ls,scipy_ls
+,dogbox,lm-scipy,trf
+prob_0,4.2e-05 (1)[2],0.01 (238.1)[1],0.0022 (52.38)
+prob_1,1e-07 (2e+06)[3],3e-10 (6000),5e-14 (1)[1]
diff --git a/fitbenchmarking/results_processing/tests/expected_results/runtime.txt b/fitbenchmarking/results_processing/tests/expected_results/runtime.txt
deleted file mode 100644
index 0bd3f32b..00000000
--- a/fitbenchmarking/results_processing/tests/expected_results/runtime.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-                scipy_ls                                 
-                  dogbox         lm-scipy             trf
-prob_0    4.2e-05 (1)[2]  0.01 (238.1)[1]  0.0022 (52.38)
-prob_1  1e-07 (2e+06)[3]     3e-10 (6000)    5e-14 (1)[1]
diff --git a/fitbenchmarking/results_processing/tests/test_tables.py b/fitbenchmarking/results_processing/tests/test_tables.py
index 3dea426b..453fa263 100644
--- a/fitbenchmarking/results_processing/tests/test_tables.py
+++ b/fitbenchmarking/results_processing/tests/test_tables.py
@@ -163,7 +163,7 @@ class GenerateTableTests(unittest.TestCase):
         fitbenchmarking/results_processing/tests/expected_results
         """
         for suffix in SORTED_TABLE_NAMES:
-            _, html, txt_table, _ = generate_table(
+            _, html, csv_table, _ = generate_table(
                 results=self.results,
                 best_results=self.best_results,
                 options=self.options,
@@ -174,10 +174,10 @@ class GenerateTableTests(unittest.TestCase):
                 suffix=suffix)
             html_table_name = os.path.join(self.expected_results_dir,
                                            f"{suffix}.html")
-            txt_table_name = os.path.join(self.expected_results_dir,
-                                          f"{suffix}.txt")
-            for f, t in zip([html_table_name, txt_table_name],
-                            [html["table"], txt_table]):
+            csv_table_name = os.path.join(self.expected_results_dir,
+                                          f"{suffix}.csv")
+            for f, t in zip([html_table_name, csv_table_name],
+                            [html["table"], csv_table]):
                 self.compare_files(f, t)
 
     def test_dropdown_html_correct(self):
@@ -290,7 +290,7 @@ class CreateResultsTableTests(unittest.TestCase):
                               unselected_minimzers={'min1': []})
         for suffix in SORTED_TABLE_NAMES:
 
-            for table_type in ['html', 'txt']:
+            for table_type in ['html', 'csv']:
                 table_name = f'{suffix}_table.{table_type}'
                 file_name = os.path.join(self.group_dir, table_name)
                 self.assertTrue(os.path.isfile(file_name),
diff --git a/fitbenchmarking/systests/linux_expected_results/all_parsers.csv b/fitbenchmarking/systests/linux_expected_results/all_parsers.csv
new file mode 100644
index 00000000..aa025e76
--- /dev/null
+++ b/fitbenchmarking/systests/linux_expected_results/all_parsers.csv
@@ -0,0 +1,7 @@
+,bumps,dfo,gofit,gsl,levmar,mantid,minuit,ralfit,scipy,scipy_ls
+,lm-bumps,dfols,regularisation: j:scipy 2-point,lmsder: j:scipy 2-point,levmar: j:scipy 2-point,Levenberg-Marquardt: j:scipy 2-point,minuit,gn: j:scipy 2-point,Nelder-Mead,lm-scipy: j:scipy 2-point
+1D sphere (synthetic neutron) IV0,239.1 (1),239.1 (1),239.1 (1),239.1 (1),239.1 (1),239.1 (1),239.1 (1),239.1 (1),239.1 (1),239.1 (1)
+DANWOOD,0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001142 (1.04),0.001098 (1),0.001098 (1),0.001098 (1)
+"DanWood, Start 1",0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001171 (1.066),0.001098 (1),0.001098 (1),0.001098 (1)
+"DanWood, Start 2",0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001098 (1),0.001183 (1.078),0.001098 (1),0.001098 (1),0.001098 (1)
+trig_noisy-fba,0.7222 (1),0.7222 (1),0.7222 (1),0.7222 (1),0.7222 (1),0.7222 (1)[2],0.7222 (1),0.7222 (1),0.7222 (1),0.7222 (1)
\ No newline at end of file
diff --git a/fitbenchmarking/systests/linux_expected_results/all_parsers.txt b/fitbenchmarking/systests/linux_expected_results/all_parsers.txt
deleted file mode 100644
index f64cb6a1..00000000
--- a/fitbenchmarking/systests/linux_expected_results/all_parsers.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-                                          bumps           dfo                           gofit                     gsl                  levmar                               mantid            minuit              ralfit         scipy                  scipy_ls
-                                       lm-bumps         dfols regularisation: j:scipy 2-point lmsder: j:scipy 2-point levmar: j:scipy 2-point Levenberg-Marquardt: j:scipy 2-point            minuit gn: j:scipy 2-point   Nelder-Mead lm-scipy: j:scipy 2-point
-1D sphere (synthetic neutron) IV0     239.1 (1)     239.1 (1)                       239.1 (1)               239.1 (1)               239.1 (1)                            239.1 (1)         239.1 (1)           239.1 (1)     239.1 (1)                 239.1 (1)
-DANWOOD                            0.001098 (1)  0.001098 (1)                    0.001098 (1)            0.001098 (1)            0.001098 (1)                         0.001098 (1)   0.001142 (1.04)        0.001098 (1)  0.001098 (1)              0.001098 (1)
-DanWood, Start 1                   0.001098 (1)  0.001098 (1)                    0.001098 (1)            0.001098 (1)            0.001098 (1)                         0.001098 (1)  0.001171 (1.066)        0.001098 (1)  0.001098 (1)              0.001098 (1)
-DanWood, Start 2                   0.001098 (1)  0.001098 (1)                    0.001098 (1)            0.001098 (1)            0.001098 (1)                         0.001098 (1)  0.001183 (1.078)        0.001098 (1)  0.001098 (1)              0.001098 (1)
-trig_noisy-fba                       0.7222 (1)    0.7222 (1)                      0.7222 (1)              0.7222 (1)              0.7222 (1)                        0.7222 (1)[2]        0.7222 (1)          0.7222 (1)    0.7222 (1)                0.7222 (1)
\ No newline at end of file
diff --git a/fitbenchmarking/systests/linux_expected_results/default_parsers.csv b/fitbenchmarking/systests/linux_expected_results/default_parsers.csv
new file mode 100644
index 00000000..b2919ba4
--- /dev/null
+++ b/fitbenchmarking/systests/linux_expected_results/default_parsers.csv
@@ -0,0 +1,5 @@
+,bumps,scipy,scipy_ls
+,lm-bumps,Nelder-Mead,lm-scipy: j:scipy 2-point
+"cubic, Start 1",3.861e-22 (1),6.267e-05 (1.623e+17)[1],3.861e-22 (1)
+"cubic, Start 2",3.861e-22 (1),7.176e-11 (1.859e+11),3.861e-22 (1)
+Problem Def 1,0.04303 (1),inf (inf)[4],inf (inf)[4]
\ No newline at end of file
diff --git a/fitbenchmarking/systests/linux_expected_results/default_parsers.txt b/fitbenchmarking/systests/linux_expected_results/default_parsers.txt
deleted file mode 100644
index ccfa08bd..00000000
--- a/fitbenchmarking/systests/linux_expected_results/default_parsers.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-                        bumps                     scipy                  scipy_ls
-                     lm-bumps               Nelder-Mead lm-scipy: j:scipy 2-point
-cubic, Start 1  3.861e-22 (1)  6.267e-05 (1.623e+17)[1]             3.861e-22 (1)
-cubic, Start 2  3.861e-22 (1)     7.176e-11 (1.859e+11)             3.861e-22 (1)
-Problem Def 1     0.04303 (1)              inf (inf)[4]              inf (inf)[4]
diff --git a/fitbenchmarking/systests/linux_expected_results/matlab.txt b/fitbenchmarking/systests/linux_expected_results/matlab.csv
similarity index 100%
rename from fitbenchmarking/systests/linux_expected_results/matlab.txt
rename to fitbenchmarking/systests/linux_expected_results/matlab.csv
diff --git a/fitbenchmarking/systests/linux_expected_results/multifit.csv b/fitbenchmarking/systests/linux_expected_results/multifit.csv
new file mode 100644
index 00000000..a489186b
--- /dev/null
+++ b/fitbenchmarking/systests/linux_expected_results/multifit.csv
@@ -0,0 +1,4 @@
+,mantid,mantid
+,Levenberg-Marquardt: j:scipy 2-point,Levenberg-Marquardt: j:scipy 3-point
+"Basic MultiFit, Dataset 1",8.16 (1),8.16 (1)
+"Basic MultiFit, Dataset 2",15.28 (1),15.28 (1)
\ No newline at end of file
diff --git a/fitbenchmarking/systests/linux_expected_results/multifit.txt b/fitbenchmarking/systests/linux_expected_results/multifit.txt
deleted file mode 100644
index 15321228..00000000
--- a/fitbenchmarking/systests/linux_expected_results/multifit.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-                                                        mantid                                     
-                          Levenberg-Marquardt: j:scipy 2-point Levenberg-Marquardt: j:scipy 3-point
-Basic MultiFit, Dataset 1                             8.16 (1)                             8.16 (1)
-Basic MultiFit, Dataset 2                            15.28 (1)                            15.28 (1)
\ No newline at end of file
diff --git a/fitbenchmarking/systests/test_regression.py b/fitbenchmarking/systests/test_regression.py
index 18f3d56a..1530d46e 100644
--- a/fitbenchmarking/systests/test_regression.py
+++ b/fitbenchmarking/systests/test_regression.py
@@ -43,7 +43,7 @@ class TestRegressionAll(TestCase):
 
         run_benchmark(self.results_dir, problem_sub_directory)
 
-        diff, msg = compare_results(problem_sub_directory, "all_parsers.txt")
+        diff, msg = compare_results(problem_sub_directory, "all_parsers.csv")
         self.assertListEqual([], diff, msg)
 
     def test_multifit_consistent(self):
@@ -57,7 +57,7 @@ class TestRegressionAll(TestCase):
                       override_software=["mantid"],
                       jac_num_method={"scipy": ["2-point", "3-point"]})
 
-        diff, msg = compare_results(problem_sub_directory, "multifit.txt")
+        diff, msg = compare_results(problem_sub_directory, "multifit.csv")
         self.assertListEqual([], diff, msg)
 
 
@@ -86,7 +86,7 @@ class TestRegressionMatlab(TestCase):
 
         run_benchmark(self.results_dir, problem_sub_directory)
 
-        diff, msg = compare_results(problem_sub_directory, "matlab.txt")
+        diff, msg = compare_results(problem_sub_directory, "matlab.csv")
         self.assertListEqual([], diff, msg)
 
 
@@ -116,7 +116,7 @@ class TestRegressionDefault(TestCase):
         run_benchmark(self.results_dir, problem_sub_directory)
 
         diff, msg = compare_results(problem_sub_directory,
-                                    "default_parsers.txt")
+                                    "default_parsers.csv")
         self.assertListEqual([], diff, msg)
 
 
@@ -176,7 +176,7 @@ def compare_results(problem_sub_directory: str, result_filename: str) -> list:
     actual_file = os.path.join(os.path.dirname(__file__),
                                'fitbenchmarking_results',
                                problem_sub_directory,
-                               'acc_table.txt')
+                               'acc_table.csv')
 
     with open(expected_file, 'r') as f:
         expected = f.readlines()