Repository: gaia-dpci/GaiaXPy

Issue:
Error instead of warning when error_correction cannot be applied (GaiaXPy 2.1.1)
The generator tool can only apply error correction over built-in systems (this is systems shipped with the package) as a correction table file is required to correct the errors. In previous versions of the package, trying to apply error correction over an additional system (one loaded by the user via the `load_additional_systems` function) would result in a user warning (`"UserWarning: System USER_AnAdditionalSystem does not have a correction table. The program will not apply error correction over this system."`), but the program would still run successfully and would apply the error correction exclusively over the systems with a correction table, leaving the systems without a correction table 'untouched'.

In the latest version of the package (2.1.1), an error is raised instead. This is problematic when a user wants to generate synthetic photometry for different systems at once but some of the systems are built-in and some others are not.

To reproduce, simply run:
```
from gaiaxpy import generate, PhotometricSystem, load_additional_systems

PhotometricSystem = load_additional_systems('/path/to/systems/')

generate(input, photometric_system=PhotometricSystem.USER_AnAdditionalSystem, error_correction=True)
```

Trace:
```
File "...gaiaxpy/generator/generator.py", line 49, in generate
  return _generate(input_object=input_object, photometric_system=ps, output_path=op,
File "...GaiaXPy/installed/2.1.1/gaiaxpy/generator/generator.py", line 92, in _generate
  validate_error_correction(internal_phot_system, error_correction)
File "...GaiaXPy/installed/2.1.1/gaiaxpy/core/generic_functions.py", line 352, in validate_error_correction
  raise ValueError('Photometry is requested for a non-built-in system, but error_correction is set to True. '
ValueError: Photometry is requested for a non-built-in system, but error_correction is set to True.
Error correction is only implemented for built-in systems.
```

* Bug reported internally by a user.

Discussion:

Patch:
diff --git a/src/gaiaxpy/core/generic_functions.py b/src/gaiaxpy/core/generic_functions.py
index 115939d..5bd204c 100644
--- a/src/gaiaxpy/core/generic_functions.py
+++ b/src/gaiaxpy/core/generic_functions.py
@@ -347,12 +347,6 @@ def format_additional_columns(additional_columns: Optional[Union[str, list, dict
         return convert_values_to_lists(additional_columns)
 
 
-def validate_error_correction(phot_system, error_correction):
-    if any(system.name.startswith(ADDITIONAL_SYSTEM_PREFIX) for system in phot_system) and error_correction:
-        raise ValueError('Photometry is requested for a non-built-in system, but error_correction is set to True. '
-                         'Error correction is only implemented for built-in systems.')
-
-
 def validate_photometric_system(photometric_system):
     """
     Ensure photometric system input isn't empty.
diff --git a/src/gaiaxpy/file_parser/parse_generic.py b/src/gaiaxpy/file_parser/parse_generic.py
index 1428933..c11e058 100644
--- a/src/gaiaxpy/file_parser/parse_generic.py
+++ b/src/gaiaxpy/file_parser/parse_generic.py
@@ -74,12 +74,12 @@ class GenericParser(object):
             str: File extension ('.csv', '.fits', or '.xml').
         """
         if not disable_info:
-            print(self.info_msg, end='\r')
+            self.print_info_msg()
         extension = _get_file_extension(file_path)
         parser = self.get_parser(extension)
         parsed_data = _cast(parser(file_path))
         if not disable_info:
-            print(self.info_msg + ' Done!', end='\r')
+            self.print_info_msg(done=True)
         return parsed_data, extension
 
     def _parse_avro(self, avro_file):
@@ -156,6 +156,12 @@ class GenericParser(object):
                                                                                    row[size_column]), axis=1)
         return df
 
+    def print_info_msg(self, done=False):
+        msg = self.info_msg
+        if done:
+            msg = msg + ' Done!'
+        print(msg, end='\r')
+
 
 def _get_file_extension(file_path):
     """
diff --git a/src/gaiaxpy/generator/config.py b/src/gaiaxpy/generator/config.py
index b544e87..b1ed89a 100644
--- a/src/gaiaxpy/generator/config.py
+++ b/src/gaiaxpy/generator/config.py
@@ -41,7 +41,7 @@ def __get_file_names_recursively(dir_path: str, show_warning: bool = False) -> l
     if len(compliant_files) == 0:
         raise ValueError('No filter files found in the given directory. Please check your files.')
     elif len(compliant_files) < len(all_files) and show_warning:
-        message = 'Some files in the directory do not correspond to filter files. The program will ignore them.'
+        message = 'Based on their names, some files in the directory do not correspond to filter files. They will be ignored.'
         print(f'UserWarning: {message}', file=sys.stderr)
     return compliant_files
 
diff --git a/src/gaiaxpy/generator/generator.py b/src/gaiaxpy/generator/generator.py
index 3efd7f3..69fde04 100644
--- a/src/gaiaxpy/generator/generator.py
+++ b/src/gaiaxpy/generator/generator.py
@@ -4,8 +4,7 @@ from typing import Union, Optional
 import pandas as pd
 
 from gaiaxpy.colour_equation.xp_filter_system_colour_equation import _apply_colour_equation
-from gaiaxpy.core.generic_functions import cast_output, format_additional_columns, validate_photometric_system, \
-    validate_error_correction
+from gaiaxpy.core.generic_functions import cast_output, format_additional_columns, validate_photometric_system
 from gaiaxpy.error_correction.error_correction import _apply_error_correction
 from gaiaxpy.input_reader.input_reader import InputReader
 from gaiaxpy.output.photometry_data import PhotometryData
@@ -89,7 +88,6 @@ def _generate(input_object: Union[list, Path, str], photometric_system: Union[li
     # Prepare systems, keep track of original systems (especially required for error_correction)
     internal_phot_system = photometric_system.copy() if isinstance(photometric_system, list) else (
         [photometric_system].copy())
-    validate_error_correction(internal_phot_system, error_correction)
     gaia_system = PhotometricSystem.Gaia_DR3_Vega
     is_gaia_in_input = __is_gaia_initially_in_systems(internal_phot_system)
     if error_correction and not is_gaia_in_input:
diff --git a/src/gaiaxpy/input_reader/archive_reader.py b/src/gaiaxpy/input_reader/archive_reader.py
index b23e017..76bb7a6 100644
--- a/src/gaiaxpy/input_reader/archive_reader.py
+++ b/src/gaiaxpy/input_reader/archive_reader.py
@@ -33,3 +33,9 @@ class ArchiveReader(object):
             gaia.login(user=user, password=password)
         else:
             pass
+
+    def show_info_msg(self, done=False):
+        msg = self.info_msg
+        if done:
+            msg = msg + ' Done!'
+        print(msg, end='\r')
diff --git a/src/gaiaxpy/input_reader/dataframe_reader.py b/src/gaiaxpy/input_reader/dataframe_reader.py
index 2703922..f96c5a4 100644
--- a/src/gaiaxpy/input_reader/dataframe_reader.py
+++ b/src/gaiaxpy/input_reader/dataframe_reader.py
@@ -67,7 +67,7 @@ class DataFrameReader(object):
 
     def read(self):
         if not self.disable_info:
-            print(self.info_msg, end='\r')
+            self.show_info_msg()
         content = self.content
         str_array_columns, np_array_columns = self.__get_parseable_columns()
         if str_array_columns:
@@ -88,10 +88,16 @@ class DataFrameReader(object):
                     data[f'{band}_covariance_matrix'] = data.apply(get_covariance_matrix, axis=1, args=(band,))
                 self.requested_columns = self.requested_columns + covariance_columns
         if not self.disable_info:
-            print(self.info_msg + ' Done!', end='\r')
+            self.show_info_msg(done=True)
         data = _cast(data)
         if self.additional_columns:
             data = rename_with_required(data, self.additional_columns)
         data = data[self.requested_columns] if self.requested_columns else data
         # No extension returned for DataFrames
         return data, None
+
+    def show_info_msg(self, done=False):
+        msg = self.info_msg
+        if done:
+            msg = msg + ' Done!'
+        print(msg, end='\r')
diff --git a/src/gaiaxpy/input_reader/list_reader.py b/src/gaiaxpy/input_reader/list_reader.py
index b496411..5959eb6 100644
--- a/src/gaiaxpy/input_reader/list_reader.py
+++ b/src/gaiaxpy/input_reader/list_reader.py
@@ -43,7 +43,7 @@ class ListReader(ArchiveReader):
         self._login(gaia)
         # ADQL query
         if not self.disable_info:
-            print(self.info_msg, end='\r')
+            self.show_info_msg()
         result = gaia.load_data(ids=sources, format='csv', data_release=_data_release, data_structure='raw',
                                 retrieval_type='XP_CONTINUOUS', avoid_datatype_check=True)
         try:
@@ -52,6 +52,6 @@ class ListReader(ArchiveReader):
         except (KeyError, IndexError):
             raise ValueError('No continuous raw data found for the given sources.')
         if not self.disable_info:
-            print(self.info_msg + ' Done!', end='\r')
+            self.show_info_msg(done=True)
         return DataFrameReader(data, function_name, self.truncation, additional_columns=self.additional_columns,
                                disable_info=True).read()
diff --git a/src/gaiaxpy/input_reader/query_reader.py b/src/gaiaxpy/input_reader/query_reader.py
index c293c28..512a444 100644
--- a/src/gaiaxpy/input_reader/query_reader.py
+++ b/src/gaiaxpy/input_reader/query_reader.py
@@ -56,7 +56,7 @@ class QueryReader(ArchiveReader):
         self._login(gaia)
         # ADQL query
         if not self.disable_info:
-            print(self.info_msg, end='\r')
+            self.show_info_msg()
         job = gaia.launch_job_async(query, dump_to_file=False)
         query_result = job.get_results()
         result = gaia.load_data(ids=self.get_srcids(query_result), format='csv', data_release=_data_release,
@@ -67,6 +67,6 @@ class QueryReader(ArchiveReader):
         except KeyError:
             raise ValueError('No continuous raw data found for the requested query.')
         if not self.disable_info:
-            print(self.info_msg + ' Done!', end='\r')
+            self.show_info_msg(done=True)
         return DataFrameReader(data, function_name, self.truncation, additional_columns=self.additional_columns,
                                disable_info=True).read()
diff --git a/src/gaiaxpy/spectrum/multi_synthetic_photometry.py b/src/gaiaxpy/spectrum/multi_synthetic_photometry.py
index 45fc968..dd5e258 100644
--- a/src/gaiaxpy/spectrum/multi_synthetic_photometry.py
+++ b/src/gaiaxpy/spectrum/multi_synthetic_photometry.py
@@ -3,6 +3,8 @@ multi_synthetic_photometry.py
 ====================================
 Module to represent a synthetic photometry in multiple photometric systems.
 """
+import re
+from typing import List
 
 import pandas as pd
 
@@ -39,13 +41,25 @@ class MultiSyntheticPhotometry(object):
         self.mags, self.fluxes, self.errors = _generate_variables(photometries)
 
     def _generate_output_df(self):
+        def _get_column_sublist(_label: str, _df: pd.DataFrame) -> List:
+            """
+            Get the columns in a DataFrame that correspond to a particular system.
+
+            Args:
+                _label: A system label.
+                _df: A DataFrame containing photometry data.
+
+            Returns:
+                A list of column names.
+            """
+            _pattern = fr'{_label}_(flux_error|flux|mag)_[A-Za-z0-9]+'
+            return list(filter(lambda col: re.match(_pattern, col), _df.columns))
+
         photometries_df = self._photometries_to_df()
         # Reorder DataFrame columns
         phot_system_labels = [phot_system.get_system_label() for phot_system in self.photometric_system]
-        reordered_columns = ['source_id']
-        for label in phot_system_labels:
-            column_sublist = [column for column in photometries_df.columns if column.startswith(f'{label}_')]
-            reordered_columns.extend(column_sublist)
+        reordered_columns = ['source_id'] + [col for label in phot_system_labels for col in _get_column_sublist(
+            label, photometries_df)]
         photometries_df = photometries_df[reordered_columns]
         return photometries_df
 


Test patch:
diff --git a/tests/test_generator/test_generator.py b/tests/test_generator/test_generator.py
index daadc27..62faed4 100644
--- a/tests/test_generator/test_generator.py
+++ b/tests/test_generator/test_generator.py
@@ -1,27 +1,33 @@
+import re
+import sys
 from collections import Counter
+from io import StringIO
 
 import pandas as pd
 import pandas.testing as pdt
 import pytest
 
-from gaiaxpy import generate, PhotometricSystem
+from gaiaxpy import generate, PhotometricSystem, remove_additional_systems, load_additional_systems
 from gaiaxpy.file_parser.cast import _cast
 from tests.files.paths import missing_bp_csv_file, mean_spectrum_fits_file, gen_missing_band_sol_path
+from tests.test_generator.generator_paths import additional_filters_dir
 
 _rtol, _atol = 1e-23, 1e-23
 
 
-@pytest.fixture(scope='module')
-def systems_list():
-    systems = [PhotometricSystem.Els_Custom_W09_S2, PhotometricSystem.Euclid_VIS, PhotometricSystem.Gaia_2,
-               PhotometricSystem.Gaia_DR3_Vega, PhotometricSystem.Halpha_Custom_AB, PhotometricSystem.H_Custom,
-               PhotometricSystem.Hipparcos_Tycho, PhotometricSystem.HST_ACSWFC, PhotometricSystem.HST_HUGS_Std,
-               PhotometricSystem.HST_WFC3UVIS, PhotometricSystem.HST_WFPC2, PhotometricSystem.IPHAS,
-               PhotometricSystem.JKC, PhotometricSystem.JKC_Std, PhotometricSystem.JPAS, PhotometricSystem.JPLUS,
-               PhotometricSystem.JWST_NIRCAM, PhotometricSystem.PanSTARRS1, PhotometricSystem.PanSTARRS1_Std,
-               PhotometricSystem.Pristine, PhotometricSystem.SDSS, PhotometricSystem.SDSS_Std,
-               PhotometricSystem.Sky_Mapper, PhotometricSystem.Stromgren, PhotometricSystem.Stromgren_Std,
-               PhotometricSystem.WFIRST]
+@pytest.fixture(scope='function')
+def __ps():
+    PhotometricSystem = remove_additional_systems()
+    yield PhotometricSystem
+
+
+@pytest.fixture
+def systems_list(__ps):
+    systems = [__ps.Els_Custom_W09_S2, __ps.Euclid_VIS, __ps.Gaia_2, __ps.Gaia_DR3_Vega, __ps.Halpha_Custom_AB,
+               __ps.H_Custom, __ps.Hipparcos_Tycho, __ps.HST_ACSWFC, __ps.HST_HUGS_Std, __ps.HST_WFC3UVIS,
+               __ps.HST_WFPC2, __ps.IPHAS, __ps.JKC, __ps.JKC_Std, __ps.JPAS, __ps.JPLUS, __ps.JWST_NIRCAM,
+               __ps.PanSTARRS1, __ps.PanSTARRS1_Std, __ps.Pristine, __ps.SDSS, __ps.SDSS_Std, __ps.Sky_Mapper,
+               __ps.Stromgren, __ps.Stromgren_Std, __ps.WFIRST]
     yield systems
 
 
@@ -36,12 +42,33 @@ def test_duplicate_columns(systems_list):
     assert 0 == len([item for item, count in Counter(generated_photometry.columns).items() if count > 1])
 
 
-def test_single_phot_object():
-    photometry = generate(mean_spectrum_fits_file, photometric_system=PhotometricSystem.JKC, save_file=False)
+def test_single_phot_object(__ps):
+    photometry = generate(mean_spectrum_fits_file, photometric_system=__ps.JKC, save_file=False)
     assert isinstance(photometry, pd.DataFrame)
 
 
-def test_single_phot_object_with_correction():
-    system = PhotometricSystem.JKC
+def test_single_phot_object_with_correction(__ps):
+    system = __ps.JKC
     photometry = generate(mean_spectrum_fits_file, photometric_system=system, error_correction=True, save_file=False)
     assert isinstance(photometry, pd.DataFrame)
+
+
+def test_error_correction_additional_systems(__ps):
+    __ps = load_additional_systems(additional_filters_dir)
+    additional_systems = [system for system in __ps if system.get_system_name().startswith('USER_')]
+
+    captured_output = StringIO()
+    sys.stderr = captured_output
+    _ = generate(mean_spectrum_fits_file, photometric_system=additional_systems, error_correction=True, save_file=False)
+    sys.stderr = sys.__stderr__
+    # Get the printed output
+    printed_text = captured_output.getvalue().strip().split('\n')
+    # Check if the expected output is in the captured output
+    _pattern = r'UserWarning: System \w+ does not have a correction table\. The program will not apply error correction over this system.'
+    matches = [re.match(_pattern, text) for text in printed_text]
+    # Check that match always occurs at the beginning
+    assert sum(m.start() for m in matches) == 0
+    # Check that length matches the expected length
+    matches_lens = (m.end() - m.start() for m in matches)
+    ptext_lens = (len(p) for p in printed_text)
+    assert list(matches_lens) == list(ptext_lens)
diff --git a/tests/test_generator/test_load_additional_systems.py b/tests/test_generator/test_load_additional_systems.py
index 0209089..1063aed 100644
--- a/tests/test_generator/test_load_additional_systems.py
+++ b/tests/test_generator/test_load_additional_systems.py
@@ -5,7 +5,7 @@ from tests.files.paths import with_missing_bp_csv_file
 from tests.test_generator.generator_paths import additional_filters_dir
 
 
-@pytest.fixture
+@pytest.fixture(scope='class')
 def __ps():
     PhotometricSystem = remove_additional_systems()
     yield PhotometricSystem