Repository: lightkurve/lightkurve

Issue:
Detrended flux columns in QLP lightcurves appear to have been renamed in recent sectors (56 and later)
<!-- Fill in the information below before opening an issue. -->

#### Problem description
<!-- Provide a clear and concise description of the issue. -->
The detrended columns in [QLP](https://archive.stsci.edu/hlsp/qlp) lighcurves used to be  `'kspsap_flux', 'kspsap_flux_err', 'kspsap_flux_sml', 'kspsap_flux_lag'`.
Starting from sector 56 (year5), they appear to be renamed to `'det_flux', 'det_flux_err', 'det_flux_sml', 'det_flux_lag'`.

The change causes some headaches when accessing detrended lightcurves from a mix of sectors.

It is also unclear if the change is a matter of renaming columns, or it actually signify something more significant.

At this point, it is unclear if there is anything `lightkurve` should do.


#### Example
<!-- Provide a link or minimal code snippet that demonstrates the issue. -->
One can find the list of columns from the following samples
```python
import lightkurve as lk

_sr = lk.search_lightcurve("TIC10549159", author="QLP", sector=56)  # new names
_lcs56 = _sr.download()
print('Sec 56', _lcs56.colnames)

_sr = lk.search_lightcurve("TIC855867", author="QLP", sector=55)  # old names
_lcs55 = _sr.download()
print('Sec 55', _lcs55.colnames)

```

Output:
```python
Sec 56 ['time', 'flux', 'flux_err', 'cadenceno', 'sap_flux', 'det_flux', 'det_flux_err', 'quality', 'orbitid', 'sap_x', 'sap_y', 'sap_bkg', 'sap_bkg_err', 'det_flux_sml', 'det_flux_lag', 'sys_rm_flux']
Sec 55 ['time', 'flux', 'flux_err', 'cadenceno', 'sap_flux', 'kspsap_flux', 'kspsap_flux_err', 'quality', 'orbitid', 'sap_x', 'sap_y', 'sap_bkg', 'sap_bkg_err', 'kspsap_flux_sml', 'kspsap_flux_lag']
```
#### Expected behavior
<!-- Describe the behavior you expected and how it differs from the behavior observed in the example. -->

#### Environment

-  lightkurve version (e.g. 1.0b6): v2.4.2


Comments:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 3ac43947..a85ed588 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -2,6 +2,8 @@
 =====================
 
 - Fixed memory leak in reading Lightcurve / TargetPixel FITS files in v2.4.2 [#1390]
+- Added support for changes in QLP High Level Science Product
+  in TESS sectors 56 and later. [#1392]
 
 
 2.4.2 (2023-11-03)
diff --git a/src/lightkurve/io/qlp.py b/src/lightkurve/io/qlp.py
index 8524879a..9115c90e 100644
--- a/src/lightkurve/io/qlp.py
+++ b/src/lightkurve/io/qlp.py
@@ -9,23 +9,26 @@ from ..utils import TessQualityFlags
 from .generic import read_generic_lightcurve
 
 
-def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column="kspsap_flux_err", quality_bitmask="default"):
+def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column=None, quality_bitmask="default"):
     """Returns a `~lightkurve.lightcurve.LightCurve` object given a light curve file from the MIT Quicklook Pipeline (QLP).
 
     By default, QLP's `sap_flux` column is used to populate the `flux` values,
-    and 'kspsap_flux_err' is used to populate `flux_err`. For a discussion
+    and `kspsap_flux_err` / `det_flux_err` is used to populate `flux_err`. For a discussion
     related to this choice, see https://github.com/lightkurve/lightkurve/issues/1083
 
+    For detrended flux, the columns are named with `kspsap_` prefix in sectors 1-55,
+    and `det_` prefix in sectors 56+. Column `sys_rm_flux` is available in sectors 56+.
+
     More information: https://archive.stsci.edu/hlsp/qlp
 
     Parameters
     ----------
     filename : str
         Local path or remote url of a QLP light curve FITS file.
-    flux_column : 'sap_flux', 'kspsap_flux', 'kspsap_flux_sml', 'kspsap_flux_lag', or 'sap_bkg'
+    flux_column : 'sap_flux', 'kspsap_flux', 'kspsap_flux_sml', 'kspsap_flux_lag', 'det_flux', 'det_flux_sml', 'det_flux_lag', 'sys_rm_flux', or 'sap_bkg'
         Which column in the FITS file contains the preferred flux data?
         By default the "Simple Aperture Photometry" flux (sap_flux) is used.
-    flux_err_column: 'kspsap_flux_err', or 'sap_bkg_err'
+    flux_err_column: 'kspsap_flux_err','det_flux_err', or 'sap_bkg_err'
       Which column in the FITS file contains the preferred flux_err data?
     quality_bitmask : str or int
         Bitmask (integer) which identifies the quality flag bitmask that should
@@ -39,9 +42,17 @@ def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column="kspsa
             * "hardest": removes all data that has been flagged.
               This mask is not recommended.
 
+        QLP-specific "Low precision points" (bit 13 in sectors 1-55, bit 31 in sectors 56+)
+        is included in "hard" and "hardest" bitmasks.
+
         See the `~lightkurve.utils.TessQualityFlags` class for details on the bitmasks.
     """
     lc = read_generic_lightcurve(filename, flux_column=flux_column, flux_err_column=flux_err_column, time_format="btjd")
+    if flux_err_column is None:
+        if lc.meta.get("SECTOR", 0) >= 56:
+            lc["flux_err"] = lc["det_flux_err"]
+        else:
+            lc["flux_err"] = lc["kspsap_flux_err"]
 
     # Filter out poor-quality data
     # NOTE: Unfortunately Astropy Table masking does not yet work for columns
@@ -50,6 +61,16 @@ def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column="kspsa
     quality_mask = TessQualityFlags.create_quality_mask(
         quality_array=lc["quality"], bitmask=quality_bitmask
     )
+    # QLP-specific quality_bitmask handling
+    if quality_bitmask in ["hardest", "hard"]:
+        if lc.meta.get("SECTOR", 0) >= 56:
+            qlp_low_precision_bitmask = 2 ** 30
+        else:
+            # https://archive.stsci.edu/hlsps/qlp/hlsp_qlp_tess_ffi_all_tess_v1_data-prod-desc.pdf
+            qlp_low_precision_bitmask = 2 ** 12
+        q_mask2 = TessQualityFlags.create_quality_mask(
+            quality_array=lc["quality"], bitmask=qlp_low_precision_bitmask)
+        quality_mask = quality_mask & q_mask2
     lc = lc[quality_mask]
 
     lc.meta["AUTHOR"] = "QLP"


Test patch:
diff --git a/tests/io/test_qlp.py b/tests/io/test_qlp.py
index 99420432..eaba9e2e 100644
--- a/tests/io/test_qlp.py
+++ b/tests/io/test_qlp.py
@@ -10,9 +10,19 @@ from lightkurve.io.detect import detect_filetype
 
 
 @pytest.mark.remote_data
-def test_qlp():
+@pytest.mark.parametrize(
+    "url, flux_err_colname_expected, qlp_low_precision_bitmask", [
+        ("https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HLSP/qlp/s0011/0000/0002/7755/4109/hlsp_qlp_tess_ffi_s0011-0000000277554109_tess_v01_llc.fits",
+         "KSPSAP_FLUX_ERR",  # for sectors 1 - 55
+         2**12,  # bit 13 for sectors 1 -55
+         ),
+        ("https://mast.stsci.edu/api/v0.1/Download/file/?uri=mast:HLSP/qlp/s0056/0000/0000/1054/9159/hlsp_qlp_tess_ffi_s0056-0000000010549159_tess_v01_llc.fits",
+         "DET_FLUX_ERR",  # for sectors 56+
+         2**30,  # bit 31 for sectors 56+
+         ),
+        ])
+def test_qlp(url, flux_err_colname_expected, qlp_low_precision_bitmask):
     """Can we read in QLP light curves?"""
-    url = "https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HLSP/qlp/s0011/0000/0002/7755/4109/hlsp_qlp_tess_ffi_s0011-0000000277554109_tess_v01_llc.fits"    
     with fits.open(url, mode="readonly") as hdulist:
         # Can we auto-detect a QLP file?
         assert detect_filetype(hdulist) == "QLP"
@@ -20,6 +30,23 @@ def test_qlp():
         lc = read_qlp_lightcurve(url, quality_bitmask=0)
         assert lc.meta["FLUX_ORIGIN"] == "sap_flux"
         assert_array_equal(lc.flux.value, hdulist[1].data["SAP_FLUX"])
+        assert_array_equal(lc.flux_err.value, hdulist[1].data[flux_err_colname_expected])
+
+        # Test handling of QLP-specific low-precision bitmask
+        # - the cadences marked as such will be masked out by "hard" / "hardest"
+
+        # first assure the test FITS file has cadence marked by QLP bit only
+        # to easily isolate the effects of the quality_bitmask
+        assert (lc["quality"] == qlp_low_precision_bitmask).any()
+
+        lc = read_qlp_lightcurve(url, quality_bitmask="default")
+        assert (lc["quality"] & (qlp_low_precision_bitmask)).any()
+
+        lc = read_qlp_lightcurve(url, quality_bitmask="hard")
+        assert not (lc["quality"] & (qlp_low_precision_bitmask)).any()
+
+        lc = read_qlp_lightcurve(url, quality_bitmask="hardest")
+        assert not (lc["quality"] & (qlp_low_precision_bitmask)).any()
 
 
 @pytest.mark.remote_data