Repository: chime-sps/champss_software
Pull request #149: Rednoise 2
Author: login: larskuenkel
Created: 2025-04-25T19:21:40Z | Merged: 2025-05-28T22:13:03Z

Due to changes to the git history the branch of https://github.com/chime-sps/champss_software/pull/114 was in a weird state which I did not quite now how to fix.

I cherry picked the the commits from that branch and applied them to the main branch while fixing some merge conflicts.


-     save rednoise info to pspec objects
-     apply rednoise normalization to injections POST harmonic calculations (re discussion with Scott)




Diff:
diff --git a/champss/ps-processes/ps_processes/processes/ps.py b/champss/ps-processes/ps_processes/processes/ps.py
index ff44d360..a2118ec3 100644
--- a/champss/ps-processes/ps_processes/processes/ps.py
+++ b/champss/ps-processes/ps_processes/processes/ps.py
@@ -115,6 +115,15 @@ class PowerSpectraCreation:
 
     num_threads: int
         The number of threads to run the parallel processing of the FFT process. Default = 8
+
+    save_medians: bool
+        Whether to save the rednoise information to the power spectrum.
+
+    write_medians: bool
+        Whether to write the rednoise information to a separate npz file.
+
+    write_zero_dm_medians: bool
+        Whether to write the zero DM rednoise information to an npz file.
     """
 
     tsamp = attribute(validator=instance_of(float), default=TSAMP)
@@ -142,6 +151,9 @@ class PowerSpectraCreation:
     nbit = attribute(validator=instance_of(int), default=32)
     num_threads = attribute(validator=instance_of(int), default=8)
     mp_chunk_size: bool = attribute(default=10)
+    save_medians: bool = attribute(default=False)
+    write_medians: bool = attribute(default=False)
+    write_zero_dm_medians: bool = attribute(default=False)
     static_filter = attribute(init=False)
     dynamic_filter = attribute(init=False)
 
@@ -180,7 +192,7 @@ def __attrs_post_init__(self):
         if self.barycentring_mode == "Topocentric" and self.barycentric_cleaning:
             self.barycentric_cleaning = False
 
-    def transform(self, dedisp_time_series):
+    def transform(self, dedisp_time_series, write_medians=False):
         """
         Function to perform the FFT and creating the normalised power spectrum.
 
@@ -189,6 +201,8 @@ def transform(self, dedisp_time_series):
         dedisp_time_series: DedispersedTimeSeries
             The input DedispersedTimeSeries class which have the data and
             the metadata to process
+        write_medians: bool
+            Whether to write the rednoise information to a npz file.
 
         Returns
         -------
@@ -262,7 +276,7 @@ def transform(self, dedisp_time_series):
                 )
             ]
 
-            pool.starmap(
+            median_info = pool.starmap(
                 partial(
                     self.transform_data,
                     normalise=self.normalise,
@@ -283,6 +297,38 @@ def transform(self, dedisp_time_series):
             pool.close()
             pool.join()
             log.info("Power spectra creation successful.")
+
+            if self.save_medians:
+                median_dm_indices = []
+                medians = []
+                scales = []
+
+                for info in median_info:
+                    median_dm_indices.extend(info[0])
+                    medians.extend(info[1])
+                    scales.extend(info[2])
+
+                medians = np.asarray(medians)
+                median_dm_indices = np.asarray(median_dm_indices)
+                scales = np.asarray(scales)
+
+                rn_medians = np.ones((1, medians.shape[0], medians.shape[1]))
+                rn_medians[0] = medians
+                rn_dm_indices = np.ones((1, len(median_dm_indices)))
+                rn_dm_indices[0] = median_dm_indices
+                # note that scales are saved iteratively over DM but are identical at each DM
+                # so we only need one row of scales
+                rn_scales = np.ones((1, scales.shape[1]))
+                rn_scales[0] = scales[0]
+
+            if self.write_medians:
+                medians_path = f"{os.path.abspath(observation.datapath)}/medians.npz"
+                log.info(f"Saving rednoise information to {medians_path}.")
+                np.savez(
+                    medians_path,
+                    medians=medians,
+                    scales=scales,
+                )
             # update the observation database
             if self.update_db:
                 self.update_database(
@@ -298,19 +344,39 @@ def transform(self, dedisp_time_series):
         datetimes = Time(dedisp_time_series.start_mjd, format="mjd").datetime.replace(
             tzinfo=pytz.utc
         )
-        return PowerSpectra(
-            power_spectra=power_spectra,
-            dms=dedisp_time_series.dms,
-            freq_labels=freq_labels,
-            ra=dedisp_time_series.ra,
-            dec=dedisp_time_series.dec,
-            datetimes=[datetimes],
-            num_days=1,
-            beta=beta,
-            bad_freq_indices=[bad_freq_indices],
-            obs_id=[dedisp_time_series.obs_id],
-            power_spectra_shared=power_spectra_shared,
-        )
+
+        if self.save_medians:
+            return PowerSpectra(
+                power_spectra=power_spectra,
+                dms=dedisp_time_series.dms,
+                freq_labels=freq_labels,
+                ra=dedisp_time_series.ra,
+                dec=dedisp_time_series.dec,
+                datetimes=[datetimes],
+                num_days=1,
+                beta=beta,
+                bad_freq_indices=[bad_freq_indices],
+                obs_id=[dedisp_time_series.obs_id],
+                power_spectra_shared=power_spectra_shared,
+                rn_medians=rn_medians,
+                rn_scales=rn_scales,
+                rn_dm_indices=rn_dm_indices,
+            )
+
+        else:
+            return PowerSpectra(
+                power_spectra=power_spectra,
+                dms=dedisp_time_series.dms,
+                freq_labels=freq_labels,
+                ra=dedisp_time_series.ra,
+                dec=dedisp_time_series.dec,
+                datetimes=[datetimes],
+                num_days=1,
+                beta=beta,
+                bad_freq_indices=[bad_freq_indices],
+                obs_id=[dedisp_time_series.obs_id],
+                power_spectra_shared=power_spectra_shared,
+            )
 
     @staticmethod
     def transform_data(
@@ -387,6 +453,10 @@ def transform_data(
         power_spectra = np.ndarray(
             target_shape, dtype=f"float{nbit}", buffer=shared_spectra.buf
         )
+
+        all_medians = []
+        all_scales = []
+
         for dm_index, dts in zip(dm_indices, dm_series_list):
             if normalise:
                 ts_mean = np.mean(dts)
@@ -427,9 +497,12 @@ def transform_data(
                 power_spectrum[bad_freq_indices] = np.nan
             if remove_rednoise:
                 log.debug("Normalising power spectrum with rednoise removal")
-                power_spectrum[1:] = rednoise_normalise(
+                power_spectrum[1:], medians, scale = rednoise_normalise(
                     power_spectrum[1:], **rednoise_config
                 )
+                all_medians.append(medians)
+                all_scales.append(scale)
+
             else:
                 log.debug("Normalising power spectrum")
                 power_spectrum[1:] = power_spectrum[1:] / (
@@ -446,6 +519,8 @@ def transform_data(
             power_spectra[dm_index, :] = power_spectrum
         shared_spectra.close()
 
+        return [dm_indices, all_medians, all_scales]
+
     def get_pointing_id_from_observation_id(self, obs_id):
         """Get the corresponding pointing ID from the observation ID."""
         try:
@@ -737,7 +812,7 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
             zero_dm_ts = self.zeropad_ts(zero_dm_ts, self.padded_length, self.normalise)
             zero_dm_spectrum = self.run_rfft(zero_dm_ts)
             zero_dm_power_spectrum = np.abs(zero_dm_spectrum) ** 2
-            zero_dm_power_spectrum[1:] = rednoise_normalise(
+            zero_dm_power_spectrum[1:], medians, scales = rednoise_normalise(
                 zero_dm_power_spectrum[1:], **self.rednoise_config
             )
             if self.barycentring_mode == "Fourier" and self.barycentric_cleaning:
@@ -758,7 +833,16 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
                 barycentric_cleaning=self.barycentric_cleaning,
                 beta=beta,
             )
-
+            if self.write_zero_dm_medians:
+                medians_path = (
+                    f"{os.path.abspath(observation.datapath)}/zero_dm_medians.npy"
+                )
+                log.info(f"Saving zero-DM medians to {medians_path}.")
+                np.savez(
+                    medians_path,
+                    medians=medians,
+                    scales=scales,
+                )
             # if self.update_db == False, birdies are not recorded into database
             if self.find_common_birdies and self.update_db:
                 log.info(
diff --git a/champss/ps-processes/ps_processes/processes/ps_inject.py b/champss/ps-processes/ps_processes/processes/ps_inject.py
index 3324f783..2850b911 100644
--- a/champss/ps-processes/ps_processes/processes/ps_inject.py
+++ b/champss/ps-processes/ps_processes/processes/ps_inject.py
@@ -4,16 +4,13 @@
 
 import numpy as np
 import scipy.stats as stats
-from matplotlib import pyplot as plt
-from scipy.fft import fft, rfft
+from scipy.fft import rfft
 from scipy.special import chdtri
 from sps_common.constants import DM_CONSTANT, FREQ_BOTTOM, FREQ_TOP
 from sps_common.interfaces.utilities import sigma_sum_powers
-from sps_databases import db_api
 
 log = logging.getLogger(__name__)
 
-import numpy as np
 import numpy.random as rand
 
 phis = np.linspace(0, 1, 1024)
@@ -55,7 +52,7 @@ def generate_pulse(noise=False):
             noise: bool
                 whether or not the pulse should be distorted by white noise
     """
-    u = rand.choice(np.linspace(0.01, 0.99, 1000))
+    u = rand.uniform(0.01, 0.99, 1000)
     # inverse sampling theorem for an exponential distribution with lambda = 1/15
     gamma = -15 * np.log(1 - u) / 2 / 360
     prof = lorentzian(phis, gamma)
@@ -138,6 +135,12 @@ def x_to_chi2(x, df):
         return chi2 / 2
 
 
+def get_median(xlow, xhigh, ylow, yhigh, x):
+    m = (yhigh - ylow) / (xhigh - xlow)
+
+    return m * (x - xlow) + ylow
+
+
 class Injection:
     """This class allows pulse injection."""
 
@@ -171,12 +174,10 @@ def onewrap_deltaDM(self):
         """Return the deltaDM where the dispersion smearing is one pulse period in
         duration.
         """
-        deltaDM = (
-            1 / (1.0 / FREQ_BOTTOM**2 - 1.0 / FREQ_TOP**2) / self.f / DM_CONSTANT
-        )
+        deltaDM = 1 / (1.0 / FREQ_BOTTOM**2 - 1.0 / FREQ_TOP**2) / self.f / DM_CONSTANT
         return deltaDM
 
-    def sigma_to_power(self, n_harm):
+    def sigma_to_power(self, n_harm, df):
         """
         This function converts an input Gaussian sigma to an approximately equivalent
         power. In order to do so, we have to estimate the number of summed harmonics in
@@ -304,7 +305,45 @@ def harmonics(self, prof_fft, df, N=4):
             harmonics[i * N : (i + 1) * N] = np.abs(amplitude) ** 2
 
         return bins, harmonics
+    
+    def get_rednoise_normalisation(self, inj_bins, inj_dms):
+        """
+        This function retrieves the rednoise information from the power spectrum and
+        scales the injections by the median power at each frequency location.
 
+        Inputs:
+        -------
+            inj_bins (arr): the frequency bins at which the injections occur
+            inj_dms (arr) : the dm bins at which the injections occur
+
+        Returns:
+        --------
+            normalizer (arr): the scaling factor for each injection bin
+        """
+        rn_scales = self.pspec_obj.rn_scales
+        rn_medians = self.pspec_obj.rn_medians
+        normalizer = np.zeros((len(inj_dms), len(inj_bins)))
+
+        for day in range(self.pspec_obj.num_days):
+            day_normalizer = (
+                np.ones((len(inj_dms), len(inj_bins))) / self.pspec_obj.num_days / np.log(2)
+            )
+            # day_medians = rn_medians[day]
+            day_medians = (
+                rn_medians[day] / np.min(rn_medians[day], axis=1)[:, np.newaxis]
+            )
+            day_scales = rn_scales[day]
+
+            scale_sum = np.zeros(len(day_scales) + 1)
+            scale_sum[1:] = np.cumsum(day_scales)
+            scale_sum[0] = 0
+            mid_bins = ((scale_sum[1:] + scale_sum[:-1]) / 2).astype("int")
+            for i, inj_dm in enumerate(inj_dms):
+                rn_interpolated = np.interp(inj_bins, mid_bins, day_medians[inj_dm])
+                day_normalizer[i] /= rn_interpolated
+            normalizer += day_normalizer
+        return normalizer
+    
     def predict_sigma(self, harms, bins, dm_indices, used_nharm, add_expected_mean):
         """
         This function predicts the sigma of an injection and scales it to a specific
@@ -319,6 +358,7 @@ def predict_sigma(self, harms, bins, dm_indices, used_nharm, add_expected_mean):
                 dm_indices (ndarray) : dm bins of the injection
                 used_nharm (int) : Number of harmonics that will be injected
         """
+
         # estimate sigma
         true_dm_injection_index = self.true_dm_trial - dm_indices[0]
         bins_reshaped = bins.reshape(used_nharm, -1)
@@ -400,7 +440,7 @@ def injection(self):
         df = freqs[1] - freqs[0]
         f_nyquist = freqs[-1]
         n_harm = int(np.floor(f_nyquist / self.f))
-        scaled_prof_fft, n_harm = self.sigma_to_power(n_harm)
+        scaled_prof_fft, n_harm = self.sigma_to_power(n_harm, df)
         used_nharm = len(scaled_prof_fft)
         log.info(f"Injecting {n_harm} harmonics.")
 
@@ -415,7 +455,10 @@ def injection(self):
             harms.append(harm)
 
         harms = np.asarray(harms)
-
+        harms *= self.get_rednoise_normalisation(
+            bins,
+            dm_indices,
+        )
         # estimate sigma
         (
             harms,
@@ -596,6 +639,7 @@ def main(
         injection_dict["detection_nharm"] = injection_output_dict["detection_nharm"]
         injection_dict["detection_sigma"] = injection_output_dict["detection_sigma"]
         injection_dict["injected_nharm"] = injection_output_dict["injected_nharm"]
+
         if isinstance(injection_dict["profile"], (np.ndarray, list)):
             injection_dict["profile"] = "custom_profile"
 
diff --git a/champss/ps-processes/ps_processes/processes/ps_search.py b/champss/ps-processes/ps_processes/processes/ps_search.py
index a18b3301..152d6798 100644
--- a/champss/ps-processes/ps_processes/processes/ps_search.py
+++ b/champss/ps-processes/ps_processes/processes/ps_search.py
@@ -213,6 +213,7 @@ def search(
             PowerSpectraDetectionClusters object with the properties of all the
             detections clusters from the pointing.
         """
+
         ps_length = ((len(pspec.freq_labels)) // self.num_harm) * self.num_harm
         # compute harmonic bins based on power spectra properties
         if not self.precompute_harms:
diff --git a/champss/ps-processes/ps_processes/processes/ps_stack.py b/champss/ps-processes/ps_processes/processes/ps_stack.py
index d01e02f2..f3975020 100644
--- a/champss/ps-processes/ps_processes/processes/ps_stack.py
+++ b/champss/ps-processes/ps_processes/processes/ps_stack.py
@@ -380,6 +380,43 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+
+            if type(pspec.rn_medians) != np.ndarray:
+                log.error("This power spectrum does not have rednoise info saved.")
+
+            elif "rn medians" not in h5f.keys():
+                log.error("This h5f file does not have rednoise info saved.")
+
+            else:
+                h5f_rn_medians = h5f["rn medians"]
+                h5f_rn_scales = h5f["rn scales"]
+                # set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones(
+                    (
+                        ndays,
+                        pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2]),
+                    )
+                )
+                stacked_rn_medians[
+                    : len(h5f_rn_medians), :, : h5f_rn_medians.shape[2]
+                ] = h5f_rn_medians
+                stacked_rn_medians[
+                    len(h5f_rn_medians) :, :, : pspec.rn_medians.shape[2]
+                ] = pspec.rn_medians
+                stacked_rn_scales = -1 * np.ones(
+                    (ndays, max(pspec.rn_scales.shape[1], h5f_rn_scales.shape[1]))
+                )
+                # only need one DM value per day of scales
+                stacked_rn_scales[: len(h5f_rn_scales)] = h5f_rn_scales
+                stacked_rn_scales[len(h5f_rn_scales) :] = pspec.rn_scales
+
+                del h5f["rn medians"]
+                del h5f["rn scales"]
+                h5f["rn medians"] = stacked_rn_medians
+                h5f["rn scales"] = stacked_rn_scales
+
         log.info("Stacking completed")
         if self.update_db:
             log.info(f"Updating the stack database for {pspec.ra} {pspec.dec}")
@@ -424,6 +461,7 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
         pspec_stack: PowerSpectra
             The stacked power spectra between daily/monthly or monthly/cumulative stacks.
         """
+
         with h5py.File(stack_file_path, "r") as h5f:
             # Make some check to ensure stacks are compatible with each other
             log.info("Checking for consistency between stacks")
@@ -449,6 +487,44 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
                 pspec.power_spectra[i] += h5f["power spectra"][(i,)]
             log.info(f"Updating the new {self.mode} power spectra information")
             pspec.num_days += h5f.attrs["number of days"]
+
+            if type(pspec.rn_medians) != np.ndarray:
+                log.error("This power spectrum does not have rednoise info saved.")
+
+            elif "rn medians" not in h5f.keys():
+                log.error("This h5f file does not have rednoise info saved.")
+
+            else:
+                h5f_rn_medians = h5f["rn medians"]
+                h5f_rn_scales = h5f["rn scales"]
+
+                # set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones(
+                    (
+                        ndays,
+                        pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2]),
+                    )
+                )
+                stacked_rn_medians[
+                    : len(h5f_rn_medians), :, : h5f_rn_medians.shape[2]
+                ] = h5f_rn_medians
+                stacked_rn_medians[
+                    len(h5f_rn_medians) :, :, : pspec.rn_medians.shape[2]
+                ] = pspec.rn_medians
+                stacked_rn_scales = -1 * np.ones(
+                    (ndays, max(pspec.rn_scales.shape[1], h5f_rn_scales.shape[1]))
+                )
+                # only need one DM value per day of scales
+                stacked_rn_scales[: len(h5f_rn_scales)] = h5f_rn_scales
+                stacked_rn_scales[len(h5f_rn_scales) :] = pspec.rn_scales
+
+                pspec.rn_medians = stacked_rn_medians
+                pspec.rn_scales = stacked_rn_scales
+
+            log.info("Stacking completed.")
+
             bad_freq_arrays = [
                 key for key in h5f.keys() if "bad frequency indices" in key
             ]
@@ -466,6 +542,7 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
             pspec.obs_id = (
                 np.append(h5f.attrs["observation ids"], pspec.obs_id).flatten().tolist()
             )
+
         return pspec
 
     def stack_power_spectra(self, pspec, pspec_stack):
@@ -510,6 +587,18 @@ def stack_power_spectra(self, pspec, pspec_stack):
         pspec_stack.bad_freq_indices.extend(pspec.bad_freq_indices)
         pspec_stack.datetimes.extend(pspec.datetimes)
         pspec_stack.obs_id.extend(pspec.obs_id)
+
+        try:
+            pspec_stack.rn_medians.extend(pspec.rn_medians)
+            pspec_stack.rn_scales.extend(pspec.rn_scales)
+            pspec_stack.rn_dm_indices.extend(pspec.rn_scales)
+
+        except:
+            if pspec.rn_medians is None:
+                log.error("The daily power spectrum does not have rednoise info saved.")
+            if pspec_stack.rn_medians is None:
+                log.error("The power spectra stack does not have rednoise info saved.")
+
         stack_end = time.time()
         log.debug(f"Took {stack_end - stack_start} seconds to stack power spectra")
 
diff --git a/champss/ps-processes/ps_processes/utilities/utilities.py b/champss/ps-processes/ps_processes/utilities/utilities.py
index 49b607c7..c1ee6bc1 100644
--- a/champss/ps-processes/ps_processes/utilities/utilities.py
+++ b/champss/ps-processes/ps_processes/utilities/utilities.py
@@ -5,7 +5,7 @@
 from scipy.stats import chi2, kstwo
 
 
-def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians=False):
+def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians = True):
     """
     Script to normalise power spectrum while removing rednoise. Based on presto's method
     of rednoise removal, in which a logarithmically increasing window is used at low
@@ -51,17 +51,18 @@ def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians=False):
     old_mid_bin = 0
     old_median = 1
     normalised_power_spectrum = np.zeros(shape=np.shape(power_spectrum))
-    if get_medians:
-        medians = np.zeros(shape=np.shape(power_spectrum))
+    medians = []
+    
     for bins in scale:
         mid_bin = int(start + bins / 2)
         new_median = np.nanmedian(power_spectrum[start : start + bins])
+        medians.append(new_median)
         i = 0
         while np.isnan(new_median):
             i += 1
             new_median = np.nanmedian(
                 power_spectrum[start + (i * bins) : start + ((i + 1) * bins)]
-            )
+                )
             if not np.isnan(new_median):
                 if start == 0:
                     new_median = new_median * (2**i)
@@ -77,15 +78,11 @@ def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians=False):
             normalised_power_spectrum[start : start + mid_bin] = power_spectrum[
                 start : start + mid_bin
             ] / (new_median / np.log(2))
-            if get_medians:
-                medians[start : start + mid_bin] = new_median
         elif start + bins >= ps_len:
             median_slope = np.linspace(old_median, new_median, num=ps_len - old_mid_bin)
             normalised_power_spectrum[old_mid_bin:] = power_spectrum[old_mid_bin:] / (
                 median_slope / np.log(2)
             )
-            if get_medians:
-                medians[old_mid_bin:] = median_slope
         else:
             # compute slope of the power spectra
             median_slope = np.linspace(
@@ -94,15 +91,15 @@ def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians=False):
             normalised_power_spectrum[old_mid_bin:mid_bin] = power_spectrum[
                 old_mid_bin:mid_bin
             ] / (median_slope / np.log(2))
-            if get_medians:
-                medians[old_mid_bin:mid_bin] = median_slope
+
         start += bins
         old_mid_bin = mid_bin
         old_median = new_median
-    if get_medians:
-        return normalised_power_spectrum, medians
-    return normalised_power_spectrum
 
+    if get_medians:
+        return normalised_power_spectrum, medians, scale
+    else:
+        return normalised_power_spectrum
 
 def rednoise_normalise_runmed(power_spectrum, w0=10, wmax=1000, bmax=3000):
     """
diff --git a/champss/sps-common/sps_common/interfaces/ps_processes.py b/champss/sps-common/sps_common/interfaces/ps_processes.py
index 82b9fdf8..06567a73 100644
--- a/champss/sps-common/sps_common/interfaces/ps_processes.py
+++ b/champss/sps-common/sps_common/interfaces/ps_processes.py
@@ -201,6 +201,9 @@ class PowerSpectra:
     beta = attrib(converter=float)
     bad_freq_indices = attrib(type=List[List[int]], on_setattr=validate)  # type: ignore
     obs_id = attrib(type=List[str], on_setattr=validate)  # type: ignore
+    rn_medians = attrib(type=np.ndarray, default=None)
+    rn_scales = attrib(type=np.ndarray, default=None)
+    rn_dm_indices = attrib(type=np.ndarray, default=None)
     power_spectra_shared = attrib(default=None)
 
     def __attrs_post_init__(self):
@@ -393,19 +396,45 @@ def read(cls, filename, nbit=32, use_shared_memory=False):
         else:
             beta = h5f.attrs["beta"]
 
-        return cls(
-            power_spectra=power_spectra,
-            dms=dms,
-            freq_labels=freq_labels,
-            ra=ra,
-            dec=dec,
-            datetimes=datetimes,
-            num_days=num_days,
-            beta=beta,
-            bad_freq_indices=bad_freq_indices,
-            obs_id=obs_id.astype(str).tolist(),
-            power_spectra_shared=power_spectra_shared,
-        )
+        if "rn medians" in h5f.keys():
+            rn_medians = np.ones(h5f["rn medians"].shape)
+            h5f["rn medians"].read_direct(rn_medians)
+            rn_scales = np.ones(h5f["rn scales"].shape)
+            h5f["rn scales"].read_direct(rn_scales)
+            rn_dm_indices = np.ones(h5f["rn dm indices"].shape)
+            h5f["rn dm indices"].read_direct(rn_dm_indices)
+
+            return cls(
+                power_spectra=power_spectra,
+                dms=dms,
+                freq_labels=freq_labels,
+                ra=ra,
+                dec=dec,
+                datetimes=datetimes,
+                num_days=num_days,
+                beta=beta,
+                bad_freq_indices=bad_freq_indices,
+                obs_id=obs_id.astype(str).tolist(),
+                power_spectra_shared=power_spectra_shared,
+                rn_medians=rn_medians,
+                rn_scales=rn_scales,
+                rn_dm_indices=rn_dm_indices,
+            )
+
+        else:
+            return cls(
+                power_spectra=power_spectra,
+                dms=dms,
+                freq_labels=freq_labels,
+                ra=ra,
+                dec=dec,
+                datetimes=datetimes,
+                num_days=num_days,
+                beta=beta,
+                bad_freq_indices=bad_freq_indices,
+                obs_id=obs_id.astype(str).tolist(),
+                power_spectra_shared=power_spectra_shared,
+            )
 
     def write(self, filename, nbit=32):
         """
@@ -445,10 +474,18 @@ def write(self, filename, nbit=32):
             datetimes_unix = np.asarray([date.timestamp() for date in self.datetimes])
             h5f.create_dataset("datetimes", data=datetimes_unix)
 
+            if self.rn_medians is not None:
+                h5f.create_dataset(
+                    "rn medians", data=self.rn_medians, dtype=f"f{int(nbit / 8)}"
+                )
+                h5f.create_dataset("rn scales", data=self.rn_scales)
+                h5f.create_dataset("rn dm indices", data=self.rn_dm_indices)
+
             h5f.attrs["ra"] = self.ra
             h5f.attrs["dec"] = self.dec
             h5f.attrs["observation ids"] = self.obs_id
             h5f.attrs["number of days"] = self.num_days
+
             if self.beta is None:
                 # no barycentric correction set/required
                 log.warning(


Reviews:
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:20:20Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:24:18Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:25:41Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:34:44Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:34:54Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:35:13Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:36:55Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:41:44Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:46:24Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:48:24Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:55:08Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T22:59:02Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T23:09:21Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T23:27:07Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T17:52:00Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T17:53:23Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T17:53:30Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:02:44Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:03:52Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:04:12Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:05:15Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:05:22Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:05:36Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:31:36Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:31:58Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T18:50:59Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T19:06:40Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T19:08:13Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T02:19:06Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T17:08:09Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T17:09:00Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T07:49:46Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T23:09:59Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T23:31:14Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T23:49:48Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T16:59:19Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T19:15:03Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T21:23:43Z
user: login: larskuenkel
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T21:27:47Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T17:19:36Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T17:22:34Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T17:25:19Z
user: login: reyniersquillace
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T17:28:30Z
user: login: reyniersquillace
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-28T19:58:15Z

Review comments:
diff_hunk: @@ -5,7 +5,7 @@
 from scipy.stats import chi2, kstwo
 
 
-def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians=False):
+def rednoise_normalise(power_spectrum, b0=50, bmax=100000):
path: champss/ps-processes/ps_processes/utilities/utilities.py
user: login: larskuenkel
body: I think we don't want to remove the "get_medians" argument necessarily.

I would add it back in, and add a return statement where it just returns the normalised power spectrum without the medians.
created_at: 2025-04-28T22:20:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -593,13 +654,12 @@ def main(
         injection_dict["bins"] = injection_output_dict["freq_indices"]
         injection_dict["predicted_nharm"] = injection_output_dict["predicted_nharm"]
         injection_dict["predicted_sigma"] = injection_output_dict["predicted_sigma"]
-        injection_dict["detection_nharm"] = injection_output_dict["detection_nharm"]
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: larskuenkel
body: Why were these removed?
created_at: 2025-04-28T22:24:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -758,7 +798,16 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
                 barycentric_cleaning=self.barycentric_cleaning,
                 beta=beta,
             )
-
+            if save_medians:
+                medians_path = (
+                    f"{os.path.abspath(observation.datapath)}/zero_dm_medians.npy"
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: I think it is better to call it .npz instead of .npy, since you are using np.savez and and not np.save
created_at: 2025-04-28T22:25:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -277,12 +277,35 @@ def transform(self, dedisp_time_series):
                     bad_freq_indices=bad_freq_indices,
                     shared_target_name=power_spectra_shared.name,
                     target_shape=power_spectra.shape,
+                    observation=observation,
                 ),
                 zip(dm_indices, dedisp_series_list),
             )
             pool.close()
             pool.join()
             log.info("Power spectra creation successful.")
+
+            median_dm_indices = []
+            medians = []
+            scales = []
+
+            for info in median_info:
+                median_dm_indices.extend(info[0])
+                medians.extend(info[1])
+                scales.extend(info[2])
+
+            medians = np.asarray(medians)
+            median_dm_indices = np.asarray(median_dm_indices)
+            scales = np.asarray(scales)
+
+            rn_medians = np.ones((1, medians.shape[0], medians.shape[1]))
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: The shape is 1 in the first dimension because this is the dimension where you are stacking multiple days?
created_at: 2025-04-28T22:34:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -323,11 +350,13 @@ def transform_data(
         nbit=32,
         remove_rednoise=True,
         clean_rfi=True,
-        rednoise_config=dict(b0=10, bmax=100000),
+        rednoise_config=dict(b0=10, bmax=100000, get_medians=True),
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: Currently get_medians is removed from that function.
created_at: 2025-04-28T22:34:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -323,11 +350,13 @@ def transform_data(
         nbit=32,
         remove_rednoise=True,
         clean_rfi=True,
-        rednoise_config=dict(b0=10, bmax=100000),
+        rednoise_config=dict(b0=10, bmax=100000, get_medians=True),
         zero_replace=True,
         bad_freq_indices=[],
         shared_target_name=None,
         target_shape=None,
+        observation=None,
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: These two options are not used and can be removed, right?
created_at: 2025-04-28T22:35:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -758,7 +798,16 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
                 barycentric_cleaning=self.barycentric_cleaning,
                 beta=beta,
             )
-
+            if save_medians:
+                medians_path = (
+                    f"{os.path.abspath(observation.datapath)}/zero_dm_medians.npy"
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: Also you enabled both writing the red noise values to the stacks and the DM 0 medians to individual files.
Was your intention to write both of them all the time?
created_at: 2025-04-28T22:36:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -305,6 +305,62 @@ def harmonics(self, prof_fft, df, N=4):
 
         return bins, harmonics
 
+    def rednoise_normalize(self, ps_shape, inj_bins):
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: larskuenkel
body: Maybe give this function another name to differentiate it from the other rednoise_normalize?

I won't go through the math of this function now, but I assume you are doing the right thing.
created_at: 2025-04-28T22:41:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -180,7 +180,7 @@ def __attrs_post_init__(self):
         if self.barycentring_mode == "Topocentric" and self.barycentric_cleaning:
             self.barycentric_cleaning = False
 
-    def transform(self, dedisp_time_series):
+    def transform(self, dedisp_time_series, save_medians=True):
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: Currently this argument does not control anything, right?

What I think would be good if you could create two attributes in the init of PowerSpectraCreation which control whether you save the zero dm red noise and whether you save the red noise info to the power spectra.

Because if you put those values there, then we can influence them just by changing entries in sps_config.yml
created_at: 2025-04-28T22:46:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: Is this check for backwards compatibility or are the files currently saved with different shapes?


created_at: 2025-04-28T22:48:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: If you want to check if an entry exists, you could to something like

`"if "rn_medians" in h5f.keys():`
created_at: 2025-04-28T22:55:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones((ndays, pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2])))
+                stacked_rn_medians[:len(pspec.rn_medians), :,
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: I think in this function you are usually performing prepending instead of appending, right?
I think appending is generally preferable and also performed for attributes like "datetimes"
created_at: 2025-04-28T22:59:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -449,6 +507,61 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
                 pspec.power_spectra[i] += h5f["power spectra"][(i,)]
             log.info(f"Updating the new {self.mode} power spectra information")
             pspec.num_days += h5f.attrs["number of days"]
+           
+
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+            
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+            
+            if len(pspec.rn_medians.shape) == 2:
+
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+            
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+             
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: So if we have a big stack created already, running the stacking would write a bunch of -1s to the file?
Would probably mean that all our stacks would grow a bunch. Do you know how much storage the red noise values for like 1 year for example would take?
created_at: 2025-04-28T23:09:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -305,6 +305,62 @@ def harmonics(self, prof_fft, df, N=4):
 
         return bins, harmonics
 
+    def rednoise_normalize(self, ps_shape, inj_bins):
+        """
+        A scaled down version of the rednoise_normalize function from utilities.py for
+        when we already know all the median information. Currently this is DM-secular
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: larskuenkel
body: Language question: What do you mean with "DM-secular"? Is that the same as DM-agnostic?
created_at: 2025-04-28T23:27:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -593,13 +654,12 @@ def main(
         injection_dict["bins"] = injection_output_dict["freq_indices"]
         injection_dict["predicted_nharm"] = injection_output_dict["predicted_nharm"]
         injection_dict["predicted_sigma"] = injection_output_dict["predicted_sigma"]
-        injection_dict["detection_nharm"] = injection_output_dict["detection_nharm"]
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: reyniersquillace
body: There was a weird glitch in ps_inject at some point where a bunch of lines got randomly deleted or moved around. I thought I caught them all but clearly I didn't. Thanks!
created_at: 2025-04-29T17:52:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -758,7 +798,16 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
                 barycentric_cleaning=self.barycentric_cleaning,
                 beta=beta,
             )
-
+            if save_medians:
+                medians_path = (
+                    f"{os.path.abspath(observation.datapath)}/zero_dm_medians.npy"
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: This is a holdover from an early version of saving rednoise info-- I think we should remove this entirely. We don't ever use the zero DM info. Do you think we should just not save it?
created_at: 2025-04-29T17:53:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -277,12 +277,35 @@ def transform(self, dedisp_time_series):
                     bad_freq_indices=bad_freq_indices,
                     shared_target_name=power_spectra_shared.name,
                     target_shape=power_spectra.shape,
+                    observation=observation,
                 ),
                 zip(dm_indices, dedisp_series_list),
             )
             pool.close()
             pool.join()
             log.info("Power spectra creation successful.")
+
+            median_dm_indices = []
+            medians = []
+            scales = []
+
+            for info in median_info:
+                median_dm_indices.extend(info[0])
+                medians.extend(info[1])
+                scales.extend(info[2])
+
+            medians = np.asarray(medians)
+            median_dm_indices = np.asarray(median_dm_indices)
+            scales = np.asarray(scales)
+
+            rn_medians = np.ones((1, medians.shape[0], medians.shape[1]))
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: Yes!
created_at: 2025-04-29T17:53:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -323,11 +350,13 @@ def transform_data(
         nbit=32,
         remove_rednoise=True,
         clean_rfi=True,
-        rednoise_config=dict(b0=10, bmax=100000),
+        rednoise_config=dict(b0=10, bmax=100000, get_medians=True),
         zero_replace=True,
         bad_freq_indices=[],
         shared_target_name=None,
         target_shape=None,
+        observation=None,
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: To be honest I have no memory of adding these... weird. Yes, they can be removed.
created_at: 2025-04-29T18:02:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -305,6 +305,62 @@ def harmonics(self, prof_fft, df, N=4):
 
         return bins, harmonics
 
+    def rednoise_normalize(self, ps_shape, inj_bins):
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: reyniersquillace
body: Ok, will do! 

We think we are doing the right thing. It was in the process of testing the stats on the injections that I noticed the weird stuff going on -- so I think there's something wonky somewhere, but it would be more easily debugged if I can access Robert's PR. 
created_at: 2025-04-29T18:03:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -180,7 +180,7 @@ def __attrs_post_init__(self):
         if self.barycentring_mode == "Topocentric" and self.barycentric_cleaning:
             self.barycentric_cleaning = False
 
-    def transform(self, dedisp_time_series):
+    def transform(self, dedisp_time_series, save_medians=True):
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: Great idea! On it.
created_at: 2025-04-29T18:04:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: This is because h5f seems to load (1, N) arrays as (N,) arrays. So I kept the saving as (1, N) for formalism but made sure to check when we load them in that they're the right length.
created_at: 2025-04-29T18:05:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Aha! Thanks!
created_at: 2025-04-29T18:05:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones((ndays, pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2])))
+                stacked_rn_medians[:len(pspec.rn_medians), :,
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Will fix!
created_at: 2025-04-29T18:05:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -449,6 +507,61 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
                 pspec.power_spectra[i] += h5f["power spectra"][(i,)]
             log.info(f"Updating the new {self.mode} power spectra information")
             pspec.num_days += h5f.attrs["number of days"]
+           
+
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+            
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+            
+            if len(pspec.rn_medians.shape) == 2:
+
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+            
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+             
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: On average, the length of the node points is somewhere around 30. This might add a couple of -1s to the rednoise info, but it would be on average length (nDM, ~1-10). Therefore over the course of a year the size of the medians would be (365, ~2000, ~30). I will check how much this is in GB if stored as floats. We could easily reduce the size by storing as uint8.
created_at: 2025-04-29T18:31:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -305,6 +305,62 @@ def harmonics(self, prof_fft, df, N=4):
 
         return bins, harmonics
 
+    def rednoise_normalize(self, ps_shape, inj_bins):
+        """
+        A scaled down version of the rednoise_normalize function from utilities.py for
+        when we already know all the median information. Currently this is DM-secular
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: reyniersquillace
body: Argh, leftover from previous version. I did mean that but it's no longer true. I'll change that.
created_at: 2025-04-29T18:31:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -758,7 +798,16 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
                 barycentric_cleaning=self.barycentric_cleaning,
                 beta=beta,
             )
-
+            if save_medians:
+                medians_path = (
+                    f"{os.path.abspath(observation.datapath)}/zero_dm_medians.npy"
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: larskuenkel
body: Well, currently you are only saving it into the power spectra, which are not always saved (if the data quality is too bad, they are not stacked). Also retrieving the red noise values from hundreds of TB of stacks sounds somewhat slow. One could make that faster by making a read method which reads only the red noise part of the power spectrum file. 

But then you would still need to know which stacks contain the interesting information.

I think writing some file and making it accessible via the database might be good, but that does not need to be part of this PR. Maybe leave this part there and make it so it gets controlled by a config argument (/attribute of the class) and I can code the rest up after the PR.
created_at: 2025-04-29T18:50:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: Ah, OK.
I might have a look if I see some way of directly loading the right shape so that we can maybe simplify it.
created_at: 2025-04-29T19:06:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones((ndays, pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2])))
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: Will the size of the third dimension ever change? Only if we change the red noise algorithm, right?
created_at: 2025-04-29T19:08:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: Made a quick test and i did not see the shapes changing. Did you use some other functions for this?
I think it would be good if we could make sure that the shapes are loaded consistently, then we don't need the case distinctions which makes the code shorter which makes it more easy to understand.

![image](https://github.com/user-attachments/assets/3e3df1b4-db17-47d1-9893-9fa7af8c1c19)

created_at: 2025-04-30T02:19:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones((ndays, pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2])))
+                stacked_rn_medians[:len(pspec.rn_medians), :,
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Fixed.
created_at: 2025-05-12T17:08:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Fixed.
created_at: 2025-05-12T17:08:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -5,7 +5,7 @@
 from scipy.stats import chi2, kstwo
 
 
-def rednoise_normalise(power_spectrum, b0=50, bmax=100000, get_medians=False):
+def rednoise_normalise(power_spectrum, b0=50, bmax=100000):
path: champss/ps-processes/ps_processes/utilities/utilities.py
user: login: reyniersquillace
body: Fixed!
created_at: 2025-05-13T07:49:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -758,7 +798,16 @@ def flag_periodic_rfi(self, dedisp_time_series, freq_labels, beta, observation):
                 barycentric_cleaning=self.barycentric_cleaning,
                 beta=beta,
             )
-
+            if save_medians:
+                medians_path = (
+                    f"{os.path.abspath(observation.datapath)}/zero_dm_medians.npy"
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: Added!
created_at: 2025-05-13T08:00:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -323,11 +350,13 @@ def transform_data(
         nbit=32,
         remove_rednoise=True,
         clean_rfi=True,
-        rednoise_config=dict(b0=10, bmax=100000),
+        rednoise_config=dict(b0=10, bmax=100000, get_medians=True),
         zero_replace=True,
         bad_freq_indices=[],
         shared_target_name=None,
         target_shape=None,
+        observation=None,
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: Fixed.
created_at: 2025-05-13T08:04:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -305,6 +305,62 @@ def harmonics(self, prof_fft, df, N=4):
 
         return bins, harmonics
 
+    def rednoise_normalize(self, ps_shape, inj_bins):
path: champss/ps-processes/ps_processes/processes/ps_inject.py
user: login: reyniersquillace
body: Function has been renamed. Testing will follow all other PR corrections!
created_at: 2025-05-13T08:04:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -180,7 +180,7 @@ def __attrs_post_init__(self):
         if self.barycentring_mode == "Topocentric" and self.barycentric_cleaning:
             self.barycentric_cleaning = False
 
-    def transform(self, dedisp_time_series):
+    def transform(self, dedisp_time_series, save_medians=True):
path: champss/ps-processes/ps_processes/processes/ps.py
user: login: reyniersquillace
body: Added.
created_at: 2025-05-13T08:38:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: When I take out those clauses checking for length and stack two days, the length of the medians is:

![image](https://github.com/user-attachments/assets/59903236-e523-4e62-b529-45f7e3f1e1e1)
(this should be 2, 2110, 31).

When I run with the clauses in:
![image](https://github.com/user-attachments/assets/6ac2fb95-f2d2-4a4a-b188-95aede802933)

I am reminded of the joke in Discworld about the teddy bear they have to keep in the computing room or the computer crashes for no reason they can tell...
created_at: 2025-05-13T19:50:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones((ndays, pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2])))
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Yes, we could hard code this! I just wanted to be save in case we change bmax for instance.
created_at: 2025-05-14T16:19:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,63 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+           
+            if len(h5f["rn medians"].shape) == 2:
+
+                h5f_rn_medians = np.ones((1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1]))
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2: 
+                
+                new_rn_medians = np.ones((1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1]))
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) != np.ndarray:
+                    log.error("This power spectrum does not have rednoise info saved.")
+
+            #don't know how to check this exists...
+            #elif class(h5f["rn medians"]) != 'h5py._hl.dataset.Dataset':
+            #    log.error("This h5f file does not have rednoise info saved.")
+            else:
+                #set guard value as -1 to pad
+                ndays = len(pspec.rn_medians) + len(h5f_rn_medians)
+                stacked_rn_medians = -1 * np.ones((ndays, pspec.rn_dm_indices.shape[1],
+                        max(pspec.rn_medians.shape[2], h5f_rn_medians.shape[2])))
+                stacked_rn_medians[:len(pspec.rn_medians), :,
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: I think in two lines the `:` was at the wrong place but I fixed that.
created_at: 2025-05-15T23:09:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -449,6 +513,69 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
                 pspec.power_spectra[i] += h5f["power spectra"][(i,)]
             log.info(f"Updating the new {self.mode} power spectra information")
             pspec.num_days += h5f.attrs["number of days"]
+
+            if len(h5f["rn medians"].shape) == 2:
+                h5f_rn_medians = np.ones(
+                    (1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1])
+                )
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2:
+                new_rn_medians = np.ones(
+                    (1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1])
+                )
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) is not np.ndarray:
+                log.error("This power spectrum does not have rednoise info saved.")
+
+            elif "rn medians" not in h5f.keys():
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: You added this case in after my comment but in this scheme we would never write the red noise dat if it was not already present in the data, right?
created_at: 2025-05-15T23:31:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,69 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+
+            if len(h5f["rn medians"].shape) == 2:
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: Might be good if we could try to prevent reshaping.
`if "rn medians" in h5f.keys()` might be a good idea to put somehow, because otherwise it would crash if that field does not exist, I think.
created_at: 2025-05-15T23:49:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,11 +457,14 @@ def write(self, filename, nbit=32):
                     )
             datetimes_unix = np.asarray([date.timestamp() for date in self.datetimes])
             h5f.create_dataset("datetimes", data=datetimes_unix)
-
+            h5f.create_dataset("rn medians", data=self.rn_medians)
path: champss/sps-common/sps_common/interfaces/ps_processes.py
user: login: larskuenkel
body: If we want save space we could save the rn_medians in the same dtype as the spectra, if we add `dtype=f"f{int(nbit / 8)}",` here.
created_at: 2025-05-16T19:15:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -363,6 +366,13 @@ def read(cls, filename, nbit=32, use_shared_memory=False):
             for date in datetimes_unix
         ]
 
+        rn_medians = np.ones(h5f["rn medians"].shape)
path: champss/sps-common/sps_common/interfaces/ps_processes.py
user: login: larskuenkel
body: Does this work for files without that key?
created_at: 2025-05-20T21:23:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,44 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+            
+            h5f_rn_medians = h5f["rn medians"]
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: larskuenkel
body: This will raise a KeyError if that field does not exist. (For example in all stacks that we already have on disk.)

You could move that for example in the else statement for it to work properly.
But in this setup it will only stack if both the red noise info exist. Which is not the case for existing stacks.

One other way in python of getting an attribte that may or may not exists is .get() or getattr().

So if you have a dictionary or something (h5py.File works as well). You can do h5f.get("Non existing key") to return a None if that field does not exist. You can also add a default return value if that key does not exist: h5f.get("Non existing key", 10)
For object like pspec you can do getattr(pspec, None) to do something quite similar. (We do not need to do that here since in this version of the code the attributes will always exist).


I think the easiest way of structuring our use case would be to do something like:
```
if type(pspec.rn_medians) == np.ndarray or "rn medians" in h5f.keys():
   # setup stacked array
  h5f_rn_medians = h5f.get(["rn medians"])
  if h5f_rn_medians is not None:
  # add hdf5 vals to stacked array
  if type(pspec.rn_medians) == np.ndarray:
  # add pspec vals to stacked array
```
created_at: 2025-05-20T21:27:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -449,6 +513,69 @@ def stack_power_spectra_readout(self, pspec, stack_file_path):
                 pspec.power_spectra[i] += h5f["power spectra"][(i,)]
             log.info(f"Updating the new {self.mode} power spectra information")
             pspec.num_days += h5f.attrs["number of days"]
+
+            if len(h5f["rn medians"].shape) == 2:
+                h5f_rn_medians = np.ones(
+                    (1, h5f["rn medians"].shape[0], h5f["rn medians"].shape[1])
+                )
+                h5f_rn_medians[0] = h5f["rn medians"][:]
+
+            else:
+                h5f_rn_medians = h5f["rn medians"][:]
+
+            if len(h5f["rn scales"].shape) == 1:
+                h5f_rn_scales = np.ones((1, len(h5f["rn medians"])))
+                h5f_rn_scales[0] = h5f["rn scales"][:]
+
+            else:
+                h5f_rn_scales = h5f["rn scales"][:]
+
+            if len(pspec.rn_medians.shape) == 2:
+                new_rn_medians = np.ones(
+                    (1, pspec.rn_medians.shape[0], pspec.rn_medians.shape[1])
+                )
+                new_rn_medians[0] = pspec.rn_medians
+                pspec.rn_medians = new_rn_medians
+
+            if len(pspec.rn_scales.shape) == 1:
+                new_rn_scales = np.ones((1, len(pspec.rn_scales)))
+                new_rn_scales[0] = pspec.rn_scales
+                pspec.rn_scales = new_rn_scales
+
+            if type(pspec.rn_medians) is not np.ndarray:
+                log.error("This power spectrum does not have rednoise info saved.")
+
+            elif "rn medians" not in h5f.keys():
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Ideally not-- this is here just to flag if we have accidentally done so.
created_at: 2025-05-21T17:19:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -444,11 +457,14 @@ def write(self, filename, nbit=32):
                     )
             datetimes_unix = np.asarray([date.timestamp() for date in self.datetimes])
             h5f.create_dataset("datetimes", data=datetimes_unix)
-
+            h5f.create_dataset("rn medians", data=self.rn_medians)
path: champss/sps-common/sps_common/interfaces/ps_processes.py
user: login: reyniersquillace
body: Added!
created_at: 2025-05-21T17:22:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -363,6 +366,13 @@ def read(cls, filename, nbit=32, use_shared_memory=False):
             for date in datetimes_unix
         ]
 
+        rn_medians = np.ones(h5f["rn medians"].shape)
path: champss/sps-common/sps_common/interfaces/ps_processes.py
user: login: reyniersquillace
body: Fixed!
created_at: 2025-05-21T17:25:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -380,6 +380,44 @@ def stack_power_spectra_infile(self, pspec, stack_file_path):
             ).flatten()
             del h5f.attrs["observation ids"]
             h5f.attrs["observation ids"] = list(new_obs_ids)
+            
+            h5f_rn_medians = h5f["rn medians"]
path: champss/ps-processes/ps_processes/processes/ps_stack.py
user: login: reyniersquillace
body: Fixed!
created_at: 2025-05-21T17:28:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: reyniersquillace
created_at: 2025-05-14T16:20:00Z
body: All comments in this PR have been addressed.
author_association: CONTRIBUTOR
user: login: larskuenkel
created_at: 2025-05-15T23:21:01Z
body: I took the liberty of pushing some minor changes where it was easier to move on symbol around rather than writing a comment.

(And obviously I made a mistake when trying to fix some cases where pre-commit was whining.)
author_association: CONTRIBUTOR
user: login: larskuenkel
created_at: 2025-05-28T19:56:32Z
body: Looks good to me. I'll change it so that it does not get run automatically and then we can adjust the code in a future PR
author_association: CONTRIBUTOR