Repository: andrew-saydjari/ApogeeReduction.jl
Pull request #163: Update make_runlist_all.jl
Author: login: KevinMcK95
Created: 2025-04-04T19:50:38Z | Merged: 2025-04-10T16:24:42Z

Make it so quartz and dome flats are always reduced to 1D


Diff:
diff --git a/pipeline.jl b/pipeline.jl
index a62dcbbc..921764ab 100644
--- a/pipeline.jl
+++ b/pipeline.jl
@@ -249,7 +249,7 @@ git_branch, git_commit = initalize_git(src_dir);
                 chip, df.exposure[expid], df.exptype[expid]],
             "_")
         # probably change to FITS to make astronomers happy (this JLD2, which is HDF5, is just for debugging)
-        jldsave(
+        safe_jldsave(
             joinpath(outdir, "apred/$(mjd)/" * outfname * ".jld2"); dimage, ivarimage, chisqimage,
             CRimage, saturation_image, nread_used, mjd_mid_exposure_old, mjd_mid_exposure_rough,
             mjd_mid_exposure_precise, mjd_mid_exposure, git_branch, git_commit)
@@ -300,7 +300,7 @@ git_branch, git_commit = initalize_git(src_dir);
         pix_bitmask .|= saturation_image * 2^13
 
         outfname = replace(fname, "ar2D" => "ar2Dcal")
-        jldsave(
+        safe_jldsave(
             outfname; dimage, ivarimage, pix_bitmask, nread_used,
             mjd_mid_exposure_old, mjd_mid_exposure_rough, mjd_mid_exposure_precise,
             mjd_mid_exposure, git_branch, git_commit)
diff --git a/pipeline_2d_1d.jl b/pipeline_2d_1d.jl
index 7d22c67f..c087db89 100644
--- a/pipeline_2d_1d.jl
+++ b/pipeline_2d_1d.jl
@@ -147,20 +147,25 @@ git_branch, git_commit = initalize_git(src_dir);
         regularized_trace_params = regularize_trace(trace_params)
 
         flux_1d, ivar_1d,
-        mask_1d = if parg["extraction"] == "boxcar"
-            extract_boxcar(dimage, ivarimage, pix_bitmask, regularized_trace_params)
+        mask_1d,
+        resid_flux,
+        resid_ivar = if parg["extraction"] == "boxcar"
+            extract_boxcar(
+                dimage, ivarimage, pix_bitmask, regularized_trace_params, return_resids = true)
         elseif parg["extraction"] == "optimal"
             #            extract_optimal(dimage, ivarimage, pix_bitmask, regularized_trace_params)
             extract_optimal_iter(dimage, ivarimage, pix_bitmask, regularized_trace_params,
                 med_center_to_fiber_func, x_prof_min, x_prof_max_ind, n_sub,
-                min_prof_fib, max_prof_fib, all_y_prof, all_y_prof_deriv)
+                min_prof_fib, max_prof_fib, all_y_prof, all_y_prof_deriv, return_resids = true)
         else
             error("Extraction method $(parg["extraction"]) not recognized")
         end
 
         # we probably want to append info from the fiber dictionary from alamanac into the file name
         outfname = replace(fname, "ar2D" => "ar1D")
-        jldsave(outfname; flux_1d, ivar_1d, mask_1d, git_branch, git_commit)
+        safe_jldsave(outfname; flux_1d, ivar_1d, mask_1d, git_branch, git_commit)
+        resid_outfname = replace(fname, "ar2D" => "ar2Dresiduals")
+        safe_jldsave(resid_outfname; resid_flux, resid_ivar, git_branch, git_commit)
     end
 end
 t_now = now();
diff --git a/src/ar1D.jl b/src/ar1D.jl
index bf3189e6..fa770d7b 100644
--- a/src/ar1D.jl
+++ b/src/ar1D.jl
@@ -50,13 +50,22 @@ function regularize_trace(trace_params; window_size = 101)
     regularized_trace
 end
 
-function extract_boxcar(dimage, ivarimage, pix_bitmask, trace_params; boxcar_halfwidth = 2)
+function extract_boxcar(dimage, ivarimage, pix_bitmask, trace_params;
+        boxcar_halfwidth = 2, return_resids = false)
+    if return_resids
+        resid_fluxes_2d = zeros(Float64, size(dimage))
+        resid_ivars_2d = zeros(Float64, size(dimage))
+    end
     flux_1d = extract_boxcar_core(dimage, trace_params, boxcar_halfwidth)
     var_1d = extract_boxcar_core(1 ./ ivarimage, trace_params, boxcar_halfwidth)
     ivar_1d = 1.0 ./ var_1d
     mask_1d = extract_boxcar_bitmask(pix_bitmask, trace_params, boxcar_halfwidth)
 
-    flux_1d, ivar_1d, mask_1d
+    if return_resids
+        return flux_1d, ivar_1d, mask_1d, resid_fluxes_2d, resid_ivars_2d
+    else
+        return flux_1d, ivar_1d, mask_1d
+    end
 end
 
 """
@@ -130,7 +139,8 @@ function extract_optimal_iter(dimage, ivarimage, pix_bitmask, trace_params,
         med_center_to_fiber_func, x_prof_min, x_prof_max_ind,
         n_sub, min_prof_fib, max_prof_fib, all_y_prof, all_y_prof_deriv;
         small_window_half_size = 2, fit_window_half_size = 4,
-        large_window_half_size = 12, n_max_repeat = 5, flag_thresh = 0.001)
+        large_window_half_size = 12, n_max_repeat = 5, flag_thresh = 0.001,
+        return_resids = false)
     n_xpix = size(trace_params, 1)
     n_ypix = size(dimage, 2)
     n_fibers = size(trace_params, 2)
@@ -152,6 +162,11 @@ function extract_optimal_iter(dimage, ivarimage, pix_bitmask, trace_params,
 
     new_flux_1d = zeros(Float64, n_fibers)
 
+    if return_resids
+        resid_fluxes_2d = zeros(Float64, size(dimage))
+        resid_ivars_2d = zeros(Float64, size(dimage))
+    end
+
     for xpix in 1:n_xpix
         #iterate on best-fit fluxes
         #to remove crosstalk/contribution
@@ -246,18 +261,26 @@ function extract_optimal_iter(dimage, ivarimage, pix_bitmask, trace_params,
                 end
             end
 
+            flux_1d[xpix, :] .= new_flux_1d
+            if return_resids
+                resid_fluxes_2d[xpix, :] .= dimage[xpix, :] .- new_comb_model_flux
+                resid_ivars_2d[xpix, :] .= 1 ./ (1 ./ ivarimage[xpix, :] .+ new_comb_model_var)
+            end
+
             if all(abs.(new_flux_1d .- flux_1d[xpix, :]) .< 0.01) & (repeat_ind > 1)
-                flux_1d[xpix, :] .= new_flux_1d
                 break
             end
 
-            flux_1d[xpix, :] .= new_flux_1d
             comb_model_flux .= new_comb_model_flux
             comb_model_var .= new_comb_model_var
         end
     end
 
-    flux_1d, ivar_1d, mask_1d
+    if return_resids
+        return flux_1d, ivar_1d, mask_1d, resid_fluxes_2d, resid_ivars_2d
+    else
+        return flux_1d, ivar_1d, mask_1d
+    end
 end
 
 """
diff --git a/src/cal_build/make_relFlux.jl b/src/cal_build/make_relFlux.jl
index a471f608..cc52035f 100644
--- a/src/cal_build/make_relFlux.jl
+++ b/src/cal_build/make_relFlux.jl
@@ -131,7 +131,7 @@ end
         absthrpt, relthrpt, bitmsk_relthrpt = get_relFlux(fname)
         outfname = replace(
             replace(fname, "apred"=>"$(cal_type)_flats"), "ar1Dcal" => "$(cal_type)Flux")
-        safe_jld2save(outfname; absthrpt, relthrpt, bitmsk_relthrpt)
+        safe_jldsave(outfname; absthrpt, relthrpt, bitmsk_relthrpt)
     end
 end
 
diff --git a/src/run_scripts/make_runlist_all.jl b/src/run_scripts/make_runlist_all.jl
index f0a43526..0b7082e7 100644
--- a/src/run_scripts/make_runlist_all.jl
+++ b/src/run_scripts/make_runlist_all.jl
@@ -37,7 +37,12 @@ mjd_list = keys(f[parg["tele"]])
 for tstmjd in mjd_list
     df = DataFrame(read(f[parg["tele"] * "/$(tstmjd)/exposures"]))
     df.nreadInt = parse.(Int, df.nread)
-    expindx_list = findall((df.nreadInt .> 3))
+    good_exp = (df.nreadInt .> 3) .|
+               ((df.imagetyp .== "DomeFlat") .& (df.observatory .== "apo")) .|
+               ((df.imagetyp .== "QuartzFlat") .& (df.nreadInt .== 3))
+    expindx_list = findall(good_exp)
+    #    expindx_list = findall((df.nreadInt .> 3) .| (df.imagetyp .== "DomeFlat") .|
+    #                           (df.imagetyp .== "QuartzFlat"))
     for expindx in expindx_list
         push!(darks_mjd, parse(Int, tstmjd))
         push!(darks_expid, expindx)
diff --git a/src/run_scripts/plot_all.jl b/src/run_scripts/plot_all.jl
index 09568cb7..3e34139f 100644
--- a/src/run_scripts/plot_all.jl
+++ b/src/run_scripts/plot_all.jl
@@ -71,6 +71,75 @@ else
     [parg["expid"]]
 end
 
+list2Dexp = []
+for mjd in unique_mjds
+    f = h5open(parg["outdir"] * "almanac/$(parg["runname"]).h5")
+    df = DataFrame(read(f["$(parg["tele"])/$(mjd)/exposures"]))
+    close(f)
+    function get_2d_name_partial(expid)
+        parg["outdir"] * "/apred/$(mjd)/" *
+        replace(get_1d_name(expid, df), "ar1D" => "ar2Dresidualscal") * ".jld2"
+    end
+    local2D = get_2d_name_partial.(expid_list)
+    push!(list2Dexp, local2D)
+end
+all2Da = vcat(list2Dexp...)
+
+# per chip example 2D flux residuals from 1D extraction
+for chip in ["a", "b", "c"]
+    all2D = replace.(all2Da, "_a_" => "_$(chip)_")
+
+    thread = SlackThread()
+    if length(unique_mjds) > 1
+        min_mjd, max_mjd = extrema(unique_mjds)
+        thread("Here are some example 2D residuals after 1D extraction on chip $(chip) from $(parg["tele"]) for SJD $(min_mjd) to $(max_mjd)")
+    else
+        thread("Here are some example 2D residuals after 1D extraction on chip $(chip) from $(parg["tele"]) for SJD $(unique_mjds[1])")
+    end
+    rng = MersenneTwister(351 + unique_mjds[1])
+
+    # TODO parallelize plotting
+    # we should customize this to the exposures we want to see and types of stars we want
+    nsamp = minimum([length(all2D), 20])
+    sample_exposures = sample(rng, all2D, nsamp, replace = false)
+    f = h5open(parg["outdir"] * "almanac/$(parg["runname"]).h5")
+    for exp_fname in sample_exposures
+        sname = split(exp_fname, "_")
+        tele, mjd, chiploc, expid, = sname[(end - 4):(end - 1)]
+        exptype = sname[end][begin:(end - 5)]
+        expid_num = parse(Int, last(expid, 4))
+        flux_2d = load(exp_fname, "resid_flux")
+        ivar_2d = load(exp_fname, "resid_ivar")
+        resid_zscore = flux_2d .* (ivar_2d .^ 0.5)
+
+        fig = Figure(size = (840, 800), fontsize = 24)
+        ax = Axis(fig[1, 1])
+        hm = heatmap!(ax, resid_zscore,
+            colormap = :diverging_bkr_55_10_c35_n256,
+            colorrange = (-5, 5),
+            interpolate = false
+        )
+
+        text!(ax,
+            0.5, 1.05,
+            text = "Zscore Residuals\nTele: $(tele), MJD: $(mjd), ExpID: $(expid), Chip: $(chiploc)",
+            align = (:center, :bottom),
+            space = :relative
+        )
+
+        Colorbar(fig[1, 2], hm, width = 20, height = Relative(1.0))
+        colgap!(fig.layout, 1, 20)  # Set spacing between image 1 and colorbar 1
+        data_aspect = diff(hm[1][])[1] / (diff(hm[2][])[1])
+        colsize!(fig.layout, 1, Aspect(1, data_aspect))
+        resize_to_layout!(fig)
+
+        savePath = dirNamePlots *
+                   "ar2Dresidualscal_$(tele)_$(mjd)_$(chiploc)_$(expid)_$(exptype).png"
+        save(savePath, fig, px_per_unit = 3)
+        thread("$(exp_fname)", savePath)
+    end
+end
+
 all1Da = String[] # all 1D files for chip a
 for mjd in unique_mjds
     df = h5open(parg["outdir"] * "almanac/$(parg["runname"]).h5") do f


Reviews:
user: login: andrew-saydjari
body: Can you change this to more carefully cover only the cases where nread>3 is "not" required. They are given by https://github.com/andrew-saydjari/ApogeeReduction.jl/blob/41915f2dc17c4dc569c8f9a3e79698c6ff365ddc/pipeline.jl#L160C1-L165C12
state: COMMENTED
author_association: OWNER
submitted_at: 2025-04-05T02:40:43Z

Review comments:

PR comments:
user: login: KevinMcK95
created_at: 2025-04-04T19:51:18Z
body: Running test right now with LCO 60768 to check if the quartz flats are reduced from 2D to 1D 
author_association: COLLABORATOR
user: login: codecov[bot]
created_at: 2025-04-04T19:58:56Z
body: ## [Codecov](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/163?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) Report
Attention: Patch coverage is `0%` with `17 lines` in your changes missing coverage. Please review.
> Project coverage is 6.66%. Comparing base [(`4f0e558`)](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/commit/4f0e558f554c01b4fd179d3a81dd6e47c56bc554?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) to head [(`9274d11`)](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/commit/9274d119637cd3203b13fec714715ed3b405a022?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari).
> Report is 14 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/163?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) | Patch % | Lines |
|---|---|---|
| [src/ar1D.jl](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/163?src=pr&el=tree&filepath=src%2Far1D.jl&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari#diff-c3JjL2FyMUQuamw=) | 0.00% | [17 Missing :warning: ](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/163?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) |

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff            @@
##            main    #163      +/-   ##
========================================
- Coverage   6.73%   6.66%   -0.08%     
========================================
  Files         11      11              
  Lines       1009    1021      +12     
========================================
  Hits          68      68              
- Misses       941     953      +12     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/163?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: KevinMcK95
created_at: 2025-04-04T20:14:18Z
body: Tests passed! The quartz flats were successfully reduced to 1D, no error messages appeared in the logs, and the other 1D outputs don't appear to have changed.
author_association: COLLABORATOR
user: login: github-actions[bot]
created_at: 2025-04-09T19:01:38Z
body: Found use of `jldsave` in the following locations:

```
./pipeline.jl:252:        jldsave(
./pipeline.jl:303:        jldsave(
./pipeline_2d_1d.jl:163:        jldsave(outfname; flux_1d, ivar_1d, mask_1d, git_branch, git_commit)
./pipeline_2d_1d.jl:165:        jldsave(resid_outfname; resid_flux, resid_ivar, git_branch, git_commit)
```

Please use `safe_jld2save` from src/utils.jl instead. See #165 for more information.

(This comment will not be updated as changes are pushed. Check the workflow status for the up-to-date results.)
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-09T19:04:15Z
body: Found use of `jldsave` in the following locations:

```
./pipeline.jl:252:        jldsave(
./pipeline.jl:303:        jldsave(
./pipeline_2d_1d.jl:166:        jldsave(outfname; flux_1d, ivar_1d, mask_1d, git_branch, git_commit)
./pipeline_2d_1d.jl:168:        jldsave(resid_outfname; resid_flux, resid_ivar, git_branch, git_commit)
```

Please use `safe_jld2save` from src/utils.jl instead. See #165 for more information.

(This comment will not be updated as changes are pushed. Check the workflow status for the up-to-date results.)
author_association: CONTRIBUTOR
user: login: KevinMcK95
created_at: 2025-04-09T19:15:22Z
body: > Found use of `jldsave` in the following locations:
> 
> ```
> ./pipeline.jl:252:        jldsave(
> ./pipeline.jl:303:        jldsave(
> ./pipeline_2d_1d.jl:166:        jldsave(outfname; flux_1d, ivar_1d, mask_1d, git_branch, git_commit)
> ./pipeline_2d_1d.jl:168:        jldsave(resid_outfname; resid_flux, resid_ivar, git_branch, git_commit)
> ```
> 
> Please use `safe_jld2save` from src/utils.jl instead. See #165 for more information.
> 
> (This comment will not be updated as changes are pushed. Check the workflow status for the up-to-date results.)

@andrew-saydjari @ajwheeler Is this comment valid? Am I safe to change from `jldsave` to `safe_jldsave` for the examples above where I am using a mix of strings, float arrays, and bitmask arrays? 
author_association: COLLABORATOR
user: login: andrew-saydjari
created_at: 2025-04-09T19:18:57Z
body: Yes, if you merged main into this, then Adam has changed things to a warn to prevent it breaking to do what the bot says. We will still need to run these down and decide what to do, but we can do that uniformly from the warnings in the nightly logs.

So, I vote yes, do make those changes back, unless @ajwheeler disagrees.
author_association: OWNER
user: login: ajwheeler
created_at: 2025-04-09T19:20:11Z
body: Yes, with the caveat that this will be the test that I didn't introduce a bug in #168. 
author_association: COLLABORATOR
user: login: KevinMcK95
created_at: 2025-04-09T19:27:31Z
body: Sounds good. I'll submit an APO and LCO test right now to make sure everything is playing nicely. Then I'll be happy for a review and merge
author_association: COLLABORATOR
user: login: KevinMcK95
created_at: 2025-04-09T23:03:23Z
body: @ajwheeler @andrew-saydjari Passed the "run a day at APO and LCO" test, so I'm happy with this

author_association: COLLABORATOR