Repository: andrew-saydjari/ApogeeReduction.jl
Pull request #137: added obs time to ar2D and ar2Dcal files
Author: login: KevinMcK95
Created: 2025-03-05T17:26:40Z | Merged: 2025-03-07T20:57:31Z

Added `sec_since_mjd` to the ar2D and ar2Dcal files, where mjd_zero=1858-11-17T00:00:00.000


Diff:
diff --git a/pipeline.jl b/pipeline.jl
index 4287b732..7c1ea940 100644
--- a/pipeline.jl
+++ b/pipeline.jl
@@ -113,6 +113,7 @@ flush(stdout);
     using FITSIO, HDF5, FileIO, JLD2, Glob
     using DataFrames, EllipsisNotation, StatsBase
     using ParallelDataTransfer, SIRS, ProgressMeter
+    using AstroTime: TAIEpoch, modified_julian, days
 
     src_dir = "./"
     include(src_dir * "src/ar3D.jl")
@@ -147,7 +148,7 @@ git_branch, git_commit = initalize_git(src_dir);
         rawpath = build_raw_path(
             df.observatory[expid], df.mjd[expid], chip, df.exposure[expid])
         # decompress and convert apz data format to a standard 3D cube of reads
-        cubedat, hdr = apz2cube(rawpath)
+        cubedat, hdr_dict = apz2cube(rawpath)
 
         nread_total = size(cubedat, 3)
 
@@ -163,8 +164,39 @@ git_branch, git_commit = initalize_git(src_dir);
             firstind, extractMethod
         end
 
-        tdat = @view cubedat[:, :, firstind_loc:end]
-        nread_used = size(tdat, 3) - 1
+        # Might want lastind_loc as well to truncate when all reads are
+        # saturated (important for calculating the exposure mid time)
+        lastind_loc = size(cubedat, 3)
+
+        tdat = @view cubedat[:, :, firstind_loc:lastind_loc]
+        nread_used = size(tdat, 3) - 1 #this is actually nread_used-1 (ie it is truly ndiff_used)
+
+        n_read_dropped = firstind_loc - 1
+        first_image_start_time = TAIEpoch(hdr_dict[firstind_loc]["DATE-OBS"])
+        last_image_start_time = TAIEpoch(hdr_dict[lastind_loc]["DATE-OBS"])
+        dtime_read = (hdr_dict[firstind_loc]["INTOFF"] / 1000 / 3600 / 24)days #dt_read, orig in ms, convert to days
+        dtime_delay = (hdr_dict[firstind_loc]["INTDELAY"] / 3600 / 24)days #orig in seconds, convert to days
+        exptime_est = (hdr_dict[firstind_loc]["EXPTIME"] / 3600 / 24)days
+
+        # NOT using dtime_delay because we start directly at first_image_start_time
+        # (though we might need to think about this more: not sure what dtime_delay does)
+        # REMEMBER to add half of a dtime_read to shift to center of exposure
+
+        # Like DRP outputs (we think)
+        mjd_mid_exposure_old = modified_julian(first_image_start_time
+                                               +
+                                               0.5 * exptime_est * size(tdat, 3) / nread_total)
+        # Using dread_time*nread_USED
+        mjd_mid_exposure_rough = modified_julian(first_image_start_time
+                                                 +
+                                                 dtime_read *
+                                                 (0.5 + 0.5 * (size(tdat, 3) - 1)))
+        # Using times directly from header
+        mjd_mid_exposure_precise = modified_julian(first_image_start_time + 0.5 * dtime_read
+                                                   +
+                                                   0.5 *
+                                                   (last_image_start_time - first_image_start_time))
+        mjd_mid_exposure = mjd_mid_exposure_precise
 
         ## remove 1/f correlated noise (using SIRS.jl) [some preallocs would be helpful]
         if cor1fnoise
@@ -214,7 +246,9 @@ git_branch, git_commit = initalize_git(src_dir);
             "_")
         # probably change to FITS to make astronomers happy (this JLD2, which is HDF5, is just for debugging)
         jldsave(
-            joinpath(outdir, "apred/$(mjd)/" * outfname * ".jld2"); dimage, ivarimage, chisqimage, CRimage, nread_used, git_branch, git_commit)
+            joinpath(outdir, "apred/$(mjd)/" * outfname * ".jld2"); dimage, ivarimage, chisqimage, CRimage,
+            nread_used, mjd_mid_exposure_old, mjd_mid_exposure_rough, mjd_mid_exposure_precise,
+            mjd_mid_exposure, git_branch, git_commit)
         return joinpath(outdir, "apred/$(mjd)/" * outfname * ".jld2")
     end
 
@@ -226,6 +260,10 @@ git_branch, git_commit = initalize_git(src_dir);
         dimage = load(fname, "dimage")
         ivarimage = load(fname, "ivarimage")
         nread_used = load(fname, "nread_used")
+        mjd_mid_exposure_old = load(fname, "mjd_mid_exposure_old")
+        mjd_mid_exposure_rough = load(fname, "mjd_mid_exposure_rough")
+        mjd_mid_exposure_precise = load(fname, "mjd_mid_exposure_precise")
+        mjd_mid_exposure = load(fname, "mjd_mid_exposure")
         CRimage = load(fname, "CRimage")
         chisqimage = load(fname, "chisqimage")
 
@@ -257,7 +295,9 @@ git_branch, git_commit = initalize_git(src_dir);
 
         outfname = replace(fname, "ar2D" => "ar2Dcal")
         jldsave(
-            outfname; dimage, ivarimage, pix_bitmask, nread_used, git_branch, git_commit)
+            outfname; dimage, ivarimage, pix_bitmask, nread_used,
+            mjd_mid_exposure_old, mjd_mid_exposure_rough, mjd_mid_exposure_precise,
+            mjd_mid_exposure, git_branch, git_commit)
     end
 end
 t_now = now();
diff --git a/src/ar3D.jl b/src/ar3D.jl
index cfb229db..0f29593f 100644
--- a/src/ar3D.jl
+++ b/src/ar3D.jl
@@ -6,15 +6,18 @@ using FITSIO
 
 function apz2cube(fname)
     f = FITS(fname)
-    hdr = read_header(f[2])
+    #    hdr_dict = Dict("ave" => read_header(f[2]))
+    hdr_dict = Dict()
+    hdr_dict[1] = read_header(f[3])
     avg_dcounts = read(f[2])
     cubedat = zeros(Float32, size(avg_dcounts)..., length(f) - 2) #use float bc of NaN
     cubedat[:, :, 1] .= read(f[3])
     for i in 2:(length(f) - 2)
+        hdr_dict[i] = read_header(f[i + 2])
         cubedat[:, :, i] .= read(f[i + 2]) .+ avg_dcounts .+ cubedat[:, :, i - 1]
     end
     close(f)
-    return cubedat, hdr
+    return cubedat, hdr_dict
 end
 
 # feeling against this sort of subtraction, but we do see why one might want to do it in the darks
diff --git a/src/cal_build/traceExtract.jl b/src/cal_build/traceExtract.jl
index a0fe71a6..e3660f80 100644
--- a/src/cal_build/traceExtract.jl
+++ b/src/cal_build/traceExtract.jl
@@ -304,8 +304,8 @@ function trace_extract(image_data, ivar_image, tele, mjd, chip, expid;
     #only keep peaks that are substantially above the minimum function
     relative_fluxes = (poss_local_max_fluxes .- med_min_fluxes) ./
                       (med_max_fluxes .- med_min_fluxes)
-    good_max_inds = abs.(relative_fluxes .- 1.0) .< 0.5
-
+    good_max_inds = (abs.(relative_fluxes .- 1.0) .< 0.5) .& (local_max_waves .>= 11) .&
+                    (local_max_waves .<= 2048 - 11)
     good_y_vals = local_max_waves[good_max_inds]
 
     #fit 1D gaussians to each identified peak, using offset_inds around each peak


Reviews:
user: login: andrew-saydjari
body: After much discussion, bug finding, and the addition of the "_old" option, I am very happy and think that the "_precise" option is now our absolute best guess at the mid exposure time (without flux weighting shenanigans). This should be an improvement even over the apMADGICS times IMO. Maybe we will see a benefit in the RV consistency across longer time baselines.
state: APPROVED
author_association: OWNER
submitted_at: 2025-03-07T20:57:25Z

Review comments:

PR comments:
user: login: codecov[bot]
created_at: 2025-03-05T17:35:33Z
body: ## [Codecov](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/137?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) Report
Attention: Patch coverage is `0%` with `4 lines` in your changes missing coverage. Please review.
> Project coverage is 6.57%. Comparing base [(`c25a1af`)](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/commit/c25a1af50f80b05e86a4aa864c035a7e840b4f82?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) to head [(`60882f2`)](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/commit/60882f21848dfea2235a2dd77e3988a11901c956?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari).
> Report is 25 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/137?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) | Patch % | Lines |
|---|---|---|
| [src/ar3D.jl](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/137?src=pr&el=tree&filepath=src%2Far3D.jl&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari#diff-c3JjL2FyM0Quamw=) | 0.00% | [4 Missing :warning: ](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/137?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari) |

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff            @@
##            main    #137      +/-   ##
========================================
- Coverage   6.59%   6.57%   -0.02%     
========================================
  Files         10      10              
  Lines        910     912       +2     
========================================
  Hits          60      60              
- Misses       850     852       +2     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/andrew-saydjari/ApogeeReduction.jl/pull/137?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Andrew+Saydjari).

<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: KevinMcK95
created_at: 2025-03-05T20:37:35Z
body: Changed from using `Dates` to using `AstroTime` for MJD definition. Also using the timing information from the header of the fits files directly (i.e. start time, read time, delay time). 
author_association: COLLABORATOR
user: login: KevinMcK95
created_at: 2025-03-05T23:16:24Z
body: Changed so the header times are properly using `TAIEpoch` instead of UTC (ignoring what the header claims). Provides 2 midpoint time estimates:
1. `first_image_start_time + dtime_read*(0.5 + n_read_DROPPED + 0.5 * (n_read_USED-1))`
2. `first_image_start_time + 0.5*dtime_read + 0.5*(last_image_start_time - first_image_start_time)`
author_association: COLLABORATOR
user: login: KevinMcK95
created_at: 2025-03-07T20:00:46Z
body: Added in third option for timing, based off of the DRP/old method:

3. `first_image_start_time +  0.5*exp_time*nreads_USED/nreads_TOTAL`
author_association: COLLABORATOR