Repository: UCBerkeleySETI/turbo_seti (language: python)

Issue:
Enhancement request: show differences between two dat files
Two processes in succession:
1) For each entry in dat file dat1, look for a match in dat file dat2.
2) For each entry in dat file dat2, look for a match in dat file dat1.

The comparison between dataframe entries uses the following dataframe columns:
* Coarse channel number (exact match)
* Frequency (within rtol) where rtol is the {math,numpy}.isclose() value (E.g. 0.0001 which signifies 0.01%)
* Drift rate (within rtol)



Discussion / hints:

Fix patch:
diff --git a/VERSION-HISTORY.md b/VERSION-HISTORY.md
index 2ae9bc7..28497cc 100644
--- a/VERSION-HISTORY.md
+++ b/VERSION-HISTORY.md
@@ -4,6 +4,7 @@ This file is a version history of turbo_seti amendments, beginning with version
 
 | `YYYY_MM_DD` | `Version` | `Contents` |
 | :--: | :--: | :-- |
+| 2022-05-16 | 2.3.0 | New utility (dat_diff) to show differences between two dat files (issue  #311). |
 | 2022-05-05 | 2.2.4 | Bug fix in the logic for removing duplicate hits. |
 | 2022-05-05 | 2.2.3 | Bug fix in drift rate calculation. |
 | 2022-04-04 | 2.2.2 | Performance improvement in GPU mode: Use a cupy RawKernel for the 'flt' function. |
diff --git a/setup.py b/setup.py
index 2fe8554..b2eac6c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-__version__ = "2.2.4"
+__version__ = "2.3.0"
 
 with open("turbo_seti/find_doppler/turbo_seti_version.py", "w") as fh:
     fh.write("TURBO_SETI_VERSION = '{}'\n".format(__version__))
@@ -16,8 +16,9 @@
 
 entry_points = {
     "console_scripts": [
-        "turboSETI = turbo_seti.find_doppler.seti_event:main",
-        "plotSETI = turbo_seti.find_event.run_pipelines:main",
+        "turboSETI  = turbo_seti.find_doppler.seti_event:main",
+        "dat_diff   = turbo_seti.find_doppler.dat_diff:main",
+        "plotSETI   = turbo_seti.find_event.run_pipelines:main",
         "dat_filter = turbo_seti.find_event.dat_filter:main",
     ]
 }
diff --git a/turbo_seti/find_doppler/__init__.py b/turbo_seti/find_doppler/__init__.py
index 587f490..7e3dc26 100644
--- a/turbo_seti/find_doppler/__init__.py
+++ b/turbo_seti/find_doppler/__init__.py
@@ -1,2 +1,2 @@
-from . import seti_event, helper_functions, file_writers, data_handler, merge_dats_logs
+from . import seti_event, helper_functions, file_writers, data_handler, merge_dats_logs, dat_diff
 from .find_doppler import FindDoppler
diff --git a/turbo_seti/find_doppler/dat_diff.py b/turbo_seti/find_doppler/dat_diff.py
new file mode 100644
index 0000000..587a3f9
--- /dev/null
+++ b/turbo_seti/find_doppler/dat_diff.py
@@ -0,0 +1,164 @@
+"""
+dat_diff - show differences between two dat files.
+
+Two processes in succession:
+1) For each entry in dat file dat1, look for a match in dat file dat2.
+2) For each entry in dat file dat2, look for a match in dat file dat1.
+
+The comparison between dataframe entries uses the following dataframe columns:
+* Coarse channel number (exact match)
+* Frequency (within rtol) - see numpy.isclose() or math.isclose() for definition.
+* Drift rate (within rtol)
+"""
+
+import os
+import sys
+
+from argparse import ArgumentParser
+import pandas as pd
+import numpy as np
+
+DEBUGGING = False
+SEP = r'\s+'
+
+
+def oops(msg):
+    """
+    Display an error message and exit to the O/S.
+
+    Parameters
+    ----------
+    msg : str
+        Error message for display.
+
+    Returns
+    -------
+    None.
+
+    """
+    print(f"\n*** Oops, {msg} !!")
+    sys.exit(86)
+
+
+def finder(df1, path1, df2, path2, etol):
+    """
+    For each entry in data frame df1, try to find a matching entry in data frame df2.
+
+    Parameters
+    ----------
+    df1 : Pandas DataFrame
+        Reference dataframe.
+    path1 : str
+        Path of external file where df1 was sourced from.
+    df2 : str
+        Dataframe to be searched for a match.
+    path2 : str
+        Path of external file where df2 was sourced from..
+    etol : float
+        Relative error tolerance.  See RTOL parameter in numpy.isclose() or math.isclose().
+
+    Returns
+    -------
+    None.
+
+    """
+
+    # Extract file names.
+    file1 = os.path.basename(path1)
+    file2 = os.path.basename(path2)
+    print(f"\n===== For each record in {file1}, try to find a corresponding one in {file2}.")
+    ndf1 = len(df1)
+    ndf2 = len(df2)
+    print(f"===== {ndf1} records in {file1}, {ndf2} records in {file2}.")
+
+    # Main loop.
+    notfound_count = 0
+    for ii in range(ndf1):
+        if DEBUGGING:
+            print(f"DEBUG ii={ii}")
+        drate1 = df1[1][ii]
+        snr1 = df1[2][ii]
+        freq1 = df1[3][ii]
+        cchan1 = df1[10][ii]
+        flag_match = False
+        for jj in range(ndf2):
+            if DEBUGGING:
+                print(f"DEBUG ii={ii}, jj={jj}")
+            drate2 = df2[1][jj]
+            #snr2 = df2[2][jj]
+            freq2 = df2[3][jj]
+            cchan2 = df2[10][jj]
+            if cchan1 != cchan2:
+                continue
+            if not np.isclose(drate1, drate2, rtol=etol):
+                continue
+            if not np.isclose(freq1, freq2, rtol=etol):
+                continue
+            flag_match = True
+            break
+        if not flag_match:
+            notfound_count += 1
+            print(f"{file1} {ii} not found in {file2} >  Coarse  {cchan1}  drate  {drate1}  Freq  {freq1}  SNR  {snr1}")
+
+    # The End.
+    if notfound_count == 0:
+        print(f"{file1} : {file2} > All found.")
+    else:
+        print(f"{file1} : {file2} > Not found count = {notfound_count}")
+
+
+def main(args=None):
+    """
+    Executable program entry point.
+    If args=None, then the command-line holds the list of arguments;
+    Else, args holds the list of arguments.
+
+    Parameters
+    ----------
+    args : list, optional
+        List of arguments. The default is None.
+
+    Returns
+    -------
+    None.
+
+    """
+
+    # Parse and validate arguments.
+    parser = ArgumentParser(description="Find differences between 2 DAT files.")
+    parser.add_argument("dat1", type=str,
+                        help="Path of the 1st DAT file.")
+    parser.add_argument("dat2", type=str,
+                        help="Path of the 2nd DAT file.")
+    parser.add_argument("-e", "--etol", dest="etol", type=float, default=0.0001,
+                        help="Relative error tolerance (see numpy.isclose() or math.isclose(). Default=0.0001 (0.01%%).")
+    if args is None:
+        args = parser.parse_args()
+    else:
+        args = parser.parse_args(args)
+    print(f"Relative error tolerance = {args.etol}")
+
+    # Make sure that the DAT path #1 exists.
+    # Load the data frames.
+    if not os.path.exists(args.dat1):
+        oops(f"DAT file {args.dat1} does not exist")
+    df1 = pd.read_csv(args.dat1, header=None, sep=SEP, engine="python", comment="#")
+    nrows1 = len(df1)
+    if nrows1 < 1:
+        oops(f"DAT file {args.dat1} is empty")
+
+    # Make sure that the DAT path #2 exists.
+    # Load the data frames.
+    if not os.path.exists(args.dat2):
+        oops(f"DAT file {args.dat2} does not exist")
+    df2 = pd.read_csv(args.dat2, header=None, sep=SEP, engine="python", comment="#")
+    nrows2 = len(df2)
+    if nrows2 < 1:
+        oops(f"DAT file {args.dat2} is empty")
+
+    finder(df1, args.dat1, df2, args.dat2, args.etol)
+    finder(df2, args.dat2, df1, args.dat1, args.etol)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/turbo_seti/find_doppler/turbo_seti_version.py b/turbo_seti/find_doppler/turbo_seti_version.py
index 1c513bb..c1d4f59 100644
--- a/turbo_seti/find_doppler/turbo_seti_version.py
+++ b/turbo_seti/find_doppler/turbo_seti_version.py
@@ -1,1 +1,1 @@
-TURBO_SETI_VERSION = '2.2.3'
+TURBO_SETI_VERSION = '2.3.0'


Test patch:
diff --git a/test/pipelines_util.py b/test/pipelines_util.py
index 566e1d1..caf5642 100644
--- a/test/pipelines_util.py
+++ b/test/pipelines_util.py
@@ -1,4 +1,4 @@
-r'test_pipelines utilities'
+r'test_pipelines_N.py utilities'
 
 import numpy as np
 import pandas as pd
diff --git a/test/test_dat_diff.py b/test/test_dat_diff.py
new file mode 100644
index 0000000..42dc579
--- /dev/null
+++ b/test/test_dat_diff.py
@@ -0,0 +1,39 @@
+""" Test find_doppler/dat_diff.py """
+import os
+import pytest
+from turbo_seti.find_doppler.dat_diff import main
+
+TESTDIR = os.path.split(os.path.abspath(__file__))[0]
+VOYADAT = os.path.join(TESTDIR, "Voyager1.single_coarse.fine_res.dat")
+VOYADATFLIPPED = os.path.join(TESTDIR, "Voyager1.single_coarse.fine_res.flipped.dat")
+
+
+@pytest.mark.order(index=-3)
+def test_dat_diff_help(capsys):
+
+    with pytest.raises(SystemExit) as exit_code:
+        args = ["-h"]
+        main(args)
+    out, err = capsys.readouterr()
+    print(out, err)
+    assert exit_code.type == SystemExit
+    assert exit_code.value.code == 0
+
+
+@pytest.mark.order(index=-2)
+def test_dat_diff_2_dats():
+
+    args = [VOYADAT, VOYADATFLIPPED]
+    main(args)
+
+
+@pytest.mark.order(index=-1)
+def test_dat_diff_missing_dat(capsys):
+
+    with pytest.raises(SystemExit) as exit_code:
+        args = ["nonexistent.dat", VOYADATFLIPPED]
+        main(args)
+    out, err = capsys.readouterr()
+    print(out, err)
+    assert exit_code.type == SystemExit
+    assert exit_code.value.code != 0
diff --git a/test/test_dat_filter.py b/test/test_dat_filter.py
index d8c124c..0f26a33 100644
--- a/test/test_dat_filter.py
+++ b/test/test_dat_filter.py
@@ -1,6 +1,4 @@
 r'''
-TESTDAT_filter.py
-
 Test dat_filter.
 '''
 
diff --git a/test/test_pipelines_1.py b/test/test_pipelines_1.py
index 18e1b26..3b382fc 100644
--- a/test/test_pipelines_1.py
+++ b/test/test_pipelines_1.py
@@ -1,5 +1,5 @@
 r'''
-test_pipelines.py
+test_pipelines_1.py
 
 Using the 0000.h5 Voyager 2020 set of HDF5 files
 from http://blpd14.ssl.berkeley.edu/voyager_2020/single_coarse_channel/
diff --git a/test/test_setup.py b/test/test_setup.py
index 7e694d6..68efbb0 100644
--- a/test/test_setup.py
+++ b/test/test_setup.py
@@ -1,7 +1,7 @@
-r""" Testspectra_gen functions"""
+r""" Test setup.py"""
 
 
 def test_setup():
     import os
     cmd = "python3 setup.py check"
-    os.system(cmd)
\ No newline at end of file
+    os.system(cmd)