Repository: desihub/desitransfer

Issue:
Transfer 'NightSummary' in night logs from KPNO to NERSC
With the rsynch of nightlogs/, also transfer all files that include the string 'NightSummary' from KPNO to NERSC. They do not have a '_kpno' tag, but they are only written at KPNO.

Discussion:

Patch:
diff --git a/bin/desi_tucson_transfer.sh b/bin/desi_tucson_transfer.sh
index c93e0a8..7f4e01a 100755
--- a/bin/desi_tucson_transfer.sh
+++ b/bin/desi_tucson_transfer.sh
@@ -31,7 +31,7 @@ set -o noglob
 #
 # Static data sets don't need to be updated as frequently.
 #
-static='protodesi public/epo spectro/redux/andes spectro/redux/blanc spectro/redux/cascades spectro/redux/minisv2 spectro/redux/oak1'
+static='protodesi public/epo spectro/redux/andes spectro/redux/blanc spectro/redux/cascades spectro/redux/denali spectro/redux/minisv2 spectro/redux/oak1'
 #
 # Dynamic data sets may change daily.
 #
diff --git a/doc/changes.rst b/doc/changes.rst
index e924ea0..2458a37 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,7 +5,9 @@ Change Log
 0.6.1 (unreleased)
 ------------------
 
-* No changes yet.
+* Sync ``NightSummaryYYYYMMDD`` files; adjust HPSS utility path (PR `#35`_).
+
+.. _`#35`: https://github.com/desihub/desitransfer/pull/35
 
 0.6.0 (2021-04-06)
 ------------------
diff --git a/py/desitransfer/daemon.py b/py/desitransfer/daemon.py
index 26935b4..247c987 100644
--- a/py/desitransfer/daemon.py
+++ b/py/desitransfer/daemon.py
@@ -410,8 +410,8 @@ The DESI Collaboration Account
                 os.remove(ls_file)
             except FileNotFoundError:
                 log.debug("Failed to remove %s because it didn't exist. That's OK.", ls_file)
-            cmd = ['/usr/common/mss/bin/hsi', '-O', ls_file,
-                   'ls', '-l', d.hpss]
+            cmd = [os.path.join(self.conf['common']['hpss'], 'hsi'),
+                   '-O', ls_file, 'ls', '-l', d.hpss]
             if self.tape:
                 log.debug(' '.join(cmd))
                 _, out, err = _popen(cmd)
@@ -436,7 +436,7 @@ The DESI Collaboration Account
                     start_dir = os.getcwd()
                     log.debug("os.chdir('%s')", d.destination)
                     os.chdir(d.destination)
-                    cmd = ['/usr/common/mss/bin/htar',
+                    cmd = [os.path.join(self.conf['common']['hpss'], 'htar'),
                            '-cvhf', os.path.join(d.hpss, backup_file),
                            '-H', 'crc:verify=all',
                            night]
diff --git a/py/desitransfer/data/desi_nightlog_transfer_kpno.txt b/py/desitransfer/data/desi_nightlog_transfer_kpno.txt
index 7d02c80..f8905e9 100644
--- a/py/desitransfer/data/desi_nightlog_transfer_kpno.txt
+++ b/py/desitransfer/data/desi_nightlog_transfer_kpno.txt
@@ -1,4 +1,5 @@
 *kpno*
+NightSummary????????
 DataQualityAssessment
 OperationsScientist
 OtherInput
diff --git a/py/desitransfer/data/desi_transfer_daemon.ini b/py/desitransfer/data/desi_transfer_daemon.ini
index 85de2d2..d12b90d 100644
--- a/py/desitransfer/data/desi_transfer_daemon.ini
+++ b/py/desitransfer/data/desi_transfer_daemon.ini
@@ -33,6 +33,8 @@ backup = 20
 sleep = 1
 # Path to ssh.
 ssh = /bin/ssh
+# Path for HPSS utilities.
+hpss = /usr/local/bin
 
 #
 # Log file configuration.


Test patch:
diff --git a/py/desitransfer/test/test_daemon.py b/py/desitransfer/test/test_daemon.py
index 85ac578..53d29a0 100644
--- a/py/desitransfer/test/test_daemon.py
+++ b/py/desitransfer/test/test_daemon.py
@@ -746,8 +746,9 @@ total size is 118,417,836,324  speedup is 494,367.55
             f.write(self.fake_hsi1)
         s = transfer.backup(c[0], '20190703', mock_status)
         self.assertFalse(s)
+        hsi = os.path.join(transfer.conf['common']['hpss'], 'hsi')
         mock_log.debug.assert_has_calls([call("os.remove('%s')", ls_file),
-                                         call("/usr/common/mss/bin/hsi -O %s ls -l desi/spectro/data" % ls_file),
+                                         call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
                                          call("Backup of %s already complete.", '20190703')])
         mock_rm.assert_called_once_with(ls_file)
         mock_status.assert_not_called()
@@ -787,15 +788,17 @@ total size is 118,417,836,324  speedup is 494,367.55
         mock_rm.side_effect = FileNotFoundError
         s = transfer.backup(c[0], '20190703', mock_status)
         self.assertTrue(s)
+        hsi = os.path.join(transfer.conf['common']['hpss'], 'hsi')
+        htar = os.path.join(transfer.conf['common']['hpss'], 'htar')
         mock_chdir.assert_has_calls([call('/desi/root/spectro/data'),
                                      call(self.tmp.name)])
         mock_log.info.assert_has_calls([call('No files appear to have changed in %s.', '20190703')])
         mock_log.debug.assert_has_calls([call("os.remove('%s')", ls_file),
                                          call("Failed to remove %s because it didn't exist. That's OK.", ls_file),
-                                         call("/usr/common/mss/bin/hsi -O %s ls -l desi/spectro/data" % ls_file),
+                                         call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
                                          call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
                                          call("os.chdir('%s')", '/desi/root/spectro/data'),
-                                         call('/usr/common/mss/bin/htar -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703'),
+                                         call('%s -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703' % htar),
                                          call("os.chdir('%s')", self.tmp.name)])
         mock_status.assert_not_called()
         mock_status.update.assert_not_called()
@@ -828,7 +831,8 @@ total size is 118,417,836,324  speedup is 494,367.55
             f.write(self.fake_hsi2)
         s = transfer.backup(c[0], '20190703', mock_status)
         self.assertTrue(s)
-        mock_popen.assert_has_calls([call(['/usr/common/mss/bin/htar', '-cvhf', 'desi/spectro/data/desi_spectro_data_20190703.tar', '-H', 'crc:verify=all', '20190703'])])
+        htar = os.path.join(transfer.conf['common']['hpss'], 'htar')
+        mock_popen.assert_has_calls([call([htar, '-cvhf', 'desi/spectro/data/desi_spectro_data_20190703.tar', '-H', 'crc:verify=all', '20190703'])])
         mock_status.assert_not_called()
         mock_status.update.assert_not_called()
 
@@ -861,11 +865,13 @@ total size is 118,417,836,324  speedup is 494,367.55
         mock_getcwd.return_value = 'HOME'
         s = transfer.backup(c[0], '20190703', mock_status)
         self.assertTrue(s)
+        hsi = os.path.join(transfer.conf['common']['hpss'], 'hsi')
+        htar = os.path.join(transfer.conf['common']['hpss'], 'htar')
         mock_log.debug.assert_has_calls([call("os.remove('%s')", os.path.join(self.tmp.name, 'desi_spectro_data.txt')),
-                                         call("/usr/common/mss/bin/hsi -O %s ls -l desi/spectro/data" % ls_file),
+                                         call("%s -O %s ls -l desi/spectro/data" % (hsi, ls_file)),
                                          call('/bin/rsync --dry-run --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/data/dts/exposures/raw/20190703/ /desi/root/spectro/data/20190703/'),
                                          call("os.chdir('%s')", '/desi/root/spectro/data'),
-                                         call('/usr/common/mss/bin/htar -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703'),
+                                         call('%s -cvhf desi/spectro/data/desi_spectro_data_20190703.tar -H crc:verify=all 20190703' % htar),
                                          call("os.chdir('%s')", 'HOME')])
         mock_log.warning.assert_has_calls([call('New files detected in %s!', '20190703'),
                                            call('No updated exposures in night %s detected.', '20190703')])