Repository: desihub/desitransfer

Issue:
Shift nightwatch transfer to daily engineering data transfer
Nightwatch is now primarily being run at NERSC, so there is no need for near-real-time transfers of nightwatch data produced at KPNO.

Comments:

Patch:
diff --git a/bin/desi_tucson_transfer.sh b/bin/desi_tucson_transfer.sh
index 94f1cc0..77cb2d5 100755
--- a/bin/desi_tucson_transfer.sh
+++ b/bin/desi_tucson_transfer.sh
@@ -29,7 +29,7 @@ set -o noglob
 #
 # Static data sets don't need to be updated as frequently.
 #
-static='protodesi public/ets spectro/redux/andes spectro/redux/minisv2 spectro/redux/oak1'
+static='protodesi public/epo public/ets spectro/redux/andes spectro/redux/minisv2 spectro/redux/oak1'
 #
 # Dynamic data sets may change daily.
 #
diff --git a/doc/changes.rst b/doc/changes.rst
index 81a4637..f41a9af 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,7 +5,12 @@ Change Log
 0.3.9 (unreleased)
 ------------------
 
-* No changes yet.
+* Deprecate continuous nightwatch transfers; nightwatch is now part of the
+  daily engineering transfer (PR `#21`_).
+* Allow alternate scratch directory to be chosen if :envvar:`CSCRATCH` is
+  unavailable (PR `#21`_).
+
+.. _`#21`: https://github.com/desihub/desitransfer/pull/21
 
 0.3.8 (2020-10-26)
 ------------------
diff --git a/py/desitransfer/common.py b/py/desitransfer/common.py
index 0b9bb1b..6eba487 100644
--- a/py/desitransfer/common.py
+++ b/py/desitransfer/common.py
@@ -81,6 +81,35 @@ def stamp(zone='US/Pacific'):
     return n.astimezone(tz).strftime('%Y-%m-%d %H:%M:%S %Z')
 
 
+def ensure_scratch(primary, alternate):
+    """Try an alternate temporary directory if the primary temporary directory
+    is unavilable.
+
+    Parameters
+    ----------
+    primary : :class:`str`
+        Primary temporary directory.
+    alternate : :class:`list`
+        A list of alternate directories.
+
+    Returns
+    -------
+    The first available temporary directory found.
+    """
+    if not isinstance(alternate, list):
+        alternate = [alternate]
+    try:
+        l = os.listdir(primary)
+    except FileNotFoundError:
+        for a in alternate:
+            try:
+                l = os.listdir(a)
+            except FileNotFoundError:
+                continue
+            return a
+    return primary
+
+
 def yesterday():
     """Yesterday's date in DESI "NIGHT" format, YYYYMMDD.
     """
diff --git a/py/desitransfer/daemon.py b/py/desitransfer/daemon.py
index a3b5813..7113f27 100644
--- a/py/desitransfer/daemon.py
+++ b/py/desitransfer/daemon.py
@@ -24,7 +24,7 @@ from socket import getfqdn
 from tempfile import TemporaryFile
 from pkg_resources import resource_filename
 from desiutil.log import get_logger
-from .common import dir_perm, file_perm, rsync, yesterday, empty_rsync
+from .common import dir_perm, file_perm, rsync, yesterday, empty_rsync, ensure_scratch
 from .status import TransferStatus
 from . import __version__ as dtVersion
 
@@ -94,7 +94,7 @@ class TransferDaemon(object):
                                             self.conf[s].getlist('expected_files'),
                                             self.conf[s]['checksum_file'])
                             for s in self.sections]
-        self.scratch = self.conf['common']['scratch']
+        self.scratch = ensure_scratch(self.conf['common']['scratch'], self.conf['common']['alternate_scratch'].split(','))
         self._configure_log(options.debug)
         return
 
@@ -365,7 +365,7 @@ The DESI Collaboration Account
             #
             pass
         else:
-            log.error('rsync problem detected!')
+            log.error('rsync problem detected for %s/%s!', night, exposure)
             log.debug("status.update('%s', '%s', 'rsync', failure=True)", night, exposure)
             status.update(night, exposure, 'rsync', failure=True)
 
diff --git a/py/desitransfer/daily.py b/py/desitransfer/daily.py
index 642825d..0ce4c30 100644
--- a/py/desitransfer/daily.py
+++ b/py/desitransfer/daily.py
@@ -11,6 +11,7 @@ import subprocess as sub
 import sys
 import time
 from argparse import ArgumentParser
+from pkg_resources import resource_filename
 from .common import dir_perm, file_perm, rsync, stamp
 from . import __version__ as dtVersion
 
@@ -24,20 +25,26 @@ class DailyDirectory(object):
         Source directory.
     destination : :class:`str`
         Desitination directory.
+    extra : :class:`list`, optional
+        Extra :command:`rsync` arguments to splice into command.
+    dirlinks : :class:`bool`, optional
+        If ``True``, convert source links into linked directory.
     """
 
-    def __init__(self, source, destination):
+    def __init__(self, source, destination, extra=[], dirlinks=False):
         self.source = source
         self.destination = destination
         self.log = self.destination + '.log'
+        self.extra = extra
+        self.dirlinks = dirlinks
 
-    def transfer(self, apache=True):
+    def transfer(self, permission=True):
         """Data transfer operations for a single destination directory.
 
         Parameters
         ----------
-        apache : :class:`bool`
-            If ``True`` set file ACLs for Apache httpd access.
+        permission : :class:`bool`, optional
+            If ``True``, set permissions for DESI collaboration access.
 
         Returns
         -------
@@ -45,6 +52,11 @@ class DailyDirectory(object):
             The status returned by :command:`rsync`.
         """
         cmd = rsync(self.source, self.destination)
+        if not self.dirlinks:
+            cmd[cmd.index('--copy-dirlinks')] = '--links'
+        if self.extra:
+            for i, e in enumerate(self.extra):
+                cmd.insert(cmd.index('--omit-dir-times') + 1 + i, e)
         with open(self.log, 'ab') as l:
             l.write(("DEBUG: desi_daily_transfer %s\n" % dtVersion).encode('utf-8'))
             l.write(("DEBUG: %s\n" % stamp()).encode('utf-8'))
@@ -54,8 +66,8 @@ class DailyDirectory(object):
             status = p.wait()
         if status == 0:
             self.lock()
-            if apache:
-                s = self.apache()
+            if permission:
+                s = self.permission()
         return status
 
     def lock(self):
@@ -66,8 +78,8 @@ class DailyDirectory(object):
             for f in filenames:
                 os.chmod(os.path.join(dirpath, f), file_perm)
 
-    def apache(self):
-        """Grant apache/www read access.
+    def permission(self):
+        """Set permissions for DESI collaboration access.
 
         In theory this should not change any permissions set by
         :meth:`~DailyDirectory.lock`.
@@ -90,16 +102,20 @@ def _config():
     """Wrap configuration so that module can be imported without
     environment variables set.
     """
+    nightwatch_exclude = resource_filename('desitransfer',
+                                           'data/desi_nightwatch_transfer_exclude.txt')
     engineering = os.path.realpath(os.path.join(os.environ['DESI_ROOT'],
                                                 'engineering'))
     spectro = os.path.realpath(os.path.join(os.environ['DESI_ROOT'],
                                             'spectro'))
     return [DailyDirectory('/exposures/desi/sps',
                            os.path.join(engineering, 'spectrograph', 'sps')),
-            # DailyDirectory('/exposures/nightwatch',
-            #                os.path.join(spectro, 'nightwatch', 'kpno')),
+            DailyDirectory('/exposures/nightwatch',
+                           os.path.join(spectro, 'nightwatch', 'kpno'),
+                           extra=['--exclude-from', nightwatch_exclude]),
             DailyDirectory('/data/dts/exposures/lost+found',
-                           os.path.join(spectro, 'staging', 'lost+found')),
+                           os.path.join(spectro, 'staging', 'lost+found'),
+                           dirlinks=True),
             # DailyDirectory('/data/fxc',
             #                os.path.join(engineering, 'fxc')),
             DailyDirectory('/data/focalplane/logs/calib_logs',
@@ -127,8 +143,6 @@ def _options(*args):
     """
     desc = "Transfer non-critical DESI data from KPNO to NERSC."
     prsr = ArgumentParser(description=desc)
-    prsr.add_argument('-A', '--no-apache', action='store_false', dest='apache',
-                      help='Do not set ACL for Apache httpd access.')
     # prsr.add_argument('-b', '--backup', metavar='H', type=int, default=20,
     #                   help='UTC time in hours to trigger HPSS backups (default %(default)s:00 UTC).')
     # prsr.add_argument('-d', '--debug', action='store_true',
@@ -142,8 +156,8 @@ def _options(*args):
                       help="Exit the script when FILE is detected (default %(default)s).")
     # prsr.add_argument('-n', '--nersc', default='cori', metavar='NERSC_HOST',
     #                   help="Trigger DESI pipeline on this NERSC system (default %(default)s).")
-    # prsr.add_argument('-P', '--no-pipeline', action='store_false', dest='pipeline',
-    #                   help="Only transfer files, don't start the DESI pipeline.")
+    prsr.add_argument('-P', '--no-permission', action='store_false', dest='permission',
+                      help='Do not set permissions for DESI collaboration access.')
     prsr.add_argument('-s', '--sleep', metavar='H', type=int, default=24,
                       help='In daemon mode, sleep H hours before checking for new data (default %(default)s hours).')
     # prsr.add_argument('-S', '--shadow', action='store_true',
@@ -167,7 +181,7 @@ def main():
             print("INFO: %s detected, shutting down daily transfer script." % options.kill)
             return 0
         for d in _config():
-            status = d.transfer(apache=options.apache)
+            status = d.transfer(permission=options.permission)
             if status != 0:
                 print("ERROR: rsync problem detected for {0.source} -> {0.destination}!".format(d))
                 return status
diff --git a/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt b/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt
index 2f1f1fa..446d0a0 100644
--- a/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt
+++ b/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt
@@ -1,3 +1,9 @@
-*/preproc*.fits
-*/qsky-*.fits
-*/*.tmp
+preproc*.fits
+qsky-*.fits
+*.tmp
+nightwatch.*
+nightwatch-debug.*
+nightwatch-webapp.*
+webapp.log
+redux
+test
diff --git a/py/desitransfer/data/desi_transfer_daemon.ini b/py/desitransfer/data/desi_transfer_daemon.ini
index c1a8ade..222baed 100644
--- a/py/desitransfer/data/desi_transfer_daemon.ini
+++ b/py/desitransfer/data/desi_transfer_daemon.ini
@@ -22,7 +22,7 @@ checksum_file = checksum-{exposure}.sha256sum
 [common]
 # Use this directory for temporary files.
 scratch = ${CSCRATCH}
-# scratch = ${HOME}/tmp
+alternate_scratch = ${HOME}/tmp
 # The presence of this file indicates checksums are being computed.
 checksum_lock = /tmp/checksum-running
 # UTC time in hours to look for delayed files.


Test patch:
diff --git a/py/desitransfer/test/test_common.py b/py/desitransfer/test/test_common.py
index 1642616..8be6d5e 100644
--- a/py/desitransfer/test/test_common.py
+++ b/py/desitransfer/test/test_common.py
@@ -6,7 +6,8 @@ import datetime
 import os
 import unittest
 from unittest.mock import patch
-from ..common import dir_perm, file_perm, empty_rsync, rsync, stamp, yesterday, today
+from tempfile import TemporaryDirectory
+from ..common import dir_perm, file_perm, empty_rsync, rsync, stamp, ensure_scratch, yesterday, today
 
 
 class TestCommon(unittest.TestCase):
@@ -22,10 +23,14 @@ class TestCommon(unittest.TestCase):
         pass
 
     def setUp(self):
-        pass
+        """Create a temporary directory to simulate CSCRATCH.
+        """
+        self.tmp = TemporaryDirectory()
 
     def tearDown(self):
-        pass
+        """Clean up temporary directory.
+        """
+        self.tmp.cleanup()
 
     def test_permissions(self):
         """Ensure that file and directory permissions do not change.
@@ -72,6 +77,19 @@ total size is 118,417,836,324  speedup is 494,367.55
         s = stamp('US/Arizona')
         self.assertEqual(s, '2019-07-03 05:00:00 MST')
 
+    def test_ensure_scratch(self):
+        """Test ensure_scratch.
+        """
+        tmp = self.tmp.name
+        t = ensure_scratch(tmp, ['/foo', '/bar'])
+        self.assertEqual(t, tmp)
+        t = ensure_scratch('/foo', tmp)
+        self.assertEqual(t, tmp)
+        t = ensure_scratch('/foo', ['/bar', tmp])
+        self.assertEqual(t, tmp)
+        t = ensure_scratch('/foo', ['/bar', '/abcdefg', tmp])
+        self.assertEqual(t, tmp)
+
     @patch('desitransfer.common.dt')
     def test_yesterday(self, mock_dt):
         """Test yesterday's date.
diff --git a/py/desitransfer/test/test_daemon.py b/py/desitransfer/test/test_daemon.py
index 0944a75..fa0551e 100644
--- a/py/desitransfer/test/test_daemon.py
+++ b/py/desitransfer/test/test_daemon.py
@@ -288,7 +288,7 @@ class TestDaemon(unittest.TestCase):
         mock_popen.assert_called_once_with(['/bin/rsync', '--verbose', '--recursive',
                                             '--copy-dirlinks', '--times', '--omit-dir-times',
                                             'dts:/data/dts/exposures/raw/20190703/00000127/', '/desi/root/spectro/staging/raw/20190703/00000127/'])
-        mock_log.error.assert_called_once_with('rsync problem detected!')
+        mock_log.error.assert_called_once_with('rsync problem detected for %s/%s!', '20190703', '00000127')
         mock_status.update.assert_called_once_with('20190703', '00000127', 'rsync', failure=True)
         #
         # Actually run the pipeline
diff --git a/py/desitransfer/test/test_daily.py b/py/desitransfer/test/test_daily.py
index 863f9e9..18feddd 100644
--- a/py/desitransfer/test/test_daily.py
+++ b/py/desitransfer/test/test_daily.py
@@ -37,6 +37,8 @@ class TestDaily(unittest.TestCase):
             self.assertEqual(c[0].source, '/exposures/desi/sps')
             self.assertEqual(c[0].destination, os.path.join(os.environ['DESI_ROOT'],
                                                             'engineering', 'spectrograph', 'sps'))
+            self.assertEqual(c[1].extra[0], '--exclude-from')
+            self.assertTrue(c[2].dirlinks)
 
     def test_options(self):
         """Test command-line arguments.
@@ -45,7 +47,7 @@ class TestDaily(unittest.TestCase):
                           ['desi_daily_transfer', '--daemon', '--kill',
                            os.path.expanduser('~/stop_daily_transfer')]):
             options = _options()
-            self.assertTrue(options.apache)
+            self.assertTrue(options.permission)
             self.assertEqual(options.sleep, 24)
             self.assertTrue(options.daemon)
             self.assertEqual(options.kill,
@@ -69,7 +71,7 @@ class TestDaily(unittest.TestCase):
                              call().__enter__(),
                              call().write(('DEBUG: desi_daily_transfer {}\n'.format(dtVersion)).encode('utf-8')),
                              call().write(b'DEBUG: 2019-07-03\n'),
-                             call().write(b'DEBUG: /bin/rsync --verbose --recursive --copy-dirlinks --times --omit-dir-times dts:/src/d0/ /dst/d0/\n'),
+                             call().write(b'DEBUG: /bin/rsync --verbose --recursive --links --times --omit-dir-times dts:/src/d0/ /dst/d0/\n'),
                              call().flush(),
                              call().__exit__(None, None, None)])
         mock_walk.assert_called_once_with('/dst/d0')
@@ -79,6 +81,30 @@ class TestDaily(unittest.TestCase):
 
     @patch('os.walk')
     @patch('os.chmod')
+    @patch('subprocess.Popen')
+    @patch('desitransfer.daily.stamp')
+    @patch('builtins.open', new_callable=mock_open)
+    def test_transfer_extra(self, mo, mock_stamp, mock_popen, mock_chmod, mock_walk):
+        """Test the transfer functions in DailyDirectory.transfer() with extra options.
+        """
+        mock_walk.return_value = [('/dst/d0', [], ['f1', 'f2'])]
+        mock_stamp.return_value = '2019-07-03'
+        mock_popen().wait.return_value = 0
+        d = DailyDirectory('/src/d0', '/dst/d0', extra=['--exclude-from', 'foo'])
+        d.transfer()
+        mo.assert_has_calls([call('/dst/d0.log', 'ab'),
+                             call().__enter__(),
+                             call().write(('DEBUG: desi_daily_transfer {}\n'.format(dtVersion)).encode('utf-8')),
+                             call().write(b'DEBUG: 2019-07-03\n'),
+                             call().write(b'DEBUG: /bin/rsync --verbose --recursive --links --times --omit-dir-times --exclude-from foo dts:/src/d0/ /dst/d0/\n'),
+                             call().flush(),
+                             call().__exit__(None, None, None)])
+        mock_walk.assert_called_once_with('/dst/d0')
+        mock_chmod.assert_has_calls([call('/dst/d0', 1512),
+                                     call('/dst/d0/f1', 288),
+                                     call('/dst/d0/f2', 288)])
+    @patch('os.walk')
+    @patch('os.chmod')
     def test_lock(self, mock_chmod, mock_walk):
         """Test the lock functions in DailyDirectory.lock().
         """
@@ -98,12 +124,12 @@ class TestDaily(unittest.TestCase):
 
     @patch('subprocess.Popen')
     @patch('builtins.open', new_callable=mock_open)
-    def test_apache(self, mo, mock_popen):
-        """Test granting apache/www permissions.
+    def test_permission(self, mo, mock_popen):
+        """Test granting permissions.
         """
         mock_popen().wait.return_value = 0
         d = DailyDirectory('/src/d0', '/dst/d0')
-        d.apache()
+        d.permission()
         mo.assert_has_calls([call('/dst/d0.log', 'ab'),
                              call().__enter__(),
                              call().write(b'DEBUG: fix_permissions.sh /dst/d0\n'),