Repository: desihub/desitransfer

Issue:
Support nightwatch transfers
Currently nightwatch transfers are handled "by hand" on dtn01.nersc.gov.  Incorporate nightwatch transfers into desitransfer and make the transfer code more robust and maintainable.

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 8f3434e..5d2e708 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,9 +25,9 @@ addons:
             - texlive-latex-extra
             - dvipng
 python:
-    - 3.5
+    # - 3.5
     - 3.6
-    # - 3.7
+    - 3.7
 env:
     global:
         # The following versions are the 'default' for tests, unless
diff --git a/bin/desi_common_init.sh b/bin/desi_common_init.sh
new file mode 100644
index 0000000..9babe53
--- /dev/null
+++ b/bin/desi_common_init.sh
@@ -0,0 +1,113 @@
+# Standalone init script to run any shell command
+# Sean McManus 2015-11-17
+# Adapted for DESI: Benjamin Weaver 2018-06-27
+# Migrated to desitransfer: 2019-08-05
+#
+# DESI Environment: Moved this to crontab
+# source /global/common/software/desi/desi_environment.sh ${DESIMODULES_VERSION}
+#
+# Common variables.
+#
+NICE='nice -n 19'
+THISHOST=$(hostname -s)
+#
+# The existence of this file will shut down data transfers.
+#
+kill_switch=${HOME}/stop_desi_transfer
+#
+# Determine the version of pgrep to select options.
+#
+PGREP_VERSION=$(pgrep -V | sed -r 's/[^0-9]+([0-9]+)\.([0-9]+)\.([0-9]+)[^0-9]+/\1\2/')
+PGREP_OPTIONS='-a -f'
+[[ ${PGREP_VERSION} == "32" ]] && PGREP_OPTIONS='-l -f'
+#
+#
+#
+start() {
+    if [[ -f ${kill_switch} ]]; then
+        echo "${kill_switch} detected, will not attempt to start ${PRGFILE}."
+        return 0
+    fi
+    if [[ -n "$(pgrep ${PGREP_OPTIONS} ${PRGFILE} 2> /dev/null | grep ${DESITRANSFER})" ]]; then
+        echo "${THISHOST} ${PRGFILE} is already started."
+        return 1
+    fi
+    #
+    # Daemonize: You must disconnect stdin, stdout, and stderr, and make it ignore the hangup signal (SIGHUP).
+    #
+    nohup ${NICE} ${PROGRAM} ${PRGOPTS}  &>/dev/null &
+    #
+    # Alternatively, use double background
+    #
+    # (/bin/bash -c "echo $$ >${PIDFILE} && exec ${NICE} ${PROGRAM} ${PRGOPTS}" &) &
+    if [[ $? == 0 ]]; then
+        echo "${PRGFILE} started."
+        return 0
+    else
+        echo "Failed to start ${PRGFILE}."
+        return 1
+    fi
+}
+#
+#
+#
+kill_it() {
+    local PRGFILE=$(basename $1)
+    local SIGNAL='SIGTERM'
+    local PPLIST=$(pgrep -f ${PRGFILE} -d ' ')
+    for PID in ${PPLIST}; do
+        local CHILDPIDS=$(pgrep -P ${PID} -d ' ')
+        echo killing ${PRGFILE} ${PID} child processes: ${CHILDPIDS}
+        while true; do
+            kill -s SIGTERM ${PID} ${CHILDPIDS} >& /dev/null
+            if [[ $? != 0 ]]; then
+                break
+            fi
+            sleep 2
+            echo -n '.'
+        done
+    done
+    #
+    # Save the big hammer for last.
+    #
+    pkill -f -9 ${PRGFILE}
+}
+#
+#
+#
+stop() {
+    if [[ -z "$(pgrep ${PGREP_OPTIONS} ${PRGFILE} 2> /dev/null | grep ${DESITRANSFER})" ]]; then
+        echo "${THISHOST} ${PRGFILE} is not running."
+        return 1
+    fi
+    echo -n "Stopping ${PRGFILE}..."
+    kill_it ${PRGFILE} #>& /dev/null
+    # if [[ $? != 0 ]]; then
+    #     echo "Operation not permitted."
+    #     return 1
+    # fi
+    echo -e "\n${PRGFILE} stopped."
+    # rm -f ${PIDFILE} 2> /dev/null
+    return 0
+}
+#
+#
+#
+status() {
+    if [[ -n "$(pgrep ${PGREP_OPTIONS} ${PRGFILE} 2> /dev/null | grep ${DESITRANSFER})" ]]; then
+        echo "${THISHOST} ${PRGFILE} is running."
+        return 0
+    else
+        echo "${THISHOST} ${PRGFILE} is not running"
+        return 0
+    fi
+}
+#
+#
+#
+restart() {
+    stop
+    sleep 1
+    start
+    return $?
+}
diff --git a/bin/desi_nightwatch_init.sh b/bin/desi_nightwatch_init.sh
new file mode 100755
index 0000000..bbdcd21
--- /dev/null
+++ b/bin/desi_nightwatch_init.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Program or script you want to run
+#
+PROGRAM=${DESITRANSFER}/bin/desi_nightwatch_transfer
+PRGFILE=$(basename ${PROGRAM})
+PRGDIR=$(dirname ${PROGRAM})
+#
+# Command line options for PRGFILE
+#
+if [[ -z "${NERSC_HOST}" ]]; then
+    PRGOPTS='--debug --no-apache'
+else
+    PRGOPTS='--debug'
+fi
+#
+# Common initialization code.
+#
+source ${PRGDIR}/desi_common_init.sh
+#
+# Main program.
+#
+case "$1" in
+    start | stop | status | restart)
+        $1
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|status|restart}"
+        exit 2
+        ;;
+esac
+exit $?
diff --git a/bin/desi_nightwatch_transfer b/bin/desi_nightwatch_transfer
new file mode 100755
index 0000000..68f1afa
--- /dev/null
+++ b/bin/desi_nightwatch_transfer
@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+from sys import exit
+from desitransfer.nightwatch import main
+exit(main())
diff --git a/bin/desi_transfer_init.sh b/bin/desi_transfer_init.sh
index a41e79e..bf6bf83 100755
--- a/bin/desi_transfer_init.sh
+++ b/bin/desi_transfer_init.sh
@@ -1,120 +1,24 @@
 #!/bin/bash
-# Standalone init script to run any shell command
-# Sean McManus 2015-11-17
-# Adapted for DESI: Benjamin Weaver 2018-06-27
-# Migrated to desitransfer: 2019-08-05
 #
-# DESI Environment: Moved this to crontab
-# source /global/common/software/desi/desi_environment.sh ${DESIMODULES_VERSION}
 # Program or script you want to run
-PROGRAM=${DESITRANSFER}/bin/desi_transfer_daemon
-# Command line options for PRGFILE
-PRGOPTS="--no-pipeline"
-NICE="nice -n 19"
-# LOCKDIR=${CSCRATCH}/run
-#
-# rm -rf ${LOCKDIR}; mkdir -p ${LOCKDIR}
 #
+PROGRAM=${DESITRANSFER}/bin/desi_transfer_daemon
 PRGFILE=$(basename ${PROGRAM})
-# PIDFILE=${LOCKDIR}/${PRGFILE}.pid
-#
-THISHOST=$(hostname -s)
-#
-# The existence of this file will shut down data transfers.
-kill_switch=${HOME}/stop_desi_transfer
-#
-#
-#
-start() {
-    if [[ -f ${kill_switch} ]]; then
-        echo "${kill_switch} detected, will not attempt to start ${PRGFILE}."
-        return 0
-    fi
-    if [[ -n "$(pgrep --list-full --full ${PRGFILE} 2> /dev/null | grep ${DESITRANSFER})" ]]; then
-        echo "${THISHOST} ${PRGFILE} is already started."
-        return 1
-    fi
-    #
-    # Daemonize: You must disconnect stdin, stdout, and stderr, and make it ignore the hangup signal (SIGHUP).
-    #
-    nohup ${NICE} ${PROGRAM} ${PRGOPTS}  &>/dev/null &
-    #
-    # Alternatively, use double background
-    #
-    # (/bin/bash -c "echo $$ >${PIDFILE} && exec ${NICE} ${PROGRAM} ${PRGOPTS}" &) &
-    if [[ $? == 0 ]]; then
-        echo "${PRGFILE} started."
-        return 0
-    else
-        echo "Failed to start ${PRGFILE}."
-        return 1
-    fi
-}
-#
-#
-#
-kill_it() {
-    local PRGFILE=$(basename $1)
-    local SIGNAL="SIGTERM"
-    local PPLIST=$(pgrep --full ${PRGFILE} --delimiter ' ')
-    for PID in ${PPLIST}; do
-        local CHILDPIDS=$(pgrep --parent ${PID} --delimiter ' ')
-        echo killing ${PRGFILE} ${PID} child processes: ${CHILDPIDS}
-        while true; do
-            kill -s SIGTERM ${PID} ${CHILDPIDS} >& /dev/null
-            if [[ $? != 0 ]]; then
-                break
-            fi
-            sleep 2
-            echo -n "."
-        done
-    done
-    #
-    # Save the big hammer for last.
-    #
-    pkill --full --signal 9 ${PRGFILE}
-}
-#
-#
-#
-stop() {
-    if [[ -z "$(pgrep --list-full --full ${PRGFILE} 2> /dev/null | grep ${DESITRANSFER})" ]]; then
-        echo "${THISHOST} ${PRGFILE} is not running."
-        return 1
-    fi
-    echo -n "Stopping ${PRGFILE}..."
-    kill_it ${PRGFILE} #>& /dev/null
-    # if [[ $? != 0 ]]; then
-    #     echo "Operation not permitted."
-    #     return 1
-    # fi
-    echo -e "\n${PRGFILE} stopped."
-    # rm -f ${PIDFILE} 2> /dev/null
-    return 0
-}
-#
-#
-#
-status() {
-    if [[ -n "$(pgrep --list-full --full ${PRGFILE} 2> /dev/null | grep ${DESITRANSFER})" ]]; then
-        echo "${THISHOST} ${PRGFILE} is running."
-        return 0
-    else
-        echo "${THISHOST} ${PRGFILE} is not running"
-        return 0
-    fi
-}
+PRGDIR=$(dirname ${PROGRAM})
 #
+# Command line options for PRGFILE
 #
+if [[ -z "${NERSC_HOST}" ]]; then
+    PRGOPTS='--no-pipeline --no-backup'
+else
+    PRGOPTS='--no-pipeline'
+fi
 #
-restart() {
-    stop
-    sleep 1
-    start
-    return $?
-}
+# Common initialization code.
 #
+source ${PRGDIR}/desi_common_init.sh
 #
+# Main program.
 #
 case "$1" in
     start | stop | status | restart)
diff --git a/doc/api.rst b/doc/api.rst
index 9ee155f..faca8ae 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -14,5 +14,8 @@ desitransfer API
 .. automodule:: desitransfer.daily
     :members:
 
+.. automodule:: desitransfer.nightwatch
+    :members:
+
 .. automodule:: desitransfer.status
     :members:
diff --git a/doc/changes.rst b/doc/changes.rst
index 0ac8bad..0cb2bec 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,7 +5,10 @@ Change Log
 0.3.5 (unreleased)
 ------------------
 
-* No changes yet.
+* Support direct KPNO to Tucson transfers when NERSC is shut down (PR `#12`_).
+* Move nightwatch transfer script into this package.
+
+.. _`#12`: https://github.com/desihub/desitransfer/pull/12
 
 0.3.4 (2020-01-10)
 ------------------
diff --git a/py/desitransfer/common.py b/py/desitransfer/common.py
index 1e3939c..0b9bb1b 100644
--- a/py/desitransfer/common.py
+++ b/py/desitransfer/common.py
@@ -85,3 +85,9 @@ def yesterday():
     """Yesterday's date in DESI "NIGHT" format, YYYYMMDD.
     """
     return (dt.datetime.now() - dt.timedelta(seconds=86400)).strftime('%Y%m%d')
+
+
+def today():
+    """Today's date in DESI "NIGHT" format, YYYYMMDD.
+    """
+    return (dt.datetime.now() - dt.timedelta(7/24+0.5)).strftime('%Y%m%d')
diff --git a/py/desitransfer/daemon.py b/py/desitransfer/daemon.py
index 3c62742..a3b5813 100644
--- a/py/desitransfer/daemon.py
+++ b/py/desitransfer/daemon.py
@@ -41,6 +41,8 @@ def _options():
     """
     desc = "Transfer DESI raw data files."
     prsr = ArgumentParser(description=desc)
+    prsr.add_argument('-B', '--no-backup', action='store_false', dest='backup',
+                      help="Skip NERSC HPSS backups.")
     prsr.add_argument('-c', '--configuration', metavar='FILE',
                       help="Read configuration from FILE.")
     prsr.add_argument('-d', '--debug', action='store_true',
@@ -74,6 +76,7 @@ class TransferDaemon(object):
         else:
             self._ini = options.configuration
         self.test = options.shadow
+        self.tape = options.backup
         self.run = options.pipeline
         getlist = lambda x: x.split(',')
         getdict = lambda x: dict([tuple(i.split(':')) for i in x.split(',')])
@@ -236,6 +239,7 @@ The DESI Collaboration Account
         if now >= self.conf['common'].getint('backup'):
             s = self.backup(d, yst)
             if s:
+                log.debug("status.update('%s', 'all', 'backup')", yst)
                 status.update(yst, 'all', 'backup')
 
     def exposure(self, d, link, status):
@@ -280,6 +284,7 @@ The DESI Collaboration Account
         # Transfer complete.
         #
         if rsync_status == '0':
+            log.debug("status.update('%s', '%s', 'rsync')", night, exposure)
             status.update(night, exposure, 'rsync')
             #
             # Check permissions.
@@ -306,6 +311,7 @@ The DESI Collaboration Account
             # Did we pass checksums?
             #
             if checksum_status == 0:
+                log.debug("status.update('%s', '%s', 'checksum')", night, exposure)
                 status.update(night, exposure, 'checksum')
                 #
                 # Set up DESI_SPECTRO_DATA.
@@ -336,17 +342,22 @@ The DESI Collaboration Account
                     cmd = self.pipeline(night, exposure)
                     if not self.test:
                         _, out, err = _popen(cmd)
+                    log.debug("status.update('%s', '%s', 'pipeline')", night, exposure)
                     status.update(night, exposure, 'pipeline')
                     for k in ('flats', 'arcs', 'science'):
-                        if os.path.exists(os.path.join(destination_exposure, '{0}-{1}-{2}.done'.format(k, night, exposure))):
+                        if os.path.exists(os.path.join(destination_exposure,
+                                                       '{0}-{1}-{2}.done'.format(k, night, exposure))):
                             cmd = self.pipeline(night, exposure, command=k)
                             if not self.test:
                                 _, out, err = _popen(cmd)
+                            log.debug("status.update('%s', '%s', 'pipeline', last='%s')",
+                                      night, exposure, k)
                             status.update(night, exposure, 'pipeline', last=k)
                 else:
                     log.info("%s/%s appears to be test data. Skipping pipeline activation.", night, exposure)
             else:
                 log.error("Checksum problem detected for %s/%s!", night, exposure)
+                log.debug("status.update('%s', '%s', 'checksum', failure=True)", night, exposure)
                 status.update(night, exposure, 'checksum', failure=True)
         elif rsync_status == 'done':
             #
@@ -355,6 +366,7 @@ The DESI Collaboration Account
             pass
         else:
             log.error('rsync problem detected!')
+            log.debug("status.update('%s', '%s', 'rsync', failure=True)", night, exposure)
             status.update(night, exposure, 'rsync', failure=True)
 
     def catchup(self, d, night):
@@ -427,14 +439,17 @@ The DESI Collaboration Account
                 log.debug("Failed to remove %s because it didn't exist. That's OK.", ls_file)
             cmd = ['/usr/common/mss/bin/hsi', '-O', ls_file,
                    'ls', '-l', d.hpss]
-            _, out, err = _popen(cmd)
+            if self.tape:
+                _, out, err = _popen(cmd)
+                with open(ls_file) as l:
+                    data = l.read()
+                backup_files = [l.split()[-1] for l in data.split('\n') if l]
+            else:
+                backup_files = []
+            backup_file = hpss_file + '_' + night + '.tar'
             #
             # Both a .tar and a .tar.idx file should be present.
             #
-            with open(ls_file) as l:
-                data = l.read()
-            backup_files = [l.split()[-1] for l in data.split('\n') if l]
-            backup_file = hpss_file + '_' + night + '.tar'
             if backup_file in backup_files and backup_file + '.idx' in backup_files:
                 log.debug("Backup of %s already complete.", night)
                 return False
@@ -456,19 +471,22 @@ The DESI Collaboration Account
                 #
                 # Issue HTAR command.
                 #
-                start_dir = os.getcwd()
-                log.debug("os.chdir('%s')", d.destination)
-                os.chdir(d.destination)
-                cmd = ['/usr/common/mss/bin/htar',
-                       '-cvhf', os.path.join(d.hpss, backup_file),
-                       '-H', 'crc:verify=all',
-                       night]
-                if self.test:
-                    log.debug(' '.join(cmd))
+                if self.tape:
+                    start_dir = os.getcwd()
+                    log.debug("os.chdir('%s')", d.destination)
+                    os.chdir(d.destination)
+                    cmd = ['/usr/common/mss/bin/htar',
+                           '-cvhf', os.path.join(d.hpss, backup_file),
+                           '-H', 'crc:verify=all',
+                           night]
+                    if self.test:
+                        log.debug(' '.join(cmd))
+                    else:
+                        _, out, err = _popen(cmd)
+                    log.debug("os.chdir('%s')", start_dir)
+                    os.chdir(start_dir)
                 else:
-                    _, out, err = _popen(cmd)
-                log.debug("os.chdir('%s')", start_dir)
-                os.chdir(start_dir)
+                    log.info('Tape backup disabled by user request.')
                 return True
         else:
             log.warning("No data from %s detected, skipping HPSS backup.", night)
diff --git a/py/desitransfer/daily.py b/py/desitransfer/daily.py
index d27b1f7..d363156 100644
--- a/py/desitransfer/daily.py
+++ b/py/desitransfer/daily.py
@@ -31,9 +31,14 @@ class DailyDirectory(object):
         self.destination = destination
         self.log = self.destination + '.log'
 
-    def transfer(self):
+    def transfer(self, apache=True):
         """Data transfer operations for a single destination directory.
 
+        Parameters
+        ----------
+        apache : :class:`bool`
+            If ``True`` set file ACLs for Apache httpd access.
+
         Returns
         -------
         :class:`int`
@@ -49,7 +54,8 @@ class DailyDirectory(object):
             status = p.wait()
         if status == 0:
             self.lock()
-            s = self.apache()
+            if apache:
+                s = self.apache()
         return status
 
     def lock(self):
@@ -121,6 +127,8 @@ def _options(*args):
     """
     desc = "Transfer non-critical DESI data from KPNO to NERSC."
     prsr = ArgumentParser(description=desc)
+    prsr.add_argument('-A', '--no-apache', action='store_false', dest='apache',
+                      help='Do not set ACL for Apache httpd access.')
     # prsr.add_argument('-b', '--backup', metavar='H', type=int, default=20,
     #                   help='UTC time in hours to trigger HPSS backups (default %(default)s:00 UTC).')
     # prsr.add_argument('-d', '--debug', action='store_true',
@@ -159,7 +167,7 @@ def main():
             print("INFO: %s detected, shutting down daily transfer script." % options.kill)
             return 0
         for d in _config():
-            status = d.transfer()
+            status = d.transfer(apache=options.apache)
             if status != 0:
                 print("ERROR: rsync problem detected for ${0.source} -> ${0.destination}!".format(d))
                 return status
diff --git a/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt b/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt
new file mode 100644
index 0000000..2f1f1fa
--- /dev/null
+++ b/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt
@@ -0,0 +1,3 @@
+*/preproc*.fits
+*/qsky-*.fits
+*/*.tmp
diff --git a/py/desitransfer/data/desi_nightwatch_transfer_include.txt b/py/desitransfer/data/desi_nightwatch_transfer_include.txt
new file mode 100644
index 0000000..0662ad6
--- /dev/null
+++ b/py/desitransfer/data/desi_nightwatch_transfer_include.txt
@@ -0,0 +1,3 @@
+nights.html
+nightlinks.js
+qa-lastexp.html
diff --git a/py/desitransfer/data/desi_transfer_status.js b/py/desitransfer/data/desi_transfer_status.js
index 5d88efc..a9779d1 100644
--- a/py/desitransfer/data/desi_transfer_status.js
+++ b/py/desitransfer/data/desi_transfer_status.js
@@ -38,14 +38,16 @@ $(function() {
             var s = Exposure.stages;
             for (var k = 0; k < this.exposures.length; k++) {
                 for (var l = 0; l < s.length; l++) {
-                    if (!this.exposures[k].stage[s[l]].success) {
-                        //
-                        // It's not successful, but is it complete?
-                        //
-                        if (this.exposures[k].stage[s[l]].stamp > 0) {
-                            return "btn-danger";
-                        } else {
-                            return "btn-warning";
+                    if (Exposure.display[l]) {
+                        if (!this.exposures[k].stage[s[l]].success) {
+                            //
+                            // It's not successful, but is it complete?
+                            //
+                            if (this.exposures[k].stage[s[l]].stamp > 0) {
+                                return "btn-danger";
+                            } else {
+                                return "btn-warning";
+                            }
                         }
                     }
                 }
@@ -117,6 +119,7 @@ $(function() {
         var E = Exposure, Ep = Exposure.prototype;
         E.padding = 8;
         E.stages = ["rsync", "checksum", "pipeline", "backup"];
+        E.display = [true, true, false, true];
         //
         // Pad integers out to 8 characters.
         //
@@ -131,7 +134,8 @@ $(function() {
         Ep.header = function() {
             var h = "<thead><tr><th class=\"text-uppercase\">exposure</th>";
             for (var k = 0; k < Exposure.stages.length; k++) {
-                h += "<th class=\"text-uppercase\">" + Exposure.stages[k] + "</th>";
+                if (Exposure.display[k])
+                    h += "<th class=\"text-uppercase\">" + Exposure.stages[k] + "</th>";
             }
             h += "<th class=\"text-uppercase\">comment</th></tr></thead>";
             return h;
@@ -143,14 +147,16 @@ $(function() {
             var r = "<tr id=\"e" + this.toString() +"\">" +
                     "<td>" + this.pad() + "</td>";
             for (var k = 0; k < Exposure.stages.length; k++) {
-                var c = "table-warning";
-                var stamp = "INCOMPLETE";
-                if (this.stage[Exposure.stages[k]].stamp != 0) {
-                    c = this.stage[Exposure.stages[k]].success ? "table-success" : "table-danger";
-                    var d = new Date(this.stage[Exposure.stages[k]].stamp);
-                    stamp = d.toISOString();
+                if (Exposure.display[k]) {
+                    var c = "table-warning";
+                    var stamp = "INCOMPLETE";
+                    if (this.stage[Exposure.stages[k]].stamp != 0) {
+                        c = this.stage[Exposure.stages[k]].success ? "table-success" : "table-danger";
+                        var d = new Date(this.stage[Exposure.stages[k]].stamp);
+                        stamp = d.toISOString();
+                    }
+                    r +=  "<td class=\"" + c + "\">" + stamp + "</td>";
                 }
-                r +=  "<td class=\"" + c + "\">" + stamp + "</td>";
             }
             r += "<td>" + this.l + "</td></tr>";
             return r;
diff --git a/py/desitransfer/nightwatch.py b/py/desitransfer/nightwatch.py
new file mode 100644
index 0000000..8946bde
--- /dev/null
+++ b/py/desitransfer/nightwatch.py
@@ -0,0 +1,251 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+# -*- coding: utf-8 -*-
+"""
+desitransfer.nightwatch
+=======================
+
+Sync KPNO nightwatch.  This module will hopefully be integrated into
+the standard transfer daemon.
+
+Run as desi@dtn01.nersc.gov.
+
+Catchup on a specific night::
+
+    NIGHT=20200124 && rsync -rlvt --exclude-from ${DESITRANSFER}/py/desitransfer/data/desi_nightwatch_transfer_exclude.txt dts:/exposures/nightwatch/${NIGHT}/ /global/cfs/cdirs/desi/spectro/nightwatch/kpno/${NIGHT}/
+
+
+Typical startup sequence (bash shell)::
+
+    source /global/common/software/desi/desi_environment.sh datatran
+    module load desitransfer
+    nohup nice -19 ${DESITRANSFER}/bin/desi_nightwatch_transfer &> /dev/null &
+    tail -f ${DESI_ROOT}/spectro/nightwatch/desi_nightwatch_transfer.log
+
+The above sequence is for starting by hand.  A cronjob on dtn01 should ensure
+that the script is running.
+"""
+import logging
+import os
+import stat
+import subprocess as sub
+import time
+from argparse import ArgumentParser
+from logging.handlers import RotatingFileHandler, SMTPHandler
+from pkg_resources import resource_filename
+from socket import getfqdn
+from tempfile import TemporaryFile
+from desiutil.log import get_logger
+from .common import rsync, today
+# from .daemon import _popen
+from . import __version__ as dtVersion
+
+
+log = None
+
+
+def _options():
+    """Parse command-line options for :command:`desi_nightwatch_transfer`.
+
+    Returns
+    -------
+    :class:`argparse.Namespace`
+        The parsed command-line options.
+    """
+    desc = "Transfer DESI nightwatch data files."
+    prsr = ArgumentParser(description=desc)
+    prsr.add_argument('-A', '--no-apache', action='store_false', dest='apache',
+                      help='Do not set ACL for Apache httpd access.')
+    # prsr.add_argument('-B', '--no-backup', action='store_false', dest='backup',
+    #                   help="Skip NERSC HPSS backups.")
+    # prsr.add_argument('-c', '--configuration', metavar='FILE',
+    #                   help="Read configuration from FILE.")
+    prsr.add_argument('-d', '--debug', action='store_true',
+                      help='Set log level to DEBUG.')
+    prsr.add_argument('-k', '--kill', metavar='FILE',
+                      default=os.path.join(os.environ['HOME'], 'stop_desi_transfer'),
+                      help="Exit the script when FILE is detected (default %(default)s).")
+    # prsr.add_argument('-P', '--no-pipeline', action='store_false', dest='pipeline',
+    #                   help="Only transfer files, don't start the DESI pipeline.")
+    # prsr.add_argument('-S', '--shadow', action='store_true',
+    #                   help='Observe the actions of another data transfer script but do not make any changes.')
+    prsr.add_argument('-s', '--sleep', metavar='M', type=int, default=10,
+                      help='Sleep M minutes before checking for new data (default %(default)s minutes).')
+    prsr.add_argument('-V', '--version', action='version',
+                      version='%(prog)s {0}'.format(dtVersion))
+    return prsr.parse_args()
+
+
+def _configure_log(debug):
+    """Re-configure the default logger returned by ``desiutil.log``.
+
+    Parameters
+    ----------
+    debug : :class:`bool`
+        If ``True`` set the log level to ``DEBUG``.
+    """
+    global log
+    # conf = self.conf['logging']
+    log = get_logger(timestamp=True)
+    h = log.parent.handlers[0]
+    handler = RotatingFileHandler(os.path.join(os.environ['DESI_ROOT'], 'spectro', 'nightwatch', 'desi_nightwatch_transfer.log'),
+                                  maxBytes=100000000,
+                                  backupCount=100)
+    handler.setFormatter(h.formatter)
+    log.parent.removeHandler(h)
+    log.parent.addHandler(handler)
+    if debug:
+        log.setLevel(logging.DEBUG)
+    email_from = os.environ['USER'] + '@' + getfqdn()
+    handler2 = SMTPHandler('localhost', email_from, ['baweaver@lbl.gov',],
+                           'Critical error reported by desi_nightwatch_transfer!')
+    fmt = """Greetings,
+
+At %(asctime)s, desi_nightwatch_transfer failed with this message:
+
+%(message)s
+
+Kia ora koutou,
+The DESI Collaboration Account
+"""
+    formatter2 = logging.Formatter(fmt, datefmt='%Y-%m-%d %H:%M:%S %Z')
+    handler2.setFormatter(formatter2)
+    handler2.setLevel(logging.CRITICAL)
+    log.parent.addHandler(handler2)
+
+
+def _popen(command):
+    """Simple wrapper for :class:`subprocess.Popen` to avoid repeated code.
+
+    Parameters
+    ----------
+    command : :class:`list`
+        Command to pass to :class:`subprocess.Popen`.
+
+    Returns
+    -------
+    :func:`tuple`
+        The returncode, standard output and standard error.
+    """
+    log.debug(' '.join(command))
+    with TemporaryFile() as tout, TemporaryFile() as terr:
+        p = sub.Popen(command, stdout=tout, stderr=terr)
+        p.wait()
+        tout.seek(0)
+        out = tout.read()
+        terr.seek(0)
+        err = terr.read()
+    return (str(p.returncode), out.decode('utf-8'), err.decode('utf-8'))
+
+
+def main():
+    """Entry point for :command:`desi_nightwatch_transfer`.
+
+    Returns
+    -------
+    :class:`int`
+        An integer suitable for passing to :func:`sys.exit`.
+    """
+    options = _options()
+    _configure_log(options.debug)
+    errcount = 0
+    wait = options.sleep*60
+    source = '/exposures/nightwatch'
+    basedir = os.path.join(os.environ['DESI_ROOT'], 'spectro', 'nightwatch')
+    kpnodir = os.path.join(basedir, 'kpno')
+    # syncdir = os.path.join(basedir, 'sync')
+    exclude = resource_filename('desitransfer', 'data/desi_nightwatch_transfer_exclude.txt')
+    include = resource_filename('desitransfer', 'data/desi_nightwatch_transfer_include.txt')
+    with open(include) as i:
+        top_level_files = i.read().strip().split('\n')
+    log.debug(', '.join(top_level_files))
+    top_level_files_mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+    while True:
+        log.info('Starting nightwatch transfer loop; desitransfer version = %s.',
+                 dtVersion)
+        if os.path.exists(options.kill):
+            log.info("%s detected, shutting down nightwatch daemon.",
+                     options.kill)
+            return 0
+        night = today()
+        t0 = time.time()
+        #
+        # First check if there is any data for tonight yet.
+        #
+        log.info('Checking for nightwatch data from %s.', night)
+        cmd = ['/bin/rsync', 'dts:{0}/'.format(source)]
+        status, out, err = _popen(cmd)
+        found = False
+        if status != '0':
+            errcount += 1
+            log.error('Getting file list for %s; trying again in %d minutes.', night, options.sleep)
+            time.sleep(wait)
+            continue
+        for line in out.split('\n'):
+            if line.endswith(night):
+                log.info(line)
+                found = True
+        if not found:
+            log.info('No nightwatch data found for %s; trying again in %d minutes.', night, options.sleep)
+            time.sleep(wait)
+            continue
+        #
+        # Sync per-night directory.
+        #
+        nightdir = os.path.join(kpnodir, night)
+        cmd = rsync(os.path.join(source, night), nightdir)
+        cmd.insert(cmd.index('--omit-dir-times') + 1, '--exclude-from')
+        cmd.insert(cmd.index('--exclude-from') + 1, exclude)
+        log.info('Syncing %s.', night)
+        status, out, err = _popen(cmd)
+        if status != '0':
+            errcount += 1
+            log.error('Syncing %s.', night)
+        #
+        # Correct the permissions.
+        #
+        if options.apache:
+            if os.path.exists(nightdir):
+                log.info('Fixing permissions for Apache.')
+                cmd = ['fix_permissions.sh', '-a', nightdir]
+                status, out, err = _popen(cmd)
+                if status != '0':
+                    errcount += 1
+                    log.error('Fixing permissions for %s.', nightdir)
+            else:
+                log.info('No data yet for night %s.', night)
+        else:
+            log.info("Skipping permission changes at user request.")
+        #
+        # Sync the top level files; skip the logs.
+        #
+        log.info('Syncing top level html/js files.')
+        cmd = ['/bin/rsync', '--verbose', '--links', '--times', '--files-from',
+               include,
+               'dts:{0}/'.format(source),
+               '{0}/'.format(kpnodir)]
+        status, out, err = _popen(cmd)
+        if status != '0':
+            errcount += 1
+            log.error('Syncing top level html files.')
+        #
+        # Hack: just add world read to those top level files since fix_permissions.sh
+        # is recursive and we don't want to redo all nights.
+        #
+        for filename in top_level_files:
+            log.debug("os.chmod('%s', 0o%o)",
+                      os.path.join(kpnodir, filename),
+                      top_level_files_mode)
+            os.chmod(os.path.join(kpnodir, filename), top_level_files_mode)
+        #
+        # Check for accumulated errors.
+        #
+        if errcount > 10:
+            log.critical('Transfer error count exceeded, shutting down.')
+            return 1
+        #
+        # If all that took less than 10 minutes, sleep a bit.
+        #
+        dt = time.time() - t0
+        if dt < wait:
+            log.info('Sleeping for a bit.')
+            time.sleep(wait - dt)
diff --git a/py/desitransfer/status.py b/py/desitransfer/status.py
index e92998f..2964d2f 100644
--- a/py/desitransfer/status.py
+++ b/py/desitransfer/status.py
@@ -97,6 +97,11 @@ class TransferStatus(object):
         last : :class:`str`, optional
             Mark this exposure as the last of a given type for the night
             ('arcs', 'flats', 'science').
+
+        Returns
+        -------
+        :class:`int`
+            The number of updates performed.
         """
         ts = int(time.time() * 1000)  # Convert to milliseconds for JS.
         i = int(night)
@@ -115,6 +120,12 @@ class TransferStatus(object):
                 if last or update:
                     self.status[il[0]] = r
                     rows = []
+                else:
+                    #
+                    # Rare edge case: daemon is in shadow/test mode and there
+                    # are untransferred files.
+                    #
+                    return 0
             else:
                 rows = [r, ]
         for row in rows:
@@ -131,6 +142,10 @@ class TransferStatus(object):
             pass
         with open(self.json, 'w') as j:
             json.dump(self.status, j, indent=None, separators=(',', ':'))
+        r = len(rows)
+        if r == 0:
+            return 1
+        return r
 
     def find(self, night, exposure=None, stage=None):
         """Find status entries that match `night`, etc.


Test patch:
diff --git a/py/desitransfer/test/test_common.py b/py/desitransfer/test/test_common.py
index 7eebe65..1642616 100644
--- a/py/desitransfer/test/test_common.py
+++ b/py/desitransfer/test/test_common.py
@@ -6,7 +6,7 @@ import datetime
 import os
 import unittest
 from unittest.mock import patch
-from ..common import dir_perm, file_perm, empty_rsync, rsync, stamp, yesterday
+from ..common import dir_perm, file_perm, empty_rsync, rsync, stamp, yesterday, today
 
 
 class TestCommon(unittest.TestCase):
@@ -74,13 +74,22 @@ total size is 118,417,836,324  speedup is 494,367.55
 
     @patch('desitransfer.common.dt')
     def test_yesterday(self, mock_dt):
-        """Test timestamp.
+        """Test yesterday's date.
         """
         mock_dt.datetime.now.return_value = datetime.datetime(2019, 7, 3, 12, 0, 0)
         mock_dt.timedelta.return_value = datetime.timedelta(seconds=86400)
         y = yesterday()
         self.assertEqual(y, '20190702')
 
+    @patch('desitransfer.common.dt')
+    def test_today(self, mock_dt):
+        """Test today's date.
+        """
+        mock_dt.datetime.now.return_value = datetime.datetime(2019, 7, 3, 5, 0, 0)
+        mock_dt.timedelta.return_value = datetime.timedelta(7/24+0.5)
+        y = today()
+        self.assertEqual(y, '20190702')
+
 
 def test_suite():
     """Allows testing of only this module with the command::
diff --git a/py/desitransfer/test/test_daemon.py b/py/desitransfer/test/test_daemon.py
index ab51215..c81f254 100644
--- a/py/desitransfer/test/test_daemon.py
+++ b/py/desitransfer/test/test_daemon.py
@@ -532,6 +532,37 @@ desi_spectro_data_20190702.tar.idx
         mock_log.warning.assert_has_calls([call('New files detected in %s!', '20190703')])
         mock_rsync.assert_called_with('/data/dts/exposures/raw', '/desi/root/spectro/data', '20190703', False)
 
+    @patch('desitransfer.daemon.rsync_night')
+    @patch('desitransfer.daemon.empty_rsync')
+    @patch('desitransfer.daemon._popen')
+    @patch('os.remove')
+    @patch('os.path.isdir')
+    @patch('desitransfer.daemon.log')
+    @patch.object(TransferDaemon, '_configure_log')
+    def test_TransferDaemon_no_backup(self, mock_cl, mock_log, mock_isdir, mock_rm, mock_popen, mock_empty, mock_rsync):
+        """Test disabling HPSS backups.
+        """
+        fake_hsi1 = """desi_spectro_data_20190703.tar
+desi_spectro_data_20190703.tar.idx
+"""
+        fake_hsi2 = """desi_spectro_data_20190702.tar
+desi_spectro_data_20190702.tar.idx
+"""
+        with patch.dict('os.environ',
+                        {'CSCRATCH': self.tmp.name,
+                         'DESI_ROOT': '/desi/root',
+                         'DESI_SPECTRO_DATA': '/desi/root/spectro/data'}):
+            with patch.object(sys, 'argv', ['desi_transfer_daemon', '--debug', '--shadow', '--no-backup']):
+                options = _options()
+            transfer = TransferDaemon(options)
+        c = transfer.directories
+        mock_isdir.return_value = True
+        mock_popen.return_value = ('0', '', '')
+        s = transfer.backup(c[0], '20190703')
+        self.assertTrue(s)
+        mock_isdir.assert_called_once_with('/desi/root/spectro/data/20190703')
+        mock_log.info.assert_has_calls([call('Tape backup disabled by user request.')])
+
     @patch('desitransfer.daemon.TemporaryFile')
     @patch('subprocess.Popen')
     @patch('desitransfer.daemon.log')
diff --git a/py/desitransfer/test/test_daily.py b/py/desitransfer/test/test_daily.py
index 9089131..b884108 100644
--- a/py/desitransfer/test/test_daily.py
+++ b/py/desitransfer/test/test_daily.py
@@ -45,6 +45,7 @@ class TestDaily(unittest.TestCase):
                           ['desi_daily_transfer', '--daemon', '--kill',
                            os.path.expanduser('~/stop_daily_transfer')]):
             options = _options()
+            self.assertTrue(options.apache)
             self.assertEqual(options.sleep, 24)
             self.assertTrue(options.daemon)
             self.assertEqual(options.kill,
diff --git a/py/desitransfer/test/test_nightwatch.py b/py/desitransfer/test/test_nightwatch.py
new file mode 100644
index 0000000..ba79a26
--- /dev/null
+++ b/py/desitransfer/test/test_nightwatch.py
@@ -0,0 +1,88 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+# -*- coding: utf-8 -*-
+"""Test desitransfer.nightwatch.
+"""
+# import datetime
+import logging
+import os
+# import shutil
+import sys
+import unittest
+from tempfile import TemporaryDirectory
+from unittest.mock import call, patch, MagicMock
+from pkg_resources import resource_filename
+from ..nightwatch import (_options, _popen, _configure_log)
+
+class TestNightwatch(unittest.TestCase):
+    """Test desitransfer.nightwatch.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        pass
+
+    @classmethod
+    def tearDownClass(cls):
+        pass
+
+    def setUp(self):
+        """Create a temporary directory to simulate CSCRATCH.
+        """
+        self.tmp = TemporaryDirectory()
+
+    def tearDown(self):
+        """Clean up temporary directory.
+        """
+        self.tmp.cleanup()
+
+    def test_options(self):
+        """Test command-line arguments.
+        """
+        with patch.object(sys, 'argv', ['desi_nightwatch_transfer', '--debug']):
+            options = _options()
+            self.assertTrue(options.debug)
+            self.assertEqual(options.kill,
+                             os.path.join(os.environ['HOME'],
+                                          'stop_desi_transfer'))
+
+    @patch('desitransfer.nightwatch.SMTPHandler')
+    @patch('desitransfer.nightwatch.RotatingFileHandler')
+    @patch('desitransfer.nightwatch.get_logger')
+    @patch('desitransfer.nightwatch.log')  # Needed to restore the module-level log object after test.
+    def test_configure_log(self, mock_log, gl, rfh, smtp):
+        """Test logging configuration.
+        """
+        with patch.dict('os.environ',
+                        {'CSCRATCH': self.tmp.name,
+                         'DESI_ROOT': '/desi/root',
+                         'DESI_SPECTRO_DATA': '/desi/root/spectro/data'}):
+            with patch.object(sys, 'argv', ['desi_nightwatch_transfer', '--debug']):
+                options = _options()
+            _configure_log(options)
+        rfh.assert_called_once_with('/desi/root/spectro/nightwatch/desi_nightwatch_transfer.log',
+                                    backupCount=100, maxBytes=100000000)
+        gl.assert_called_once_with(timestamp=True)
+        gl().setLevel.assert_called_once_with(logging.DEBUG)
+
+    @patch('desitransfer.nightwatch.TemporaryFile')
+    @patch('subprocess.Popen')
+    @patch('desitransfer.nightwatch.log')
+    def test_popen(self, mock_log, mock_popen, mock_temp):
+        """Test Popen wrapper.
+        """
+        mock_file = mock_temp().__enter__.return_value = MagicMock()
+        mock_file.read.return_value = b'MOCK'
+        proc = mock_popen.return_value = MagicMock()
+        proc.returncode = 0
+        pp = _popen(['foo', 'bar'])
+        self.assertEqual(pp, ('0', 'MOCK', 'MOCK'))
+        mock_log.debug.assert_called_once_with('foo bar')
+        mock_popen.assert_called_once_with(['foo', 'bar'], stdout=mock_file, stderr=mock_file)
+
+
+def test_suite():
+    """Allows testing of only this module with the command::
+
+        python setup.py test -m <modulename>
+    """
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)
diff --git a/py/desitransfer/test/test_status.py b/py/desitransfer/test/test_status.py
index 23adbe1..8c1bcfd 100644
--- a/py/desitransfer/test/test_status.py
+++ b/py/desitransfer/test/test_status.py
@@ -96,8 +96,9 @@ class TestStatus(unittest.TestCase):
             s = TransferStatus(d)
             self.assertTrue(os.path.exists(os.path.join(d, 'desi_transfer_status.json.bad')))
             self.assertListEqual(s.status, [])
-            self.assertListEqual(os.listdir(d), ['desi_transfer_status.json.bad',
-                                                 'desi_transfer_status.json'])
+            self.assertSetEqual(frozenset(os.listdir(d)),
+                                frozenset(['desi_transfer_status.json.bad',
+                                           'desi_transfer_status.json']))
         mock_log.error.assert_called_once_with('Malformed JSON file detected: %s; saving original file as %s.',
                                                os.path.join(d, 'desi_transfer_status.json'),
                                                os.path.join(d, 'desi_transfer_status.json.bad'))
@@ -117,8 +118,9 @@ class TestStatus(unittest.TestCase):
             s = TransferStatus(d)
             self.assertTrue(os.path.exists(os.path.join(d, 'desi_transfer_status.json.bad')))
             self.assertListEqual(s.status, [])
-            self.assertListEqual(os.listdir(d), ['desi_transfer_status.json.bad',
-                                                 'desi_transfer_status.json'])
+            self.assertSetEqual(frozenset(os.listdir(d)),
+                                frozenset(['desi_transfer_status.json.bad',
+                                           'desi_transfer_status.json']))
         mock_print.assert_has_calls([call('ERROR: Malformed JSON file detected: %s; saving original file as %s.' % (os.path.join(d, 'desi_transfer_status.json'),
                                                                                                                     os.path.join(d, 'desi_transfer_status.json.bad'))),
                                      call("DEBUG: shutil.copy2('%s', '%s')" % (os.path.join(d, 'desi_transfer_status.json'),
@@ -137,18 +139,27 @@ class TestStatus(unittest.TestCase):
             with open(js, 'w') as f:
                 json.dump(st, f, indent=None, separators=(',', ':'))
             s = TransferStatus(d)
-            s.update('20200703', '12345678', 'checksum')
+            r = s.update('20200703', '12345678', 'checksum')
             self.assertTrue(os.path.exists(js + '.bak'))
+            self.assertEqual(r, 1)
             self.assertEqual(s.status[0], [20200703, 12345678, 'checksum', True, '', 1565300090000])
-            s.update('20200703', '12345680', 'rsync')
+            r = s.update('20200703', '12345680', 'rsync')
+            self.assertEqual(r, 1)
             self.assertEqual(s.status[0], [20200703, 12345680, 'rsync', True, '', 1565300090000])
-            s.update('20200703', '12345678', 'rsync', failure=True)
+            r = s.update('20200703', '12345678', 'rsync', failure=True)
+            self.assertEqual(r, 1)
             self.assertEqual(s.status[2], [20200703, 12345678, 'rsync', False, '', 1565300090000])
-            s.update('20200703', '12345681', 'pipeline')
+            r = s.update('20200703', '12345678', 'rsync', failure=True)
+            self.assertEqual(r, 0)
+            self.assertEqual(s.status[2], [20200703, 12345678, 'rsync', False, '', 1565300090000])
+            r = s.update('20200703', '12345681', 'pipeline')
+            self.assertEqual(r, 1)
             self.assertEqual(s.status[0], [20200703, 12345681, 'pipeline', True, '', 1565300090000])
-            s.update('20200703', '12345681', 'pipeline', last='arcs')
+            r = s.update('20200703', '12345681', 'pipeline', last='arcs')
+            self.assertEqual(r, 1)
             self.assertEqual(s.status[0], [20200703, 12345681, 'pipeline', True, 'arcs', 1565300090000])
-            s.update('20200703', 'all', 'backup')
+            r = s.update('20200703', 'all', 'backup')
+            self.assertEqual(r, 4)
             b = [i[3] for i in s.status if i[2] == 'backup']
             self.assertTrue(all(b))
             self.assertEqual(len(b), 4)