Repository: desihub/desitransfer

Issue:
Compact transfer status data
Consider reformatting the transfer status data, `desi_transfer_status.json`.  A preliminary test showed that it could be reduced in size by 50% with a simple reformat.  For example, this converts the data into a dictionary indexed by `[night][exposure]`:
```python
import os
import json
with open('desi_transfer_status.json') as j:
    data = json.load(j)

statuses = {'rsync': 0, 'checksum': 1, 'backup': 2}
nights = dict()
for row in data:
    if row[0] in nights:
        if row[1] in nights[row[0]]:
            nights[row[0]][row[1]].append([statuses[row[2]], int(row[3]), row[5]])
        else:
            nights[row[0]][row[1]] = [[statuses[row[2]], int(row[3]), row[5]]]
    else:
        nights[row[0]] = {row[1]: [[statuses[row[2]], int(row[3]), row[5]]]}

with open('nights.json', 'w') as j:
    json.dump(nights, j, indent=None, separators=(',', ':'))
```

Discussion:

Patch:
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 5575c79..721cce9 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -17,7 +17,7 @@ jobs:
             fail-fast: true
             matrix:
                 os: [ubuntu-latest]
-                python-version: [3.6, 3.7, 3.8, 3.9]
+                python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
 
         steps:
             - name: Checkout code
@@ -43,7 +43,7 @@ jobs:
             fail-fast: true
             matrix:
                 os: [ubuntu-latest]
-                python-version: [3.8]
+                python-version: ['3.9']
 
         steps:
             - name: Checkout code
@@ -74,7 +74,7 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: [3.8]
+                python-version: ['3.9']
 
         steps:
             - name: Checkout code
@@ -97,7 +97,7 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: [3.8]
+                python-version: ['3.9']
 
         steps:
             - name: Checkout code
diff --git a/bin/desi_spectro_data_unpack.sh b/bin/desi_spectro_data_unpack.sh
new file mode 100755
index 0000000..ddc0afe
--- /dev/null
+++ b/bin/desi_spectro_data_unpack.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Help!
+#
+function usage() {
+    local execName=$(basename $0)
+    (
+    echo "${execName} [-d DIR] [-h] [-I] [-t] [-v]"
+    echo ""
+    echo "Unpack DESI data tarballs."
+    echo ""
+    echo " -d DIR = Tarballs are on DIR, e.g. \${SCRATCH}"
+    echo "     -h = Print this message and exit."
+    echo "     -I = Ignore checksum errors on tarballs."
+    echo "     -t = Test mode.  Do not make any changes. Implies -v."
+    echo "     -v = Verbose mode. Print extra information."
+    echo ""
+    ) >&2
+}
+#
+# Make sure we are running on datatran.
+#
+if [[ "${NERSC_HOST}" != "datatran" ]]; then
+    echo "ERROR: This script should be executed on a DTN node!" >&2
+    exit 1
+fi
+#
+# Get options.
+#
+src_dir=${PWD}
+dst_dir=/global/cfs/cdirs/desi/spectro/staging/raw
+test=false
+verbose=false
+ignore_errors=false
+while getopts d:hItv argname; do
+    case ${argname} in
+        d) src_dir=${OPTARG} ;;
+        h) usage; exit 0 ;;
+        I) ignore_errors=true ;;
+        t) test=true; verbose=true ;;
+        v) verbose=true ;;
+        *) usage; exit 1 ;;
+    esac
+done
+shift $((OPTIND - 1))
+#
+# Set up environment.
+#
+[[ -z "${DESIUTIL}" ]] && source /global/common/software/desi/desi_environment.sh datatran-test
+[[ -z "${DESITRANSFER}" ]] && module load desitransfer/main
+#
+# Find tarballs.
+#
+declare -a nights
+for kk in ${src_dir}/*.sha256sum; do
+    k=$(basename ${kk})
+    d=$(dirname ${kk})
+    if [[ "${k}" == "*.sha256sum" ]]; then
+        echo "ERROR: No tarballs found!" >&2
+        exit 1
+    fi
+    #
+    # Verify tarball.
+    #
+    night=$(cut -d. -f1 <<<${k})
+    ${verbose} && echo "(cd ${d} && sha256sum --quiet --check ${k})"
+    ${test}    || (cd ${d} && sha256sum --quiet --check ${k})
+    if [[ $? != 0 ]]; then
+        if ${ignore_errors}; then
+            echo "WARNING: Checksum failure when verifying ${night}.tar, continuing anyway." >&2
+        else
+            echo "ERROR: Checksum failure when verifying ${night}.tar!" >&2
+            exit 1
+        fi
+    fi
+    #
+    # Unpack tarball.
+    #
+    ${verbose} && echo tar -xf ${d}/${night}.tar -C ${dst_dir}
+    ${test}    || tar -xf ${d}/${night}.tar -C ${dst_dir}
+    #
+    # Set permissions.
+    #
+    ${verbose} && echo "find ${night} -type f -exec chmod 0440 {} ;"
+    ${test}    || find ${night} -type f -exec chmod 0440 \{\} \;
+    ${verbose} && echo "find ${night} -type d -exec chmod 2550 {} ;"
+    ${test}    || find ${night} -type d -exec chmod 2550 \{\} \;
+    ${verbose} && echo chmod u+w ${night}
+    ${test}    || chmod u+w ${night}
+    #
+    # Move into place.
+    #
+    ${verbose} && echo mv ${night} ../../data
+    ${test}    || mv ${night} ../../data
+    ${verbose} && echo chmod u-w ../../data/${night}
+    ${test}    || chmod u-w ../../data/${night}
+    #
+    # Clean up.
+    #
+    ${verbose} && echo rm ${d}/${night}.tar ${d}/${night}.sha256sum
+    ${test}    || rm ${d}/${night}.tar ${d}/${night}.sha256sum
+    ${verbose} && echo "nights+=( ${night} )"
+    nights+=( ${night} )
+done
+#
+# Update transfer status after all nights are in place.
+#
+for night in "${nights[@]}"; do
+    for e in ../../data/${night}/*; do
+        expid=$(basename ${e})
+        ${verbose} && echo desi_transfer_status ${night} ${expid} rsync
+        ${test}    || desi_transfer_status ${night} ${expid} rsync
+        ${verbose} && echo "(cd ${e} && sha256sum --quiet --check checksum-${expid}.sha256sum)"
+        ${test}    || (cd ${e} && sha256sum --quiet --check checksum-${expid}.sha256sum)
+        if [[ $? == 0 ]]; then
+            ${verbose} && echo desi_transfer_status ${night} ${expid} checksum
+            ${test}    || desi_transfer_status ${night} ${expid} checksum
+        else
+            ${verbose} && echo desi_transfer_status --failure ${night} ${expid} checksum
+            ${test}    || desi_transfer_status --failure ${night} ${expid} checksum
+        fi
+    done
+    ${verbose} && echo desi_transfer_status ${night} all backup
+    ${test}    || desi_transfer_status ${night} all backup
+done
diff --git a/bin/desi_transfer_status_restore b/bin/desi_transfer_status_restore
index 4766fc8..37fbb2e 100755
--- a/bin/desi_transfer_status_restore
+++ b/bin/desi_transfer_status_restore
@@ -45,19 +45,18 @@ def main():
     """
     bt = backup_times()
     nights = os.listdir(os.environ['DESI_SPECTRO_DATA'])
-    status = list()
+    status = dict()
     for night in nights:
+        status[night] = dict()
         if night != 'README.html':
             exposures = os.listdir(os.path.join(os.environ['DESI_SPECTRO_DATA'], night))
             for exp in exposures:
                 rt = int(os.stat(os.path.join(os.environ['DESI_SPECTRO_DATA'], night, exp)).st_mtime * 1000)
-                status.append([int(night), int(exp), 'rsync', True, '', rt])
-                status.append([int(night), int(exp), 'checksum', True, '', rt])
+                status[night][exposure] = [[0, 1, rt], [1, 1, rt]]
                 try:
-                    status.append([int(night), int(exp), 'backup', True, '', bt[night]])
+                    status[night][exposure].append([2, 1, bt[night]])
                 except KeyError:
                     pass
-    status = sorted(status, key=lambda x: x[0]*10000000 + x[1], reverse=True)
     with open('desi_transfer_status_restore.json', 'w') as j:
         json.dump(status, j, indent=None, separators=(',', ':'))
     return 0
diff --git a/doc/changes.rst b/doc/changes.rst
index 719da9d..9c7ca6f 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -2,10 +2,17 @@
 Change Log
 ==========
 
-0.7.3 (unreleased)
+1.0.0 (unreleased)
 ------------------
 
-* No changes yet.
+* *Planned*: refactor package to deprecate ``setup.py``.
+
+0.8.0 (unreleased)
+------------------
+
+* Use a more compact format for raw data transfer status (PR `#49`_).
+
+.. _`#49`: https://github.com/desihub/desitransfer/pull/49
 
 0.7.2 (2022-08-11)
 ------------------
diff --git a/py/desitransfer/_version.py b/py/desitransfer/_version.py
index c0e68ca..a3da054 100644
--- a/py/desitransfer/_version.py
+++ b/py/desitransfer/_version.py
@@ -1,1 +1,1 @@
-__version__ = '0.7.3.dev404'
+__version__ = '0.8.0.dev404'
diff --git a/py/desitransfer/data/desi_transfer_status.html b/py/desitransfer/data/desi_transfer_status.html
index 762d3d7..5a4f590 100644
--- a/py/desitransfer/data/desi_transfer_status.html
+++ b/py/desitransfer/data/desi_transfer_status.html
@@ -5,13 +5,16 @@
         <meta charset="utf-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
         <link rel="stylesheet" type="text/css"
-            href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
-            integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO"
+            href="https://cdn.jsdelivr.net/npm/bootstrap@4.6.2/dist/css/bootstrap.min.css"
+            integrity="sha384-xOolHFLEh07PJGoPkLv1IbcEPTNtaed2xpHsD9ESMhqIYd0nLMwNLD69Npy4HI+N"
             crossorigin="anonymous" />
-        <!-- <link rel="stylesheet" type="text/css" media="screen" href="dts_status.css" /> -->
         <script type="text/javascript"
-            src="https://code.jquery.com/jquery-3.3.1.min.js"
-            integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8="
+            src="https://code.jquery.com/jquery-3.6.1.min.js"
+            integrity="sha256-o88AwQnZB+VDvE9tvIXrMQaPlFFSUTR+nldQm1LuPXQ="
+            crossorigin="anonymous"></script>
+        <script type="text/javascript"
+            src="https://cdn.jsdelivr.net/npm/bootstrap@4.6.2/dist/js/bootstrap.bundle.min.js"
+            integrity="sha384-Fy6S3B9q64WdZWQUiU+q4/2Lc9npb8tCaSX9FK7E8HnRr0Jz8D6OP9dO5Vg3Q9ct"
             crossorigin="anonymous"></script>
         <script type="text/javascript" src="desi_transfer_status.js"></script>
         <style type="text/css">
@@ -19,12 +22,30 @@
         </style>
     </head>
     <body>
-        <div class="container">
+        <nav class="navbar navbar-light navbar-expand-lg bg-light">
+            <div class="container">
+                <a class="navbar-brand" href="https://www.desi.lbl.gov/">DESI</a>
+                <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
+                    <span class="navbar-toggler-icon"></span>
+                </button>
+                <div class="collapse navbar-collapse" id="navbarCollapse">
+                    <ul class="navbar-nav mr-auto">
+                        <li class="nav-item active"><a class="nav-link" href="./index.html">&#127968; Home <span class="sr-only">(current)</span></a></li>
+                    </ul>
+                </div><!-- end class="collapse navbar-collapse" -->
+            </div><!-- end class="container" -->
+        </nav><!-- end class="navbar" -->
+        <main role="main" class="container">
             <div class="row">
-                <div class="col-12">
+                <div class="col">
                     <h1>DESI Transfer Status</h1>
+                </div><!-- end class="col" -->
+            </div><!-- end class="row" -->
+            <div class="row">
+                <div class="col">
                     <form>
-                        <fieldset><legend>Display Mode</legend>
+                        <fieldset id="years"></fieldset>
+                        <fieldset id="display"><legend>Display Mode</legend>
                             <div class="form-check form-check-inline">
                                 <input class="form-check-input displayMode" type="radio" name="displayMode" id="display10" value="display10" checked="checked" />
                                 <label class="form-check-label displayMode" for="display10">Display Last 10 Nights</label>
@@ -35,9 +56,14 @@
                             </div>
                         </fieldset>
                     </form>
-                </div><!-- end class="col-12" -->
+                </div><!-- end class="col" -->
+            </div><!-- end class="row" -->
+            <div class="row">
+                <div class="col">
+                    <h2 id="displayTitle"></h2>
+                </div><!-- end class="col" -->
             </div><!-- end class="row" -->
             <div id="content"></div>
-        </div><!-- end class="container" -->
+        </main>
     </body>
 </html>
diff --git a/py/desitransfer/data/desi_transfer_status.js b/py/desitransfer/data/desi_transfer_status.js
index 8afebf3..bd15f26 100644
--- a/py/desitransfer/data/desi_transfer_status.js
+++ b/py/desitransfer/data/desi_transfer_status.js
@@ -7,8 +7,8 @@ $(function() {
             this.n = n;
             this.exposures = [];
             this.div = $("<div/>", {"id": ""+this.n, "class": "row"});
-            this.buttons = $("<div/>", {"class": "col-4"});
-            this.table = $("<div/>", {"class": "col-8"});
+            this.buttons = $("<div/>", {"class": "col-md-4"});
+            this.table = $("<div/>", {"class": "col-md-8"});
         }
         var N = Night, Np = Night.prototype;
         //
@@ -19,7 +19,7 @@ $(function() {
             if (i == -1) {
                 this.exposures.push(E);
             } else {
-                this.exposures[i].addStage(E);
+                alert("Trying to add duplicate exposure " + E.e + " to Night " + this.n + "!");
             }
         };
         //
@@ -84,12 +84,13 @@ $(function() {
         // Table of individual exposures.
         //
         Np.table_rows = function() {
-            var r = "<table id=\"t" + this.n + "\" class=\"table table-borderless table-sm\"style=\"display:none;\">" +
-                    this.exposures[0].header() + "<tbody>";
+            var r = $("<table/>", {"id": "t" + this.n, "class": "table table-borderless table-sm", "style": "display:none;"});
+            r.append(this.exposures[0].header());
+            var body = $("<tbody/>");
             for (var k = 0; k < this.exposures.length; k++) {
-                r += this.exposures[k].row();
+                body.append(this.exposures[k].row());
             }
-            r += "</tbody></table>";
+            r.append(body);
             return r;
         };
         return Night;
@@ -98,23 +99,31 @@ $(function() {
         //
         // Constructor.
         //
-        function Exposure(r) {
-            this.n = r[0];
-            this.e = r[1];
+        function Exposure(n, e, r) {
+            this.n = n;
+            this.e = e;
             this.stage = {};
-            for (var k = 0; k < Exposure.stages.length; k++) {
-                this.stage[Exposure.stages[k]] = {"success": false, "stamp": 0};
+            for (var i = 0; i < r.length; i++) {
+                var s = Exposure.stages[r[i][0]];
+                if (this.stage.hasOwnProperty(s)) {
+                    if (r[i][2] > this.stage[s].stamp) {
+                        this.stage[s].success = r[i][1] == 1;
+                        this.stage[s].stamp = r[i][2];
+                    }
+                } else {
+                    this.stage[s] = {"success": r[i][1] == 1, "stamp": r[i][2]};
+                }
             }
-            if (Exposure.stages.indexOf(r[2]) == -1) {
-                alert("Invalid stage '" + r[2] + "' in " + r[0] + "/" + r[1] + "!");
-            } else {
-                this.stage[r[2]] = {"success": r[3], "stamp": r[5]};
+            for (var j = 0; j < Exposure.stages.length; j++) {
+                var s = Exposure.stages[j];
+                if (!this.stage.hasOwnProperty(s)) {
+                    this.stage[s] = {"success": false, "stamp": 0};
+                }
             }
-            // this.c = this.status ? "bg-success" : "bg-danger";
-            // this.l = r[4].length > 0 ? " Last " + r[4] + " exposure." : "";
         }
         var E = Exposure, Ep = Exposure.prototype;
         E.padding = 8;
+        E.rawBaseURL = "https://data.desi.lbl.gov/desi/spectro/data/";
         E.stages = ["rsync", "checksum", "backup"];
         //
         // Pad integers out to 8 characters.
@@ -125,23 +134,31 @@ $(function() {
             return pe.join("");
         };
         //
+        // URL for actual raw data.
+        //
+        Ep.rawURL = function() {
+            return Exposure.rawBaseURL + this.n + "/" + this.pad() + "/";
+        };
+        //
         // Header for status table.
         //
         Ep.header = function() {
-            var h = "<thead><tr><th class=\"text-uppercase\">exposure</th>";
+            var h = $("<thead/>");
+            var r = $("<tr/>");
+            r.append($("<th/>", {"class": "text-uppercase"}).html("exposure"));
             for (var k = 0; k < Exposure.stages.length; k++) {
-                h += "<th class=\"text-uppercase\">" + Exposure.stages[k] + "</th>";
+                r.append($("<th/>", {"class": "text-uppercase"}).html(Exposure.stages[k]));
             }
-            // h += "<th class=\"text-uppercase\">comment</th></tr></thead>";
-            h += "</tr></thead>";
+            h.append(r);
             return h;
         };
         //
         // Row in the status table.
         //
         Ep.row = function() {
-            var r = "<tr id=\"e" + this.toString() +"\">" +
-                    "<td>" + this.pad() + "</td>";
+            var r = $("<tr/>", {"id": "e" + this.toString()});
+            var link = $("<a/>", {"href": this.rawURL()}).html(this.pad());
+            r.append($("<td/>").html(link));
             for (var k = 0; k < Exposure.stages.length; k++) {
                 var c = "table-warning";
                 var stamp = "INCOMPLETE";
@@ -150,38 +167,11 @@ $(function() {
                     var d = new Date(this.stage[Exposure.stages[k]].stamp);
                     stamp = d.toISOString();
                 }
-                r +=  "<td class=\"" + c + "\">" + stamp + "</td>";
+                r.append($("<td/>", {"class": c}).html("<small>" + stamp + "</small>"));
             }
-            // r += "<td>" + this.l + "</td></tr>";
-            r += "</tr>";
             return r;
         };
         //
-        // Add additional stage data to an existing exposure.
-        //
-        Ep.addStage = function(stage) {
-            if (stage.n == this.n && stage.e == this.e) {
-                for (var k = 0; k < Exposure.stages.length; k++) {
-                    var s = Exposure.stages[k]
-                    //
-                    // Does stage have a defined timestamp?
-                    //
-                    if (stage.stage[s].stamp != 0) {
-                        //
-                        // Is it more recent?  This will also be true if this
-                        // has an undefined timestamp.
-                        //
-                        if (stage.stage[s].stamp > this.stage[s].stamp) {
-                            this.stage[s].success = stage.stage[s].success;
-                            this.stage[s].stamp = stage.stage[s].stamp;
-                        }
-                    }
-                }
-            } else {
-                alert("Can't add " + stage.toString() + " to " + this.toString() + "!");
-            }
-        };
-        //
         // Format night/exposure.
         //
         Ep.toString = function() {
@@ -189,19 +179,32 @@ $(function() {
         };
         return Exposure;
     })();
+    //
+    // Current year.
+    //
+    var d = new Date();
+    var currentYear = "" + d.getFullYear();
+    //
+    // Object to store status data.
+    //
     var Status = {
         //
         // Display status.
         //
         displayAll: false,
         //
+        // Display year.
+        //
+        displayYear: currentYear,
+        //
         // Raw data read from JSON file.
         //
-        raw: [],
+        // raw: [],
+        raw: {currentYear: {}},
         //
         // List of Night objects.
         //
-        nights: [],
+        nights: {currentYear: []},
         //
         // If a night already exists in the nights array, return the index.
         //
@@ -210,44 +213,53 @@ $(function() {
                 if (this.nights[k].n == n) return k;
             }
             return -1;
+        },
+        //
+        // Per-year data file.
+        //
+        dataFile: function() {
+            return "desi_transfer_status_" + this.displayYear + ".json";
         }
     };
     //
     // Main display function.
     //
-    display = function() {
+    var display = function() {
+        var default_display = 10;
         $("#content").empty();
+        var h2 = "Showing " +
+                 (Status.displayAll ? "all" : "most recent 10") +
+                 " Nights from " +
+                 Status.displayYear;
+        $("#displayTitle").html(h2);
+        var nights = Status.nights[Status.displayYear];
+        nights = [];
         var night;
-        Status.nights = [];
-        var N_nights = 0;
-        var N_display = 10;
-        for (var k = 0; k < Status.raw.length; k++) {
-            var n = Status.raw[k][0];
-            if (Status.hasNight(n) == -1) {
-                //
-                // Finish previous night
-                //
-                if (Status.nights.length > 0) {
-                    night.finish();
-                    if (!Status.displayAll && N_nights >= N_display) break;
-                }
+        var raw = Status.raw[Status.displayYear];
+        var all_nights = Object.keys(raw).sort().reverse();
+        // alert(all_nights.join(","));
+        var n_display = Status.displayAll ? all_nights.length : default_display;
+        if (all_nights.length < default_display) n_display = all_nights.length;
+        for (var k = 0; k < n_display; k++) {
+            var n = all_nights[k];
+            //
+            // Start a new night
+            //
+            night = new Night(n);
+            nights.push(night);
+            var exposures = Object.keys(raw[n]).sort().reverse();
+            for (var l = 0; l < exposures.length; l++) {
                 //
-                // Start a new night
+                // Add exposure to existing night.
                 //
-                night = new Night(n);
-                Status.nights.push(night);
-                N_nights += 1;
+                var e = new Exposure(n, exposures[l], raw[n][exposures[l]]);
+                night.addExposure(e);
             }
             //
-            // Add exposure to existing night.
+            // Finish the current night.
             //
-            var e = new Exposure(Status.raw[k]);
-            night.addExposure(e);
+            night.finish();
         }
-        //
-        // Finish the final night
-        //
-        if (Status.displayAll) night.finish();
     };
     //
     // Display Mode.
@@ -255,6 +267,38 @@ $(function() {
     $(".displayMode").change(function() {
         return Status.displayAll = $("input[name=displayMode]:checked").val() === "displayAll";
     }).change(display);
-    $.getJSON("desi_transfer_status.json", {}, function(data) { Status.raw = data; }).always(display);
+    //
+    // Dynamically generate year selection.
+    //
+    var years = function() {
+        var firstYear = 2018;
+        $("#years").empty();
+        $("#years").append($("<legend/>").html("Display Year"));
+        for (var year = currentYear; year >= firstYear; year--) {
+            var d = $("<div/>", {"class": "form-check form-check-inline"});
+            var io = {"class": "form-check-input displayYear", "type": "radio", "name": "displayYear", "id": "display" + year, "value": "" + year}
+            if (year == currentYear) io["checked"] = "checked";
+            var i = $("<input/>", io);
+            var l = $("<label/>", {"class": "form-check-label displayYear", "for": "display" + year}).html("" + year);
+            d.append(i);
+            d.append(l);
+            $("#years").append(d);
+        }
+    };
+    years();
+    $(".displayYear").change(function() {
+        Status.displayYear = $("input[name=displayYear]:checked").val();
+        if (Status.raw.hasOwnProperty(Status.displayYear)) {
+            display();
+        } else {
+            Status.nights[Status.displayYear] = [];
+            $.getJSON(Status.dataFile(), {}, function(data) {Status.raw[Status.displayYear] = data;}).done(display);
+        }
+        return true;
+    });
+    //
+    // Load initial data.
+    //
+    $.getJSON(Status.dataFile(), {}, function(data) {Status.raw[Status.displayYear] = data;}).done(display);
     return true;
 });
diff --git a/py/desitransfer/status.py b/py/desitransfer/status.py
index 27c8034..6e82637 100644
--- a/py/desitransfer/status.py
+++ b/py/desitransfer/status.py
@@ -11,8 +11,10 @@ import os
 import shutil
 import sys
 import time
+from datetime import date
 from argparse import ArgumentParser
 from pkg_resources import resource_filename
+from desiutil.log import log, DEBUG
 from . import __version__ as dtVersion
 
 
@@ -23,24 +25,38 @@ class TransferStatus(object):
     ----------
     directory : :class:`str`
         Retrieve and store JSON-encoded transfer status data in `directory`.
+    install : :class:`bool`, optional
+        If ``True``, install HTML and JS files.
+    year : :class:`str` or :class:`int`
+        Update records belonging to `year`. If not set, the current
+        year is assumed.
     """
 
-    def __init__(self, directory):
+    def __init__(self, directory, install=False, year=None):
+        self._stages = {'rsync': 0, 'checksum': 1, 'backup': 2}
         self.directory = directory
-        self.json = os.path.join(self.directory, 'desi_transfer_status.json')
-        self.status = list()
-        if not os.path.exists(self.directory):
-            # log.debug("os.makedirs('%s')", self.directory)
-            os.makedirs(self.directory)
+        self.status = dict()
+        if year is None:
+            self.current_year = str(date.today().year)
+        else:
+            self.current_year = str(year)
+        self.first_year = "2018"
+        self.json = os.path.join(self.directory,
+                                 f'desi_transfer_status_{self.current_year}.json')
+        if not os.path.exists(self.directory) or install:
+            log.debug("os.makedirs('%s', exist_ok=True)", self.directory)
+            os.makedirs(self.directory, exist_ok=True)
             for ext in ('html', 'js'):
                 src = resource_filename('desitransfer',
                                         'data/desi_transfer_status.' + ext)
                 if ext == 'html':
+                    log.debug("shutil.copyfile('%s', '%s')", src,
+                              os.path.join(self.directory, 'index.html'))
                     shutil.copyfile(src,
                                     os.path.join(self.directory, 'index.html'))
                 else:
+                    log.debug("shutil.copy('%s', '%s')", src, self.directory)
                     shutil.copy(src, self.directory)
-            return
         try:
             with open(self.json) as j:
                 try:
@@ -57,31 +73,16 @@ class TransferStatus(object):
         This function will save the malformed file to a .bad file for
         later analysis, and write an empty array to a new status file.
         """
-        from .daemon import log
         bad = self.json + '.bad'
-        m = "Malformed JSON file detected: %s; saving original file as %s."
-        try:
-            log.error(m, self.json, bad)
-        except AttributeError:
-            # If the status code is running stand-alone, the log object
-            # will be None.
-            print("ERROR: " + (m % (self.json, bad)))
-        m = "shutil.copy2('%s', '%s')"
-        try:
-            log.debug(m, self.json, bad)
-        except AttributeError:
-            print("DEBUG: " + (m % (self.json, bad)))
+        log.error("Malformed JSON file detected: %s; saving original file as %s.", self.json, bad)
+        log.debug("shutil.copy2('%s', '%s')", self.json, bad)
         shutil.copy2(self.json, bad)
-        m = "Writing empty array to %s."
-        try:
-            log.info(m, self.json)
-        except AttributeError:
-            print("INFO: " + (m % (self.json,)))
+        log.info("Writing empty array to %s.", self.json)
         with open(self.json, 'w') as j:
-            j.write('[]')
+            j.write('{}')
         return
 
-    def update(self, night, exposure, stage, failure=False, last=''):
+    def update(self, night, exposure, stage, failure=False):
         """Update the transfer status.
 
         Parameters
@@ -94,9 +95,6 @@ class TransferStatus(object):
             Stage of data transfer ('rsync', 'checksum', 'backup', ...).
         failure : :class:`bool`, optional
             Indicate failure.
-        last : :class:`str`, optional
-            Mark this exposure as the last of a given type for the night
-            ('arcs', 'flats', 'science').
 
         Returns
         -------
@@ -104,21 +102,28 @@ class TransferStatus(object):
             The number of updates performed.
         """
         ts = int(time.time() * 1000)  # Convert to milliseconds for JS.
-        i = int(night)
         success = not failure
+        row = [self._stages[stage], int(success), ts]
         if exposure == 'all':
-            unique_ie = frozenset([self.status[k][1] for k in self.find(i)])
-            rows = [[i, ie, stage, success, last, ts]
-                    for ie in unique_ie]
+            rows = list()
+            for expid in self.status[night]:
+                log.debug("self.status['%s']['%s'].insert(0, [%d, %d, %d])", night, expid, row[0], row[1], row[2])
+                self.status[night][expid].insert(0, row)
+                rows.append(row)
         else:
-            ie = int(exposure)
-            r = [i, ie, stage, success, last, ts]
-            il = self.find(i, ie, stage)
+            expid = str(int(exposure))
+            if night not in self.status:
+                log.debug("self.status['%s'] = {'%s': []}", night, expid)
+                self.status[night] = {expid: []}
+            log.debug("il = self.find('%s', '%s', '%s')", night, expid, stage)
+            il = self.find(night, expid, stage)
             if il:
-                update = ((ts >= self.status[il[0]][5]) and
-                          (success is not self.status[il[0]][3]))
-                if last or update:
-                    self.status[il[0]] = r
+                old_row = self.status[night][expid][il[0]]
+                log.debug("self.status['%s']['%s'][%d] = [%d, %d, %d]", night, expid, il[0], old_row[0], old_row[1], old_row[2])
+                update = (ts >= old_row[2]) and (int(success) != old_row[1])
+                if update:
+                    log.debug("self.status['%s']['%s'][%d] = [%d, %d, %d]", night, expid, il[0], row[0], row[1], row[2])
+                    self.status[night][expid][il[0]] = row
                     rows = []
                 else:
                     #
@@ -127,15 +132,18 @@ class TransferStatus(object):
                     #
                     return 0
             else:
-                rows = [r, ]
-        for row in rows:
-            self.status.insert(0, row)
-        self.status = sorted(self.status, key=lambda x: x[0]*10000000 + x[1],
-                             reverse=True)
+                try:
+                    log.debug("self.status['%s']['%s'].insert(0, [%d, %d, %d])", night, expid, row[0], row[1], row[2])
+                    self.status[night][expid].insert(0, row)
+                except KeyError:
+                    log.debug("self.status['%s']['%s'] = [%d, %d, %d]", night, expid, row[0], row[1], row[2])
+                    self.status[night][expid] = [row]
+                rows = [row, ]
         #
         # Copy the original file before modifying.
         # This will overwrite any existing .bak file
         #
+        log.debug("shutil.copy2('%s', '%s')", self.json, self.json + '.bak')
         try:
             shutil.copy2(self.json, self.json + '.bak')
         except FileNotFoundError:
@@ -161,20 +169,45 @@ class TransferStatus(object):
 
         Returns
         -------
-        :class:`list`
-            A list of the *indexes* of matching status entries.
+        :class:`list` or class:`dict`
+            If only `night` is set, return a :class:`dict` containing
+            information on all exposures for that `night`. If `exposure`
+            is not set, return a :class:`dict` keyed by exposure containing
+            all data matching `stage` for that night. If `stage` is not set,
+            return a :class:`list` containing *indexes* pointing to
+            all data about that exposure. If both `exposure` and `stage` are set,
+            return a :class:`list` of *indexes* pointing to the data for `exposure`
+            filtered on `stage`.
+
+        Raises
+        ------
+        :exc:`KeyError`
+            If `night` is not yet defined.
         """
         if exposure is None and stage is None:
-            return [k for k, r in enumerate(self.status) if r[0] == night]
+            try:
+                return self.status[night]
+            except KeyError:
+                raise KeyError(f"Undefined night = '{night}'!")
         elif exposure is None:
-            return [k for k, r in enumerate(self.status) if r[0] == night
-                    and r[2] == stage]
+            e = dict()
+            for expid in self.status[night]:
+                e[expid] = [k for k, r in enumerate(self.status[night][expid]) if r[0] == self._stages[stage]]
+            return e
         elif stage is None:
-            return [k for k, r in enumerate(self.status) if r[0] == night
-                    and r[1] == exposure]
+            try:
+                log.debug("e = self.status['%s']['%s']", night, exposure)
+                e = self.status[night][exposure]
+            except KeyError:
+                log.debug("e = self.status['%s']['%s'] = list()", night, exposure)
+                e = self.status[night][exposure] = list()
+            return e
         else:
-            return [k for k, r in enumerate(self.status) if r[0] == night
-                    and r[1] == exposure and r[2] == stage]
+            try:
+                r = [k for k, r in enumerate(self.status[night][exposure]) if r[0] == self._stages[stage]]
+            except KeyError:
+                r = list()
+            return r
 
 
 def _options():
@@ -190,16 +223,15 @@ def _options():
     prsr.add_argument('-d', '--directory', dest='directory', metavar='DIR',
                       default=os.path.join(os.environ['DESI_ROOT'],
                                            'spectro', 'staging', 'status'),
-                      help="Install and update files in DIR.")
+                      help="Install and update files in DIR (default %(default)s).")
     prsr.add_argument('-f', '--failure', action='store_true', dest='failure',
                       help='Indicate that the transfer failed somehow.')
     prsr.add_argument('-i', '--install', action='store_true', dest='install',
                       help='Ensure that HTML and related files are in place.')
-    prsr.add_argument('-l', '--last', dest='last', default='',
-                      choices=['flats', 'arcs', 'science'],
-                      help='Indicate that a certain set of exposures is complete.')
     prsr.add_argument('-V', '--version', action='version',
                       version='%(prog)s {0}'.format(dtVersion))
+    prsr.add_argument('-v', '--verbose', action='store_true',
+                      help='Print debugging information.')
     prsr.add_argument('night', type=int, metavar='YYYYMMDD',
                       help="Night of observation.")
     prsr.add_argument('expid', metavar='EXPID',
@@ -218,10 +250,33 @@ def main():
     :class:`int`
         An integer suitable for passing to :func:`sys.exit`.
     """
-    # global log
+    global log
     options = _options()
-    # log = get_logger()
-    st = TransferStatus(options.directory)
-    st.update(options.night, options.expid, options.stage,
-              options.failure, options.last)
+    if options.verbose:
+        log.setLevel(DEBUG)
+    log.debug("st = TransferStatus('%s', install=%s, year='%s')", options.directory, options.install, str(options.night)[0:4])
+    st = TransferStatus(options.directory, install=options.install, year=str(options.night)[0:4])
+    log.debug("st.update('%s', '%s', '%s', %s)", str(options.night), options.expid, options.stage, options.failure)
+    st.update(str(options.night), options.expid, options.stage, options.failure)
     return 0
+
+
+# import os
+# import json
+# with open('desi_transfer_status.json') as j:
+#     data = json.load(j)
+#
+# statuses = {'rsync': 0, 'checksum': 1, 'backup': 2}
+# for year in ('2022', '2021', '2020', '2019', '2018', '2017'):
+#     nights = dict()
+#     for row in data:
+#         if str(row[0]).startswith(year):
+#             if row[0] in nights:
+#                 if row[1] in nights[row[0]]:
+#                     nights[row[0]][row[1]].append([statuses[row[2]], int(row[3]), row[5]])
+#                 else:
+#                     nights[row[0]][row[1]] = [[statuses[row[2]], int(row[3]), row[5]]]
+#             else:
+#                 nights[row[0]] = {row[1]: [[statuses[row[2]], int(row[3]), row[5]]]}
+#     with open(f'nights_{year}.json', 'w') as j:
+#         json.dump(nights, j, indent=None, separators=(',', ':'))
diff --git a/py/desitransfer/tucson.py b/py/desitransfer/tucson.py
index 3be2fe2..7b188f2 100644
--- a/py/desitransfer/tucson.py
+++ b/py/desitransfer/tucson.py
@@ -9,6 +9,7 @@ Entry point for :command:`desi_tucson_transfer`.
 import logging
 import os
 import subprocess as sub
+import time
 from argparse import ArgumentParser
 from socket import getfqdn
 from logging.handlers import SMTPHandler
@@ -228,12 +229,22 @@ def main():
     #
     # Wait for daily KPNO -> NERSC transfer to finish.
     #
+    try:
+        sleepy_time = int(options.sleep)
+    except ValueError:
+        suffix = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
+        for s in suffix:
+            if options.sleep.endswith(s):
+                try:
+                    sleepy_time = int(options.sleep[0:-1])*suffix[s]
+                except ValueError:
+                    log.error("Invalid value for sleep interval: '%s'!", options.sleep)
+                    return 1
     log.debug("requests.get('%s')", os.environ['DESISYNC_STATUS_URL'])
     if not options.test:
         while requests.get(os.environ['DESISYNC_STATUS_URL']).status_code != 200:
-            if options.debug:
-                log.info("Daily transfer incomplete, sleeping %s.", options.sleep)
-                time.sleep()
+            log.debug("Daily transfer incomplete, sleeping %s.", options.sleep)
+            time.sleep(sleepy_time)
     #
     # Main transfer
     #
diff --git a/requirements.txt b/requirements.txt
index 753685d..7a5fc11 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 setuptools
 requests
 pytz
-git+https://github.com/desihub/desiutil.git@3.2.2
+git+https://github.com/desihub/desiutil.git@3.2.6


Test patch:
diff --git a/py/desitransfer/test/test_common.py b/py/desitransfer/test/test_common.py
index 3ac959d..2d0c152 100644
--- a/py/desitransfer/test/test_common.py
+++ b/py/desitransfer/test/test_common.py
@@ -175,11 +175,3 @@ total size is 118,417,836,324  speedup is 494,367.55
         # mock_datetime.return_value = datetime(2021, 7, 3, 13, 0, 0, tzinfo=MST)
         i = idle_time(tz='US/Pacific')
         self.assertEqual(i, -3600)
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
diff --git a/py/desitransfer/test/test_daemon.py b/py/desitransfer/test/test_daemon.py
index e96efd4..b31a982 100644
--- a/py/desitransfer/test/test_daemon.py
+++ b/py/desitransfer/test/test_daemon.py
@@ -1160,11 +1160,3 @@ total size is 118,417,836,324  speedup is 494,367.55
                                                   '1', '20190703')
         mock_log.error.assert_has_calls([call('rsync STDOUT = \n%s', 'stdout'),
                                          call('rsync STDERR = \n%s', 'stderr')])
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
diff --git a/py/desitransfer/test/test_daily.py b/py/desitransfer/test/test_daily.py
index b26dd15..f472f42 100644
--- a/py/desitransfer/test/test_daily.py
+++ b/py/desitransfer/test/test_daily.py
@@ -167,11 +167,3 @@ class TestDaily(unittest.TestCase):
                                      call(['fix_permissions.sh', '/dst/d0'],
                                           stdout=mo(), stderr=-2),
                                      call().wait()])
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
diff --git a/py/desitransfer/test/test_nightlog.py b/py/desitransfer/test/test_nightlog.py
index 6098579..c7d3a79 100644
--- a/py/desitransfer/test/test_nightlog.py
+++ b/py/desitransfer/test/test_nightlog.py
@@ -62,11 +62,3 @@ class TestNightlog(unittest.TestCase):
                                     backupCount=100, maxBytes=100000000)
         gl.assert_called_once_with(timestamp=True)
         gl().setLevel.assert_called_once_with(logging.DEBUG)
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
diff --git a/py/desitransfer/test/test_nightwatch.py b/py/desitransfer/test/test_nightwatch.py
index ade1540..9ebb52e 100644
--- a/py/desitransfer/test/test_nightwatch.py
+++ b/py/desitransfer/test/test_nightwatch.py
@@ -64,11 +64,3 @@ class TestNightwatch(unittest.TestCase):
                                     backupCount=100, maxBytes=100000000)
         gl.assert_called_once_with(timestamp=True)
         gl().setLevel.assert_called_once_with(logging.DEBUG)
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
diff --git a/py/desitransfer/test/test_status.py b/py/desitransfer/test/test_status.py
index 8c1bcfd..5216c06 100644
--- a/py/desitransfer/test/test_status.py
+++ b/py/desitransfer/test/test_status.py
@@ -19,7 +19,8 @@ class TestStatus(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        pass
+        cls.fake_status = {"20200703": {"12345677": [[0, 1, 1565300073000]],
+                                        "12345678": [[0, 1, 1565300074664]]}}
 
     @classmethod
     def tearDownClass(cls):
@@ -56,168 +57,171 @@ class TestStatus(unittest.TestCase):
             self.assertEqual(s.directory, d)
             self.assertTrue(os.path.isdir(d))
             self.assertListEqual(os.listdir(d), [])
-            self.assertListEqual(s.status, [])
+            self.assertListEqual(list(s.status.keys()), [])
         #
         # Directory with JSON file.
         #
-        st = [[20200703, 12345678, 'rsync', True, '', 1565300074664],
-              [20200703, 12345677, 'rsync', True, '', 1565300073000]]
         with TemporaryDirectory() as d:
-            js = os.path.join(d, 'desi_transfer_status.json')
+            js = os.path.join(d, 'desi_transfer_status_2020.json')
             with open(js, 'w') as f:
-                json.dump(st, f, indent=None, separators=(',', ':'))
-            s = TransferStatus(d)
+                json.dump(self.fake_status, f, indent=None, separators=(',', ':'))
+            s = TransferStatus(d, year='2020')
             self.assertEqual(s.directory, d)
             self.assertTrue(os.path.isdir(d))
-            self.assertListEqual(os.listdir(d), ['desi_transfer_status.json'])
-            self.assertListEqual(s.status[0], st[0])
-            self.assertListEqual(s.status[1], st[1])
+            self.assertListEqual(os.listdir(d), ['desi_transfer_status_2020.json'])
+            self.assertListEqual(sorted(list(s.status.keys())),
+                                 sorted(list(self.fake_status.keys())))
+            self.assertListEqual(sorted(list(s.status['20200703'].keys())),
+                                 sorted(list(self.fake_status['20200703'].keys())))
         #
         # New directory.
         #
         d = '/desi/spectro/status'
-        # with patch('desitransfer.status.log') as l:
-        with patch('os.makedirs') as m:
-            with patch('shutil.copy') as cp:
-                with patch('shutil.copyfile') as cf:
-                    s = TransferStatus(d)
-        # l.debug.assert_called_once_with("os.makedirs('%s')", d)
-        m.assert_called_once_with(d)
+        with patch('desitransfer.status.log') as l:
+            with patch('os.makedirs') as m:
+                with patch('shutil.copy') as cp:
+                    with patch('shutil.copyfile') as cf:
+                        s = TransferStatus(d)
+        l.debug.assert_has_calls([call("os.makedirs('%s', exist_ok=True)", d),
+                                  call("shutil.copyfile('%s', '%s')", h, os.path.join(d, 'index.html')),
+                                  call("shutil.copy('%s', '%s')", j, d)])
+        m.assert_called_once_with(d, exist_ok=True)
         cp.assert_called_once_with(j, d)
         cf.assert_called_once_with(h, os.path.join(d, 'index.html'))
 
-    @patch('desitransfer.daemon.log')
+    @patch('desitransfer.status.log')
     def test_TransferStatus_handle_malformed_with_log(self, mock_log):
         """Test handling of malformed JSON files.
         """
         bad = resource_filename('desitransfer.test', 't/bad.json')
         with TemporaryDirectory() as d:
-            shutil.copy(bad, os.path.join(d, 'desi_transfer_status.json'))
-            s = TransferStatus(d)
-            self.assertTrue(os.path.exists(os.path.join(d, 'desi_transfer_status.json.bad')))
-            self.assertListEqual(s.status, [])
+            shutil.copy(bad, os.path.join(d, 'desi_transfer_status_2020.json'))
+            s = TransferStatus(d, year=2020)
+            self.assertTrue(os.path.exists(os.path.join(d, 'desi_transfer_status_2020.json.bad')))
+            self.assertListEqual(list(s.status.keys()), [])
             self.assertSetEqual(frozenset(os.listdir(d)),
-                                frozenset(['desi_transfer_status.json.bad',
-                                           'desi_transfer_status.json']))
+                                frozenset(['desi_transfer_status_2020.json.bad',
+                                           'desi_transfer_status_2020.json']))
         mock_log.error.assert_called_once_with('Malformed JSON file detected: %s; saving original file as %s.',
-                                               os.path.join(d, 'desi_transfer_status.json'),
-                                               os.path.join(d, 'desi_transfer_status.json.bad'))
+                                               os.path.join(d, 'desi_transfer_status_2020.json'),
+                                               os.path.join(d, 'desi_transfer_status_2020.json.bad'))
         mock_log.debug.assert_called_once_with("shutil.copy2('%s', '%s')",
-                                               os.path.join(d, 'desi_transfer_status.json'),
-                                               os.path.join(d, 'desi_transfer_status.json.bad'))
+                                               os.path.join(d, 'desi_transfer_status_2020.json'),
+                                               os.path.join(d, 'desi_transfer_status_2020.json.bad'))
         mock_log.info.assert_called_once_with('Writing empty array to %s.',
-                                              os.path.join(d, 'desi_transfer_status.json'))
-
-    @patch('builtins.print')
-    def test_TransferStatus_handle_malformed_without_log(self, mock_print):
-        """Test handling of malformed JSON files (no log object).
-        """
-        bad = resource_filename('desitransfer.test', 't/bad.json')
-        with TemporaryDirectory() as d:
-            shutil.copy(bad, os.path.join(d, 'desi_transfer_status.json'))
-            s = TransferStatus(d)
-            self.assertTrue(os.path.exists(os.path.join(d, 'desi_transfer_status.json.bad')))
-            self.assertListEqual(s.status, [])
-            self.assertSetEqual(frozenset(os.listdir(d)),
-                                frozenset(['desi_transfer_status.json.bad',
-                                           'desi_transfer_status.json']))
-        mock_print.assert_has_calls([call('ERROR: Malformed JSON file detected: %s; saving original file as %s.' % (os.path.join(d, 'desi_transfer_status.json'),
-                                                                                                                    os.path.join(d, 'desi_transfer_status.json.bad'))),
-                                     call("DEBUG: shutil.copy2('%s', '%s')" % (os.path.join(d, 'desi_transfer_status.json'),
-                                                                               os.path.join(d, 'desi_transfer_status.json.bad'))),
-                                     call("INFO: Writing empty array to %s." % (os.path.join(d, 'desi_transfer_status.json'),))])
+                                              os.path.join(d, 'desi_transfer_status_2020.json'))
 
     @patch('time.time')
     def test_TransferStatus_update(self, mock_time):
         """Test status reporting mechanism updates.
         """
         mock_time.return_value = 1565300090
-        st = [[20200703, 12345678, 'rsync', True, '', 1565300074664],
-              [20200703, 12345677, 'rsync', True, '', 1565300073000]]
         with TemporaryDirectory() as d:
-            js = os.path.join(d, 'desi_transfer_status.json')
+            js = os.path.join(d, 'desi_transfer_status_2020.json')
             with open(js, 'w') as f:
-                json.dump(st, f, indent=None, separators=(',', ':'))
-            s = TransferStatus(d)
-            r = s.update('20200703', '12345678', 'checksum')
+                json.dump(self.fake_status, f, indent=None, separators=(',', ':'))
+            s = TransferStatus(d, year=2020)
+            r = s.update('20200703', '12345677', 'checksum')
             self.assertTrue(os.path.exists(js + '.bak'))
             self.assertEqual(r, 1)
-            self.assertEqual(s.status[0], [20200703, 12345678, 'checksum', True, '', 1565300090000])
+            self.assertEqual(s.status['20200703']['12345677'][0], [1, 1, 1565300090000])
+
+            r = s.update('20200703', '12345678', 'checksum', failure=True)
+            self.assertEqual(r, 1)
+            self.assertEqual(s.status['20200703']['12345678'][0], [1, 0, 1565300090000])
+
             r = s.update('20200703', '12345680', 'rsync')
             self.assertEqual(r, 1)
-            self.assertEqual(s.status[0], [20200703, 12345680, 'rsync', True, '', 1565300090000])
-            r = s.update('20200703', '12345678', 'rsync', failure=True)
+            self.assertEqual(s.status['20200703']['12345680'][0], [0, 1, 1565300090000])
+
+            r = s.update('20200703', '12345678', 'checksum')
             self.assertEqual(r, 1)
-            self.assertEqual(s.status[2], [20200703, 12345678, 'rsync', False, '', 1565300090000])
-            r = s.update('20200703', '12345678', 'rsync', failure=True)
-            self.assertEqual(r, 0)
-            self.assertEqual(s.status[2], [20200703, 12345678, 'rsync', False, '', 1565300090000])
-            r = s.update('20200703', '12345681', 'pipeline')
+            self.assertEqual(s.status['20200703']['12345678'][0], [1, 1, 1565300090000])
+
+            r = s.update('20200703', '12345680', 'checksum')
             self.assertEqual(r, 1)
-            self.assertEqual(s.status[0], [20200703, 12345681, 'pipeline', True, '', 1565300090000])
-            r = s.update('20200703', '12345681', 'pipeline', last='arcs')
+            self.assertEqual(s.status['20200703']['12345680'][0], [1, 1, 1565300090000])
+
+            r = s.update('20200703', '12345681', 'rsync')
             self.assertEqual(r, 1)
-            self.assertEqual(s.status[0], [20200703, 12345681, 'pipeline', True, 'arcs', 1565300090000])
+            self.assertEqual(s.status['20200703']['12345681'][0], [0, 1, 1565300090000])
+
+            r = s.update('20200703', '12345681', 'checksum')
+            self.assertEqual(r, 1)
+            self.assertEqual(s.status['20200703']['12345681'][0], [1, 1, 1565300090000])
+
             r = s.update('20200703', 'all', 'backup')
             self.assertEqual(r, 4)
-            b = [i[3] for i in s.status if i[2] == 'backup']
-            self.assertTrue(all(b))
-            self.assertEqual(len(b), 4)
+            c = 0
+            for e in s.status['20200703']:
+                for r in s.status['20200703'][e]:
+                    if r[0] == 2:
+                        self.assertEqual(r[1], 1)
+                        c += 1
+            self.assertEqual(c, 4)
 
     @patch('time.time')
     def test_TransferStatus_update_empty(self, mock_time):
         """Test status reporting mechanism updates (with no initial JSON file).
         """
         mock_time.return_value = 1565300090
-        # st = [[20200703, 12345678, 'rsync', True, '', 1565300074664],
-        #       [20200703, 12345677, 'rsync', True, '', 1565300073000]]
         with TemporaryDirectory() as d:
-            js = os.path.join(d, 'desi_transfer_status.json')
-            # with open(js, 'w') as f:
-            #     json.dump(st, f, indent=None, separators=(',', ':'))
-            s = TransferStatus(d)
+            js = os.path.join(d, 'desi_transfer_status_2020.json')
+            s = TransferStatus(d, year=2020)
             s.update('20200703', '12345678', 'checksum')
             self.assertFalse(os.path.exists(js + '.bak'))
-            self.assertEqual(s.status[0], [20200703, 12345678, 'checksum', True, '', 1565300090000])
+            self.assertEqual(s.status['20200703']['12345678'][0], [1, 1, 1565300090000])
             s.update('20200703', '12345680', 'rsync')
             self.assertTrue(os.path.exists(js + '.bak'))
-            self.assertEqual(s.status[0], [20200703, 12345680, 'rsync', True, '', 1565300090000])
+            self.assertEqual(s.status['20200703']['12345680'][0], [0, 1, 1565300090000])
             s.update('20200703', '12345678', 'checksum', failure=True)
-            self.assertEqual(s.status[1], [20200703, 12345678, 'checksum', False, '', 1565300090000])
-            s.update('20200703', '12345681', 'pipeline')
-            self.assertEqual(s.status[0], [20200703, 12345681, 'pipeline', True, '', 1565300090000])
-            s.update('20200703', '12345681', 'pipeline', last='arcs')
-            self.assertEqual(s.status[0], [20200703, 12345681, 'pipeline', True, 'arcs', 1565300090000])
+            self.assertEqual(s.status['20200703']['12345678'][0], [1, 0, 1565300090000])
+            s.update('20200703', '12345681', 'rsync')
+            self.assertEqual(s.status['20200703']['12345681'][0], [0, 1, 1565300090000])
             s.update('20200703', 'all', 'backup')
-            b = [i[3] for i in s.status if i[2] == 'backup']
-            self.assertTrue(all(b))
-            self.assertEqual(len(b), 3)
+            c = 0
+            for e in s.status['20200703']:
+                for r in s.status['20200703'][e]:
+                    if r[0] == 2:
+                        self.assertEqual(r[1], 1)
+                        c += 1
+            self.assertEqual(c, 3)
+
+    @patch('time.time')
+    def test_TransferStatus_edge_case(self, mock_time):
+        """Test edge case when desitransfer.daemon is running in test mode.
+        """
+        mock_time.return_value = 1565300000
+        with TemporaryDirectory() as d:
+            js = os.path.join(d, 'desi_transfer_status_2020.json')
+            with open(js, 'w') as f:
+                json.dump(self.fake_status, f, indent=None, separators=(',', ':'))
+            s = TransferStatus(d, year=2020)
+            r = s.update("20200703", "12345677", "rsync", failure=True)
+            self.assertEqual(r, 0)
+            self.assertEqual(s.status['20200703']['12345677'][0], [0, 1, 1565300073000])
 
     def test_TransferStatus_find(self):
         """Test status search.
         """
-        st = [[20200703, 12345678, 'checksum', True, '', 1565300074664],
-              [20200703, 12345678, 'rsync', True, '', 1565300074664],
-              [20200703, 12345677, 'checksum', True, '', 1565300073000],
-              [20200703, 12345677, 'rsync', True, '', 1565300073000]]
+        st = {"20200703": {"12345677": [[0, 1, 1565300073000], [1, 0, 1565300073000]],
+                           "12345678": [[0, 1, 1565300074664], [1, 0, 1565300074664]]}}
         with TemporaryDirectory() as d:
-            js = os.path.join(d, 'desi_transfer_status.json')
+            js = os.path.join(d, 'desi_transfer_status_2020.json')
             with open(js, 'w') as f:
                 json.dump(st, f, indent=None, separators=(',', ':'))
-            s = TransferStatus(d)
-            i = s.find(20200702)
-            self.assertEqual(len(i), 0)
-            i = s.find(20200703)
-            self.assertEqual(len(i), 4)
-            i = s.find(20200703, 12345678)
+            s = TransferStatus(d, year=2020)
+            with self.assertRaises(KeyError) as cm:
+                i = s.find('20200702')
+            self.assertEqual(cm.exception.args[0], "Undefined night = '20200702'!")
+            i = s.find('20200703')
             self.assertEqual(len(i), 2)
-            i = s.find(20200703, stage='checksum')
+            i = s.find('20200703', '12345678')
             self.assertEqual(len(i), 2)
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
+            i = s.find('20200703', '12345679')
+            self.assertEqual(len(i), 0)
+            i = s.find('20200703', stage='checksum')
+            self.assertEqual(len(i), 3)
+            self.assertListEqual(i["12345677"], [1])
+            self.assertListEqual(i["12345678"], [1])
+            self.assertListEqual(i["12345679"], [])
diff --git a/py/desitransfer/test/test_top_level.py b/py/desitransfer/test/test_top_level.py
index 7037c87..e278abb 100644
--- a/py/desitransfer/test/test_top_level.py
+++ b/py/desitransfer/test/test_top_level.py
@@ -31,11 +31,3 @@ class TestTopLevel(unittest.TestCase):
         """Ensure the version conforms to PEP386/PEP440.
         """
         self.assertRegex(theVersion, self.versionre)
-
-
-def test_suite():
-    """Allows testing of only this module with the command::
-
-        python setup.py test -m <modulename>
-    """
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)