Repository: biocommons/biocommons.seqrepo

Issue:
simplify pulling new releases
Mirroring seqrepo requires something like

    rsync -HRavP rsync.biocommons.org::seqrepo/2016082[78] /tmp/seqrepo/

or

    rsync -HavP --link-dest=/tmp/seqrepo/20160827/ rsync.biocommons.org::seqrepo/20160828/ /tmp/seqrepo/20160828/

Use `seqrepo` cli to simplify this process. This will likely force a rethink about the distinction between seqrepo **root** (*e.g.,* `/usr/local/share/seqrepo`) and seqrepo **dir** (`/usr/local/share/seqrepo/master`).





Discussion:

Patch:
diff --git a/README.rst b/README.rst
index 0efa0ad..9a20600 100644
--- a/README.rst
+++ b/README.rst
@@ -71,14 +71,15 @@ On Ubuntu 16.04::
 
   $ sudo apt install -y python3-dev gcc zlib1g-dev tabix
   $ pip install seqrepo
-  $ rsync -HRavP rsync.biocommons.org::seqrepo/20160828 /usr/local/share/seqrepo/
-  $ seqrepo -d /usr/local/share/seqrepo/20160828 start-shell
-  seqrepo 0.2.3.dev2+neeca95d3ae6e.d20160830
-  root directory: /opt/seqrepo/20160828, 7.9 GB
+  $ seqrepo pull -i 20160906
+  $ seqrepo -i 20160906 show-status 
+  seqrepo 0.2.3.post3.dev8+nb8298bd62283
+  root directory: /usr/local/share/seqrepo/20160906, 7.9 GB
   backends: fastadir (schema 1), seqaliasdb (schema 1) 
-  sequences: 773511 sequences, 93005806376 residues, 189 files
-  aliases: 5572724 aliases, 5473237 current, 9 namespaces, 773511 sequences
+  sequences: 773587 sequences, 93051609959 residues, 192 files
+  aliases: 5579572 aliases, 5480085 current, 26 namespaces, 773587 sequences
 
+  $ seqrepo -i 20160906 start-shell
   In [1]: sr["NC_000001.11"][780000:780020]
   Out[1]: 'TGGTGGCACGCGCTTGTAGT'
 
diff --git a/biocommons/seqrepo/cli.py b/biocommons/seqrepo/cli.py
index a38eaac..02b606b 100644
--- a/biocommons/seqrepo/cli.py
+++ b/biocommons/seqrepo/cli.py
@@ -1,10 +1,13 @@
 """command line interface to a local SeqRepo repository
 
+SeqRepo is a non-redundant, compressed, journalled, file-based storage
+for biological sequences
+
 https://github.com/biocommons/biocommons.seqrepo
 
-Typical usage is via the `seqrepo` script::
+Try::
 
-  $ seqrepo --help 
+  $ seqrepo --help
 
 """
 
@@ -20,6 +23,7 @@ import re
 import shutil
 import stat
 import sys
+import subprocess
 import tempfile
 
 import bioutils.assemblies
@@ -33,17 +37,23 @@ from .py2compat import commonpath, gzip_open_encoded, makedirs
 
 def parse_arguments():
     top_p = argparse.ArgumentParser(
-        description=__doc__.split("\n\n")[0], formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        epilog="seqrepo " + __version__ +
-        ". See https://github.com/biocommons/biocommons.seqrepo for more information")
-    top_p.add_argument("--dir", "-d", help="seqrepo data directory; created by init", required=True)
-    top_p.add_argument("--verbose", "-v", action="count", default=0, help="be verbose; multiple accepted")
+        description=__doc__.split("\n\n")[0],
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        epilog="seqrepo " + __version__ + ". See https://github.com/biocommons/biocommons.seqrepo for more information"
+        )
+    top_p.add_argument("--root-directory", "-r", default="/usr/local/share/seqrepo", 
+                       help="seqrepo root directory")
+    top_p.add_argument("--instance-name", "-i", default="master", 
+                       help="name of seqrepo instance within root-directory")
+    top_p.add_argument("--verbose", "-v", action="count", default=0,
+                       help="be verbose; multiple accepted")
     top_p.add_argument("--version", action="version", version=__version__)
 
     subparsers = top_p.add_subparsers(title='subcommands')
 
     # add-assembly-names
-    ap = subparsers.add_parser("add-assembly-names", help="add assembly aliases (from bioutils.assemblies) to existing sequences")
+    ap = subparsers.add_parser("add-assembly-names",
+                               help="add assembly aliases (from bioutils.assemblies) to existing sequences")
     ap.set_defaults(func=add_assembly_names)
 
     # export
@@ -51,7 +61,7 @@ def parse_arguments():
     ap.set_defaults(func=export)
 
     # init
-    ap = subparsers.add_parser("init", help="initialize bsa directory")
+    ap = subparsers.add_parser("init", help="initialize seqrepo directory")
     ap.set_defaults(func=init)
 
     # load
@@ -67,6 +77,24 @@ def parse_arguments():
         required=True,
         help="namespace name (e.g., ncbi, ensembl, lrg)", )
 
+    # pull
+    ap = subparsers.add_parser("pull", help="pull incremental update from seqrepo mirror")
+    ap.set_defaults(func=pull)
+    ap.add_argument("--instance-name", "-i", required=False,
+                    help="instance name to pull; latest if not specified")
+    ap.add_argument(
+        "--rsync-exe",
+        default="/usr/bin/rsync",
+        help="path to rsync executable")
+    ap.add_argument(
+        "--remote-host",
+        default="dl.biocommons.org",
+        help="rsync server host")
+    ap.add_argument(
+        "--dry-run", "-n",
+        default=False,
+        action="store_true")
+
     # show-status
     ap = subparsers.add_parser("show-status", help="show seqrepo status")
     ap.set_defaults(func=show_status)
@@ -84,7 +112,7 @@ def parse_arguments():
     ap.set_defaults(func=start_shell)
 
     # upgrade
-    ap = subparsers.add_parser("upgrade", help="upgrade bsa database and directory")
+    ap = subparsers.add_parser("upgrade", help="upgrade seqrepo database and directory")
     ap.set_defaults(func=upgrade)
 
     opts = top_p.parse_args()
@@ -94,14 +122,14 @@ def parse_arguments():
 def add_assembly_names(opts):
     """add assembly names as aliases to existing sequences
 
-    Specifically, associate aliases like GRCh37:HSCHRUN_RANDOM_CTG42 with (existing) ncbi:NT_167243.1
+    Specifically, associate aliases like GRCh37.p9:1 with (existing) 
     """
     logger = logging.getLogger(__name__)
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
     sr = SeqRepo(opts.dir, writeable=True)
     ncbi_alias_map = {r["alias"]: r["seq_id"] for r in sr.aliases.find_aliases(namespace="ncbi", current_only=False)}
     namespaces = [r["namespace"] for r in sr.aliases._db.execute("select distinct namespace from seqalias")]
     assemblies = bioutils.assemblies.get_assemblies()
-    import IPython; IPython.embed()	  ### TODO: Remove IPython.embed()
     assemblies_to_load = sorted([k for k in assemblies if k not in namespaces])
     logger.info("{} assemblies to load".format(len(assemblies_to_load)))
     for assy_name in tqdm.tqdm(assemblies_to_load, unit="assembly"):
@@ -120,7 +148,7 @@ def add_assembly_names(opts):
                                    namespace=assy_name,
                                    alias=s["name"])
         sr.commit()
-         
+
 
 def export(opts):
     def convert_alias_records_to_ns_dict(records):
@@ -135,6 +163,7 @@ def export(opts):
         for i in range(0, len(seq), line_width):
             yield seq[i:i + line_width]
 
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
     sr = SeqRepo(opts.dir)
     for srec,arecs in sr:
         nsad = convert_alias_records_to_ns_dict(arecs)
@@ -145,6 +174,7 @@ def export(opts):
 
 
 def init(opts):
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
     if os.path.exists(opts.dir) and len(os.listdir(opts.dir)) > 0:
         raise IOError("{opts.dir} exists and is not empty".format(opts=opts))
     sr = SeqRepo(opts.dir, writeable=True)  # flake8: noqa
@@ -155,6 +185,7 @@ def load(opts):
     disable_bar = logger.getEffectiveLevel() < logging.WARNING
     defline_re = re.compile("(?P<namespace>gi|ref)\|(?P<alias>[^|]+)")
 
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
     sr = SeqRepo(opts.dir, writeable=True)
 
     n_seqs_seen = n_seqs_added = n_aliases_added = 0
@@ -193,7 +224,53 @@ def load(opts):
             n_aliases_added += n_aa
 
 
+def pull(opts):
+    instance_name_re = re.compile('^201\d{5}$')  # smells like a datestamp
+    #instance_name_re = re.compile('^[89]\d+$')  # debugging
+    def _get_remote_instances(opts):
+        line_re = re.compile(r'd[-rwx]{9}\s+[\d,]+ \d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} (.+)')
+        lines = subprocess.check_output([opts.rsync_exe, "--no-motd", opts.remote_host + "::seqrepo"]).decode().splitlines()[1:]
+        dirs = (m.group(1) for m in (line_re.match(l) for l in lines) if m)
+        return list(filter(instance_name_re.match, dirs))
+    def _get_local_instances(opts):
+        return list(filter(instance_name_re.match, os.listdir(opts.root_directory)))
+
+    logger = logging.getLogger(__name__)
+
+    remote_instances = _get_remote_instances(opts)
+    if opts.instance_name:
+        instance_name = opts.instance_name
+        if instance_name not in remote_instances:
+            raise KeyError("{}: not in list of remote instance names".format(instance_name))
+    else:
+        instance_name = sorted(remote_instances)[-1]
+        logger.info("most recent seqrepo instance is " + instance_name)
+
+    local_instances = _get_local_instances(opts)
+    if instance_name in local_instances:
+        logger.warn("{}: instance already exists; skipping".format(instance_name))
+        return
+
+    tmp_dir = tempfile.mkdtemp(dir=opts.root_directory, prefix=instance_name + ".")
+    os.rmdir(tmp_dir)          # let rsync create it the directory
+
+    cmd = [opts.rsync_exe, "-aHP", "--no-motd"]
+    if local_instances:
+        latest_local_instance = sorted(local_instances)[-1]
+        cmd += ["--link-dest=" + os.path.join(opts.root_directory, latest_local_instance) + "/"]
+    cmd += ["{h}::seqrepo/{i}/".format(h=opts.remote_host, i=instance_name),
+           tmp_dir]
+
+    logger.debug("Running: " + " ".join(cmd))
+    if not opts.dry_run:
+        subprocess.check_call(cmd)
+        dst_dir = os.path.join(opts.root_directory, instance_name)
+        os.rename(tmp_dir, dst_dir)
+        logger.info("{}: successfully updated ({})".format(instance_name, dst_dir))
+
+
 def show_status(opts):
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
     tot_size = sum(os.path.getsize(os.path.join(dirpath,filename))
                        for dirpath, dirnames, filenames in os.walk(opts.dir)
                        for filename in filenames)
@@ -216,17 +293,14 @@ def snapshot(opts):
 
     """
     logger = logging.getLogger(__name__)
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
 
-    src_dir = os.path.realpath(opts.dir)
     dst_dir = opts.destination_directory
-
-    if dst_dir.startswith("/"):
-        # use as-is
-        pass
-    else:
+    if not dst_dir.startswith("/"):
         # interpret dst_dir as relative to parent dir of opts.dir
-        dst_dir = os.path.join(src_dir, '..', dst_dir)
+        dst_dir = os.path.join(opts.root_directory, dst_dir)
 
+    src_dir = os.path.realpath(opts.dir)
     dst_dir = os.path.realpath(dst_dir)
 
     if commonpath([src_dir, dst_dir]).startswith(src_dir):
@@ -285,12 +359,13 @@ def snapshot(opts):
 
 
 def start_shell(opts):
-    sr = show_status(opts)
+    #sr = show_status(opts)
     import IPython
-    IPython.embed(display_banner=False)
-
+    IPython.embed(header="seqrepo " + __version__ + 
+                  "\nhttps://github.com/biocommons/biocommons.seqrepo/")
 
 def upgrade(opts):
+    opts.dir = os.path.join(opts.root_directory, opts.instance_name)
     sr = SeqRepo(opts.dir, writeable=True)
     print("upgraded to schema version {}".format(sr.seqinfo.schema_version()))
 
diff --git a/doc/cli.rst b/doc/cli.rst
index 18b49d7..a65494f 100644
--- a/doc/cli.rst
+++ b/doc/cli.rst
@@ -5,10 +5,10 @@ seqrepo includes a command line interface for loading, fetching, and exporting s
   
 This documentation assumes that the seqrepo base directory is::
 
-  SEQREPO=/usr/local/share/seqrepo
+  SEQREPO_ROOT=/usr/local/share/seqrepo
 
-Current convention is to add sequences to `$SEQREPO/master`, then
-snapshot this to a dated directory like `$SEQREPO/20160828`.  (This
+Current convention is to add sequences to `$SEQREPO_ROOT/master`, then
+snapshot this to a dated directory like `$SEQREPO_ROOT/20160828`.  (This
 convention is conceptually similar to source code development on a
 master branch with tags.)
 
diff --git a/doc/mirror.rst b/doc/mirror.rst
index d74f50b..b666407 100644
--- a/doc/mirror.rst
+++ b/doc/mirror.rst
@@ -2,11 +2,11 @@ Fetching existing sequence repositories
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 A public instance of seqrepo with dated snapshots is available at on
-seqrepo.biocommons.org.
+dl.biocommons.org.
 
 You can list available snapshots like so::
 
-  $ rsync rsync.biocommons.org::seqrepo                                                                                                                                                                                            
+  $ rsync dl.biocommons.org::seqrepo                                                                                                                                                                                            
   This is the rsync service for tools and data from biocommons.org
   This service is hosted by Invitae (https://invitae.com/).
   
@@ -17,7 +17,7 @@ You can list available snapshots like so::
 You may mirror the entire seqrepo archive or a specific snapshot, as
 shown below::
   
-  $ rsync -HavP rsync.biocommons.org::seqrepo/20160828/ /tmp/seqrepo/20160828/
+  $ rsync -HavP dl.biocommons.org::seqrepo/20160828/ /tmp/seqrepo/20160828/
   
   This is the rsync service for tools and data from biocommons.org
   This service is hosted by Invitae (https://invitae.com/).
@@ -31,7 +31,7 @@ shown below::
 If you have a previous snapshot, you should invoke rsync like this in
 order to hard link unchanged files::
 
-  $ rsync -HavP --link-dest=/tmp/seqrepo/20160827/ rsync.biocommons.org::seqrepo/20160828/ /tmp/seqrepo/20160828/
+  $ rsync -HavP --link-dest=/tmp/seqrepo/20160827/ dl.biocommons.org::seqrepo/20160828/ /tmp/seqrepo/20160828/
 
 
 If seqrepo is already installed, you may check the repo status with::
diff --git a/sbin/seqrepo-push b/sbin/seqrepo-push
new file mode 100755
index 0000000..7e86ac2
--- /dev/null
+++ b/sbin/seqrepo-push
@@ -0,0 +1,28 @@
+#!/bin/bash
+# push a seqrepo instance to an rsync server
+# https://github.com/biocommons/biocommons.seqrepo/
+# Used by Reece to push new instances -- likely not relevant for anyone else
+
+# minion is an ssh alias
+host=minion
+
+# seqrepo root directory
+seqrepo_root=/usr/local/share/seqrepo
+
+if [ "$#" != "2" ]; then
+    echo "Usage: $0 <previous_instance> <push_instance>" 1>&2
+    exit 1
+fi
+
+prev_instance="$1"; shift
+push_instance="$1"; shift
+
+echo "prev_instance=${prev_instance}"
+echo "push_instance=${push_instance}"
+
+(
+    set -ex
+    cd $seqrepo_root
+    rsync -avHP "$push_instance/" --link-dest="../$prev_instance/" minion:"dl.biocommons.org/seqrepo/$push_instance.tmp/"
+    ssh minion /bin/mv "dl.biocommons.org/seqrepo/$push_instance.tmp/" "dl.biocommons.org/seqrepo/$push_instance/"
+)


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index b94bb7e..87b09b3 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -15,9 +15,10 @@ def opts():
     test_data_dir = os.path.join(test_dir, 'data')
 
     opts = MockOpts()
-    opts.dir = os.path.join(tempfile.mkdtemp(prefix="seqrepo_pytest_"), "seqrepo")
+    opts.root_directory = os.path.join(tempfile.mkdtemp(prefix="seqrepo_pytest_"), "seqrepo")
     opts.fasta_files = [os.path.join(test_data_dir, "sequences.fa.gz")]
     opts.namespace = "test"
+    opts.instance_name = "test"
     opts.verbose = 0
     return opts