Repository: DataBiosphere/terra-notebook-utils (language: python)

Issue:
Use manifest for batch copy, introduce CLI arg



Discussion / hints:

Fix patch:
diff --git a/Makefile b/Makefile
index 3643cad9..dde667e5 100644
--- a/Makefile
+++ b/Makefile
@@ -34,6 +34,10 @@ $(tests): %.py : mypy lint
 dev_scripts/test_installed_cli.py:
 	python dev_scripts/test_installed_cli.py
 
+# Test the wdl script locally using miniwdl
+wdl_tests/copy_batch.wdl:
+	miniwdl run --verbose --copy-input-files wdl_tests/copy_batch.wdl --input wdl_tests/copy_batch_input.json
+
 version: terra_notebook_utils/version.py
 
 terra_notebook_utils/version.py: setup.py
@@ -51,4 +55,4 @@ sdist: clean
 install: build
 	pip install --upgrade dist/*.whl
 
-.PHONY: test lint mypy tests clean build install dev_scripts/test_installed_cli.py
+.PHONY: test lint mypy tests clean build install dev_scripts/test_installed_cli.py wdl_tests/copy_batch.wdl
diff --git a/common.mk b/common.mk
index 0d9b7468..49cd6092 100644
--- a/common.mk
+++ b/common.mk
@@ -2,6 +2,12 @@
 
 SHELL=/bin/bash -eo pipefail
 
+# needed by miniwdl
+# TODO: uncomment this when shellcheck can be made available in ci/cd
+# ifeq ($(shell which shellcheck),)
+# $(error Please install shellcheck using "apt-get install shellcheck" or "brew install shellcheck")
+# endif
+
 release_major:
 	$(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\d)+\.(\d)+\.(\d+)+/; print "v@{[$$1+1]}.0.0"'))
 	$(MAKE) release
diff --git a/requirements-dev.txt b/requirements-dev.txt
index f54eb6ee..7b3cda81 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -3,4 +3,5 @@ coverage
 mypy >= 0.800, < 0.900
 twine
 google-crc32c >= 1.0.0
+miniwdl
 -r requirements.txt


Test patch:
diff --git a/wdl_tests/copy_batch.wdl b/wdl_tests/copy_batch.wdl
new file mode 100644
index 00000000..c4815f3c
--- /dev/null
+++ b/wdl_tests/copy_batch.wdl
@@ -0,0 +1,132 @@
+version 1.0
+
+workflow drs {
+    meta {
+        description: "This workflow tests downloading DRS URIs using terra-notebook-utils."
+        tags: "DRS"
+        authors: "M. Baumann, L. Blauvelt, B. Hannafious"
+    }
+    parameter_meta {
+        drs_uris: "Array of DRS URIs to be downloaded"
+    }
+    input {
+        Array[String] drs_uris
+    }
+
+    call create_manifest {
+        input: drs_uris=drs_uris
+    }
+    call tnu_download {
+        input:
+            manifest=create_manifest.tnu_manifest
+    }
+}
+
+task create_manifest {
+    meta {
+        description: "Creates a manifest mapping DRS URIs to local/gs schema appropriate for use with terra-notebook-utils."
+    }
+    parameter_meta {
+        drs_uris: "Array of DRS URIs to be downloaded"
+        cpu: "runtime parameter - number of CPUs "
+        memory: "runtime parameter - amount of memory to allocate in GB. Default is: 16"
+        boot_disk: "runtime parameter - amount of boot disk space to allocate in GB. Default is: 50"
+        disk: "runtime parameter - amount of disk space to allocate in GB. Default is: 128"
+    }
+    input {
+        Array[String] drs_uris
+        Int? cpu
+        Int? memory
+        Int? boot_disk
+        Int? disk
+
+        String tnu_manifest_filename = "./manifest.json"
+    }
+    command <<<
+        set -eux o pipefail
+
+        # Identify the runtime environment; Check Linux version
+        cat /etc/issue
+        uname -a
+
+        # Commands that could be added to the Dockerfile
+        apt-get update
+        apt-get install -yq --no-install-recommends apt-utils jq
+
+        # Create tnu manifest
+        echo '"~{sep='" "' drs_uris}"' | jq -s '[.[] as $item | {drs_uri: $item, dst: "."}]' > ~{tnu_manifest_filename}
+    >>>
+
+    output {
+        File tnu_manifest = tnu_manifest_filename
+    }
+
+    runtime {
+        docker: "broadinstitute/cromwell-drs-localizer:61"
+        cpu: select_first([cpu, "4"])
+        memory: select_first([memory,"16"]) + " GB"
+        disks: "local-disk " + select_first([disk, "128"]) + " HDD"
+        bootDiskSizeGb: select_first([boot_disk,"30"])
+    }
+}
+
+task tnu_download {
+    meta {
+        description: "This task tests downloading DRS URIs using terra-notebook-utils."
+    }
+    parameter_meta {
+        manifest: "Manifest mapping DRS URIs to local or gs destinations."
+        cpu: "runtime parameter - number of CPUs"
+        memory: "runtime parameter - amount of memory to allocate in GB. Default is: 16"
+        boot_disk: "runtime parameter - amount of boot disk space to allocate in GB. Default is: 50"
+        disk: "runtime parameter - amount of disk space to allocate in GB. Default is: 128"
+    }
+    input {
+        File manifest
+        Int? cpu
+        Int? memory
+        Int? boot_disk
+        Int? disk
+    }
+    command <<<
+        set -eux o pipefail
+
+        CURRENT_DIR=$(pwd)
+
+        # Where all non-tnu downloads go (wget, gsutil, and curl)
+        # TMP_DL_DIR=/cromweLl_root/speedtest3crdws3s
+        # mkdir -p ${TMP_DL_DIR}
+
+        # Identify the runtime environment; Check Linux version
+        cat /etc/issue
+        uname -a
+
+        # Commands that could be added to the Dockerfile
+        apt-get update
+        apt-get install -yq --no-install-recommends apt-utils git
+
+        # Install terra-notebook-utils
+        python -m pip install git+https://github.com/DataBiosphere/terra-notebook-utils
+        python -m pip show terra-notebook-utils
+
+        # Download the files in the manifest
+        start_time=`date +%s`
+        time tnu drs copy-batch --workspace terra-notebook-utils-tests --workspace-namespace firecloud-cgl --manifest ~{manifest}
+        tnu_exit_status=$?
+        echo "tnu exit status: "$tnu_exit_status
+        end_time=`date +%s`
+        total_time="$(($end_time-$start_time))"
+        # Check final disk usage after downloading
+        df -h
+
+        # TODO: verify that the downloaded files exist
+    >>>
+
+    runtime {
+        docker: "broadinstitute/cromwell-drs-localizer:61"
+        cpu: select_first([cpu, "4"])
+        memory: select_first([memory,"16"]) + " GB"
+        disks: "local-disk " + select_first([disk, "128"]) + " HDD"
+        bootDiskSizeGb: select_first([boot_disk,"30"])
+    }
+}
diff --git a/wdl_tests/copy_batch_input.json b/wdl_tests/copy_batch_input.json
new file mode 100644
index 00000000..5b0bfbba
--- /dev/null
+++ b/wdl_tests/copy_batch_input.json
@@ -0,0 +1,7 @@
+{
+  "drs_uris": [
+    "drs://dg.4503/5ec0e501-432e-4cad-808d-1a4e9100b7de",
+    "drs://dg.4503/6ffc2f59-2596-405c-befd-9634dc0ed837",
+    "drs://dg.4503/0f26beeb-d468-405e-abb7-412eb7bf8b19"
+  ]
+}