Repository: ORNL/DataFed (language: python)

Issue:
Add running repo instance to CI
# Description

This PR adds the repo and gcs instances to the CI pipeline, the pipeline will stand up these instances, create the repo form, then from a separate location use a client to run the end-to-end tests. 


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml
index 24c268d7f..7bf00b5dc 100644
--- a/.github/workflows/format-check.yml
+++ b/.github/workflows/format-check.yml
@@ -28,9 +28,9 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get install -y python3-sphinx sphinx-rtd-theme-common
-          pip install sphinx_rtd_theme recommonmark flake8 black pytest pytest-cov
+          pip install sphinx_rtd_theme recommonmark flake8 black pytest pytest-cov black[jupyter]
       - name: Black formatting checks
-        run: black --check .
+        run: black  --exclude '/jupyter_notebooks/' --check .
       - name: Lint with flake8
         run: |
           flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --max-line-length 100 .
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d57dd3c2d..666a0159a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,16 +3,22 @@ stages:
   - trigger-infrastructure
   - signal
   - clear-docker-cache
-  - build-deploy-base
+  - build-base
   - provision-client
   - build
   - end-to-end-setup-arango
   - end-to-end-setup
   - end-to-end-test
-  - end-to-end-teardown
+
+# WARNING
+#
+# Do not put if else statements without if and else, gitlab yaml does not like
+# this kind of scripting
+#
+# [ -f file_path ] && ok do this....
 
 variables:
-  HARBOR_USER: 'robot$$datafed+harbor_datafed_gitlab_ci_registry'
+  HARBOR_USER: 'robot$$datafed+harbor_datafed_gitlab_ci_registry_2'
   REGISTRY: 'camden.ornl.gov'
   DATAFED_DEPENDENCIES_INSTALL_PATH: "/shared/install"
 ################################################################################
@@ -106,125 +112,42 @@ clear-python-client-cache:
     - docker system prune -f
     - ./scripts/ci_purge_images.sh
 
-
 ################################################################################
-# STAGE: build-deploy-base
+# STAGE: build-base
 ################################################################################
-# Building base containers the base containers should have all the required
-# dependencies it should not be necessary to build them every time. Ideally, we
-# would rarely need to rebuild the dependencies and we could just reuse the
-# container "golden base" images already stored in the registry.
-#
-# NOTE GitLab is set to remove images after a month, so they will be forced to
-# rebuild at least once a month.
-build-ws-base:
-  needs: ["clear-core-cache"]
-  stage: build-deploy-base
-  variables:
-    IMAGE_TAG: "datafed/ws-base"
-    GIT_STRATEGY: clone
-  tags:
-    - ci-datafed-core
-    - docker
-  script:
-    - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
-    - docker system prune -f
-    - docker build -f web/docker/Dockerfile.web-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
-    - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest"
-
-build-core-base:
-  needs: ["clear-core-cache"]
-  stage: build-deploy-base
+# build runtime and depedencies containers
+build-dependencies:
+  stage: build-base
   variables:
-    IMAGE_TAG: "datafed/core-base"
+    IMAGE_TAG: "datafed/dependencies"
     GIT_STRATEGY: clone
   tags:
-    - ci-datafed-core
     - docker
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - docker system prune -f
-    - docker build -f core/docker/Dockerfile.core-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
+    - docker build -f docker/Dockerfile.dependencies -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest"
 
-build-repo-base:
-  needs: ["clear-repo-cache"]
+build-runtime:
+  stage: build-base
   variables:
-    IMAGE_TAG: "datafed/repo-base"
+    IMAGE_TAG: "datafed/runtime"
     GIT_STRATEGY: clone
-  stage: build-deploy-base
   tags:
-    - ci-datafed-repo
     - docker
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - docker system prune -f
-    - docker build -f repository/docker/Dockerfile.repo-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
+    - docker build -f docker/Dockerfile.runtime -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest"
 
-build-python-client-base:
-  needs: ["clear-python-client-cache"]
-  variables:
-    IMAGE_TAG: "datafed/python-client-base"
-    GIT_STRATEGY: clone
-  stage: build-deploy-base
-  tags:
-    - docker
-    - ci-datafed-client
-  script:
-    - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
-    - docker system prune -f
-    - docker build -f python/docker/Dockerfile.python-client-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
-    - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest"
-
-build-gcs-base:
-  needs: ["clear-repo-cache"]
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-    GIT_STRATEGY: clone
-    DISTRO: "ubuntu-20.04"
-    GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal"
-    IMAGE_TAG2: "datafed/gcs-authz-base"
-    IMAGE_TAG3: "datafed/gcs-authz"
-  stage: build-deploy-base
-  tags:
-    - ci-datafed-globus
-    - docker
-  rules:
-    - changes:
-        - scripts/dependency_versions.sh
-    - when: manual
-  script:
-    - version=$(cat ./scripts/dependency_versions.sh | grep DATAFED_GCS_SUBMODULE_VERSION)
-    - version=${version//\"/}
-    - export $version
-    - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
-    - docker system prune -f
-    - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - >
-      if ! docker pull "${GCS_TAG}:${DATAFED_GCS_SUBMODULE_VERSION}-${BRANCH_LOWER}:latest"; then
-        cd external/globus-connect-server-deploy/docker
-        git checkout "${DATAFED_GCS_SUBMODULE_VERSION}"
-        docker build --progress plain --tag "$GCS_TAG:${DATAFED_GCS_SUBMODULE_VERSION}" - < "./docker-files/Dockerfile.${DISTRO}"
-        #LATEST_IMAGE=$(docker images | grep ${DISTRO} | head -n1 | awk '{print $3}')
-        #docker tag "$LATEST_IMAGE" "$GCS_TAG:${DATAFED_GCS_SUBMODULE_VERSION}" 
-        docker push "$GCS_TAG:${DATAFED_GCS_SUBMODULE_VERSION}"
-        docker tag "$GCS_TAG:${DATAFED_GCS_SUBMODULE_VERSION}" "${GCS_TAG}-${BRANCH_LOWER}:latest" 
-        docker push "${GCS_TAG}-${BRANCH_LOWER}:latest"
-        cd ../../../ # back too root of project
-      else
-        echo "Docker image already exists in the registry. Skipping build."
-      fi 
-
-################################################################################
-# STAGE: provision client
-################################################################################
+#################################################################################
+## STAGE: provision client
+#################################################################################
 provision-client:
-
   needs: ["signal"]
   variables:
     GIT_STRATEGY: clone
@@ -236,19 +159,19 @@ provision-client:
   script:
     - ./scripts/generate_datafed.sh
     - ./scripts/install_client_dependencies.sh
-  rules:
-    - changes:
-        - scripts/generate_datafed.sh
-        - scripts/install_client_dependencies.sh
-        - scripts/dependency_install_functions.sh
-        - scripts/dependency_versions.sh
- 
+      #  rules:
+      #    - changes:
+      #        - scripts/generate_datafed.sh
+      #        - scripts/install_client_dependencies.sh
+      #        - scripts/dependency_install_functions.sh
+      #        - scripts/dependency_versions.sh
+
 ################################################################################
 # STAGE: build
 ################################################################################
 # Building containers for running services
 build-ws:
-  needs: ["build-ws-base"]
+  needs: ["build-dependencies", "build-runtime"]
   stage: build
   variables:
     IMAGE_TAG: "datafed/ws"
@@ -258,25 +181,22 @@ build-ws:
     - docker
   rules:
     - changes:
-        - dockerfiles/**/*
+        - docker/**/*
         - scripts/**/*
         - web/**/*
         - common/proto/**/*
         - .gitlab-ci.yml
       when: on_success
-    - exists:
-      - "build-ws-base"
-      when: on_success
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker build --build-arg BASE_WEB_IMAGE="${REGISTRY}/${IMAGE_TAG}-base-${BRANCH_LOWER}:latest" -f web/docker/Dockerfile.web.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}" .
+    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" -f web/docker/Dockerfile -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}"
 
 build-core:
-  needs: ["build-core-base"]
+  needs: ["build-dependencies", "build-runtime"]
   stage: build
   variables:
     IMAGE_TAG: "datafed/core"
@@ -286,147 +206,149 @@ build-core:
     - docker
   rules:
     - changes:
-        - dockerfiles/**/*
+        - docker/**/*
         - scripts/**/*
-        - common/**/*
         - core/**/*
+        - common/**/*
         - CMakeLists.txt
         - cmake/**/*
         - .gitlab-ci.yml
       when: on_success
-    - exists:
-      - "build-core-base"
-      when: on_success
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
-    - env
     - ./scripts/generate_datafed.sh
-    - cat ./config/datafed.sh
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker build --build-arg BASE_CORE_IMAGE="${REGISTRY}/${IMAGE_TAG}-base-${BRANCH_LOWER}:latest" -f core/docker/Dockerfile.core.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}" .
+    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" -f core/docker/Dockerfile -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}"
 
-build-foxx:
+build-repo:
+  needs: ["build-dependencies", "build-runtime"]
   stage: build
-  needs: ["signal"]
   variables:
+    IMAGE_TAG: "datafed/repo"
     GIT_STRATEGY: clone
-    DATAFED_DATABASE_PASSWORD: "${CI_DATAFED_DATABASE_PASSWORD}"
-    DATAFED_ZEROMQ_SYSTEM_SECRET: "${CI_DATAFED_ZEROMQ_SYSTEM_SECRET}"
   tags:
-    - ci-datafed-arango
+    - ci-datafed-repo
+    - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - web/**/*
+        - common/proto/**/*
+        - .gitlab-ci.yml
+      when: on_success
   script:
-    - arangod --version
-    - sudo systemctl status arangodb3
+    - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
+    - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
-    - env
-    - cat ./config/datafed.sh
-    - >
-      cmake -S. -B build
-      -DENABLE_FOXX_TESTS=ON
-      -DBUILD_CORE_SERVER=OFF
-      -DBUILD_COMMON=OFF
-      -DBUILD_WEB_SERVER=OFF
-      -DBUILD_DOCS=OFF
-      -DBUILD_PYTHON_CLIENT=OFF
-      -DBUILD_TESTS=ON
-      -DINSTALL_FOXX=ON
-    - cmake --build build
-    - cmake --build build --target install
-    - cmake --build build --target test
+    - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
+    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" -f repository/docker/Dockerfile -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
+    - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}"
 
-build-repo:
+build-gcs-base:
+  needs: ["build-dependencies", "build-runtime"]
+  stage: build
   variables:
-    IMAGE_TAG: "datafed/repo"
+    IMAGE_TAG: "datafed/gcs-base"
+    GIT_SUBMODULE_STRATEGY: recursive
     GIT_STRATEGY: clone
-  needs: ["build-repo-base"]
-  stage: build
   tags:
-    - ci-datafed-repo
+    - ci-datafed-globus
     - docker
   rules:
     - changes:
-        - dockerfiles/**/*
+        - docker/**/*
         - scripts/**/*
         - common/**/*
-        - repository/**/*
+        - .gitlab-ci.yml
         - CMakeLists.txt
         - cmake/**/*
-        - .gitlab-ci.yml
-      when: on_success
-    - exists:
-      - "build-repo-base"
       when: on_success
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
-    - ./scripts/generate_datafed.sh
+    - source "scripts/dependency_versions.sh"
+    - cd "external/globus-connect-server-deploy/docker"
+    - git checkout "$DATAFED_GCS_SUBMODULE_VERSION"
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker build --build-arg BASE_REPO_IMAGE="${REGISTRY}/${IMAGE_TAG}-base-${BRANCH_LOWER}:latest" -f repository/docker/Dockerfile.repo.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}" .
+    - docker build --progress plain -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" - < "./docker-files/Dockerfile.ubuntu-20.04"
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}"
 
-build-gcs-authz:
+build-gcs:
+  needs: ["build-dependencies", "build-runtime", "build-gcs-base"]
   stage: build
-  needs: ["build-gcs-base"]
   variables:
-    IMAGE_TAG: "datafed/gcs-authz"
+    IMAGE_TAG: "datafed/gcs"
     GIT_STRATEGY: clone
   tags:
     - ci-datafed-globus
     - docker
   rules:
     - changes:
-        - repository/gridftp/**/*
+        - docker/**/*
         - scripts/**/*
         - common/**/*
         - .gitlab-ci.yml
         - CMakeLists.txt
         - cmake/**/*
-      when: always
-    - exists:
-      - "build-gcs-base"
+        - repository/docker/entrypoint_authz.sh
+        - repository/docker/Dockerfile.gcs
       when: on_success
-    - when: manual
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker build --build-arg BASE_GCS_IMAGE="${REGISTRY}/datafed/gcs-ubuntu-focal-${BRANCH_LOWER}:latest" -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}" .
+    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" --build-arg GCS_IMAGE="${REGISTRY}/datafed/gcs-base-${BRANCH_LOWER}:latest" -f repository/docker/Dockerfile.gcs -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}"
 
-build-python-client:
-  needs: ["build-python-client-base"]
+build-python-client-base:
+  needs: ["clear-python-client-cache"]
   variables:
-    IMAGE_TAG: "datafed/python-client"
+    IMAGE_TAG: "datafed/python-client-base"
     GIT_STRATEGY: clone
   stage: build
   tags:
     - docker
     - ci-datafed-client
+  script:
+    - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
+    - docker system prune -f
+    - docker build -f python/docker/Dockerfile.python-client-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
+    - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
+    - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest"
+
+
+# Build foxx
+build-foxx:
+  needs: ["build-dependencies", "build-runtime"]
+  stage: build
+  variables:
+    IMAGE_TAG: "datafed/foxx"
+    GIT_STRATEGY: clone
+  tags:
+    - docker
   rules:
     - changes:
-        - python/**/*
+        - docker/**/*
         - scripts/**/*
-        - common/**/*
+        - web/**/*
+        - common/proto/**/*
         - .gitlab-ci.yml
-        - CMakeLists.txt
-        - cmake/**/*
-      when: always
-    - exists:
-      - "build-python-client-base"
       when: on_success
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - docker build --build-arg BASE_PYTHON_CLIENT_IMAGE="${REGISTRY}/${IMAGE_TAG}-base-${BRANCH_LOWER}:latest" -f python/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}" .
+    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" -f docker/Dockerfile.foxx -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
     - docker push "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}"
 
+
 ################################################################################
-# STAGE: Integration
+# STAGE: End to end setup arango
 ################################################################################
 
 end-to-end-arango-setup:
@@ -439,68 +361,61 @@ end-to-end-arango-setup:
     - arangod --version
     - ./scripts/run_arango_service.sh
 
-end-to-end-ws-setup:
+end-to-end-foxx-setup:
   variables:
-    IMAGE_TAG: "datafed/ws-"
+    IMAGE_TAG: "datafed/foxx-"
     GIT_STRATEGY: clone
     HOST_LOG_FILE_PATH: "/shared/logs"
     CONTAINER_LOG_FILE_PATH: "/datafed/logs"
-    DATAFED_WEB_KEY_DIR: "/shared/keys"
-    DATAFED_WEB_CERT_NAME: "cert.crt"
-    DATAFED_WEB_KEY_NAME: "cert.key"
-    DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_KEY_DIR}/${DATAFED_WEB_CERT_NAME}"
-    DATAFED_WEB_CSR_PATH: "${DATAFED_WEB_KEY_DIR}/cert.csr"
-    DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_DIR}/${DATAFED_WEB_KEY_NAME}"
-  stage: end-to-end-setup
-  needs: ["end-to-end-core-setup"]
-  #  dependencies: 
-  #  - end-to-end-core-setup
+    DATAFED_DATABASE_HOST: "$CI_DATAFED_DATABASE_HOST"
+    RUN_FILE: "run_foxx.sh"
+  stage: end-to-end-setup-arango
+  needs: ["end-to-end-arango-setup"]
   tags:
-    - ci-datafed-core
     - docker
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
-    - echo "$BRANCH_LOWER"
     - mkdir -p "$HOST_LOG_FILE_PATH"
-    - mkdir -p "${DATAFED_WEB_KEY_DIR}"
-    - ./scripts/ci_setup_web_certs.sh
     - chmod o+w "${HOST_LOG_FILE_PATH}"
+    - USER_ID=$(id -u)
     - chown gitlab-runner "$HOST_LOG_FILE_PATH"
+    - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
-    - docker login  "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - USER_ID=$(id -u)
-    - GROUP_ID=$(id -g)
-    - CORE_ADDRESS=$(hostname -I | awk '{print $1}')
-    - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-core-key.pub
-    - env
-    - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}"
+    - env > env_file
+    - mkdir foxx_tmp
+    - ls -la foxx_tmp
+    - if [ -f foxx_tmp/.foxx_is_installed ]; then rm foxx_tmp/.foxx_is_installed; fi
+    - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
+    - ./scripts/container_stop.sh -n "foxx-" -p
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
-    - echo "#!/bin/bash" > run_web.sh
-    - echo "docker run -d \\" >> run_web.sh
-    - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh
-    - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh
-    - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh
-    - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh
-    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh
-    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh
-    - echo "-e DATAFED_WEB_CERT_PATH=\"/datafed/install/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh
-    - echo "-e DATAFED_WEB_KEY_PATH=\"/datafed/install/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh
-    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh
-    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"$CORE_ADDRESS:7513\" \\" >> run_web.sh
-    - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh
-    - echo "-p 443:443 \\" >> run_web.sh
-    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh
-    - echo "-v \"/shared/datafed-core-key.pub:/datafed/install/keys/datafed-core-key.pub\" \\" >> run_web.sh
-    - echo "-v \"${DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh
-    - echo "-v \"${DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" " >> run_web.sh
-    - chmod +x run_web.sh
-    - ./run_web.sh
-    - sleep 30
-    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" 
+    - echo "#!/bin/bash" > "${RUN_FILE}"
+    - echo "docker run -d \\" >> "${RUN_FILE}"
+    - echo "--name \"foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> "${RUN_FILE}"
+    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> "${RUN_FILE}"
+    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> "${RUN_FILE}"
+    - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> "${RUN_FILE}"
+    - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> "${RUN_FILE}"
+    - echo "-e DATAFED_DATABASE_HOST=\"$CI_DATAFED_DATABASE_HOST\" \\" >> "${RUN_FILE}"
+    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> "${RUN_FILE}"
+    - echo "-e UID=\"$USER_ID\" \\" >> "${RUN_FILE}"
+    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> "${RUN_FILE}"
+    - echo "-v \"./foxx_tmp:/tmp\" \\" >> "${RUN_FILE}"
+    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> "${RUN_FILE}"
+    - chmod +x "${RUN_FILE}"
+    - "./${RUN_FILE}"
+    - sleep 10
+      # Make sure container is running immediately after because it is meant to
+      # be ephermal anyway, this is not the same for the other containers
+    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest"
+    - while [ ! -f "foxx_tmp/.foxx_is_installed" ]; do echo "Waiting for foxx_tmp/.foxx_is_installed"; sleep 10; done
+    - ./scripts/ci_database_health_check.sh
+  after_script:
+    - rm -rf foxx_tmp
+
+################################################################################
+# STAGE: End to end setup
+################################################################################
 
-# Note here we use network=host to run the docker container this is
-# because we do not need it to communicate with an external port 
 end-to-end-core-setup:
   variables:
     IMAGE_TAG: "datafed/core-"
@@ -509,7 +424,7 @@ end-to-end-core-setup:
     CONTAINER_LOG_FILE_PATH: "/datafed/logs"
     DATAFED_DATABASE_HOST: "$CI_DATAFED_DATABASE_HOST"
   stage: end-to-end-setup
-  needs: ["end-to-end-arango-setup"]
+  needs: ["end-to-end-arango-setup", "end-to-end-foxx-setup"]
   tags:
     - ci-datafed-core
     - docker
@@ -521,11 +436,12 @@ end-to-end-core-setup:
     - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - echo "$BRANCH_LOWER"
     - ./scripts/generate_datafed.sh
+    - env > env_file
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}"
+    - ./scripts/container_stop.sh -n "core-" -p
     - ./scripts/ci_database_health_check.sh
-    - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-core-key.pub
-    - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-core-key.priv
+    - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub
+    - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - echo "#!/bin/bash" > run_core.sh
     - echo "docker run -d \\" >> run_core.sh
@@ -542,13 +458,71 @@ end-to-end-core-setup:
     - echo "-p 7513:7513 \\" >> run_core.sh
     - echo "-p 7512:7512 \\" >> run_core.sh
     - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh
-    - echo "-v \"/shared/datafed-core-key.pub\":/datafed/install/keys/datafed-core-key.pub \\" >> run_core.sh
-    - echo "-v \"/shared/datafed-core-key.priv\":/datafed/install/keys/datafed-core-key.priv \\" >> run_core.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_core.sh
+    - echo "-v \"/shared/keys/datafed-core-key.pub\":/opt/datafed/keys/datafed-core-key.pub \\" >> run_core.sh
+    - echo "-v \"/shared/keys/datafed-core-key.priv\":/opt/datafed/keys/datafed-core-key.priv \\" >> run_core.sh
+    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_core.sh
     - chmod +x run_core.sh
     - ./run_core.sh
     - sleep 10
-    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" 
+    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" 
+
+end-to-end-ws-setup:
+  variables:
+    IMAGE_TAG: "datafed/ws-"
+    GIT_STRATEGY: clone
+    HOST_LOG_FILE_PATH: "/shared/logs"
+    CONTAINER_LOG_FILE_PATH: "/datafed/logs"
+    DATAFED_WEB_KEY_DIR: "/shared/keys"
+    DATAFED_WEB_CERT_NAME: "cert.crt"
+    DATAFED_WEB_KEY_NAME: "cert.key"
+    DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_KEY_DIR}/${DATAFED_WEB_CERT_NAME}"
+    DATAFED_WEB_CSR_PATH: "${DATAFED_WEB_KEY_DIR}/cert.csr"
+    DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_DIR}/${DATAFED_WEB_KEY_NAME}"
+  stage: end-to-end-setup
+  needs : ["end-to-end-core-setup"]
+  tags:
+    - ci-datafed-core
+    - docker
+  script:
+    - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
+    - echo "$BRANCH_LOWER"
+    - mkdir -p "$HOST_LOG_FILE_PATH"
+    - mkdir -p "${DATAFED_WEB_KEY_DIR}"
+    - ./scripts/ci_setup_web_certs.sh
+    - chmod o+w "${HOST_LOG_FILE_PATH}"
+    - chown gitlab-runner "$HOST_LOG_FILE_PATH"
+    - ./scripts/generate_datafed.sh
+    - docker login  "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
+    - USER_ID=$(id -u)
+    - GROUP_ID=$(id -g)
+    - CORE_ADDRESS=$(hostname -I | awk '{print $1}')
+    - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub
+    - env
+    - ./scripts/container_stop.sh -n "ws-" -p 
+    - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
+    - echo "#!/bin/bash" > run_web.sh
+    - echo "docker run -d \\" >> run_web.sh
+    - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh
+    - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh
+    - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh
+    - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh
+    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh
+    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh
+    - echo "-e DATAFED_WEB_CERT_PATH=\"/opt/datafed/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh
+    - echo "-e DATAFED_WEB_KEY_PATH=\"/opt/datafed/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh
+    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh
+    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"$CORE_ADDRESS:7513\" \\" >> run_web.sh
+    - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh
+    - echo "-p 443:443 \\" >> run_web.sh
+    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh
+    - echo "-v \"/shared/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub\" \\" >> run_web.sh
+    - echo "-v \"${DATAFED_WEB_CERT_PATH}:/opt/datafed/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh
+    - echo "-v \"${DATAFED_WEB_KEY_PATH}:/opt/datafed/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh
+    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\" " >> run_web.sh
+    - chmod +x run_web.sh
+    - ./run_web.sh
+    - sleep 30
+    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" 
 
 # Repo server currently will crash on startup if it cannot connect to the core
 # server.
@@ -558,8 +532,9 @@ end-to-end-repo-setup:
     GIT_STRATEGY: clone
     HOST_LOG_FILE_PATH: "/shared/logs"
     CONTAINER_LOG_FILE_PATH: "/datafed/logs"
+    DATAFED_HOST_COLLECTION_MOUNT: "/shared/collections"
   stage: end-to-end-setup
-  needs: ["end-to-end-core-setup"]
+  needs: ["end-to-end-ws-setup"]
   tags:
     - ci-datafed-globus
     - docker
@@ -567,43 +542,58 @@ end-to-end-repo-setup:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
     - mkdir -p "$HOST_LOG_FILE_PATH"
+    - if [ ! -d "${DATAFED_HOST_COLLECTION_MOUNT}" ]; then mkdir -p "${DATAFED_HOST_COLLECTION_MOUNT}"; fi
     - chmod o+w "${HOST_LOG_FILE_PATH}"
     - USER_ID=$(id -u)
     - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - ./scripts/generate_datafed.sh
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}"
+    - ./scripts/container_stop.sh -n "repo-" -p
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub
     - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv
     - echo "#!/bin/bash" > run_repo.sh
-    - echo "docker run \\" >> run_repo.sh
+    - echo "docker run -d \\" >> run_repo.sh
     - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh
     - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh
     - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh
     - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh
     - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_repo.sh
+    - echo "-e DATAFED_HTTPS_SERVER_PORT=\"443\" \\" >> run_repo.sh
     - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_repo.sh
+    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"${CI_DATAFED_DOMAIN}:7513\" \\" >> run_repo.sh
     - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_repo.sh
+    - echo "-e DATAFED_GCS_COLLECTION_ROOT_PATH=\"/mnt/datafed\" \\" >> run_repo.sh
     - echo "-e UID=\"$USER_ID\" \\" >> run_repo.sh
+    - echo "-p 9000:9000 \\" >> run_repo.sh
     - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh
-    - echo "-v \"/shared/datafed-repo-key.pub\":/datafed/install/keys/datafed-repo-key.pub \\" >> run_repo.sh
-    - echo "-v \"/shared/datafed-repo-key.priv\":/datafed/install/keys/datafed-repo-key.priv \\" >> run_repo.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_repo.sh
+    - echo "-v \"${DATAFED_HOST_COLLECTION_MOUNT}:/mnt/datafed\" \\" >> run_repo.sh
+    - echo "-v \"/shared/datafed-repo-key.pub\":/opt/datafed/keys/datafed-repo-key.pub \\" >> run_repo.sh
+    - echo "-v \"/shared/datafed-repo-key.priv\":/opt/datafed/keys/datafed-repo-key.priv \\" >> run_repo.sh
+    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_repo.sh
     - chmod +x run_repo.sh
     - ./run_repo.sh
     - sleep 10
-    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" 
+    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" 
 
 # Requires setting up Globus Connect Server, requires firewall exceptions on
 # the machine running this.
+# Note we need the certificates to be available on the gcs-authz container
+# if it is meant to be run on the same machine as the metadata services
+# because the Apache web server can then route traffic appropriately, if 
+# run separate from the metadata services it should not be needed.
+# NOTE it should also run after the repo service because when the form is
+# generated, it requires the repo server public key.
 end-to-end-gcs-authz-setup:
   variables:
-    IMAGE_TAG: "datafed/gcs-authz-"
+    IMAGE_TAG: "datafed/gcs-"
     HOST_LOG_FILE_PATH: "/shared/logs"
     CONTAINER_LOG_FILE_PATH: "/datafed/logs"
     GIT_STRATEGY: clone
+    DATAFED_HOST_COLLECTION_MOUNT: "/shared/collections"
+    DATAFED_GLOBUS_DIR: "/shared/globus"
   stage: end-to-end-setup
+  needs: ["end-to-end-repo-setup"]
   tags:
     - ci-datafed-globus
     - docker
@@ -611,37 +601,62 @@ end-to-end-gcs-authz-setup:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
     - mkdir -p "$HOST_LOG_FILE_PATH"
+    - mkdir -p "${DATAFED_GLOBUS_DIR}"
+    - cp "${CI_DATAFED_GCS_DEPLOYMENT_KEY}" "${DATAFED_GLOBUS_DIR}/deployment-key.json"
+    - cp "${CI_DATAFED_GCS_CLIENT_CRED}" "${DATAFED_GLOBUS_DIR}/client_cred.json"
     - USER_ID=$(id -u)
     - chmod o+w "${HOST_LOG_FILE_PATH}"
     - chown gitlab-runner "$HOST_LOG_FILE_PATH"
     - ./scripts/generate_datafed.sh
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}"
-    - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}"
+    - ./scripts/container_stop.sh -n "gcs-authz" -p
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub
     - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv
     - echo "#!/bin/bash" > run_globus.sh
-    - echo "docker run \\" >> run_globus.sh
+    - echo "docker run -d \\" >> run_globus.sh
     - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh
+    - echo "--network host \\" >> run_globus.sh
     - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh
     - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh
     - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh
     - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh
     - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh
+    - echo "-e DATAFED_HTTPS_SERVER_PORT=\"443\" \\" >> run_globus.sh
+    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_globus.sh
+    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"${CI_DATAFED_DOMAIN}:7513\" \\" >> run_globus.sh
+    - echo "-e DATAFED_GCS_ROOT_NAME=\"${CI_DATAFED_GCS_ROOT_NAME}\" \\" >> run_globus.sh
+    - echo "-e DATAFED_GCS_COLLECTION_ROOT_PATH=\"/mnt/datafed\" \\" >> run_globus.sh
+    - echo "-e DATAFED_GLOBUS_SUBSCRIPTION=\"${CI_DATAFED_GLOBUS_SUBSCRIPTION}\" \\" >> run_globus.sh
+    - echo "-e DATAFED_GLOBUS_CONTROL_PORT=\"443\" \\" >> run_globus.sh
+    - echo "-e DATAFED_REPO_USER=\"datafed\" \\" >> run_globus.sh
+    - echo "-e DATAFED_AUTHZ_USER=\"datafed\" \\" >> run_globus.sh
     - echo "-e UID=\"$USER_ID\" \\" >> run_globus.sh
+    - echo "-e BUILD_WITH_METADATA_SERVICES=\"FALSE\" \\" >> run_globus.sh
+    - echo "-e DATAFED_REPO_ID_AND_DIR=\"${CI_DATAFED_REPO_ID_AND_DIR}\" \\" >> run_globus.sh
+    - echo "-e DATAFED_GCS_IP=\"${CI_DATAFED_GCS_IP}\" \\" >> run_globus.sh
+    - echo "-e DATAFED_REPO_DOMAIN=\"${CI_DATAFED_REPO_DOMAIN}\" \\" >> run_globus.sh
+    - echo "-v \"${DATAFED_GLOBUS_DIR}:/opt/datafed/globus\" \\" >> run_globus.sh
     - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh
-    - echo "-v \"/shared/datafed-repo-key.pub\":/datafed/install/keys/datafed-repo-key.pub \\" >> run_globus.sh
-    - echo "-v \"/shared/datafed-repo-key.priv\":/datafed/install/keys/datafed-repo-key.priv \\" >> run_globus.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_globus.sh
+    - echo "-v \"${DATAFED_HOST_COLLECTION_MOUNT}:/mnt/datafed\" \\" >> run_globus.sh
+    - echo "-v \"/shared/datafed-repo-key.pub\":/opt/datafed/keys/datafed-repo-key.pub \\" >> run_globus.sh
+    - echo "-v \"/shared/datafed-repo-key.priv\":/opt/datafed/keys/datafed-repo-key.priv \\" >> run_globus.sh
+    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_globus.sh
+    - if [ -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; then rm "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; fi
     - chmod +x run_globus.sh
     - ./run_globus.sh
-    - sleep 10
-    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" 
+    - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done
+    - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"
+    - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest"
+    - cp ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh .
+    - cp ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.json .
+  artifacts:
+    paths:
+      - "${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"
+      - "${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.json"
 
-################################################################################
-# Actual Testing
-################################################################################
-end-to-end-client-test:
+
+end_to_end_client-test:
   variables:
     GIT_STRATEGY: clone
     DATAFED_DATABASE_HOST: "${CI_DATAFED_DATABASE_HOST}"
@@ -656,13 +671,19 @@ end-to-end-client-test:
     DATAFED_USER99_GLOBUS_ACCESS_TOKEN: "${CI_DATAFED_USER99_GLOBUS_ACCESS_TOKEN}"
     DATAFED_USER99_GLOBUS_UUID: "${CI_DATAFED_USER99_GLOBUS_UUID}"
     DATAFED_ZEROMQ_SYSTEM_SECRET: "${CI_DATAFED_ZEROMQ_SYSTEM_SECRET}"
+    DATAFED_DOMAIN: "${CI_DATAFED_DOMAIN}"
+    DATAFED_PYTHON_CLIENT_ALLOW_SELF_SIGNED_CERTS: "TRUE"
   stage: end-to-end-test
+  dependencies:
+    - end-to-end-gcs-authz-setup
+  needs: ["end-to-end-gcs-authz-setup"]
   tags:
     - ci-datafed-client
   script:
+    - export DATAFED_REPO_FORM_PATH="$(pwd)/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.json"
+    - env > env_file
+    - echo "Testing"
     - ./scripts/generate_datafed.sh
-    - env
-    - cat ./config/datafed.sh
     - >
       cmake -S. -B build
       -DENABLE_FOXX_TESTS=OFF
@@ -677,4 +698,3 @@ end-to-end-client-test:
     - cmake --build build
     - cmake --build build --target pydatafed
     - cmake --build build --target test
-
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f3a3b766..33a53f90a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@
 4. [912] - Adding working compose instance for core metadata services.
 5. [937] - Working metadata services running together as part of CI
 6. [946] - Added docker compose for DataFed Repository and for Metadata Services
+7. [955] - Adds repo pieces to CI with working end-to-end tests
 
 ## PATCH Bug fixes/Technical Debt/Documentation
 
diff --git a/compose/all/compose.yml b/compose/all/compose.yml
index 9dd3b1f4f..85fc568e8 100644
--- a/compose/all/compose.yml
+++ b/compose/all/compose.yml
@@ -136,6 +136,7 @@ services:
       HOST_HOSTNAME: "localhost"
       DATAFED_AUTHZ_USER: "datafed"
       DATAFED_GCS_IP: "${DATAFED_GCS_IP}"
+      DATAFED_REPO_DOMAIN: "${DATAFED_REPO_DOMAIN}"
     network_mode: host
     image: datafed-gcs:latest
     volumes:
diff --git a/docker/Dockerfile.foxx b/docker/Dockerfile.foxx
index c1a7726c8..9c4dcbaa9 100644
--- a/docker/Dockerfile.foxx
+++ b/docker/Dockerfile.foxx
@@ -46,12 +46,12 @@ RUN mkdir -p ${DATAFED_DIR}
 RUN mkdir -p ${DATAFED_INSTALL_PATH}
 RUN mkdir -p ${DATAFED_DEFAULT_LOG_PATH}
 
-RUN curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key; apt-key add - < Release.key
+RUN curl -OL https://download.arangodb.com/arangodb311/DEBIAN/Release.key; apt-key add - < Release.key
 
-# RUN echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list
+# RUN echo 'deb https://download.arangodb.com/arangodb311/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list
 # If trust the site but the gpg key has expired can add [trusted=yes]
 # i.e.
-RUN echo 'deb [trusted=yes] https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list
+RUN echo 'deb [trusted=yes] https://download.arangodb.com/arangodb311/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list
 RUN apt update
 RUN apt install arangodb3-client -y
 RUN arangosh --version
@@ -60,4 +60,4 @@ USER root
 
 WORKDIR ${BUILD_DIR}
 
-ENTRYPOINT ${BUILD_DIR}/docker/entrypoint_foxx.sh
+ENTRYPOINT /datafed/source/docker/entrypoint_foxx.sh
diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh
index ed8387eb9..43e145e8f 100755
--- a/docker/entrypoint_foxx.sh
+++ b/docker/entrypoint_foxx.sh
@@ -18,12 +18,13 @@ PROJECT_ROOT=$(realpath "${SOURCE}/../")
 install_flag="/tmp/.foxx_is_installed"
 if [ ! -f "$install_flag" ]
 then
+  echo "Installing foxx."
   log_path="$DATAFED_DEFAULT_LOG_PATH"
   if [ ! -d "${log_path}" ]
   then
     su -c "mkdir -p ${log_path}" datafed
   fi
-
+  
   # It should be fine to run this as root because it is an ephemeral container anyway
   cd "${PROJECT_ROOT}"
   # Check to see if foxx has previously been installed
@@ -49,7 +50,12 @@ then
   "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" --build build --target install
   
   touch "$install_flag"
+  chown "$UID":"$UID" "$install_flag"
+
+else
+  echo "/tmp/.foxx_is_installed has been found skipping reinstall"
 fi
 
+echo "Sleeping"
 sleep 1000
 
diff --git a/jupyter_notebooks/0_verify.ipynb b/jupyter_notebooks/0_verify.ipynb
index 63c42c469..4b5b215b5 100644
--- a/jupyter_notebooks/0_verify.ipynb
+++ b/jupyter_notebooks/0_verify.ipynb
@@ -1,226 +1,199 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# DataFed Checklist"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import getpass\n",
-    "import subprocess\n",
-    "from platform import platform\n",
-    "import sys"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 0. Machine information:"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Python version:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "sys.version_info"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "platform()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 1. Verify DataFed installation:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "try:\n",
-    "    # This package is not part of anaconda and may need to be installed.\n",
-    "    from datafed.CommandLib import API\n",
-    "except ImportError:\n",
-    "    print('datafed not found. Installing from pip.')\n",
-    "    subprocess.call([sys.executable, \"-m\", \"pip\", \"install\", 'datafed'])\n",
-    "    from datafed.CommandLib import API\n",
-    "\n",
-    "from datafed import version as df_ver\n",
-    "\n",
-    "if not df_ver.startswith('1.1'):\n",
-    "    print('Attempting to update DataFed.')\n",
-    "    subprocess.call([sys.executable, \"-m\", \"pip\", \"install\", '--upgrade', 'datafed'])\n",
-    "    print('Please restart the python kernel or upgrade manually to V 1.1.0:1 if you are repeatedly seeing this message via'\n",
-    "          '\\n\\tpip install --upgrade datafed')\n",
-    "else: \n",
-    "    df_api = API()\n",
-    "    print('Success! You have DataFed: ' + df_ver)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 2. Verify user authentication: "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if df_api.getAuthUser():\n",
-    "    print('Success! You have been authenticated into DataFed as: ' + df_api.getAuthUser())\n",
-    "else:\n",
-    "    print('You have not authenticated into DataFed Client')\n",
-    "    print('Please follow instructions in the \"Basic Configuration\" section in the link below to authenticate yourself:')\n",
-    "    print('https://ornl.github.io/DataFed/user/client/install.html#basic-configuration')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 3. Ensure Globus Endpoint is set:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if not df_api.endpointDefaultGet():\n",
-    "    print('Please follow instructions in the link below to find your Globus Endpoint:')\n",
-    "    print('https://ornl.github.io/DataFed/system/getting_started.html#install-identify-globus-endpoint')\n",
-    "    endpoint = input('\\nPlease enter either the Endpoint UUID or Legacy Name for your Globus Endpoint: ')\n",
-    "    df_api.endpointDefaultSet(endpoint)\n",
-    "    \n",
-    "print('Your default Globus Endpoint in DataFed is:\\n' + df_api.endpointDefaultGet())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 4. Test Globus Endpoint:\n",
-    "And in the case of Windows - write access to the folder:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dget_resp = df_api.dataGet('d/35437908',\n",
-    "                           os.path.abspath('.'), \n",
-    "                           wait=True)\n",
-    "dget_resp"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if dget_resp[0].task[0].status == 3:\n",
-    "    print('Success! Downloaded a test file to your location. Removing the file now')\n",
-    "    os.remove('35437908.md5sum')\n",
-    "else:\n",
-    "    if dget_resp[0].task[0].msg == \"globus connect offline\":\n",
-    "        print('You need to activate your Globus Endpoint and/or ensure Globus Connect Personal is running.\\n'\n",
-    "              'Please visit https://globus.org to activate your Endpoint')\n",
-    "    elif dget_resp[0].task[0].msg == \"permission denied\":\n",
-    "        print('Globus does not have write access to this directory. \\n'\n",
-    "              'If you are using Globus Connect Personal, ensure that this notebook runs within'\n",
-    "              'one of the directories where Globus has write access. You may consider moving this'\n",
-    "              'notebook to a valid directory or add this directory to the Globus Connect Personal settings')\n",
-    "    else:\n",
-    "        NotImplementedError('Get in touch with us or consider looking online to find a solution to this problem:\\n' + dget_resp[0].task[0].msg)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### (Optional) for Windows - test for Admin priveleges\n",
-    "Admin priveleges may be necessary to remove files or directories using Python:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import ctypes, os\n",
-    "try:\n",
-    "    is_admin = os.getuid() == 0\n",
-    "except AttributeError:\n",
-    "    is_admin = ctypes.windll.shell32.IsUserAnAdmin() != 0\n",
-    "    \n",
-    "is_admin"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
+    "cells": [
+        {"cell_type": "markdown", "metadata": {}, "source": ["# DataFed Checklist"]},
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "import getpass\n",
+                "import subprocess\n",
+                "from platform import platform\n",
+                "import sys",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["## 0. Machine information:"],
+        },
+        {"cell_type": "markdown", "metadata": {}, "source": ["Python version:"]},
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["sys.version_info"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["platform()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["## 1. Verify DataFed installation:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "try:\n",
+                "    # This package is not part of anaconda and may need to be installed.\n",
+                "    from datafed.CommandLib import API\n",
+                "except ImportError:\n",
+                "    print('datafed not found. Installing from pip.')\n",
+                '    subprocess.call([sys.executable, "-m", "pip", "install", \'datafed\'])\n',
+                "    from datafed.CommandLib import API\n",
+                "\n",
+                "from datafed import version as df_ver\n",
+                "\n",
+                "if not df_ver.startswith('1.1'):\n",
+                "    print('Attempting to update DataFed.')\n",
+                '    subprocess.call([sys.executable, "-m", "pip", "install", \'--upgrade\', \'datafed\'])\n',
+                "    print('Please restart the python kernel or upgrade manually to V 1.1.0:1 if you are repeatedly seeing this message via'\n",
+                "          '\\n\\tpip install --upgrade datafed')\n",
+                "else: \n",
+                "    df_api = API()\n",
+                "    print('Success! You have DataFed: ' + df_ver)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["## 2. Verify user authentication: "],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "if df_api.getAuthUser():\n",
+                "    print('Success! You have been authenticated into DataFed as: ' + df_api.getAuthUser())\n",
+                "else:\n",
+                "    print('You have not authenticated into DataFed Client')\n",
+                "    print('Please follow instructions in the \"Basic Configuration\" section in the link below to authenticate yourself:')\n",
+                "    print('https://ornl.github.io/DataFed/user/client/install.html#basic-configuration')",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["## 3. Ensure Globus Endpoint is set:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "if not df_api.endpointDefaultGet():\n",
+                "    print('Please follow instructions in the link below to find your Globus Endpoint:')\n",
+                "    print('https://ornl.github.io/DataFed/system/getting_started.html#install-identify-globus-endpoint')\n",
+                "    endpoint = input('\\nPlease enter either the Endpoint UUID or Legacy Name for your Globus Endpoint: ')\n",
+                "    df_api.endpointDefaultSet(endpoint)\n",
+                "    \n",
+                "print('Your default Globus Endpoint in DataFed is:\\n' + df_api.endpointDefaultGet())",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## 4. Test Globus Endpoint:\n",
+                "And in the case of Windows - write access to the folder:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dget_resp = df_api.dataGet('d/35437908',\n",
+                "                           os.path.abspath('.'), \n",
+                "                           wait=True)\n",
+                "dget_resp",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "if dget_resp[0].task[0].status == 3:\n",
+                "    print('Success! Downloaded a test file to your location. Removing the file now')\n",
+                "    os.remove('35437908.md5sum')\n",
+                "else:\n",
+                '    if dget_resp[0].task[0].msg == "globus connect offline":\n',
+                "        print('You need to activate your Globus Endpoint and/or ensure Globus Connect Personal is running.\\n'\n",
+                "              'Please visit https://globus.org to activate your Endpoint')\n",
+                '    elif dget_resp[0].task[0].msg == "permission denied":\n',
+                "        print('Globus does not have write access to this directory. \\n'\n",
+                "              'If you are using Globus Connect Personal, ensure that this notebook runs within'\n",
+                "              'one of the directories where Globus has write access. You may consider moving this'\n",
+                "              'notebook to a valid directory or add this directory to the Globus Connect Personal settings')\n",
+                "    else:\n",
+                "        NotImplementedError('Get in touch with us or consider looking online to find a solution to this problem:\\n' + dget_resp[0].task[0].msg)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### (Optional) for Windows - test for Admin priveleges\n",
+                "Admin priveleges may be necessary to remove files or directories using Python:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import ctypes, os\n",
+                "try:\n",
+                "    is_admin = os.getuid() == 0\n",
+                "except AttributeError:\n",
+                "    is_admin = ctypes.windll.shell32.IsUserAnAdmin() != 0\n",
+                "    \n",
+                "is_admin",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.8.3",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 2,
 }
diff --git a/jupyter_notebooks/1_Basics.ipynb b/jupyter_notebooks/1_Basics.ipynb
index 9ca898758..114aa73e3 100644
--- a/jupyter_notebooks/1_Basics.ipynb
+++ b/jupyter_notebooks/1_Basics.ipynb
@@ -1,344 +1,310 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Getting Started with DataFed\n",
-    "In this notebook we will be going over basic concepts such as ``contexts``, ``projects``, understanding how DataFed responds to function calls, etc. \n",
-    "\n",
-    "To get started, we need to import only the ``API`` class from ``CommandLab`` in the datafed package. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Next, we need to instantiate the ``API`` class as:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# First DataFed function\n",
-    "Let's try out the DataFed API by simply asking DataFed for a list of all projects that we are part of using the ``projectList()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pl_resp = df_api.projectList()\n",
-    "print(pl_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Jupyter Tip: </span>\n",
-    "> <span style=\"color:blue\"> With the cursor just past the starting parenthesis of ``projectList(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>\n",
-    "\n",
-    "## DataFed Messages\n",
-    "\n",
-    "DataFed responds using ``Google Protocol Buffer`` or ``protobuf`` messages\n",
-    "\n",
-    "Let's take a closer look at this response:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(type(pl_resp), len(pl_resp))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "As you can see, the reply is a tuple containing two objects, namely the protobuf message reply itself, `[0]` and the type of reply received, `ListingReply` at `[1]`. We can confirm this by checking the response type:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "type(pl_resp[0])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Contexts\n",
-    "DataFed allows us to work within multiple different \"data spaces\" – such as our own ``Personal Data``, and those of our ``Project``s. Let's find out what ``context`` DataFed automatically put us into using ``getContext()``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df_api.getContext())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "By default, DataFed sets our working space or ``context`` to our own ``Personal Data`` (`u/username`). \n",
-    "\n",
-    "### Specifying contexts:\n",
-    "If we want to take a look at the ``root`` collection of the Training Project, we need to specify that using the `context` keyword argument:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df_api.collectionView('root', context='p/trn001'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here's what we get when we give the same `collectionView` request without the project context:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df_api.collectionView('root'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Subscripting and Iterating messages\n",
-    "Let us take a look at the contents of the Project (its ``root`` Collection) using ``collectionItemsList()``"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ls_resp = df_api.collectionItemsList('root', context='p/trn001')\n",
-    "print(ls_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Much like the ``projectList()``, we get a ``ListingReply`` in this case as well\n",
-    "\n",
-    "### Subscripting\n",
-    "The listing reply `item` behaves similarly to a python list in terms of subscriptability. For example, in order to access the title of the last item in the collection listing, we can use indexing:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ls_resp[0].item[-1].title"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Iterating\n",
-    "These messages also mirror python lists in their iterability. We can iterate through the items in this listing and use the subscripting capability to only extract the ``id`` and ``alias`` fields of each of the collections"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "for record in ls_resp[0].item:\n",
-    "    print(record.id, '\\t', record.alias)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Aliases and IDs\n",
-    "\n",
-    "Let's try taking a closer look at the ``PROJSHARE`` collection using its ``alias``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('projshare')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The above request failed because we asked DataFed to look for a Collection with alias: ``projshare`` without specifying the ``context``. Naturally, DataFed assumed that we meant our own ``Personal Data`` rather than the training ``Project``. \n",
-    "\n",
-    "If we want to address an item by its ``alias``, we need to be careful about its ``context`` since:\n",
-    "\n",
-    "**An ``alias`` is unique only within a given ``context``** such as ``Personal Data`` or a ``Project``\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Correct the above function call to view ``projshare`` collection: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Alternatively, we can view the correct collection by referring to it using its ``id``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('c/34559108')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The above command worked even though we did not specify a ``context`` because:\n",
-    "\n",
-    "**``ID``s are unique across DataFed and do not need a ``context``**<br><br><br>\n",
-    "\n",
-    "## Setting Context:\n",
-    "Having to specify the context for every function call can be tiring if we are sure we are working within a single context. \n",
-    "\n",
-    "We can set the context via the ``setContext()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> ``setContext()`` is valid within the scope of a single python session. You would need to call the function each time you instantiate the DataFed ``CommandLib.API`` class. E.g. - at the top of every notebook </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Let's attempt to view the ``projshare`` Collection via its ``alias`` **without** specifying the ``context`` keyword argument:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('projshare')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Using the DataFed API's `collectionView()`, extract the create time (`ct`) of your own personal collection within the training project or ``projshare``. <br><br> <b> Bonus: </b> Consider using the built-in `df_api.timestampToStr()` function or `datetime.datetime.fromtimestamp` from the `datetime` package to convert the unix formatted time to a human readable string </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "create_time = "
-   ]
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python [default]",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Getting Started with DataFed\n",
+                "In this notebook we will be going over basic concepts such as ``contexts``, ``projects``, understanding how DataFed responds to function calls, etc. \n",
+                "\n",
+                "To get started, we need to import only the ``API`` class from ``CommandLab`` in the datafed package. ",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["from datafed.CommandLib import API"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["Next, we need to instantiate the ``API`` class as:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# First DataFed function\n",
+                "Let's try out the DataFed API by simply asking DataFed for a list of all projects that we are part of using the ``projectList()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["pl_resp = df_api.projectList()\n", "print(pl_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Jupyter Tip: </span>\n',
+                '> <span style="color:blue"> With the cursor just past the starting parenthesis of ``projectList(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>\n',
+                "\n",
+                "## DataFed Messages\n",
+                "\n",
+                "DataFed responds using ``Google Protocol Buffer`` or ``protobuf`` messages\n",
+                "\n",
+                "Let's take a closer look at this response:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(type(pl_resp), len(pl_resp))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "As you can see, the reply is a tuple containing two objects, namely the protobuf message reply itself, `[0]` and the type of reply received, `ListingReply` at `[1]`. We can confirm this by checking the response type:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["type(pl_resp[0])"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Contexts\n",
+                'DataFed allows us to work within multiple different "data spaces" – such as our own ``Personal Data``, and those of our ``Project``s. Let\'s find out what ``context`` DataFed automatically put us into using ``getContext()``:',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(df_api.getContext())"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "By default, DataFed sets our working space or ``context`` to our own ``Personal Data`` (`u/username`). \n",
+                "\n",
+                "### Specifying contexts:\n",
+                "If we want to take a look at the ``root`` collection of the Training Project, we need to specify that using the `context` keyword argument:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(df_api.collectionView('root', context='p/trn001'))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Here's what we get when we give the same `collectionView` request without the project context:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(df_api.collectionView('root'))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Subscripting and Iterating messages\n",
+                "Let us take a look at the contents of the Project (its ``root`` Collection) using ``collectionItemsList()``",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "ls_resp = df_api.collectionItemsList('root', context='p/trn001')\n",
+                "print(ls_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Much like the ``projectList()``, we get a ``ListingReply`` in this case as well\n",
+                "\n",
+                "### Subscripting\n",
+                "The listing reply `item` behaves similarly to a python list in terms of subscriptability. For example, in order to access the title of the last item in the collection listing, we can use indexing:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["ls_resp[0].item[-1].title"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Iterating\n",
+                "These messages also mirror python lists in their iterability. We can iterate through the items in this listing and use the subscripting capability to only extract the ``id`` and ``alias`` fields of each of the collections",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "for record in ls_resp[0].item:\n",
+                "    print(record.id, '\\t', record.alias)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Aliases and IDs\n",
+                "\n",
+                "Let's try taking a closer look at the ``PROJSHARE`` collection using its ``alias``:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('projshare')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "The above request failed because we asked DataFed to look for a Collection with alias: ``projshare`` without specifying the ``context``. Naturally, DataFed assumed that we meant our own ``Personal Data`` rather than the training ``Project``. \n",
+                "\n",
+                "If we want to address an item by its ``alias``, we need to be careful about its ``context`` since:\n",
+                "\n",
+                "**An ``alias`` is unique only within a given ``context``** such as ``Personal Data`` or a ``Project``\n",
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Correct the above function call to view ``projshare`` collection: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Alternatively, we can view the correct collection by referring to it using its ``id``:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('c/34559108')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "The above command worked even though we did not specify a ``context`` because:\n",
+                "\n",
+                "**``ID``s are unique across DataFed and do not need a ``context``**<br><br><br>\n",
+                "\n",
+                "## Setting Context:\n",
+                "Having to specify the context for every function call can be tiring if we are sure we are working within a single context. \n",
+                "\n",
+                "We can set the context via the ``setContext()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> ``setContext()`` is valid within the scope of a single python session. You would need to call the function each time you instantiate the DataFed ``CommandLib.API`` class. E.g. - at the top of every notebook </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Let's attempt to view the ``projshare`` Collection via its ``alias`` **without** specifying the ``context`` keyword argument:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('projshare')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Using the DataFed API\'s `collectionView()`, extract the create time (`ct`) of your own personal collection within the training project or ``projshare``. <br><br> <b> Bonus: </b> Consider using the built-in `df_api.timestampToStr()` function or `datetime.datetime.fromtimestamp` from the `datetime` package to convert the unix formatted time to a human readable string </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["create_time = "],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python [default]",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.5.5",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/1_Basics_with_Solutions.ipynb b/jupyter_notebooks/1_Basics_with_Solutions.ipynb
index 59505d359..da1293f95 100644
--- a/jupyter_notebooks/1_Basics_with_Solutions.ipynb
+++ b/jupyter_notebooks/1_Basics_with_Solutions.ipynb
@@ -1,357 +1,323 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Getting Started with DataFed\n",
-    "In this notebook we will be going over basic concepts such as ``contexts``, ``projects``, understanding how DataFed responds to function calls, etc. \n",
-    "\n",
-    "To get started, we need to import only the ``API`` class from ``CommandLab`` in the datafed package. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Next, we need to instantiate the ``API`` class as:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# First DataFed function\n",
-    "Let's try out the DataFed API by simply asking DataFed for a list of all projects that we are part of using the ``projectList()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pl_resp = df_api.projectList()\n",
-    "print(pl_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Jupyter Tip: </span>\n",
-    "> <span style=\"color:blue\"> With the cursor just past the starting parenthesis of ``projectList(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>\n",
-    "\n",
-    "## DataFed Messages\n",
-    "\n",
-    "DataFed responds using ``Google Protocol Buffer`` or ``protobuf`` messages\n",
-    "\n",
-    "Let's take a closer look at this response:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(type(pl_resp), len(pl_resp))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "As you can see, the reply is a tuple containing two objects, namely the protobuf message reply itself, `[0]` and the type of reply received, `ListingReply` at `[1]`. We can confirm this by checking the response type:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "type(pl_resp[0])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Contexts\n",
-    "DataFed allows us to work within multiple different \"data spaces\" – such as our own ``Personal Data``, and those of our ``Project``s. Let's find out what ``context`` DataFed automatically put us into using ``getContext()``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df_api.getContext())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "By default, DataFed sets our working space or ``context`` to our own ``Personal Data`` (`u/username`). \n",
-    "\n",
-    "### Specifying contexts:\n",
-    "If we want to take a look at the ``root`` collection of the Training Project, we need to specify that using the `context` keyword argument:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df_api.collectionView('root', context='p/trn001'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here's what we get when we give the same `collectionView` request without the project context:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(df_api.collectionView('root'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Subscripting and Iterating messages\n",
-    "Let us take a look at the contents of the Project (its ``root`` Collection) using ``collectionItemsList()``"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ls_resp = df_api.collectionItemsList('root', context='p/trn001')\n",
-    "print(ls_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Much like the ``projectList()``, we get a ``ListingReply`` in this case as well\n",
-    "\n",
-    "### Subscripting\n",
-    "The listing reply `item` behaves similarly to a python list in terms of subscriptability. For example, in order to access the title of the last item in the collection listing, we can use indexing:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ls_resp[0].item[-1].title"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Iterating\n",
-    "These messages also mirror python lists in their iterability. We can iterate through the items in this listing and use the subscripting capability to only extract the ``id`` and ``alias`` fields of each of the collections"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "for record in ls_resp[0].item:\n",
-    "    print(record.id, '\\t', record.alias)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Aliases and IDs\n",
-    "\n",
-    "Let's try taking a closer look at the ``PROJSHARE`` collection using its ``alias``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('projshare')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The above request failed because we asked DataFed to look for a Collection with alias: ``projshare`` without specifying the ``context``. Naturally, DataFed assumed that we meant our own ``Personal Data`` rather than the training ``Project``. \n",
-    "\n",
-    "If we want to address an item by its ``alias``, we need to be careful about its ``context`` since:\n",
-    "\n",
-    "**An ``alias`` is unique only within a given ``context``** such as ``Personal Data`` or a ``Project``\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Correct the above function call to view ``projshare`` collection: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('projshare', context='p/trn001)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Alternatively, we can view the correct collection by referring to it using its ``id``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('c/34559108')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The above command worked even though we did not specify a ``context`` because:\n",
-    "\n",
-    "**``ID``s are unique across DataFed and do not need a ``context``**<br><br><br>\n",
-    "\n",
-    "## Setting Context:\n",
-    "Having to specify the context for every function call can be tiring if we are sure we are working within a single context. \n",
-    "\n",
-    "We can set the context via the ``setContext()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> ``setContext()`` is valid within the scope of a single python session. You would need to call the function each time you instantiate the DataFed ``CommandLib.API`` class. E.g. - at the top of every notebook </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Let's attempt to view the ``projshare`` Collection via its ``alias`` **without** specifying the ``context`` keyword argument:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.collectionView('projshare')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Using the DataFed API's `collectionView()`, extract the create time (`ct`) of your own personal collection within the training project or ``projshare``. <br><br> <b> Bonus: </b> Consider using the built-in `df_api.timestampToStr()` function or `datetime.datetime.fromtimestamp` from the `datetime` package to convert the unix formatted time to a human readable string </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "personal_collection_id = ?\n",
-    "create_time = df_api.collectionView(personal_collection_id)[0].coll[0].ct\n",
-    "\n",
-    "# Bonus:\n",
-    "df_api.timestampToStr(create_time)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Getting Started with DataFed\n",
+                "In this notebook we will be going over basic concepts such as ``contexts``, ``projects``, understanding how DataFed responds to function calls, etc. \n",
+                "\n",
+                "To get started, we need to import only the ``API`` class from ``CommandLab`` in the datafed package. ",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["from datafed.CommandLib import API"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["Next, we need to instantiate the ``API`` class as:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# First DataFed function\n",
+                "Let's try out the DataFed API by simply asking DataFed for a list of all projects that we are part of using the ``projectList()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["pl_resp = df_api.projectList()\n", "print(pl_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Jupyter Tip: </span>\n',
+                '> <span style="color:blue"> With the cursor just past the starting parenthesis of ``projectList(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>\n',
+                "\n",
+                "## DataFed Messages\n",
+                "\n",
+                "DataFed responds using ``Google Protocol Buffer`` or ``protobuf`` messages\n",
+                "\n",
+                "Let's take a closer look at this response:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(type(pl_resp), len(pl_resp))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "As you can see, the reply is a tuple containing two objects, namely the protobuf message reply itself, `[0]` and the type of reply received, `ListingReply` at `[1]`. We can confirm this by checking the response type:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["type(pl_resp[0])"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Contexts\n",
+                'DataFed allows us to work within multiple different "data spaces" – such as our own ``Personal Data``, and those of our ``Project``s. Let\'s find out what ``context`` DataFed automatically put us into using ``getContext()``:',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(df_api.getContext())"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "By default, DataFed sets our working space or ``context`` to our own ``Personal Data`` (`u/username`). \n",
+                "\n",
+                "### Specifying contexts:\n",
+                "If we want to take a look at the ``root`` collection of the Training Project, we need to specify that using the `context` keyword argument:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(df_api.collectionView('root', context='p/trn001'))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Here's what we get when we give the same `collectionView` request without the project context:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(df_api.collectionView('root'))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Subscripting and Iterating messages\n",
+                "Let us take a look at the contents of the Project (its ``root`` Collection) using ``collectionItemsList()``",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "ls_resp = df_api.collectionItemsList('root', context='p/trn001')\n",
+                "print(ls_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Much like the ``projectList()``, we get a ``ListingReply`` in this case as well\n",
+                "\n",
+                "### Subscripting\n",
+                "The listing reply `item` behaves similarly to a python list in terms of subscriptability. For example, in order to access the title of the last item in the collection listing, we can use indexing:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["ls_resp[0].item[-1].title"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Iterating\n",
+                "These messages also mirror python lists in their iterability. We can iterate through the items in this listing and use the subscripting capability to only extract the ``id`` and ``alias`` fields of each of the collections",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "for record in ls_resp[0].item:\n",
+                "    print(record.id, '\\t', record.alias)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Aliases and IDs\n",
+                "\n",
+                "Let's try taking a closer look at the ``PROJSHARE`` collection using its ``alias``:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('projshare')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "The above request failed because we asked DataFed to look for a Collection with alias: ``projshare`` without specifying the ``context``. Naturally, DataFed assumed that we meant our own ``Personal Data`` rather than the training ``Project``. \n",
+                "\n",
+                "If we want to address an item by its ``alias``, we need to be careful about its ``context`` since:\n",
+                "\n",
+                "**An ``alias`` is unique only within a given ``context``** such as ``Personal Data`` or a ``Project``\n",
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Correct the above function call to view ``projshare`` collection: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('projshare', context='p/trn001)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Alternatively, we can view the correct collection by referring to it using its ``id``:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('c/34559108')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "The above command worked even though we did not specify a ``context`` because:\n",
+                "\n",
+                "**``ID``s are unique across DataFed and do not need a ``context``**<br><br><br>\n",
+                "\n",
+                "## Setting Context:\n",
+                "Having to specify the context for every function call can be tiring if we are sure we are working within a single context. \n",
+                "\n",
+                "We can set the context via the ``setContext()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> ``setContext()`` is valid within the scope of a single python session. You would need to call the function each time you instantiate the DataFed ``CommandLib.API`` class. E.g. - at the top of every notebook </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Let's attempt to view the ``projshare`` Collection via its ``alias`` **without** specifying the ``context`` keyword argument:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.collectionView('projshare')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Using the DataFed API\'s `collectionView()`, extract the create time (`ct`) of your own personal collection within the training project or ``projshare``. <br><br> <b> Bonus: </b> Consider using the built-in `df_api.timestampToStr()` function or `datetime.datetime.fromtimestamp` from the `datetime` package to convert the unix formatted time to a human readable string </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "personal_collection_id = ?\n",
+                "create_time = df_api.collectionView(personal_collection_id)[0].coll[0].ct\n",
+                "\n",
+                "# Bonus:\n",
+                "df_api.timestampToStr(create_time)\n",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.7.9",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/2_Data_Records.ipynb b/jupyter_notebooks/2_Data_Records.ipynb
index c228fb9f6..c2ab45fc1 100644
--- a/jupyter_notebooks/2_Data_Records.ipynb
+++ b/jupyter_notebooks/2_Data_Records.ipynb
@@ -1,362 +1,342 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 2. Data Records\n",
-    "In this notebook, we will be going over how to create, add metadata and other contextual information, edit, establish relationships between ``Data Records`` - the fundamental unit in DataFed\n",
-    "\n",
-    "## Setting up:\n",
-    "Just as in the previous notebook, we start be instantiating the ``datafed.CommandLib.API`` class to communicate with DataFed:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Specifying context:\n",
-    "Since we want to work within the ``context`` of the Training Project:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "To begin with, you will be working within your own private collection whose ``alias`` is the same as your DataFed username.\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Enter your username into the ``parent_collection`` variable </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parent_collection = ?"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Creating Data Records:\n",
-    "Data Records can hold a whole lot of contextual information about the raw data in them. One key component is the scientific metadata. Ideally, we would get this metadata from the headers of the raw data file or some other log file that was generated along with the raw data. \n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> DataFed expects scientific metadata to be specified **like** a python dictionary. </span>\n",
-    "\n",
-    "For now, let's set up some dummy metadata:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parameters = {\n",
-    "              'a': 4,\n",
-    "              'b': [1, 2, -4, 7.123],\n",
-    "              'c': 'Something important',\n",
-    "              'd': {'x': 14, # Can use nested dictionaries\n",
-    "                    'y': -19\n",
-    "                   } \n",
-    "              }"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> DataFed **currently** encodes metadata in JSON strings. The next version will accept python dictonaries as is.</span>\n",
-    "\n",
-    "For now, we will need to convert the metadata to a JSON string using the ``dumps()`` function in the ``json`` package"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "json.dumps(parameters)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We use the `dataCreate()` function to make our new record, and the `json.dumps()` function to format the python dictionary to JSON:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dc_resp = df_api.dataCreate('my important data',\n",
-    "                            metadata=json.dumps(parameters),\n",
-    "                            parent_id=parent_collection, # The parent collection, whose alias is your username\n",
-    "                            )\n",
-    "dc_resp"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### <span style=\"color:blue\"> **Note:** </span> \n",
-    "> <span style=\"color:blue\">  In the future, the ``dataCreate()`` function would by default return only the ``ID`` of the record instead of such a verbose response if it successfully created the Data Record. We expect to be able to continue to get this verbose response through an optional argument. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\">  Exercise: </span>\n",
-    "<span style=\"color:green\"> Extract the ``ID`` of the data record from the message returned from ``dataCreate()`` for future use: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "record_id = ?\n",
-    "print(record_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Data Records and the information in them are not static and can always be modified at any time\n",
-    "## Updating Data Records\n",
-    "Let's add some additional metadata and change the title of our record:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "du_resp = df_api.dataUpdate(record_id,\n",
-    "                            title='Some new title for the data',\n",
-    "                            metadata=json.dumps({'appended_metadata': True})\n",
-    "                            )\n",
-    "print(du_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### <span style=\"color:blue\"> **Note:** </span> \n",
-    "> <span style=\"color:blue\"> In the future, the dataUpdate() command would return only an acknowledgement of the successful execution of the data update. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Viewing Data Records\n",
-    "We can get full information about a data record  including the complete metadata via the ``dataView()`` function. Let us use this function to verify that the changes have been incorporated:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dv_resp = df_api.dataView(record_id)\n",
-    "print(dv_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### <span style=\"color:blue\"> **Note:** </span> \n",
-    "> <span style=\"color:blue\"> Record metadata is always stored as a JSON string. </span>\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Try isolating the updated metadata and converting it to a python dictionary. </span>\n",
-    "\n",
-    "Hint - ``json.loads()`` is the opposite of ``json.dumps()``"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In the first update, we **merged** new metadata with the existing metadata within the record. However ``dataUpdate()`` is also capable of replacing the metadata as well. \n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Now try to **replace** the metadata. <br><br>Hint: look at the `metadata_set` keyword argument in the docstrings. </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> Tip: </span>\n",
-    "> <span style=\"color:blue\"> With the cursor just past the starting parenthesis of ``dataUpdate(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "new_metadata = ?\n",
-    "du_resp = df_api.dataUpdate(record_id,\n",
-    "                            ...\n",
-    "                            )\n",
-    "dv_resp = df_api.dataView(record_id)\n",
-    "print(json.loads(dv_resp[0].data[0].metadata))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Provenance\n",
-    "Along with in-depth, detailed scientific metadata describing each data record, DataFed also provides a very handy tool for tracking data provenance, i.e. recording the relationships between Data Records which can be used to track the history, lineage, and origins of a data object. "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Create a new record meant to hold some processed version of the first data record. <br> **Caution**: Make sure to create it in the correct Collection.</span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "new_params = {}\n",
-    "\n",
-    "dc2_resp = df_api.dataCreate( ...\n",
-    "                             )\n",
-    "\n",
-    "clean_rec_id = dc2_resp[0].data[0].id\n",
-    "print(clean_rec_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Specifying Relationships\n",
-    "Now that we have two records, we can specify the second record's relationship to the first by adding a **dependency** via the ``deps_add`` keyword argument of the ``dataUpdate()`` function.\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note: </span>\n",
-    "> <span style=\"color:blue\"> As the documentation for ``dataUpdate()`` will reveal, dependencies must be specified as a ``list`` of relationships. Each relationship is expressed as a ``list`` where the first item is a dependency type (a string) and the second is the data record (also a string). </span>\n",
-    "\n",
-    "DataFed currently supports three relationship types:\n",
-    "* `der` - Is derived from\n",
-    "* `comp` - Is comprised of\n",
-    "* `ver` - Is new version of"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dep_resp = df_api.dataUpdate(clean_rec_id, \n",
-    "                             deps_add=[[\"der\", record_id]])\n",
-    "print(dep_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Take a look at the records on the DataFed Web Portal in order to see a graphical representation of the data provenance. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\">1. Create a new data record to hold a figure in your journal article. <br>2. Extract the record ID. <br>3. Now establish a provenance link between this figure record and the processed data record we just created. You may try out a different dependency type if you like. <br>4. Take a look at the DataFed web portal to see the update to the Provenance of the records</span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python [default]",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# 2. Data Records\n",
+                "In this notebook, we will be going over how to create, add metadata and other contextual information, edit, establish relationships between ``Data Records`` - the fundamental unit in DataFed\n",
+                "\n",
+                "## Setting up:\n",
+                "Just as in the previous notebook, we start be instantiating the ``datafed.CommandLib.API`` class to communicate with DataFed:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["from datafed.CommandLib import API"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Specifying context:\n",
+                "Since we want to work within the ``context`` of the Training Project:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "To begin with, you will be working within your own private collection whose ``alias`` is the same as your DataFed username.\n",
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Enter your username into the ``parent_collection`` variable </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["parent_collection = ?"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Creating Data Records:\n",
+                "Data Records can hold a whole lot of contextual information about the raw data in them. One key component is the scientific metadata. Ideally, we would get this metadata from the headers of the raw data file or some other log file that was generated along with the raw data. \n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> DataFed expects scientific metadata to be specified **like** a python dictionary. </span>\n',
+                "\n",
+                "For now, let's set up some dummy metadata:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "parameters = {\n",
+                "              'a': 4,\n",
+                "              'b': [1, 2, -4, 7.123],\n",
+                "              'c': 'Something important',\n",
+                "              'd': {'x': 14, # Can use nested dictionaries\n",
+                "                    'y': -19\n",
+                "                   } \n",
+                "              }",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> DataFed **currently** encodes metadata in JSON strings. The next version will accept python dictonaries as is.</span>\n',
+                "\n",
+                "For now, we will need to convert the metadata to a JSON string using the ``dumps()`` function in the ``json`` package",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["import json\n", "json.dumps(parameters)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We use the `dataCreate()` function to make our new record, and the `json.dumps()` function to format the python dictionary to JSON:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dc_resp = df_api.dataCreate('my important data',\n",
+                "                            metadata=json.dumps(parameters),\n",
+                "                            parent_id=parent_collection, # The parent collection, whose alias is your username\n",
+                "                            )\n",
+                "dc_resp",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '#### <span style="color:blue"> **Note:** </span> \n',
+                '> <span style="color:blue">  In the future, the ``dataCreate()`` function would by default return only the ``ID`` of the record instead of such a verbose response if it successfully created the Data Record. We expect to be able to continue to get this verbose response through an optional argument. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green">  Exercise: </span>\n',
+                '<span style="color:green"> Extract the ``ID`` of the data record from the message returned from ``dataCreate()`` for future use: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["record_id = ?\n", "print(record_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Data Records and the information in them are not static and can always be modified at any time\n",
+                "## Updating Data Records\n",
+                "Let's add some additional metadata and change the title of our record:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "du_resp = df_api.dataUpdate(record_id,\n",
+                "                            title='Some new title for the data',\n",
+                "                            metadata=json.dumps({'appended_metadata': True})\n",
+                "                            )\n",
+                "print(du_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '#### <span style="color:blue"> **Note:** </span> \n',
+                '> <span style="color:blue"> In the future, the dataUpdate() command would return only an acknowledgement of the successful execution of the data update. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Viewing Data Records\n",
+                "We can get full information about a data record  including the complete metadata via the ``dataView()`` function. Let us use this function to verify that the changes have been incorporated:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["dv_resp = df_api.dataView(record_id)\n", "print(dv_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '#### <span style="color:blue"> **Note:** </span> \n',
+                '> <span style="color:blue"> Record metadata is always stored as a JSON string. </span>\n',
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Try isolating the updated metadata and converting it to a python dictionary. </span>\n',
+                "\n",
+                "Hint - ``json.loads()`` is the opposite of ``json.dumps()``",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In the first update, we **merged** new metadata with the existing metadata within the record. However ``dataUpdate()`` is also capable of replacing the metadata as well. \n",
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Now try to **replace** the metadata. <br><br>Hint: look at the `metadata_set` keyword argument in the docstrings. </span>\n',
+                "\n",
+                '### <span style="color:blue"> Tip: </span>\n',
+                '> <span style="color:blue"> With the cursor just past the starting parenthesis of ``dataUpdate(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "new_metadata = ?\n",
+                "du_resp = df_api.dataUpdate(record_id,\n",
+                "                            ...\n",
+                "                            )\n",
+                "dv_resp = df_api.dataView(record_id)\n",
+                "print(json.loads(dv_resp[0].data[0].metadata))",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Provenance\n",
+                "Along with in-depth, detailed scientific metadata describing each data record, DataFed also provides a very handy tool for tracking data provenance, i.e. recording the relationships between Data Records which can be used to track the history, lineage, and origins of a data object. ",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Create a new record meant to hold some processed version of the first data record. <br> **Caution**: Make sure to create it in the correct Collection.</span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "new_params = {}\n",
+                "\n",
+                "dc2_resp = df_api.dataCreate( ...\n",
+                "                             )\n",
+                "\n",
+                "clean_rec_id = dc2_resp[0].data[0].id\n",
+                "print(clean_rec_id)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Specifying Relationships\n",
+                "Now that we have two records, we can specify the second record's relationship to the first by adding a **dependency** via the ``deps_add`` keyword argument of the ``dataUpdate()`` function.\n",
+                "\n",
+                '### <span style="color:blue"> Note: </span>\n',
+                '> <span style="color:blue"> As the documentation for ``dataUpdate()`` will reveal, dependencies must be specified as a ``list`` of relationships. Each relationship is expressed as a ``list`` where the first item is a dependency type (a string) and the second is the data record (also a string). </span>\n',
+                "\n",
+                "DataFed currently supports three relationship types:\n",
+                "* `der` - Is derived from\n",
+                "* `comp` - Is comprised of\n",
+                "* `ver` - Is new version of",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dep_resp = df_api.dataUpdate(clean_rec_id, \n",
+                '                             deps_add=[["der", record_id]])\n',
+                "print(dep_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Take a look at the records on the DataFed Web Portal in order to see a graphical representation of the data provenance. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green">1. Create a new data record to hold a figure in your journal article. <br>2. Extract the record ID. <br>3. Now establish a provenance link between this figure record and the processed data record we just created. You may try out a different dependency type if you like. <br>4. Take a look at the DataFed web portal to see the update to the Provenance of the records</span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python [default]",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.5.5",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/2_Data_Records_with_Solutions.ipynb b/jupyter_notebooks/2_Data_Records_with_Solutions.ipynb
index bfdadea37..501df70bf 100644
--- a/jupyter_notebooks/2_Data_Records_with_Solutions.ipynb
+++ b/jupyter_notebooks/2_Data_Records_with_Solutions.ipynb
@@ -1,374 +1,354 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 2. Data Records\n",
-    "In this notebook, we will be going over how to create, add metadata and other contextual information, edit, establish relationships between ``Data Records`` - the fundamental unit in DataFed\n",
-    "\n",
-    "## Setting up:\n",
-    "Just as in the previous notebook, we start be instantiating the ``datafed.CommandLib.API`` class to communicate with DataFed:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Specifying context:\n",
-    "Since we want to work within the ``context`` of the Training Project:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "To begin with, you will be working within your own private collection whose ``alias`` is the same as your DataFed username.\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Enter your username into the ``parent_collection`` variable </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parent_collection = ?"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Creating Data Records:\n",
-    "Data Records can hold a whole lot of contextual information about the raw data in them. One key component is the scientific metadata. Ideally, we would get this metadata from the headers of the raw data file or some other log file that was generated along with the raw data. \n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> DataFed expects scientific metadata to be specified **like** a python dictionary. </span>\n",
-    "\n",
-    "For now, let's set up some dummy metadata:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parameters = {\n",
-    "              'a': 4,\n",
-    "              'b': [1, 2, -4, 7.123],\n",
-    "              'c': 'Something important',\n",
-    "              'd': {'x': 14, # Can use nested dictionaries\n",
-    "                    'y': -19\n",
-    "                   } \n",
-    "              }"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> DataFed **currently** encodes metadata in JSON strings. The next version will accept python dictonaries as is.</span>\n",
-    "\n",
-    "For now, we will need to convert the metadata to a JSON string using the ``dumps()`` function in the ``json`` package"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "json.dumps(parameters)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We use the `dataCreate()` function to make our new record, and the `json.dumps()` function to format the python dictionary to JSON:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dc_resp = df_api.dataCreate('my important data',\n",
-    "                            metadata=json.dumps(parameters),\n",
-    "                            parent_id=parent_collection, # The parent collection, whose alias is your username\n",
-    "                            )\n",
-    "dc_resp"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### <span style=\"color:blue\"> **Note:** </span> \n",
-    "> <span style=\"color:blue\">  In the future, the ``dataCreate()`` function would by default return only the ``ID`` of the record instead of such a verbose response if it successfully created the Data Record. We expect to be able to continue to get this verbose response through an optional argument. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\">  Exercise: </span>\n",
-    "<span style=\"color:green\"> Extract the ``ID`` of the data record from the message returned from ``dataCreate()`` for future use: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "record_id = dc_resp[0].data[0].id\n",
-    "print(record_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Data Records and the information in them are not static and can always be modified at any time\n",
-    "## Updating Data Records\n",
-    "Let's add some additional metadata and change the title of our record:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "du_resp = df_api.dataUpdate(record_id,\n",
-    "                            title='Some new title for the data',\n",
-    "                            metadata=json.dumps({'appended_metadata': True})\n",
-    "                            )\n",
-    "print(du_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### <span style=\"color:blue\"> **Note:** </span> \n",
-    "> <span style=\"color:blue\"> In the future, the dataUpdate() command would return only an acknowledgement of the successful execution of the data update. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Viewing Data Records\n",
-    "We can get full information about a data record  including the complete metadata via the ``dataView()`` function. Let us use this function to verify that the changes have been incorporated:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dv_resp = df_api.dataView(record_id)\n",
-    "print(dv_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### <span style=\"color:blue\"> **Note:** </span> \n",
-    "> <span style=\"color:blue\"> Record metadata is always stored as a JSON string. </span>\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Try isolating the updated metadata and converting it to a python dictionary. </span>\n",
-    "\n",
-    "Hint - ``json.loads()`` is the opposite of ``json.dumps()``"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metadata = json.loads(dv_resp[0].data[0].metadata)\n",
-    "print(metadata)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In the first update, we **merged** new metadata with the existing metadata within the record. However ``dataUpdate()`` is also capable of replacing the metadata as well. \n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Now try to **replace** the metadata. <br><br>Hint: look at the `metadata_set` keyword argument in the docstrings. </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> Tip: </span>\n",
-    "> <span style=\"color:blue\"> With the cursor just past the starting parenthesis of ``dataUpdate(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "new_metadata = {\"key\": \"value\", \"E\": 'mc^2'}\n",
-    "du_resp = df_api.dataUpdate(record_id,\n",
-    "                            metadata = json.dumps(new_metadata),\n",
-    "                            metadata_set=True)\n",
-    "dv_resp = df_api.dataView(record_id)\n",
-    "print(json.loads(dv_resp[0].data[0].metadata))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Provenance\n",
-    "Along with in-depth, detailed scientific metadata describing each data record, DataFed also provides a very handy tool for tracking data provenance, i.e. recording the relationships between Data Records which can be used to track the history, lineage, and origins of a data object. "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Create a new record meant to hold some processed version of the first data record. <br> **Caution**: Make sure to create it in the correct Collection.</span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "new_params = {\"hello\": \"world\", \"counting\": [1, 2, 3, 4]}\n",
-    "\n",
-    "dc2_resp = df_api.dataCreate( \"Subsequent Record\",\n",
-    "                             metadata=json.dumps(new_params),\n",
-    "                             parent_id=parent_collection)\n",
-    "\n",
-    "clean_rec_id = dc2_resp[0].data[0].id\n",
-    "print(clean_rec_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Specifying Relationships\n",
-    "Now that we have two records, we can specify the second record's relationship to the first by adding a **dependency** via the ``deps_add`` keyword argument of the ``dataUpdate()`` function.\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note: </span>\n",
-    "> <span style=\"color:blue\"> As the documentation for ``dataUpdate()`` will reveal, dependencies must be specified as a ``list`` of relationships. Each relationship is expressed as a ``list`` where the first item is a dependency type (a string) and the second is the data record (also a string). </span>\n",
-    "\n",
-    "DataFed currently supports three relationship types:\n",
-    "* `der` - Is derived from\n",
-    "* `comp` - Is comprised of\n",
-    "* `ver` - Is new version of"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dep_resp = df_api.dataUpdate(clean_rec_id, \n",
-    "                             deps_add=[[\"der\", record_id]])\n",
-    "print(dep_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\"> Take a look at the records on the DataFed Web Portal in order to see a graphical representation of the data provenance. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise: </span>\n",
-    "<span style=\"color:green\">1. Create a new data record to hold a figure in your journal article. <br>2. Extract the record ID. <br>3. Now establish a provenance link between this figure record and the processed data record we just created. You may try out a different dependency type if you like. <br>4. Take a look at the DataFed web portal to see the update to the Provenance of the records</span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 1\n",
-    "reply = df_api.dataCreate('Figure 1', parent_id=parent_collection)\n",
-    "# 2 \n",
-    "fig_id = reply[0].data[0].id\n",
-    "# 3 \n",
-    "provenance_link = df_api.dataUpdate(fig_id, deps_add=[('comp', clean_rec_id)])\n",
-    "print(provenance_link[0].data[0])"
-   ]
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.9"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# 2. Data Records\n",
+                "In this notebook, we will be going over how to create, add metadata and other contextual information, edit, establish relationships between ``Data Records`` - the fundamental unit in DataFed\n",
+                "\n",
+                "## Setting up:\n",
+                "Just as in the previous notebook, we start be instantiating the ``datafed.CommandLib.API`` class to communicate with DataFed:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["from datafed.CommandLib import API"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Specifying context:\n",
+                "Since we want to work within the ``context`` of the Training Project:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "To begin with, you will be working within your own private collection whose ``alias`` is the same as your DataFed username.\n",
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Enter your username into the ``parent_collection`` variable </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["parent_collection = ?"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Creating Data Records:\n",
+                "Data Records can hold a whole lot of contextual information about the raw data in them. One key component is the scientific metadata. Ideally, we would get this metadata from the headers of the raw data file or some other log file that was generated along with the raw data. \n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> DataFed expects scientific metadata to be specified **like** a python dictionary. </span>\n',
+                "\n",
+                "For now, let's set up some dummy metadata:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "parameters = {\n",
+                "              'a': 4,\n",
+                "              'b': [1, 2, -4, 7.123],\n",
+                "              'c': 'Something important',\n",
+                "              'd': {'x': 14, # Can use nested dictionaries\n",
+                "                    'y': -19\n",
+                "                   } \n",
+                "              }",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> DataFed **currently** encodes metadata in JSON strings. The next version will accept python dictonaries as is.</span>\n',
+                "\n",
+                "For now, we will need to convert the metadata to a JSON string using the ``dumps()`` function in the ``json`` package",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["import json\n", "json.dumps(parameters)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We use the `dataCreate()` function to make our new record, and the `json.dumps()` function to format the python dictionary to JSON:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dc_resp = df_api.dataCreate('my important data',\n",
+                "                            metadata=json.dumps(parameters),\n",
+                "                            parent_id=parent_collection, # The parent collection, whose alias is your username\n",
+                "                            )\n",
+                "dc_resp",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '#### <span style="color:blue"> **Note:** </span> \n',
+                '> <span style="color:blue">  In the future, the ``dataCreate()`` function would by default return only the ``ID`` of the record instead of such a verbose response if it successfully created the Data Record. We expect to be able to continue to get this verbose response through an optional argument. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green">  Exercise: </span>\n',
+                '<span style="color:green"> Extract the ``ID`` of the data record from the message returned from ``dataCreate()`` for future use: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["record_id = dc_resp[0].data[0].id\n", "print(record_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Data Records and the information in them are not static and can always be modified at any time\n",
+                "## Updating Data Records\n",
+                "Let's add some additional metadata and change the title of our record:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "du_resp = df_api.dataUpdate(record_id,\n",
+                "                            title='Some new title for the data',\n",
+                "                            metadata=json.dumps({'appended_metadata': True})\n",
+                "                            )\n",
+                "print(du_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '#### <span style="color:blue"> **Note:** </span> \n',
+                '> <span style="color:blue"> In the future, the dataUpdate() command would return only an acknowledgement of the successful execution of the data update. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Viewing Data Records\n",
+                "We can get full information about a data record  including the complete metadata via the ``dataView()`` function. Let us use this function to verify that the changes have been incorporated:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["dv_resp = df_api.dataView(record_id)\n", "print(dv_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '#### <span style="color:blue"> **Note:** </span> \n',
+                '> <span style="color:blue"> Record metadata is always stored as a JSON string. </span>\n',
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Try isolating the updated metadata and converting it to a python dictionary. </span>\n',
+                "\n",
+                "Hint - ``json.loads()`` is the opposite of ``json.dumps()``",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "metadata = json.loads(dv_resp[0].data[0].metadata)\n",
+                "print(metadata)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In the first update, we **merged** new metadata with the existing metadata within the record. However ``dataUpdate()`` is also capable of replacing the metadata as well. \n",
+                "\n",
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Now try to **replace** the metadata. <br><br>Hint: look at the `metadata_set` keyword argument in the docstrings. </span>\n',
+                "\n",
+                '### <span style="color:blue"> Tip: </span>\n',
+                '> <span style="color:blue"> With the cursor just past the starting parenthesis of ``dataUpdate(``, simultaneously press the ``Shift`` and ``Tab`` keys once, twice, or four times to view more of the documentation about the function. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                'new_metadata = {"key": "value", "E": \'mc^2\'}\n',
+                "du_resp = df_api.dataUpdate(record_id,\n",
+                "                            metadata = json.dumps(new_metadata),\n",
+                "                            metadata_set=True)\n",
+                "dv_resp = df_api.dataView(record_id)\n",
+                "print(json.loads(dv_resp[0].data[0].metadata))",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Provenance\n",
+                "Along with in-depth, detailed scientific metadata describing each data record, DataFed also provides a very handy tool for tracking data provenance, i.e. recording the relationships between Data Records which can be used to track the history, lineage, and origins of a data object. ",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Create a new record meant to hold some processed version of the first data record. <br> **Caution**: Make sure to create it in the correct Collection.</span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                'new_params = {"hello": "world", "counting": [1, 2, 3, 4]}\n',
+                "\n",
+                'dc2_resp = df_api.dataCreate( "Subsequent Record",\n',
+                "                             metadata=json.dumps(new_params),\n",
+                "                             parent_id=parent_collection)\n",
+                "\n",
+                "clean_rec_id = dc2_resp[0].data[0].id\n",
+                "print(clean_rec_id)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Specifying Relationships\n",
+                "Now that we have two records, we can specify the second record's relationship to the first by adding a **dependency** via the ``deps_add`` keyword argument of the ``dataUpdate()`` function.\n",
+                "\n",
+                '### <span style="color:blue"> Note: </span>\n',
+                '> <span style="color:blue"> As the documentation for ``dataUpdate()`` will reveal, dependencies must be specified as a ``list`` of relationships. Each relationship is expressed as a ``list`` where the first item is a dependency type (a string) and the second is the data record (also a string). </span>\n',
+                "\n",
+                "DataFed currently supports three relationship types:\n",
+                "* `der` - Is derived from\n",
+                "* `comp` - Is comprised of\n",
+                "* `ver` - Is new version of",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dep_resp = df_api.dataUpdate(clean_rec_id, \n",
+                '                             deps_add=[["der", record_id]])\n',
+                "print(dep_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green"> Take a look at the records on the DataFed Web Portal in order to see a graphical representation of the data provenance. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise: </span>\n',
+                '<span style="color:green">1. Create a new data record to hold a figure in your journal article. <br>2. Extract the record ID. <br>3. Now establish a provenance link between this figure record and the processed data record we just created. You may try out a different dependency type if you like. <br>4. Take a look at the DataFed web portal to see the update to the Provenance of the records</span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "# 1\n",
+                "reply = df_api.dataCreate('Figure 1', parent_id=parent_collection)\n",
+                "# 2 \n",
+                "fig_id = reply[0].data[0].id\n",
+                "# 3 \n",
+                "provenance_link = df_api.dataUpdate(fig_id, deps_add=[('comp', clean_rec_id)])\n",
+                "print(provenance_link[0].data[0])",
+            ],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.7.9",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/3_Data_Transfer.ipynb b/jupyter_notebooks/3_Data_Transfer.ipynb
index d3a03ebe6..bacc83551 100644
--- a/jupyter_notebooks/3_Data_Transfer.ipynb
+++ b/jupyter_notebooks/3_Data_Transfer.ipynb
@@ -1,515 +1,488 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 3. Transferring Data in DataFed\n",
-    "In this notebook, we will be going over uploading to & downloading from Data Records and monitoring data transfer tasks.\n",
-    "\n",
-    "## Before we begin:\n",
-    "Import necessary libraries"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "import time\n",
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Instantiate the DataFed API and set ``context`` to the Training project:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()\n",
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Enter your username to work within your personal Collection. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parent_collection = # your username here"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Copy over the ID for the record you created in the previous notebook. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "record_id = ?"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Uploading raw data\n",
-    "Shortly, we will learn how to upload data to the record we just created. For demonstration purposes, we will just create a simple text file and use this as the raw data for the Data Record\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> DataFed does not impose any restrictions on the file extension / format for the raw data </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with open('./raw_data.txt', mode='w') as file_handle:\n",
-    "    file_handle.write('This is some data')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Always ensure that your Globus endpoint is active and that your files are located in a directory that is visible to the Globus Endpoint </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## `dataPut()`\n",
-    "Uploading data files to DataFed is done using the `dataPut` command."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "put_resp = df_api.dataPut(record_id,\n",
-    "                          './raw_data.txt',\n",
-    "                          wait=True, # Waits until transfer completes.\n",
-    "                          )\n",
-    "print(put_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We get two components in the response:\n",
-    "* Information about the Data Record, data was uploaded to\n",
-    "* Information about the data transfer ``task`` - more on this later\n",
-    "\n",
-    "The ``dataPut()`` method **initiates a Globus transfer** on our behalf from the machine **wherever** the file was present to wherever the default data repository is located. In this case, the file was in our local file system and on the same machine where we are executing the command.\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> The above data file was specified by its relative local path, so DataFed used our pre-configured default Globus endpoint to find the data file. As long as we have the id for any *active* Globus endpoint that we have authenticated access to, we can transfer data from that endpoint with its full absolute file path – even if the file system is not attached ot the local machine. Look for more information on this in later examples. </span>\n",
-    "\n",
-    "Let's view the data record now that we've uploaded our data. Pay attention to the ``ext`` and ``source`` fields which should now populated:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dv_resp = df_api.dataView(record_id)\n",
-    "print(dv_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Downloading raw data\n",
-    "DataFed is also capable of getting data stored in a DataFed repository and placing it in the local or other Globus-visible filesystem via the ``dataGet()`` function. \n",
-    "\n",
-    "Let us download the content in the data record we have been working on so far for demonstration purposes\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> The current version of DataFed has a bug where ``dataGet()`` only accepts a list of Data Record or Collection IDs. Until the next version, users are recommended to put their singular ID into a list for ``dataGet().`` </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "get_resp = df_api.dataGet([record_id], # currently only accepts a list of IDs / aliases\n",
-    "                          '.', # directory where data should be downloaded\n",
-    "                          orig_fname=False, # do not name file by its original name\n",
-    "                          wait=True, # Wait until Globus transfer completes\n",
-    "                         )\n",
-    "print(get_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In the response we only get back information about the data transfer ``task`` - more on this shortly\n",
-    "\n",
-    "The bug in ``dataGet()`` also reveals its capability to **download multiple data records or even Collections.**\n",
-    "\n",
-    "Let's confirm that the data has been downloaded successfully:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "os.listdir('.')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "expected_file_name = os.path.join('.', record_id.split('d/')[-1]) + '.json'\n",
-    "print('Does a file with this name: ' + expected_file_name + ' exist?')\n",
-    "print(os.path.exists(expected_file_name))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Tasks\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> A DataFed task may itself contain / be responsible for several Globus file transfers, potentially from / to multiple locations. </span>\n",
-    "    \n",
-    "DataFed makes it possible to check on the status of transfer tasks in an easy and programmatic manner.\n",
-    "\n",
-    "Before we learn more about tasks, first lets try to get the ``id`` of the task in ``get_resp`` from the recent ``dataGet()`` function call:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "task_id = get_resp[0].task[0].id\n",
-    "print(task_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Viewing Tasks\n",
-    "We can get more information about a given transfer via the `taskView()` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "task_resp = df_api.taskView(task_id)\n",
-    "print(task_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We get a new kind of message - a ``TaskDataReply``. \n",
-    "Key fields to keep an eye on:\n",
-    "* ``status``\n",
-    "* ``msg``\n",
-    "* ``source``\n",
-    "* ``dest``\n",
-    "\n",
-    "If we are interested in monitoring tasks, triggering activities or subsequent steps of workflows based on transfers, we would need to know how to get the ``status`` property from the ``TaskDataReply``: "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "task_resp[0].task[0].status"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Even though the message above says `TS_SUCCEEDED`, we see that this task status codes to the integer `3`.\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> Cheat sheet for interpreting task statuses: <br> ``2``: in progress <br> ``3``: complete <br> anything else - problem  </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> A future version of DataFed may change the nature of the output / type for the status property. In general, the exact return object types and nomenclature may evolve with DataFed. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Listing Tasks\n",
-    "We can request a listing of all our recently initiated tasks:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.taskList()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The output of this listing would be very helpful for the Exercise below"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Example scenario - Simulations\n",
-    "Let's say that we want to run a series of simulations where one or more parameters are varied and each simulation is run with a unique set of parameters. Let's also assume that our eventual goal is to build a surrogate model for the computationally expensive simulation using machine learning. So, we **want to capture the metadata and data associated with the series of simulations** to train the machine learning model later on. \n",
-    "\n",
-    "We have set up skeleton functions and code snippets to help you mimic the data management for such a simulation. We would like you to take what you have learnt so far and fill in the blanks\n",
-    "<br>\n",
-    "\n",
-    "\n",
-    "### Fake simulation\n",
-    "Here, we have simulated a computationally \"expensive\" simulation that simply sleeps for a few seconds."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def expensive_simulation():\n",
-    "    time.sleep(5)\n",
-    "    # Yes, this simulation is deterministic and always results in the same result:\n",
-    "    path_to_results = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
-    "    # The simulation uses the same combination of parameters\n",
-    "    metadata = {'a': 1, 'b': 2, 'c': 3.14}\n",
-    "    return path_to_results, metadata"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Define a function that: <br> 1. creates a new Data Record with the provided metadata (as a dictionary) and other details, <br> 2. extracts the record id, <br> 3. puts the raw data into the record, <br> 4. extracts and returns the task ID. <br><br> Feel free to print any messages that may help you track things. </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Pay attention to the ``wait`` keyword argument when putting the raw data into record </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def capture_data(simulation_index, # integer - counter to signify the Nth simulation in the series\n",
-    "                 metadata, # dictionary - combination of parameters used for this simulation\n",
-    "                 raw_data_path, # string - Path to the raw data file that needs to be put into the receord\n",
-    "                 parent_collection=parent_collection, # string - Collection to create this Data Record into\n",
-    "                ):\n",
-    "    # 1. Create a new Data Record with the metadata and use the simulation index to provide a unique title\n",
-    "\n",
-    "    # 2. Extract the record ID from the response\n",
-    "    \n",
-    "    # 3. Put the raw data into this record:\n",
-    "    \n",
-    "    # 4. Extract the ID for the data transfer task\n",
-    "\n",
-    "    # 5. Return the task ID"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Try out this function to make sure it works. See what it does on the **DataFed web portal**. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "capture_data(14, {'param_1': 456, 'param_2': 'Hello', 'param_3': [1, -2.34, 54]})"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> We will want a simple function to monitor the status of all the data upload tasks. Define a function that accepts a list of task IDs and returns their status after looking them up on DataFed </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def check_xfer_status(task_ids):\n",
-    "    # put singular task ID into a list\n",
-    "    if isinstance(task_ids, str):\n",
-    "        task_ids = [task_ids]\n",
-    "    \n",
-    "    # Create a list to hold the statuses of each of the tasks\n",
-    "        \n",
-    "    # Iterate over each of the task IDs\n",
-    "    \n",
-    "        # For each task ID, get detailed information about it\n",
-    "    \n",
-    "        # Extract the task status from the detailed information\n",
-    "    \n",
-    "        # Append this status to the list of statuses\n",
-    "        \n",
-    "    # Return the list of statuses"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Try out your function using the IDs of the recent ``dataPut()`` and ``dataGet()`` functions. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "check_xfer_status(...)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Run the series of simulations:\n",
-    "Use the three functions defined above to mimic the process of exploring a parameter space using simulations, where for each iteration, we: <br> 1. run a simulation, <br> 2. capture the data + metadata into DataFed, <br> 3. monitor the data upload tasks."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "xfer_tasks = list()\n",
-    "for ind in range(3):\n",
-    "    print('Starting simulation #{}'.format(ind))\n",
-    "    # Run the simulation.\n",
-    "    path_to_results, metadata = expensive_simulation()\n",
-    "    # Capture the data and metadata into DataFed\n",
-    "    task_id = capture_data(ind, metadata, path_to_results)\n",
-    "    # Append the task ID for this data upload into xfer_tasks\n",
-    "    xfer_tasks.append(task_id)\n",
-    "    # Print out the status of the data transfers\n",
-    "    print('Transfer status(es): {}'.format(check_xfer_status(xfer_tasks)))\n",
-    "    print('')\n",
-    "\n",
-    "print('Simulations complete')\n",
-    "time.wait(10)\n",
-    "print('Transfer status(es): {}'.format(check_xfer_status(xfer_tasks)))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> What happens if you set the ``wait`` in ``dataPut()`` to ``True``? </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> Users are recommended to perform data orchestration (especially large data movement - upload / download) operations outside the scope of heavy / parallel computation operations in order to avoid wasting precious wall time on compute clusters.</span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python [default]",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# 3. Transferring Data in DataFed\n",
+                "In this notebook, we will be going over uploading to & downloading from Data Records and monitoring data transfer tasks.\n",
+                "\n",
+                "## Before we begin:\n",
+                "Import necessary libraries",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import json\n",
+                "import os\n",
+                "import time\n",
+                "from datafed.CommandLib import API",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Instantiate the DataFed API and set ``context`` to the Training project:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()\n", "df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Enter your username to work within your personal Collection. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["parent_collection = # your username here"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Copy over the ID for the record you created in the previous notebook. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["record_id = ?"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Uploading raw data\n",
+                "Shortly, we will learn how to upload data to the record we just created. For demonstration purposes, we will just create a simple text file and use this as the raw data for the Data Record\n",
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> DataFed does not impose any restrictions on the file extension / format for the raw data </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "with open('./raw_data.txt', mode='w') as file_handle:\n",
+                "    file_handle.write('This is some data')",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Always ensure that your Globus endpoint is active and that your files are located in a directory that is visible to the Globus Endpoint </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## `dataPut()`\n",
+                "Uploading data files to DataFed is done using the `dataPut` command.",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "put_resp = df_api.dataPut(record_id,\n",
+                "                          './raw_data.txt',\n",
+                "                          wait=True, # Waits until transfer completes.\n",
+                "                          )\n",
+                "print(put_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We get two components in the response:\n",
+                "* Information about the Data Record, data was uploaded to\n",
+                "* Information about the data transfer ``task`` - more on this later\n",
+                "\n",
+                "The ``dataPut()`` method **initiates a Globus transfer** on our behalf from the machine **wherever** the file was present to wherever the default data repository is located. In this case, the file was in our local file system and on the same machine where we are executing the command.\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> The above data file was specified by its relative local path, so DataFed used our pre-configured default Globus endpoint to find the data file. As long as we have the id for any *active* Globus endpoint that we have authenticated access to, we can transfer data from that endpoint with its full absolute file path – even if the file system is not attached ot the local machine. Look for more information on this in later examples. </span>\n',
+                "\n",
+                "Let's view the data record now that we've uploaded our data. Pay attention to the ``ext`` and ``source`` fields which should now populated:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["dv_resp = df_api.dataView(record_id)\n", "print(dv_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Downloading raw data\n",
+                "DataFed is also capable of getting data stored in a DataFed repository and placing it in the local or other Globus-visible filesystem via the ``dataGet()`` function. \n",
+                "\n",
+                "Let us download the content in the data record we have been working on so far for demonstration purposes\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> The current version of DataFed has a bug where ``dataGet()`` only accepts a list of Data Record or Collection IDs. Until the next version, users are recommended to put their singular ID into a list for ``dataGet().`` </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "get_resp = df_api.dataGet([record_id], # currently only accepts a list of IDs / aliases\n",
+                "                          '.', # directory where data should be downloaded\n",
+                "                          orig_fname=False, # do not name file by its original name\n",
+                "                          wait=True, # Wait until Globus transfer completes\n",
+                "                         )\n",
+                "print(get_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In the response we only get back information about the data transfer ``task`` - more on this shortly\n",
+                "\n",
+                "The bug in ``dataGet()`` also reveals its capability to **download multiple data records or even Collections.**\n",
+                "\n",
+                "Let's confirm that the data has been downloaded successfully:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["os.listdir('.')"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "expected_file_name = os.path.join('.', record_id.split('d/')[-1]) + '.json'\n",
+                "print('Does a file with this name: ' + expected_file_name + ' exist?')\n",
+                "print(os.path.exists(expected_file_name))",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Tasks\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> A DataFed task may itself contain / be responsible for several Globus file transfers, potentially from / to multiple locations. </span>\n',
+                "    \n",
+                "DataFed makes it possible to check on the status of transfer tasks in an easy and programmatic manner.\n",
+                "\n",
+                "Before we learn more about tasks, first lets try to get the ``id`` of the task in ``get_resp`` from the recent ``dataGet()`` function call:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["task_id = get_resp[0].task[0].id\n", "print(task_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Viewing Tasks\n",
+                "We can get more information about a given transfer via the `taskView()` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["task_resp = df_api.taskView(task_id)\n", "print(task_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We get a new kind of message - a ``TaskDataReply``. \n",
+                "Key fields to keep an eye on:\n",
+                "* ``status``\n",
+                "* ``msg``\n",
+                "* ``source``\n",
+                "* ``dest``\n",
+                "\n",
+                "If we are interested in monitoring tasks, triggering activities or subsequent steps of workflows based on transfers, we would need to know how to get the ``status`` property from the ``TaskDataReply``: ",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["task_resp[0].task[0].status"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Even though the message above says `TS_SUCCEEDED`, we see that this task status codes to the integer `3`.\n",
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> Cheat sheet for interpreting task statuses: <br> ``2``: in progress <br> ``3``: complete <br> anything else - problem  </span>\n',
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> A future version of DataFed may change the nature of the output / type for the status property. In general, the exact return object types and nomenclature may evolve with DataFed. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Listing Tasks\n",
+                "We can request a listing of all our recently initiated tasks:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.taskList()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "The output of this listing would be very helpful for the Exercise below"
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Example scenario - Simulations\n",
+                "Let's say that we want to run a series of simulations where one or more parameters are varied and each simulation is run with a unique set of parameters. Let's also assume that our eventual goal is to build a surrogate model for the computationally expensive simulation using machine learning. So, we **want to capture the metadata and data associated with the series of simulations** to train the machine learning model later on. \n",
+                "\n",
+                "We have set up skeleton functions and code snippets to help you mimic the data management for such a simulation. We would like you to take what you have learnt so far and fill in the blanks\n",
+                "<br>\n",
+                "\n",
+                "\n",
+                "### Fake simulation\n",
+                'Here, we have simulated a computationally "expensive" simulation that simply sleeps for a few seconds.',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def expensive_simulation():\n",
+                "    time.sleep(5)\n",
+                "    # Yes, this simulation is deterministic and always results in the same result:\n",
+                "    path_to_results = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
+                "    # The simulation uses the same combination of parameters\n",
+                "    metadata = {'a': 1, 'b': 2, 'c': 3.14}\n",
+                "    return path_to_results, metadata",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Define a function that: <br> 1. creates a new Data Record with the provided metadata (as a dictionary) and other details, <br> 2. extracts the record id, <br> 3. puts the raw data into the record, <br> 4. extracts and returns the task ID. <br><br> Feel free to print any messages that may help you track things. </span>\n',
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Pay attention to the ``wait`` keyword argument when putting the raw data into record </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def capture_data(simulation_index, # integer - counter to signify the Nth simulation in the series\n",
+                "                 metadata, # dictionary - combination of parameters used for this simulation\n",
+                "                 raw_data_path, # string - Path to the raw data file that needs to be put into the receord\n",
+                "                 parent_collection=parent_collection, # string - Collection to create this Data Record into\n",
+                "                ):\n",
+                "    # 1. Create a new Data Record with the metadata and use the simulation index to provide a unique title\n",
+                "\n",
+                "    # 2. Extract the record ID from the response\n",
+                "    \n",
+                "    # 3. Put the raw data into this record:\n",
+                "    \n",
+                "    # 4. Extract the ID for the data transfer task\n",
+                "\n",
+                "    # 5. Return the task ID",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Try out this function to make sure it works. See what it does on the **DataFed web portal**. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "capture_data(14, {'param_1': 456, 'param_2': 'Hello', 'param_3': [1, -2.34, 54]})"
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> We will want a simple function to monitor the status of all the data upload tasks. Define a function that accepts a list of task IDs and returns their status after looking them up on DataFed </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def check_xfer_status(task_ids):\n",
+                "    # put singular task ID into a list\n",
+                "    if isinstance(task_ids, str):\n",
+                "        task_ids = [task_ids]\n",
+                "    \n",
+                "    # Create a list to hold the statuses of each of the tasks\n",
+                "        \n",
+                "    # Iterate over each of the task IDs\n",
+                "    \n",
+                "        # For each task ID, get detailed information about it\n",
+                "    \n",
+                "        # Extract the task status from the detailed information\n",
+                "    \n",
+                "        # Append this status to the list of statuses\n",
+                "        \n",
+                "    # Return the list of statuses",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Try out your function using the IDs of the recent ``dataPut()`` and ``dataGet()`` functions. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["check_xfer_status(...)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Run the series of simulations:\n",
+                "Use the three functions defined above to mimic the process of exploring a parameter space using simulations, where for each iteration, we: <br> 1. run a simulation, <br> 2. capture the data + metadata into DataFed, <br> 3. monitor the data upload tasks.",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "xfer_tasks = list()\n",
+                "for ind in range(3):\n",
+                "    print('Starting simulation #{}'.format(ind))\n",
+                "    # Run the simulation.\n",
+                "    path_to_results, metadata = expensive_simulation()\n",
+                "    # Capture the data and metadata into DataFed\n",
+                "    task_id = capture_data(ind, metadata, path_to_results)\n",
+                "    # Append the task ID for this data upload into xfer_tasks\n",
+                "    xfer_tasks.append(task_id)\n",
+                "    # Print out the status of the data transfers\n",
+                "    print('Transfer status(es): {}'.format(check_xfer_status(xfer_tasks)))\n",
+                "    print('')\n",
+                "\n",
+                "print('Simulations complete')\n",
+                "time.wait(10)\n",
+                "print('Transfer status(es): {}'.format(check_xfer_status(xfer_tasks)))",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> What happens if you set the ``wait`` in ``dataPut()`` to ``True``? </span>\n',
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> Users are recommended to perform data orchestration (especially large data movement - upload / download) operations outside the scope of heavy / parallel computation operations in order to avoid wasting precious wall time on compute clusters.</span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python [default]",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.5.5",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/3_Data_Transfer_with_Solutions.ipynb b/jupyter_notebooks/3_Data_Transfer_with_Solutions.ipynb
index 7393d52f3..54afa6e67 100755
--- a/jupyter_notebooks/3_Data_Transfer_with_Solutions.ipynb
+++ b/jupyter_notebooks/3_Data_Transfer_with_Solutions.ipynb
@@ -1,585 +1,548 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 3. Transferring Data in DataFed\n",
-    "In this notebook, we will be going over uploading to & downloading from Data Records and monitoring data transfer tasks.\n",
-    "\n",
-    "## Before we begin:\n",
-    "Import necessary libraries"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "import time\n",
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Instantiate the DataFed API and set ``context`` to the Training project:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()\n",
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Enter your username to work within your personal Collection. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parent_collection = \"c/34558900\" # your username here"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Copy over the ID for the record you created in the previous notebook. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "record_id = \"d/43650517\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Uploading raw data\n",
-    "Shortly, we will learn how to upload data to the record we just created. For demonstration purposes, we will just create a simple text file and use this as the raw data for the Data Record\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> DataFed does not impose any restrictions on the file extension / format for the raw data </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with open('./raw_data.txt', mode='w') as file_handle:\n",
-    "    file_handle.write('This is some data')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Always ensure that your Globus endpoint is active and that your files are located in a directory that is visible to the Globus Endpoint </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## `dataPut()`\n",
-    "Uploading data files to DataFed is done using the `dataPut` command."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "put_resp = df_api.dataPut(record_id,\n",
-    "                          './raw_data.txt',\n",
-    "                          wait=True, # Waits until transfer completes.\n",
-    "                          )\n",
-    "print(put_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We get two components in the response:\n",
-    "* Information about the Data Record, data was uploaded to\n",
-    "* Information about the data transfer ``task`` - more on this later\n",
-    "\n",
-    "The ``dataPut()`` method **initiates a Globus transfer** on our behalf from the machine **wherever** the file was present to wherever the default data repository is located. In this case, the file was in our local file system and on the same machine where we are executing the command.\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> The above data file was specified by its relative local path, so DataFed used our pre-configured default Globus endpoint to find the data file. As long as we have the id for any *active* Globus endpoint that we have authenticated access to, we can transfer data from that endpoint with its full absolute file path – even if the file system is not attached ot the local machine. Look for more information on this in later examples. </span>\n",
-    "\n",
-    "Let's view the data record now that we've uploaded our data. Pay attention to the ``ext`` and ``source`` fields which should now populated:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dv_resp = df_api.dataView(record_id)\n",
-    "print(dv_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Downloading raw data\n",
-    "DataFed is also capable of getting data stored in a DataFed repository and placing it in the local or other Globus-visible filesystem via the ``dataGet()`` function. \n",
-    "\n",
-    "Let us download the content in the data record we have been working on so far for demonstration purposes"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "get_resp = df_api.dataGet(record_id,\n",
-    "                          '.', # directory where data should be downloaded\n",
-    "                          orig_fname=False, # do not name file by its original name\n",
-    "                          wait=True, # Wait until Globus transfer completes\n",
-    "                         )\n",
-    "print(get_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In the response we only get back information about the data transfer ``task`` - more on this shortly\n",
-    "\n",
-    "The bug in ``dataGet()`` also reveals its capability to **download multiple data records or even Collections.**\n",
-    "\n",
-    "Let's confirm that the data has been downloaded successfully:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "os.listdir('.')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "expected_file_name = os.path.join('.', record_id.split('d/')[-1]) + '.txt'\n",
-    "print('Does a file with this name: ' + expected_file_name + ' exist?')\n",
-    "print(os.path.exists(expected_file_name))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Tasks\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> A DataFed task may itself contain / be responsible for several Globus file transfers, potentially from / to multiple locations. </span>\n",
-    "    \n",
-    "DataFed makes it possible to check on the status of transfer tasks in an easy and programmatic manner.\n",
-    "\n",
-    "Before we learn more about tasks, first lets try to get the ``id`` of the task in ``get_resp`` from the recent ``dataGet()`` function call:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "task_id = get_resp[0].task[0].id\n",
-    "print(task_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Viewing Tasks\n",
-    "We can get more information about a given transfer via the `taskView()` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "task_resp = df_api.taskView(task_id)\n",
-    "print(task_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We get a new kind of message - a ``TaskDataReply``. \n",
-    "Key fields to keep an eye on:\n",
-    "* ``status``\n",
-    "* ``msg``\n",
-    "* ``source``\n",
-    "* ``dest``\n",
-    "\n",
-    "If we are interested in monitoring tasks, triggering activities or subsequent steps of workflows based on transfers, we would need to know how to get the ``status`` property from the ``TaskDataReply``: "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "task_resp[0].task[0].status"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Even though the message above says `TS_SUCCEEDED`, we see that this task status codes to the integer `3`.\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> Cheat sheet for interpreting task statuses: <br> ``2``: in progress <br> ``3``: complete <br> anything else - problem  </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> A future version of DataFed may change the nature of the output / type for the status property. In general, the exact return object types and nomenclature may evolve with DataFed. </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Listing Tasks\n",
-    "We can request a listing of all our recently initiated tasks:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "df_api.taskList()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The output of this listing would be very helpful for the Exercise below"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Example scenario - Simulations\n",
-    "Let's say that we want to run a series of simulations where one or more parameters are varied and each simulation is run with a unique set of parameters. Let's also assume that our eventual goal is to build a surrogate model for the computationally expensive simulation using machine learning. So, we **want to capture the metadata and data associated with the series of simulations** to train the machine learning model later on. \n",
-    "\n",
-    "We have set up skeleton functions and code snippets to help you mimic the data management for such a simulation. We would like you to take what you have learnt so far and fill in the blanks\n",
-    "<br>\n",
-    "\n",
-    "\n",
-    "### Fake simulation\n",
-    "Here, we have simulated a computationally \"expensive\" simulation that simply sleeps for a few seconds."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def expensive_simulation():\n",
-    "    time.sleep(5)\n",
-    "    # Yes, this simulation is deterministic and always results in the same result:\n",
-    "    path_to_results = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
-    "    # The simulation uses the same combination of parameters\n",
-    "    metadata = {'a': 1, 'b': 2, 'c': 3.14}\n",
-    "    return path_to_results, metadata"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Define a function that: <br> 1. creates a new Data Record with the provided metadata (as a dictionary) and other details, <br> 2. extracts the record id, <br> 3. puts the raw data into the record, <br> 4. extracts and returns the task ID. <br><br> Feel free to print any messages that may help you track things. </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Pay attention to the ``wait`` keyword argument when putting the raw data into record </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def capture_data(simulation_index, # integer - counter to signify the Nth simulation in the series\n",
-    "                 metadata, # dictionary - combination of parameters used for this simulation\n",
-    "                 raw_data_path, # string - Path to the raw data file that needs to be put into the receord\n",
-    "                 parent_collection=parent_collection, # string - Collection to create this Data Record into\n",
-    "                ):\n",
-    "    \n",
-    "    # 1. Create a new Data Record with the metadata and use the simulation index to provide a unique title\n",
-    "    rec_resp = df_api.dataCreate('Simulation_' + str(simulation_index),\n",
-    "                                 metadata=json.dumps(metadata),\n",
-    "                                 parent_id=parent_collection)\n",
-    "    \n",
-    "    # 2. Extract the record ID from the response\n",
-    "    this_rec_id = rec_resp[0].data[0].id\n",
-    "    \n",
-    "    # 3. Put the raw data into this record:\n",
-    "    put_resp = df_api.dataPut(this_rec_id, raw_data_path, wait=False)\n",
-    "    \n",
-    "    # 4. Extract the ID for the data transfer task\n",
-    "    task_id = put_resp[0].task.id\n",
-    "    \n",
-    "    # 5. Return the task ID\n",
-    "    return task_id"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Try out this function to make sure it works. See what it does on the **DataFed web portal**. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'task/45433451'"
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "path_to_results, metadata = expensive_simulation()\n",
-    "\n",
-    "task_id = capture_data(14, metadata, path_to_results)\n",
-    "task_id"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> We will want a simple function to monitor the status of all the data upload tasks. Define a function that accepts a list of task IDs and returns their status after looking them up on DataFed </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def check_xfer_status(task_ids):\n",
-    "    \n",
-    "    # put singular task ID into a list\n",
-    "    if isinstance(task_ids, str):\n",
-    "        task_ids = [task_ids]\n",
-    "    \n",
-    "    # Create a list to hold the statuses of each of the tasks\n",
-    "    statuses = list()\n",
-    "    \n",
-    "    # Iterate over each of the task IDs\n",
-    "    for this_task_id in task_ids:\n",
-    "        \n",
-    "        # For each task ID, get detailed information about it\n",
-    "        task_resp = df_api.taskView(this_task_id)\n",
-    "        \n",
-    "        # Extract the task status from the detailed information\n",
-    "        this_status = task_resp[0].task[0].status\n",
-    "        \n",
-    "        # Append this status to the list of statuses\n",
-    "        statuses.append(this_status)\n",
-    "    \n",
-    "    # Return the list of statuses\n",
-    "    return statuses"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Try out your function using the IDs of the recent ``dataPut()`` and ``dataGet()`` functions. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[3]"
-      ]
-     },
-     "execution_count": 22,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "check_xfer_status(task_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Run the series of simulations:\n",
-    "Use the three functions defined above to mimic the process of exploring a parameter space using simulations, where for each iteration, we: <br> 1. run a simulation, <br> 2. capture the data + metadata into DataFed, <br> 3. monitor the data upload tasks."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Starting simulation #0\n",
-      "Transfer status(es): [2]\n",
-      "\n",
-      "Starting simulation #1\n",
-      "Transfer status(es): [2, 2]\n",
-      "\n",
-      "Starting simulation #2\n",
-      "Transfer status(es): [3, 2, 2]\n",
-      "\n",
-      "Simulations complete! Waiting for uploads to complete\n",
-      "\n",
-      "Transfer status(es): [3, 3, 2]\n",
-      "Transfer status(es): [3, 3, 2]\n",
-      "Transfer status(es): [3, 3, 2]\n",
-      "Transfer status(es): [3, 3, 2]\n",
-      "Transfer status(es): [3, 3, 3]\n",
-      "\n",
-      "Finished uploading all data!\n"
-     ]
-    }
-   ],
-   "source": [
-    "xfer_tasks = list()\n",
-    "for ind in range(3):\n",
-    "    print('Starting simulation #{}'.format(ind))\n",
-    "    # Run the simulation.\n",
-    "    path_to_results, metadata = expensive_simulation()\n",
-    "    # Capture the data and metadata into DataFed\n",
-    "    task_id = capture_data(ind, metadata, path_to_results)\n",
-    "    # Append the task ID for this data upload into xfer_tasks\n",
-    "    xfer_tasks.append(task_id)\n",
-    "    # Print out the status of the data transfers\n",
-    "    print('Transfer status(es): {}'.format(check_xfer_status(xfer_tasks)))\n",
-    "    print('')\n",
-    "\n",
-    "print('Simulations complete! Waiting for uploads to complete\\n')\n",
-    "\n",
-    "while True:\n",
-    "    time.sleep(1)\n",
-    "    statuses = check_xfer_status(xfer_tasks)\n",
-    "    print('Transfer status(es): {}'.format(statuses))\n",
-    "    if all([this == 3 for this in statuses]): break\n",
-    "        \n",
-    "print('\\nFinished uploading all data!')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> What happens if you set the ``wait`` in ``dataPut()`` to ``True``? </span>\n",
-    "\n",
-    "### <span style=\"color:blue\"> **Note:** </span>\n",
-    "> <span style=\"color:blue\"> Users are recommended to perform data orchestration (especially large data movement - upload / download) operations outside the scope of heavy / parallel computation operations in order to avoid wasting precious wall time on compute clusters.</span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# 3. Transferring Data in DataFed\n",
+                "In this notebook, we will be going over uploading to & downloading from Data Records and monitoring data transfer tasks.\n",
+                "\n",
+                "## Before we begin:\n",
+                "Import necessary libraries",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import json\n",
+                "import os\n",
+                "import time\n",
+                "from datafed.CommandLib import API",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Instantiate the DataFed API and set ``context`` to the Training project:"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()\n", "df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Enter your username to work within your personal Collection. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ['parent_collection = "c/34558900" # your username here'],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Copy over the ID for the record you created in the previous notebook. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ['record_id = "d/43650517"'],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Uploading raw data\n",
+                "Shortly, we will learn how to upload data to the record we just created. For demonstration purposes, we will just create a simple text file and use this as the raw data for the Data Record\n",
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> DataFed does not impose any restrictions on the file extension / format for the raw data </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "with open('./raw_data.txt', mode='w') as file_handle:\n",
+                "    file_handle.write('This is some data')",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Always ensure that your Globus endpoint is active and that your files are located in a directory that is visible to the Globus Endpoint </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## `dataPut()`\n",
+                "Uploading data files to DataFed is done using the `dataPut` command.",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "put_resp = df_api.dataPut(record_id,\n",
+                "                          './raw_data.txt',\n",
+                "                          wait=True, # Waits until transfer completes.\n",
+                "                          )\n",
+                "print(put_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We get two components in the response:\n",
+                "* Information about the Data Record, data was uploaded to\n",
+                "* Information about the data transfer ``task`` - more on this later\n",
+                "\n",
+                "The ``dataPut()`` method **initiates a Globus transfer** on our behalf from the machine **wherever** the file was present to wherever the default data repository is located. In this case, the file was in our local file system and on the same machine where we are executing the command.\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> The above data file was specified by its relative local path, so DataFed used our pre-configured default Globus endpoint to find the data file. As long as we have the id for any *active* Globus endpoint that we have authenticated access to, we can transfer data from that endpoint with its full absolute file path – even if the file system is not attached ot the local machine. Look for more information on this in later examples. </span>\n',
+                "\n",
+                "Let's view the data record now that we've uploaded our data. Pay attention to the ``ext`` and ``source`` fields which should now populated:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["dv_resp = df_api.dataView(record_id)\n", "print(dv_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Downloading raw data\n",
+                "DataFed is also capable of getting data stored in a DataFed repository and placing it in the local or other Globus-visible filesystem via the ``dataGet()`` function. \n",
+                "\n",
+                "Let us download the content in the data record we have been working on so far for demonstration purposes",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "get_resp = df_api.dataGet(record_id,\n",
+                "                          '.', # directory where data should be downloaded\n",
+                "                          orig_fname=False, # do not name file by its original name\n",
+                "                          wait=True, # Wait until Globus transfer completes\n",
+                "                         )\n",
+                "print(get_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In the response we only get back information about the data transfer ``task`` - more on this shortly\n",
+                "\n",
+                "The bug in ``dataGet()`` also reveals its capability to **download multiple data records or even Collections.**\n",
+                "\n",
+                "Let's confirm that the data has been downloaded successfully:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["os.listdir('.')"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "expected_file_name = os.path.join('.', record_id.split('d/')[-1]) + '.txt'\n",
+                "print('Does a file with this name: ' + expected_file_name + ' exist?')\n",
+                "print(os.path.exists(expected_file_name))",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Tasks\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> A DataFed task may itself contain / be responsible for several Globus file transfers, potentially from / to multiple locations. </span>\n',
+                "    \n",
+                "DataFed makes it possible to check on the status of transfer tasks in an easy and programmatic manner.\n",
+                "\n",
+                "Before we learn more about tasks, first lets try to get the ``id`` of the task in ``get_resp`` from the recent ``dataGet()`` function call:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["task_id = get_resp[0].task[0].id\n", "print(task_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Viewing Tasks\n",
+                "We can get more information about a given transfer via the `taskView()` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["task_resp = df_api.taskView(task_id)\n", "print(task_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We get a new kind of message - a ``TaskDataReply``. \n",
+                "Key fields to keep an eye on:\n",
+                "* ``status``\n",
+                "* ``msg``\n",
+                "* ``source``\n",
+                "* ``dest``\n",
+                "\n",
+                "If we are interested in monitoring tasks, triggering activities or subsequent steps of workflows based on transfers, we would need to know how to get the ``status`` property from the ``TaskDataReply``: ",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["task_resp[0].task[0].status"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Even though the message above says `TS_SUCCEEDED`, we see that this task status codes to the integer `3`.\n",
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> Cheat sheet for interpreting task statuses: <br> ``2``: in progress <br> ``3``: complete <br> anything else - problem  </span>\n',
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> A future version of DataFed may change the nature of the output / type for the status property. In general, the exact return object types and nomenclature may evolve with DataFed. </span>',
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Listing Tasks\n",
+                "We can request a listing of all our recently initiated tasks:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {"scrolled": true},
+            "outputs": [],
+            "source": ["df_api.taskList()"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "The output of this listing would be very helpful for the Exercise below"
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Example scenario - Simulations\n",
+                "Let's say that we want to run a series of simulations where one or more parameters are varied and each simulation is run with a unique set of parameters. Let's also assume that our eventual goal is to build a surrogate model for the computationally expensive simulation using machine learning. So, we **want to capture the metadata and data associated with the series of simulations** to train the machine learning model later on. \n",
+                "\n",
+                "We have set up skeleton functions and code snippets to help you mimic the data management for such a simulation. We would like you to take what you have learnt so far and fill in the blanks\n",
+                "<br>\n",
+                "\n",
+                "\n",
+                "### Fake simulation\n",
+                'Here, we have simulated a computationally "expensive" simulation that simply sleeps for a few seconds.',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def expensive_simulation():\n",
+                "    time.sleep(5)\n",
+                "    # Yes, this simulation is deterministic and always results in the same result:\n",
+                "    path_to_results = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
+                "    # The simulation uses the same combination of parameters\n",
+                "    metadata = {'a': 1, 'b': 2, 'c': 3.14}\n",
+                "    return path_to_results, metadata",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Define a function that: <br> 1. creates a new Data Record with the provided metadata (as a dictionary) and other details, <br> 2. extracts the record id, <br> 3. puts the raw data into the record, <br> 4. extracts and returns the task ID. <br><br> Feel free to print any messages that may help you track things. </span>\n',
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Pay attention to the ``wait`` keyword argument when putting the raw data into record </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 19,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def capture_data(simulation_index, # integer - counter to signify the Nth simulation in the series\n",
+                "                 metadata, # dictionary - combination of parameters used for this simulation\n",
+                "                 raw_data_path, # string - Path to the raw data file that needs to be put into the receord\n",
+                "                 parent_collection=parent_collection, # string - Collection to create this Data Record into\n",
+                "                ):\n",
+                "    \n",
+                "    # 1. Create a new Data Record with the metadata and use the simulation index to provide a unique title\n",
+                "    rec_resp = df_api.dataCreate('Simulation_' + str(simulation_index),\n",
+                "                                 metadata=json.dumps(metadata),\n",
+                "                                 parent_id=parent_collection)\n",
+                "    \n",
+                "    # 2. Extract the record ID from the response\n",
+                "    this_rec_id = rec_resp[0].data[0].id\n",
+                "    \n",
+                "    # 3. Put the raw data into this record:\n",
+                "    put_resp = df_api.dataPut(this_rec_id, raw_data_path, wait=False)\n",
+                "    \n",
+                "    # 4. Extract the ID for the data transfer task\n",
+                "    task_id = put_resp[0].task.id\n",
+                "    \n",
+                "    # 5. Return the task ID\n",
+                "    return task_id",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Try out this function to make sure it works. See what it does on the **DataFed web portal**. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 20,
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {"text/plain": ["'task/45433451'"]},
+                    "execution_count": 20,
+                    "metadata": {},
+                    "output_type": "execute_result",
+                }
+            ],
+            "source": [
+                "path_to_results, metadata = expensive_simulation()\n",
+                "\n",
+                "task_id = capture_data(14, metadata, path_to_results)\n",
+                "task_id",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> We will want a simple function to monitor the status of all the data upload tasks. Define a function that accepts a list of task IDs and returns their status after looking them up on DataFed </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 21,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "def check_xfer_status(task_ids):\n",
+                "    \n",
+                "    # put singular task ID into a list\n",
+                "    if isinstance(task_ids, str):\n",
+                "        task_ids = [task_ids]\n",
+                "    \n",
+                "    # Create a list to hold the statuses of each of the tasks\n",
+                "    statuses = list()\n",
+                "    \n",
+                "    # Iterate over each of the task IDs\n",
+                "    for this_task_id in task_ids:\n",
+                "        \n",
+                "        # For each task ID, get detailed information about it\n",
+                "        task_resp = df_api.taskView(this_task_id)\n",
+                "        \n",
+                "        # Extract the task status from the detailed information\n",
+                "        this_status = task_resp[0].task[0].status\n",
+                "        \n",
+                "        # Append this status to the list of statuses\n",
+                "        statuses.append(this_status)\n",
+                "    \n",
+                "    # Return the list of statuses\n",
+                "    return statuses",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Try out your function using the IDs of the recent ``dataPut()`` and ``dataGet()`` functions. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 22,
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {"text/plain": ["[3]"]},
+                    "execution_count": 22,
+                    "metadata": {},
+                    "output_type": "execute_result",
+                }
+            ],
+            "source": ["check_xfer_status(task_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Run the series of simulations:\n",
+                "Use the three functions defined above to mimic the process of exploring a parameter space using simulations, where for each iteration, we: <br> 1. run a simulation, <br> 2. capture the data + metadata into DataFed, <br> 3. monitor the data upload tasks.",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 23,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Starting simulation #0\n",
+                        "Transfer status(es): [2]\n",
+                        "\n",
+                        "Starting simulation #1\n",
+                        "Transfer status(es): [2, 2]\n",
+                        "\n",
+                        "Starting simulation #2\n",
+                        "Transfer status(es): [3, 2, 2]\n",
+                        "\n",
+                        "Simulations complete! Waiting for uploads to complete\n",
+                        "\n",
+                        "Transfer status(es): [3, 3, 2]\n",
+                        "Transfer status(es): [3, 3, 2]\n",
+                        "Transfer status(es): [3, 3, 2]\n",
+                        "Transfer status(es): [3, 3, 2]\n",
+                        "Transfer status(es): [3, 3, 3]\n",
+                        "\n",
+                        "Finished uploading all data!\n",
+                    ],
+                }
+            ],
+            "source": [
+                "xfer_tasks = list()\n",
+                "for ind in range(3):\n",
+                "    print('Starting simulation #{}'.format(ind))\n",
+                "    # Run the simulation.\n",
+                "    path_to_results, metadata = expensive_simulation()\n",
+                "    # Capture the data and metadata into DataFed\n",
+                "    task_id = capture_data(ind, metadata, path_to_results)\n",
+                "    # Append the task ID for this data upload into xfer_tasks\n",
+                "    xfer_tasks.append(task_id)\n",
+                "    # Print out the status of the data transfers\n",
+                "    print('Transfer status(es): {}'.format(check_xfer_status(xfer_tasks)))\n",
+                "    print('')\n",
+                "\n",
+                "print('Simulations complete! Waiting for uploads to complete\\n')\n",
+                "\n",
+                "while True:\n",
+                "    time.sleep(1)\n",
+                "    statuses = check_xfer_status(xfer_tasks)\n",
+                "    print('Transfer status(es): {}'.format(statuses))\n",
+                "    if all([this == 3 for this in statuses]): break\n",
+                "        \n",
+                "print('\\nFinished uploading all data!')",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> What happens if you set the ``wait`` in ``dataPut()`` to ``True``? </span>\n',
+                "\n",
+                '### <span style="color:blue"> **Note:** </span>\n',
+                '> <span style="color:blue"> Users are recommended to perform data orchestration (especially large data movement - upload / download) operations outside the scope of heavy / parallel computation operations in order to avoid wasting precious wall time on compute clusters.</span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.8.3",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/4_Collections_Queries.ipynb b/jupyter_notebooks/4_Collections_Queries.ipynb
index 33b6d8c03..4ab8cf8fa 100644
--- a/jupyter_notebooks/4_Collections_Queries.ipynb
+++ b/jupyter_notebooks/4_Collections_Queries.ipynb
@@ -1,561 +1,517 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 4. Collections and Queries in DataFed\n",
-    "In this notebook, we will be going over creating Collections, viewing contained items, organizing Collections, downloading Collections, and searching for data"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Before we begin:\n",
-    "Import necessary libraries"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import json\n",
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Instantiate the DataFed API and set ``context`` to the Training project"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()\n",
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Reset this variable to your username or Globus ID so that you work within your own collection by default </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parent_collection = \"breetju\" # Name of this user"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Overview:\n",
-    "In this notebook, let us assume that we are working on a machine learning problem aimed at putting together training data for a machine learning model. For illustration purposes, we will assume that we aim to train a classifier for classifying animals"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Create Collection\n",
-    "First, let us create a collection to hold all our data. \n",
-    "\n",
-    "We will be using the ``collectionCreate()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "coll_resp = df_api.collectionCreate('Image classification training data',\n",
-    "                                    parent_id=parent_collection)\n",
-    "print(coll_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In this case we got back a ``CollDataReply`` object. This is somewhat similar to what you get from ``dataCreate()`` we just saw. \n",
-    "\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Extract the ``id`` of this newly created collection: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "train_coll_id = None"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Populate with training data\n",
-    "Now that we have a place to put the training data, let us populate this collection with examples of animals\n",
-    "### Define a function to generate (fake) training data:\n",
-    "We need a function to:\n",
-    "* Create a Data Record\n",
-    "* Put data into this Data Record\n",
-    "\n",
-    "For simplicity we will use some dummy data from a public Globus Endpoint This information has been filled in for you via the ``raw_data_path`` variable\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> We have a skeleton function prepared for you along with comments to guide you. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import random\n",
-    "\n",
-    "def generate_animal_data(is_dog=True):\n",
-    "    this_animal = 'cat'\n",
-    "    if is_dog:\n",
-    "        this_animal = 'dog'\n",
-    "    # To mimic a real-life scenario, we append a number to the animal type to denote\n",
-    "    # the N-th example of a cat or dog. In this case, we use a random integer.\n",
-    "    suffix = '_' + str(random.randint(1, 1000))\n",
-    "    \n",
-    "    # Create the record here:\n",
-    "                       \n",
-    "                       \n",
-    "    # Extract the ID of the Record:\n",
-    "                       \n",
-    "    \n",
-    "    # path to the file containing the (dummy) raw data\n",
-    "    raw_data_path = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
-    "    \n",
-    "    # Put the raw data into the record you just created: \n",
-    "    \n",
-    "    \n",
-    "    # Only return the ID of the Data Record you created:\n",
-    "    return this_rec_id"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### Generate 5 examples of cats and dogs:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "cat_records = list()\n",
-    "dog_records = list()\n",
-    "for _ in range(5):\n",
-    "    dog_records.append(generate_animal_data(is_dog=True))\n",
-    "for _ in range(5):\n",
-    "    cat_records.append(generate_animal_data(is_dog=False))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(cat_records)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(dog_records)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Listing items in a Collection:\n",
-    "Let us take a look at the training data we have assembled so far using the ``colectionItemsList()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "coll_list_resp = df_api.collectionItemsList(train_coll_id)\n",
-    "print(coll_list_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> If we had several dozens, hundreds, or even thousands of items in a Collection, we would need to call ``collectionItemsList()`` multiple times by stepping up the ``offset`` keyword argument each time to get the next “page” of results. </span>\n",
-    "\n",
-    "### <span style=\"color:green\"> Discussion </span>\n",
-    "<span style=\"color:green\"> Let's say that we are only interested in finding records that have cats in this (potentially) large collection of training data. How do we go about doing that? </span>\n",
-    "\n",
-    "# Data Query / Search\n",
-    "### <span style=\"color:red\"> Caution </span>\n",
-    "> <span style=\"color:red\"> Search vocabulary is likely to change with newer versions of DataFed </span>\n",
-    "\n",
-    "Use the DataFed web interface to:\n",
-    "* Search for cats\n",
-    "* Specifically in your collection\n",
-    "* Save the query\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Saved queries can be found in the bottom of the navigation (left) pane under ``Project Data`` and ``Saved Queries`` </span>\n",
-    "\n",
-    "# Find saved queries:\n",
-    "We can list all saved queries via ``queryList()``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ql_resp = df_api.queryList()\n",
-    "print(ql_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Notice that we again recieved the familiar ``ListingReply`` object as the response\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Get the ``id`` of the desired query out of the response: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query_id = ?\n",
-    "print(query_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# View the saved query\n",
-    "Use the ``queryView()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.queryView(query_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Run the saved query\n",
-    "Use the ``queryExec()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query_resp = df_api.queryExec(query_id)\n",
-    "print(query_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Yet again, we get back the ``ListingReply`` message. \n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Extract just the ``id``s from each of the items in the message: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# First get IDs from query result\n",
-    "cat_rec_ids = ?"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We already have the ground truth in ``cat_records``. Is this the same as what we got from the query?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(set(cat_rec_ids) == set(cat_records))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Separating cats from dogs\n",
-    "Our goal now is to gather all cat Data Records into a dedicated Collection\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Create a new collection to hold the Cats record </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Extract the ``id`` for this Collection: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "cat_coll_id = ?"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Adding Items to Collection\n",
-    "Now let us add only the cat Data Records into this new collection using the ``collectionItemsUpdate()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "cup_resp = df_api.collectionItemsUpdate(cat_coll_id, add_ids=cat_rec_ids)\n",
-    "print(cup_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Unlike most DataFed functions, this function doesn't really return much\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> View the contents of the Cats Collection to make sure that all Cat Data Records are present in this Collection </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> View the contents of the main training data Collection: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Data Records can exist in **multiple** Collections just like video or songs can exist on multiple playlists </span>\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Remove the cat Data Records from the training data collection. They already exist in the \"Cats\" Collection </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> View the contents of the training data Collection. Do you see the individual cat Data Records in this collection? </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search or Organize?\n",
-    "If you could always search for your data, what is the benefit to organizing them into collections?\n",
-    "\n",
-    "# Download entire Collection\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Recall that DataFed can download arbitrarily large number of Records regardless of the physical locations of the DataFed repositories containing the data. </span>\n",
-    "\n",
-    "Let us first make sure we don't already have a directory with the desired name:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dest_dir = './cat_data'\n",
-    "\n",
-    "if os.path.exists(dest_dir):\n",
-    "    import shutil\n",
-    "    shutil.rmtree(dest_dir)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Download the entire Cat Collection with a single DataFed function call </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Let's verify that we did infact download the data:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "os.listdir(dest_dir)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## <span style=\"color:green\"> Optional Exercise </span>\n",
-    "<span style=\"color:green\">1. Create a new Collection to hold the simulation data you created in the previous notebook <br>2. Use the functions you saw above to ensure that the Data Records only exist in the Simulation Collection </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python [default]",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# 4. Collections and Queries in DataFed\n",
+                "In this notebook, we will be going over creating Collections, viewing contained items, organizing Collections, downloading Collections, and searching for data",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["## Before we begin:\n", "Import necessary libraries"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "import json\n",
+                "from datafed.CommandLib import API",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Instantiate the DataFed API and set ``context`` to the Training project"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()\n", "df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Reset this variable to your username or Globus ID so that you work within your own collection by default </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ['parent_collection = "breetju" # Name of this user'],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Overview:\n",
+                "In this notebook, let us assume that we are working on a machine learning problem aimed at putting together training data for a machine learning model. For illustration purposes, we will assume that we aim to train a classifier for classifying animals",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Create Collection\n",
+                "First, let us create a collection to hold all our data. \n",
+                "\n",
+                "We will be using the ``collectionCreate()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "coll_resp = df_api.collectionCreate('Image classification training data',\n",
+                "                                    parent_id=parent_collection)\n",
+                "print(coll_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In this case we got back a ``CollDataReply`` object. This is somewhat similar to what you get from ``dataCreate()`` we just saw. \n",
+                "\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Extract the ``id`` of this newly created collection: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["train_coll_id = None"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Populate with training data\n",
+                "Now that we have a place to put the training data, let us populate this collection with examples of animals\n",
+                "### Define a function to generate (fake) training data:\n",
+                "We need a function to:\n",
+                "* Create a Data Record\n",
+                "* Put data into this Data Record\n",
+                "\n",
+                "For simplicity we will use some dummy data from a public Globus Endpoint This information has been filled in for you via the ``raw_data_path`` variable\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> We have a skeleton function prepared for you along with comments to guide you. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import random\n",
+                "\n",
+                "def generate_animal_data(is_dog=True):\n",
+                "    this_animal = 'cat'\n",
+                "    if is_dog:\n",
+                "        this_animal = 'dog'\n",
+                "    # To mimic a real-life scenario, we append a number to the animal type to denote\n",
+                "    # the N-th example of a cat or dog. In this case, we use a random integer.\n",
+                "    suffix = '_' + str(random.randint(1, 1000))\n",
+                "    \n",
+                "    # Create the record here:\n",
+                "                       \n",
+                "                       \n",
+                "    # Extract the ID of the Record:\n",
+                "                       \n",
+                "    \n",
+                "    # path to the file containing the (dummy) raw data\n",
+                "    raw_data_path = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
+                "    \n",
+                "    # Put the raw data into the record you just created: \n",
+                "    \n",
+                "    \n",
+                "    # Only return the ID of the Data Record you created:\n",
+                "    return this_rec_id",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["#### Generate 5 examples of cats and dogs:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "cat_records = list()\n",
+                "dog_records = list()\n",
+                "for _ in range(5):\n",
+                "    dog_records.append(generate_animal_data(is_dog=True))\n",
+                "for _ in range(5):\n",
+                "    cat_records.append(generate_animal_data(is_dog=False))",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(cat_records)"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(dog_records)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Listing items in a Collection:\n",
+                "Let us take a look at the training data we have assembled so far using the ``colectionItemsList()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "coll_list_resp = df_api.collectionItemsList(train_coll_id)\n",
+                "print(coll_list_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> If we had several dozens, hundreds, or even thousands of items in a Collection, we would need to call ``collectionItemsList()`` multiple times by stepping up the ``offset`` keyword argument each time to get the next “page” of results. </span>\n',
+                "\n",
+                '### <span style="color:green"> Discussion </span>\n',
+                '<span style="color:green"> Let\'s say that we are only interested in finding records that have cats in this (potentially) large collection of training data. How do we go about doing that? </span>\n',
+                "\n",
+                "# Data Query / Search\n",
+                '### <span style="color:red"> Caution </span>\n',
+                '> <span style="color:red"> Search vocabulary is likely to change with newer versions of DataFed </span>\n',
+                "\n",
+                "Use the DataFed web interface to:\n",
+                "* Search for cats\n",
+                "* Specifically in your collection\n",
+                "* Save the query\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Saved queries can be found in the bottom of the navigation (left) pane under ``Project Data`` and ``Saved Queries`` </span>\n',
+                "\n",
+                "# Find saved queries:\n",
+                "We can list all saved queries via ``queryList()``:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["ql_resp = df_api.queryList()\n", "print(ql_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Notice that we again recieved the familiar ``ListingReply`` object as the response\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Get the ``id`` of the desired query out of the response: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["query_id = ?\n", "print(query_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["# View the saved query\n", "Use the ``queryView()`` function:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.queryView(query_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["# Run the saved query\n", "Use the ``queryExec()`` function:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "query_resp = df_api.queryExec(query_id)\n",
+                "print(query_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Yet again, we get back the ``ListingReply`` message. \n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Extract just the ``id``s from each of the items in the message: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["# First get IDs from query result\n", "cat_rec_ids = ?"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We already have the ground truth in ``cat_records``. Is this the same as what we got from the query?"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["print(set(cat_rec_ids) == set(cat_records))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Separating cats from dogs\n",
+                "Our goal now is to gather all cat Data Records into a dedicated Collection\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Create a new collection to hold the Cats record </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Extract the ``id`` for this Collection: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["cat_coll_id = ?"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Adding Items to Collection\n",
+                "Now let us add only the cat Data Records into this new collection using the ``collectionItemsUpdate()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "cup_resp = df_api.collectionItemsUpdate(cat_coll_id, add_ids=cat_rec_ids)\n",
+                "print(cup_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Unlike most DataFed functions, this function doesn't really return much\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> View the contents of the Cats Collection to make sure that all Cat Data Records are present in this Collection </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> View the contents of the main training data Collection: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Data Records can exist in **multiple** Collections just like video or songs can exist on multiple playlists </span>\n',
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Remove the cat Data Records from the training data collection. They already exist in the "Cats" Collection </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> View the contents of the training data Collection. Do you see the individual cat Data Records in this collection? </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Search or Organize?\n",
+                "If you could always search for your data, what is the benefit to organizing them into collections?\n",
+                "\n",
+                "# Download entire Collection\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Recall that DataFed can download arbitrarily large number of Records regardless of the physical locations of the DataFed repositories containing the data. </span>\n',
+                "\n",
+                "Let us first make sure we don't already have a directory with the desired name:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dest_dir = './cat_data'\n",
+                "\n",
+                "if os.path.exists(dest_dir):\n",
+                "    import shutil\n",
+                "    shutil.rmtree(dest_dir)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Download the entire Cat Collection with a single DataFed function call </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["Let's verify that we did infact download the data:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["os.listdir(dest_dir)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '## <span style="color:green"> Optional Exercise </span>\n',
+                '<span style="color:green">1. Create a new Collection to hold the simulation data you created in the previous notebook <br>2. Use the functions you saw above to ensure that the Data Records only exist in the Simulation Collection </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python [default]",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.5.5",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/4_Collections_Queries_with_Solutions.ipynb b/jupyter_notebooks/4_Collections_Queries_with_Solutions.ipynb
index 350316e3f..01b42e02c 100755
--- a/jupyter_notebooks/4_Collections_Queries_with_Solutions.ipynb
+++ b/jupyter_notebooks/4_Collections_Queries_with_Solutions.ipynb
@@ -1,871 +1,793 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 4. Collections and Queries in DataFed\n",
-    "In this notebook, we will be going over creating Collections, viewing contained items, organizing Collections, downloading Collections, and searching for data"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Before we begin:\n",
-    "Import necessary libraries"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import json\n",
-    "from datafed.CommandLib import API"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Instantiate the DataFed API and set ``context`` to the Training project"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api = API()\n",
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Reset this variable to your username or Globus ID so that you work within your own collection by default </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "parent_collection = \"somnaths\" # Name of this user"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Overview:\n",
-    "In this notebook, let us assume that we are working on a machine learning problem aimed at putting together training data for a machine learning model. For illustration purposes, we will assume that we aim to train a classifier for classifying animals"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Create Collection\n",
-    "First, let us create a collection to hold all our data. \n",
-    "\n",
-    "We will be using the ``collectionCreate()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(coll {\n",
-      "  id: \"c/43657980\"\n",
-      "  title: \"Image classification training data\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  ct: 1615405612\n",
-      "  ut: 1615405612\n",
-      "  parent_id: \"c/34558900\"\n",
-      "}\n",
-      ", 'CollDataReply')\n"
-     ]
-    }
-   ],
-   "source": [
-    "coll_resp = df_api.collectionCreate('Image classification training data',\n",
-    "                                    parent_id=parent_collection)\n",
-    "print(coll_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In this case we got back a ``CollDataReply`` object. This is somewhat similar to what you get from ``dataCreate()`` we just saw. \n",
-    "\n",
-    "Now, let's Extract the ``id`` of this newly created collection:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "c/43657980\n"
-     ]
-    }
-   ],
-   "source": [
-    "train_coll_id = coll_resp[0].coll[0].id\n",
-    "print(train_coll_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Populate with training data\n",
-    "Now that we have a place to put the training data, let us populate this collection with examples of animals\n",
-    "### Define a function to generate (fake) training data:\n",
-    "We need a function to:\n",
-    "* Create a Data Record\n",
-    "* Put data into this Data Record\n",
-    "\n",
-    "For simplicity we will use some dummy data from a public Globus Endpoint This information has been filled in for you via the ``raw_data_path`` variable. \n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> We have a skeleton function prepared for you along with comments to guide you. </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import random\n",
-    "\n",
-    "def generate_animal_data(is_dog=True):\n",
-    "    this_animal = 'cat'\n",
-    "    if is_dog:\n",
-    "        this_animal = 'dog'\n",
-    "    # Ideally, we would have more sensible titles such as \"Doberman\", \"Poodle\", etc. instead of \"Dog_234\"\n",
-    "    # To mimic a real-life scenario, we append a number to the animal type to denote\n",
-    "    # the N-th example of a cat or dog. In this case, we use a random integer.\n",
-    "    title = this_animal + '_' + str(random.randint(1, 1000))\n",
-    "    # Create the record here:\n",
-    "    rec_resp = df_api.dataCreate(title,\n",
-    "                                 metadata=json.dumps({'animal': this_animal}),\n",
-    "                                 parent_id=train_coll_id)                   \n",
-    "                       \n",
-    "    # Extract the ID of the Record:\n",
-    "    this_rec_id = rec_resp[0].data[0].id                   \n",
-    "    \n",
-    "    # path to the file containing the (dummy) raw data\n",
-    "    raw_data_path = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
-    "    \n",
-    "    # Put the raw data into the record you just created: \n",
-    "    put_resp = df_api.dataPut(this_rec_id, raw_data_path)\n",
-    "    \n",
-    "    # Only return the ID of the Data Record you created:\n",
-    "    return this_rec_id"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### Generate 5 examples of cats and dogs:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "cat_records = list()\n",
-    "dog_records = list()\n",
-    "for _ in range(5):\n",
-    "    dog_records.append(generate_animal_data(is_dog=True))\n",
-    "for _ in range(5):\n",
-    "    cat_records.append(generate_animal_data(is_dog=False))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['d/43659103', 'd/43659126', 'd/43659149', 'd/43659172', 'd/43659195']\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(cat_records)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['d/43658988', 'd/43659011', 'd/43659034', 'd/43659057', 'd/43659080']\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(dog_records)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Listing items in a Collection:\n",
-    "Let us take a look at the training data we have assembled so far using the ``colectionItemsList()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(item {\n",
-      "  id: \"d/43658988\"\n",
-      "  title: \"dog_196\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659034\"\n",
-      "  title: \"dog_57\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659080\"\n",
-      "  title: \"dog_686\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659011\"\n",
-      "  title: \"dog_689\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659057\"\n",
-      "  title: \"dog_778\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "offset: 5\n",
-      "count: 20\n",
-      "total: 10\n",
-      ", 'ListingReply')\n"
-     ]
-    }
-   ],
-   "source": [
-    "coll_list_resp = df_api.collectionItemsList(train_coll_id, offset=5)\n",
-    "print(coll_list_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> If we had several dozens, hundreds, or even thousands of items in a Collection, we would need to call ``collectionItemsList()`` multiple times by stepping up the ``offset`` keyword argument each time to get the next “page” of results. </span>\n",
-    "\n",
-    "### <span style=\"color:green\"> Discussion </span>\n",
-    "<span style=\"color:green\"> Let's say that we are only interested in finding records that have cats in this (potentially) large collection of training data. How do we go about doing that? </span>\n",
-    "\n",
-    "# Data Query / Search\n",
-    "### <span style=\"color:red\"> Caution </span>\n",
-    "> <span style=\"color:red\"> Search vocabulary is likely to change with newer versions of DataFed </span>\n",
-    "\n",
-    "Use the DataFed web interface to:\n",
-    "* Search for cats\n",
-    "* Specifically in your collection\n",
-    "* Save the query\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Saved queries can be found in the bottom of the navigation (left) pane under ``Project Data`` and ``Saved Queries`` </span>\n",
-    "\n",
-    "# Find saved queries:\n",
-    "We can list all saved queries via ``queryList()``:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(item {\n",
-      "  id: \"q/43664114\"\n",
-      "  title: \"is_cat\"\n",
-      "}\n",
-      "offset: 0\n",
-      "count: 20\n",
-      "total: 1\n",
-      ", 'ListingReply')\n"
-     ]
-    }
-   ],
-   "source": [
-    "ql_resp = df_api.queryList()\n",
-    "print(ql_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Notice that we again recieved the familiar ``ListingReply`` object as the response\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Get the ``id`` of the desired query out of the response: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "q/43664114\n"
-     ]
-    }
-   ],
-   "source": [
-    "query_id = ql_resp[0].item[0].id\n",
-    "print(query_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# View the saved query\n",
-    "Use the ``queryView()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(query {\n",
-       "   id: \"q/43664114\"\n",
-       "   title: \"is_cat\"\n",
-       "   query: \"{\\\"meta\\\":\\\"md.animal == \\\\\\\"cat\\\\\\\"\\\",\\\"scopes\\\":[{\\\"scope\\\":4,\\\"id\\\":\\\"c/43657980\\\",\\\"recurse\\\":true}]}\"\n",
-       "   owner: \"u/somnaths\"\n",
-       "   ct: 1615406400\n",
-       "   ut: 1615406400\n",
-       " },\n",
-       " 'QueryDataReply')"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df_api.queryView(query_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Run the saved query\n",
-    "Use the ``queryExec()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(item {\n",
-      "  id: \"d/43659103\"\n",
-      "  title: \"cat_209\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659126\"\n",
-      "  title: \"cat_511\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659149\"\n",
-      "  title: \"cat_341\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659172\"\n",
-      "  title: \"cat_558\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      "item {\n",
-      "  id: \"d/43659195\"\n",
-      "  title: \"cat_821\"\n",
-      "  owner: \"p/trn001\"\n",
-      "  creator: \"u/somnaths\"\n",
-      "  size: 2842.0\n",
-      "  notes: 0\n",
-      "}\n",
-      ", 'ListingReply')\n"
-     ]
-    }
-   ],
-   "source": [
-    "query_resp = df_api.queryExec(query_id)\n",
-    "print(query_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Yet again, we get back the ``ListingReply`` message. \n",
-    "\n",
-    "Now let us extract just the ``id``s from each of the items in the message:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['d/43659103', 'd/43659126', 'd/43659149', 'd/43659172', 'd/43659195']\n"
-     ]
-    }
-   ],
-   "source": [
-    "cat_rec_ids = list()\n",
-    "for record in query_resp[0].item:\n",
-    "    cat_rec_ids.append(record.id)\n",
-    "# one could also use list comprehensions to get the answer in one line:\n",
-    "# cat_rec_ids = [record.id for record in query_resp[0].item]\n",
-    "print(cat_rec_ids)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We already have the ground truth in ``cat_records``. Is this the same as what we got from the query?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "True\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(set(cat_rec_ids) == set(cat_records))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Separating cats from dogs\n",
-    "Our goal now is to gather all cat Data Records into a dedicated Collection\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Create a new collection to hold the Cats record </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "c/43666045\n"
-     ]
-    }
-   ],
-   "source": [
-    "coll_resp = df_api.collectionCreate('Cats', parent_id=train_coll_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Extract the ``id`` for this Collection: </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "c/43666045\n"
-     ]
-    }
-   ],
-   "source": [
-    "cat_coll_id = coll_resp[0].coll[0].id\n",
-    "print(cat_coll_id)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Adding Items to Collection\n",
-    "Now let us add only the cat Data Records into this new collection using the ``collectionItemsUpdate()`` function:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(, 'ListingReply')\n"
-     ]
-    }
-   ],
-   "source": [
-    "cup_resp = df_api.collectionItemsUpdate(cat_coll_id, add_ids=cat_rec_ids)\n",
-    "print(cup_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Unlike most DataFed functions, this function doesn't really return much\n",
-    "\n",
-    "Now, let us view the contents of the Cats Collection to make sure that all Cat Data Records are present in this Collection. \n",
-    "\n",
-    "Just to keep the output clean and short, we will only extract the ID and title of the items"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "d/43659103 cat_209\n",
-      "d/43659149 cat_341\n",
-      "d/43659126 cat_511\n",
-      "d/43659172 cat_558\n",
-      "d/43659195 cat_821\n"
-     ]
-    }
-   ],
-   "source": [
-    "ls_resp = df_api.collectionItemsList(cat_coll_id)\n",
-    "# Iterating through the items in the Collection and only extracting a few items:\n",
-    "for obj in ls_resp[0].item:\n",
-    "    print(obj.id, obj.title)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> View the contents of the main training data Collection. <br> You may use the snippet above if you like and modify it accordingly </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ls_resp = df_api.collectionItemsList(train_coll_id)\n",
-    "# Iterating through the items in the Collection and only extracting a few items:\n",
-    "for obj in ls_resp[0].item:\n",
-    "    print(obj.id, obj.title)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Data Records can exist in **multiple** Collections just like video or songs can exist on multiple playlists </span>\n",
-    "\n",
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Remove the cat Data Records from the training data collection. They already exist in the \"Cats\" Collection. <br> **Hint**: The function call is very similar to the function call for adding cats to the \"Cats\" collection </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "(, 'ListingReply')\n"
-     ]
-    }
-   ],
-   "source": [
-    "cup_resp = df_api.collectionItemsUpdate(train_coll_id, rem_ids=cat_rec_ids)\n",
-    "print(cup_resp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> View the contents of the training data Collection. <br> You may reuse a code snippet from an earlier cell. <br> Do you see the individual cat Data Records in this collection? </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ls_resp = df_api.collectionItemsList(train_coll_id)\n",
-    "# Iterating through the items in the Collection and only extracting a few items:\n",
-    "for obj in ls_resp[0].item:\n",
-    "    print(obj.id, obj.title)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search or Organize?\n",
-    "If you could always search for your data, what is the benefit to organizing them into collections?\n",
-    "\n",
-    "# Download entire Collection\n",
-    "\n",
-    "### <span style=\"color:blue\"> Note </span>\n",
-    "> <span style=\"color:blue\"> Recall that DataFed can download arbitrarily large number of Records regardless of the physical locations of the DataFed repositories containing the data. </span>\n",
-    "\n",
-    "Let us first make sure we don't already have a directory with the desired name:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dest_dir = './cat_data'\n",
-    "\n",
-    "if os.path.exists(dest_dir):\n",
-    "    import shutil\n",
-    "    shutil.rmtree(dest_dir)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### <span style=\"color:green\"> Exercise </span>\n",
-    "<span style=\"color:green\"> Download the entire Cat Collection with a single DataFed function call. <br> **Hint:** You may want to look at a function we used in the third notebook </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(task {\n",
-       "   id: \"task/43667334\"\n",
-       "   type: TT_DATA_GET\n",
-       "   status: TS_SUCCEEDED\n",
-       "   client: \"u/somnaths\"\n",
-       "   step: 2\n",
-       "   steps: 3\n",
-       "   msg: \"Finished\"\n",
-       "   ct: 1615407332\n",
-       "   ut: 1615407338\n",
-       "   source: \"d/43659103, d/43659126, d/43659149, d/43659172, d/43659195, ...\"\n",
-       "   dest: \"1646e89e-f4f0-11e9-9944-0a8c187e8c12/Users/syz/OneDrive - Oak Ridge National Laboratory/DataFed_Tutorial/Notebooks/cat_data\"\n",
-       " },\n",
-       " 'TaskDataReply')"
-      ]
-     },
-     "execution_count": 24,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df_api.dataGet(cat_coll_id, './cat_data', wait=True)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Let's verify that we did infact download the data:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['43659195.md5sum',\n",
-       " '43659103.md5sum',\n",
-       " '43659172.md5sum',\n",
-       " '43659126.md5sum',\n",
-       " '43659149.md5sum']"
-      ]
-     },
-     "execution_count": 25,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "os.listdir(dest_dir)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## <span style=\"color:green\"> Optional Exercise </span>\n",
-    "<span style=\"color:green\">1. Create a new Collection to hold the simulation data you created in the previous notebook <br>2. Use the functions you saw above to ensure that the Data Records only exist in the Simulation Collection </span>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "anaconda-cloud": {},
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# 4. Collections and Queries in DataFed\n",
+                "In this notebook, we will be going over creating Collections, viewing contained items, organizing Collections, downloading Collections, and searching for data",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["## Before we begin:\n", "Import necessary libraries"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "import json\n",
+                "from datafed.CommandLib import API",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Instantiate the DataFed API and set ``context`` to the Training project"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api = API()\n", "df_api.setContext('p/trn001')"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Reset this variable to your username or Globus ID so that you work within your own collection by default </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "metadata": {},
+            "outputs": [],
+            "source": ['parent_collection = "somnaths" # Name of this user'],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Overview:\n",
+                "In this notebook, let us assume that we are working on a machine learning problem aimed at putting together training data for a machine learning model. For illustration purposes, we will assume that we aim to train a classifier for classifying animals",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Create Collection\n",
+                "First, let us create a collection to hold all our data. \n",
+                "\n",
+                "We will be using the ``collectionCreate()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 4,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "(coll {\n",
+                        '  id: "c/43657980"\n',
+                        '  title: "Image classification training data"\n',
+                        '  owner: "p/trn001"\n',
+                        "  ct: 1615405612\n",
+                        "  ut: 1615405612\n",
+                        '  parent_id: "c/34558900"\n',
+                        "}\n",
+                        ", 'CollDataReply')\n",
+                    ],
+                }
+            ],
+            "source": [
+                "coll_resp = df_api.collectionCreate('Image classification training data',\n",
+                "                                    parent_id=parent_collection)\n",
+                "print(coll_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In this case we got back a ``CollDataReply`` object. This is somewhat similar to what you get from ``dataCreate()`` we just saw. \n",
+                "\n",
+                "Now, let's Extract the ``id`` of this newly created collection:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 5,
+            "metadata": {},
+            "outputs": [
+                {"name": "stdout", "output_type": "stream", "text": ["c/43657980\n"]}
+            ],
+            "source": [
+                "train_coll_id = coll_resp[0].coll[0].id\n",
+                "print(train_coll_id)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Populate with training data\n",
+                "Now that we have a place to put the training data, let us populate this collection with examples of animals\n",
+                "### Define a function to generate (fake) training data:\n",
+                "We need a function to:\n",
+                "* Create a Data Record\n",
+                "* Put data into this Data Record\n",
+                "\n",
+                "For simplicity we will use some dummy data from a public Globus Endpoint This information has been filled in for you via the ``raw_data_path`` variable. \n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> We have a skeleton function prepared for you along with comments to guide you. </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 6,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import random\n",
+                "\n",
+                "def generate_animal_data(is_dog=True):\n",
+                "    this_animal = 'cat'\n",
+                "    if is_dog:\n",
+                "        this_animal = 'dog'\n",
+                '    # Ideally, we would have more sensible titles such as "Doberman", "Poodle", etc. instead of "Dog_234"\n',
+                "    # To mimic a real-life scenario, we append a number to the animal type to denote\n",
+                "    # the N-th example of a cat or dog. In this case, we use a random integer.\n",
+                "    title = this_animal + '_' + str(random.randint(1, 1000))\n",
+                "    # Create the record here:\n",
+                "    rec_resp = df_api.dataCreate(title,\n",
+                "                                 metadata=json.dumps({'animal': this_animal}),\n",
+                "                                 parent_id=train_coll_id)                   \n",
+                "                       \n",
+                "    # Extract the ID of the Record:\n",
+                "    this_rec_id = rec_resp[0].data[0].id                   \n",
+                "    \n",
+                "    # path to the file containing the (dummy) raw data\n",
+                "    raw_data_path = 'sdss#public/uufs/chpc.utah.edu/common/home/sdss/dr10/apogee/spectro/data/55574/55574.md5sum'\n",
+                "    \n",
+                "    # Put the raw data into the record you just created: \n",
+                "    put_resp = df_api.dataPut(this_rec_id, raw_data_path)\n",
+                "    \n",
+                "    # Only return the ID of the Data Record you created:\n",
+                "    return this_rec_id",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["#### Generate 5 examples of cats and dogs:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 7,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "cat_records = list()\n",
+                "dog_records = list()\n",
+                "for _ in range(5):\n",
+                "    dog_records.append(generate_animal_data(is_dog=True))\n",
+                "for _ in range(5):\n",
+                "    cat_records.append(generate_animal_data(is_dog=False))",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 8,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "['d/43659103', 'd/43659126', 'd/43659149', 'd/43659172', 'd/43659195']\n"
+                    ],
+                }
+            ],
+            "source": ["print(cat_records)"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 9,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "['d/43658988', 'd/43659011', 'd/43659034', 'd/43659057', 'd/43659080']\n"
+                    ],
+                }
+            ],
+            "source": ["print(dog_records)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Listing items in a Collection:\n",
+                "Let us take a look at the training data we have assembled so far using the ``colectionItemsList()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 11,
+            "metadata": {"scrolled": true},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "(item {\n",
+                        '  id: "d/43658988"\n',
+                        '  title: "dog_196"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659034"\n',
+                        '  title: "dog_57"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659080"\n',
+                        '  title: "dog_686"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659011"\n',
+                        '  title: "dog_689"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659057"\n',
+                        '  title: "dog_778"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "offset: 5\n",
+                        "count: 20\n",
+                        "total: 10\n",
+                        ", 'ListingReply')\n",
+                    ],
+                }
+            ],
+            "source": [
+                "coll_list_resp = df_api.collectionItemsList(train_coll_id, offset=5)\n",
+                "print(coll_list_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> If we had several dozens, hundreds, or even thousands of items in a Collection, we would need to call ``collectionItemsList()`` multiple times by stepping up the ``offset`` keyword argument each time to get the next “page” of results. </span>\n',
+                "\n",
+                '### <span style="color:green"> Discussion </span>\n',
+                '<span style="color:green"> Let\'s say that we are only interested in finding records that have cats in this (potentially) large collection of training data. How do we go about doing that? </span>\n',
+                "\n",
+                "# Data Query / Search\n",
+                '### <span style="color:red"> Caution </span>\n',
+                '> <span style="color:red"> Search vocabulary is likely to change with newer versions of DataFed </span>\n',
+                "\n",
+                "Use the DataFed web interface to:\n",
+                "* Search for cats\n",
+                "* Specifically in your collection\n",
+                "* Save the query\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Saved queries can be found in the bottom of the navigation (left) pane under ``Project Data`` and ``Saved Queries`` </span>\n',
+                "\n",
+                "# Find saved queries:\n",
+                "We can list all saved queries via ``queryList()``:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 12,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "(item {\n",
+                        '  id: "q/43664114"\n',
+                        '  title: "is_cat"\n',
+                        "}\n",
+                        "offset: 0\n",
+                        "count: 20\n",
+                        "total: 1\n",
+                        ", 'ListingReply')\n",
+                    ],
+                }
+            ],
+            "source": ["ql_resp = df_api.queryList()\n", "print(ql_resp)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Notice that we again recieved the familiar ``ListingReply`` object as the response\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Get the ``id`` of the desired query out of the response: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 13,
+            "metadata": {},
+            "outputs": [
+                {"name": "stdout", "output_type": "stream", "text": ["q/43664114\n"]}
+            ],
+            "source": ["query_id = ql_resp[0].item[0].id\n", "print(query_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["# View the saved query\n", "Use the ``queryView()`` function:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 14,
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "(query {\n",
+                            '   id: "q/43664114"\n',
+                            '   title: "is_cat"\n',
+                            '   query: "{\\"meta\\":\\"md.animal == \\\\\\"cat\\\\\\"\\",\\"scopes\\":[{\\"scope\\":4,\\"id\\":\\"c/43657980\\",\\"recurse\\":true}]}"\n',
+                            '   owner: "u/somnaths"\n',
+                            "   ct: 1615406400\n",
+                            "   ut: 1615406400\n",
+                            " },\n",
+                            " 'QueryDataReply')",
+                        ]
+                    },
+                    "execution_count": 14,
+                    "metadata": {},
+                    "output_type": "execute_result",
+                }
+            ],
+            "source": ["df_api.queryView(query_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["# Run the saved query\n", "Use the ``queryExec()`` function:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 15,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "(item {\n",
+                        '  id: "d/43659103"\n',
+                        '  title: "cat_209"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659126"\n',
+                        '  title: "cat_511"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659149"\n',
+                        '  title: "cat_341"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659172"\n',
+                        '  title: "cat_558"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        "item {\n",
+                        '  id: "d/43659195"\n',
+                        '  title: "cat_821"\n',
+                        '  owner: "p/trn001"\n',
+                        '  creator: "u/somnaths"\n',
+                        "  size: 2842.0\n",
+                        "  notes: 0\n",
+                        "}\n",
+                        ", 'ListingReply')\n",
+                    ],
+                }
+            ],
+            "source": [
+                "query_resp = df_api.queryExec(query_id)\n",
+                "print(query_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Yet again, we get back the ``ListingReply`` message. \n",
+                "\n",
+                "Now let us extract just the ``id``s from each of the items in the message:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 16,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "['d/43659103', 'd/43659126', 'd/43659149', 'd/43659172', 'd/43659195']\n"
+                    ],
+                }
+            ],
+            "source": [
+                "cat_rec_ids = list()\n",
+                "for record in query_resp[0].item:\n",
+                "    cat_rec_ids.append(record.id)\n",
+                "# one could also use list comprehensions to get the answer in one line:\n",
+                "# cat_rec_ids = [record.id for record in query_resp[0].item]\n",
+                "print(cat_rec_ids)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "We already have the ground truth in ``cat_records``. Is this the same as what we got from the query?"
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 17,
+            "metadata": {},
+            "outputs": [
+                {"name": "stdout", "output_type": "stream", "text": ["True\n"]}
+            ],
+            "source": ["print(set(cat_rec_ids) == set(cat_records))"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Separating cats from dogs\n",
+                "Our goal now is to gather all cat Data Records into a dedicated Collection\n",
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Create a new collection to hold the Cats record </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 18,
+            "metadata": {},
+            "outputs": [
+                {"name": "stdout", "output_type": "stream", "text": ["c/43666045\n"]}
+            ],
+            "source": [
+                "coll_resp = df_api.collectionCreate('Cats', parent_id=train_coll_id)"
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Extract the ``id`` for this Collection: </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 19,
+            "metadata": {},
+            "outputs": [
+                {"name": "stdout", "output_type": "stream", "text": ["c/43666045\n"]}
+            ],
+            "source": ["cat_coll_id = coll_resp[0].coll[0].id\n", "print(cat_coll_id)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Adding Items to Collection\n",
+                "Now let us add only the cat Data Records into this new collection using the ``collectionItemsUpdate()`` function:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 20,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": ["(, 'ListingReply')\n"],
+                }
+            ],
+            "source": [
+                "cup_resp = df_api.collectionItemsUpdate(cat_coll_id, add_ids=cat_rec_ids)\n",
+                "print(cup_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "Unlike most DataFed functions, this function doesn't really return much\n",
+                "\n",
+                "Now, let us view the contents of the Cats Collection to make sure that all Cat Data Records are present in this Collection. \n",
+                "\n",
+                "Just to keep the output clean and short, we will only extract the ID and title of the items",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 21,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "d/43659103 cat_209\n",
+                        "d/43659149 cat_341\n",
+                        "d/43659126 cat_511\n",
+                        "d/43659172 cat_558\n",
+                        "d/43659195 cat_821\n",
+                    ],
+                }
+            ],
+            "source": [
+                "ls_resp = df_api.collectionItemsList(cat_coll_id)\n",
+                "# Iterating through the items in the Collection and only extracting a few items:\n",
+                "for obj in ls_resp[0].item:\n",
+                "    print(obj.id, obj.title)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> View the contents of the main training data Collection. <br> You may use the snippet above if you like and modify it accordingly </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "ls_resp = df_api.collectionItemsList(train_coll_id)\n",
+                "# Iterating through the items in the Collection and only extracting a few items:\n",
+                "for obj in ls_resp[0].item:\n",
+                "    print(obj.id, obj.title)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Data Records can exist in **multiple** Collections just like video or songs can exist on multiple playlists </span>\n',
+                "\n",
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Remove the cat Data Records from the training data collection. They already exist in the "Cats" Collection. <br> **Hint**: The function call is very similar to the function call for adding cats to the "Cats" collection </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 22,
+            "metadata": {},
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": ["(, 'ListingReply')\n"],
+                }
+            ],
+            "source": [
+                "cup_resp = df_api.collectionItemsUpdate(train_coll_id, rem_ids=cat_rec_ids)\n",
+                "print(cup_resp)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> View the contents of the training data Collection. <br> You may reuse a code snippet from an earlier cell. <br> Do you see the individual cat Data Records in this collection? </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "ls_resp = df_api.collectionItemsList(train_coll_id)\n",
+                "# Iterating through the items in the Collection and only extracting a few items:\n",
+                "for obj in ls_resp[0].item:\n",
+                "    print(obj.id, obj.title)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "## Search or Organize?\n",
+                "If you could always search for your data, what is the benefit to organizing them into collections?\n",
+                "\n",
+                "# Download entire Collection\n",
+                "\n",
+                '### <span style="color:blue"> Note </span>\n',
+                '> <span style="color:blue"> Recall that DataFed can download arbitrarily large number of Records regardless of the physical locations of the DataFed repositories containing the data. </span>\n',
+                "\n",
+                "Let us first make sure we don't already have a directory with the desired name:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 23,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "dest_dir = './cat_data'\n",
+                "\n",
+                "if os.path.exists(dest_dir):\n",
+                "    import shutil\n",
+                "    shutil.rmtree(dest_dir)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '### <span style="color:green"> Exercise </span>\n',
+                '<span style="color:green"> Download the entire Cat Collection with a single DataFed function call. <br> **Hint:** You may want to look at a function we used in the third notebook </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 24,
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "(task {\n",
+                            '   id: "task/43667334"\n',
+                            "   type: TT_DATA_GET\n",
+                            "   status: TS_SUCCEEDED\n",
+                            '   client: "u/somnaths"\n',
+                            "   step: 2\n",
+                            "   steps: 3\n",
+                            '   msg: "Finished"\n',
+                            "   ct: 1615407332\n",
+                            "   ut: 1615407338\n",
+                            '   source: "d/43659103, d/43659126, d/43659149, d/43659172, d/43659195, ..."\n',
+                            '   dest: "1646e89e-f4f0-11e9-9944-0a8c187e8c12/Users/syz/OneDrive - Oak Ridge National Laboratory/DataFed_Tutorial/Notebooks/cat_data"\n',
+                            " },\n",
+                            " 'TaskDataReply')",
+                        ]
+                    },
+                    "execution_count": 24,
+                    "metadata": {},
+                    "output_type": "execute_result",
+                }
+            ],
+            "source": ["df_api.dataGet(cat_coll_id, './cat_data', wait=True)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": ["Let's verify that we did infact download the data:"],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 25,
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "['43659195.md5sum',\n",
+                            " '43659103.md5sum',\n",
+                            " '43659172.md5sum',\n",
+                            " '43659126.md5sum',\n",
+                            " '43659149.md5sum']",
+                        ]
+                    },
+                    "execution_count": 25,
+                    "metadata": {},
+                    "output_type": "execute_result",
+                }
+            ],
+            "source": ["os.listdir(dest_dir)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '## <span style="color:green"> Optional Exercise </span>\n',
+                '<span style="color:green">1. Create a new Collection to hold the simulation data you created in the previous notebook <br>2. Use the functions you saw above to ensure that the Data Records only exist in the Simulation Collection </span>',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [],
+        },
+    ],
+    "metadata": {
+        "anaconda-cloud": {},
+        "kernelspec": {
+            "display_name": "Python 3",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.8.3",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4,
 }
diff --git a/jupyter_notebooks/5_Collaborative_Exercise.ipynb b/jupyter_notebooks/5_Collaborative_Exercise.ipynb
index bd5abaa2b..a3e1bd71e 100644
--- a/jupyter_notebooks/5_Collaborative_Exercise.ipynb
+++ b/jupyter_notebooks/5_Collaborative_Exercise.ipynb
@@ -1,177 +1,161 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Collaborative Exercise: <span style=\"color:teal\"> DataFed Zoo </span>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In this exercise, you are encouraged to refer back to the examples and exercises in previous notebooks, feel free to re-use as much code as you'd like. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datafed.CommandLib import API\n",
-    "df_api = API()\n",
-    "df_api.setContext('p/trn001')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Animal Generator: Create 5 records\n",
-    "Similar to the cat/dog creator, define a function that creates a record with the title \"animal\" + \"_\" + \"random numeric suffix\". \n",
-    "\n",
-    "The key difference is that your record metadata **must** contain a standardized field called \"height\".\n",
-    "\n",
-    "Here is sample code for generating random animal heights to 2 decimal places: \n",
-    "\n",
-    "    import random \n",
-    "    metadata = {\"height\": round(random.uniform(min_height, max_height), 2)\n",
-    "\n",
-    "These records should be created in the 'projshare' collection, to which everyone has access.\n",
-    "\n",
-    "Use your function to create 5 new records for the animal of your choice."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def animal_generator(...):\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Search for New Animals: Create a Query\n",
-    "Take a quick look at the 'projshare' collection (either using the API) or the web interface, and pick another animal someone else has made records for, *or* another user whose data you'd like to use. \n",
-    "\n",
-    "Use the web interface to create a query that searches the data for the records made for that animal, or for that user's records.\n",
-    "\n",
-    "Run your query below:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query_results = df_api.queryExec(...)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Isolate New Animals: Find the Record IDs\n",
-    "Iterate over the query_results protobuf message to create a list of IDs for the records that matched your search criteria."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "rec_ids = list(...)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Gather Height Data: Extract Metadata from the Records\n",
-    "Use the dataView() command to get and extract the height metadata for each of the records in your list. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "heights = list()\n",
-    "for rec_id in rec_ids:\n",
-    "    ...\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Make New Data: Find the Average Height\n",
-    "Find the average animal height, and create a new record for the with this value in its metadata, i.e. `metadata={\"average_height\": x}`. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "average_height = ?\n",
-    "\n",
-    "avg_rec = df_api.dataCreate(...)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Track Provenance: Add References to the Input Data Records\n",
-    "Using the `deps_add` keyword argument, update your average record so that it shows which records its data was derived from.\n",
-    "\n",
-    "The deps_add argument is an array of lists or tuples whose elements are: [0] the type of relationship, and [1] the id or alias of the record to whom the relationship will be connected. \n",
-    "For example:\n",
-    "\n",
-    "    deps_add = [('der', record_id_1), ('comp', record_id_2), ...]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_api.dataUpdate(avg_rec, ...)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python [default]",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.5.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                '# Collaborative Exercise: <span style="color:teal"> DataFed Zoo </span>'
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In this exercise, you are encouraged to refer back to the examples and exercises in previous notebooks, feel free to re-use as much code as you'd like. "
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "from datafed.CommandLib import API\n",
+                "df_api = API()\n",
+                "df_api.setContext('p/trn001')",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Animal Generator: Create 5 records\n",
+                'Similar to the cat/dog creator, define a function that creates a record with the title "animal" + "_" + "random numeric suffix". \n',
+                "\n",
+                'The key difference is that your record metadata **must** contain a standardized field called "height".\n',
+                "\n",
+                "Here is sample code for generating random animal heights to 2 decimal places: \n",
+                "\n",
+                "    import random \n",
+                '    metadata = {"height": round(random.uniform(min_height, max_height), 2)\n',
+                "\n",
+                "These records should be created in the 'projshare' collection, to which everyone has access.\n",
+                "\n",
+                "Use your function to create 5 new records for the animal of your choice.",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["def animal_generator(...):\n", "    "],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Search for New Animals: Create a Query\n",
+                "Take a quick look at the 'projshare' collection (either using the API) or the web interface, and pick another animal someone else has made records for, *or* another user whose data you'd like to use. \n",
+                "\n",
+                "Use the web interface to create a query that searches the data for the records made for that animal, or for that user's records.\n",
+                "\n",
+                "Run your query below:",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["query_results = df_api.queryExec(...)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Isolate New Animals: Find the Record IDs\n",
+                "Iterate over the query_results protobuf message to create a list of IDs for the records that matched your search criteria.",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["rec_ids = list(...)"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Gather Height Data: Extract Metadata from the Records\n",
+                "Use the dataView() command to get and extract the height metadata for each of the records in your list. ",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["heights = list()\n", "for rec_id in rec_ids:\n", "    ...\n"],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Make New Data: Find the Average Height\n",
+                'Find the average animal height, and create a new record for the with this value in its metadata, i.e. `metadata={"average_height": x}`. ',
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "average_height = ?\n",
+                "\n",
+                "avg_rec = df_api.dataCreate(...)",
+            ],
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Track Provenance: Add References to the Input Data Records\n",
+                "Using the `deps_add` keyword argument, update your average record so that it shows which records its data was derived from.\n",
+                "\n",
+                "The deps_add argument is an array of lists or tuples whose elements are: [0] the type of relationship, and [1] the id or alias of the record to whom the relationship will be connected. \n",
+                "For example:\n",
+                "\n",
+                "    deps_add = [('der', record_id_1), ('comp', record_id_2), ...]",
+            ],
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": ["df_api.dataUpdate(avg_rec, ...)"],
+        },
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python [default]",
+            "language": "python",
+            "name": "python3",
+        },
+        "language_info": {
+            "codemirror_mode": {"name": "ipython", "version": 3},
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.5.5",
+        },
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5,
 }
diff --git a/python/datafed_pkg/datafed/CommandLib.py b/python/datafed_pkg/datafed/CommandLib.py
index a4016e52b..3f4ad8098 100644
--- a/python/datafed_pkg/datafed/CommandLib.py
+++ b/python/datafed_pkg/datafed/CommandLib.py
@@ -13,7 +13,7 @@
 import json as jsonlib
 import time
 import pathlib
-import wget
+import requests
 from . import SDMS_Auth_pb2 as auth
 from . import SDMS_pb2 as sdms
 from . import MessageLib
@@ -2694,6 +2694,11 @@ def _setSaneDefaultOptions(self):
             opts["server_port"] = 7512
             save = True
 
+        if "allow_self_signed_certs" not in opts:
+            self.cfg.set("allow_self_signed_certs", False)
+            opts["allow_self_signed_certs"] = False
+            save = True
+
         if "server_pub_key_file" not in opts:
             serv_key_file = None
 
@@ -2722,7 +2727,28 @@ def _setSaneDefaultOptions(self):
                 if not os.path.exists(serv_key_file):
                     # Make default server pub key file
                     url = "https://" + opts["server_host"] + "/datafed-core-key.pub"
-                    wget.download(url, out=serv_key_file)
+
+                    # Path where the downloaded file will be saved
+                    output_path = serv_key_file
+
+                    try:
+                        # Make the request and allow self-signed certificates if needed
+                        if opts["allow_self_signed_certs"]:
+                            response = requests.get(url, verify=False, stream=True)
+                        else:
+                            response = requests.get(url, verify=True, stream=True)
+                        # Check if the request was successful
+                        response.raise_for_status()
+
+                        # Open the file in binary write mode and write the response content
+                        with open(output_path, "wb") as file:
+                            for chunk in response.iter_content(chunk_size=8192):
+                                if chunk:  # Filter out keep-alive new chunks
+                                    file.write(chunk)
+                        print("File downloaded successfully.")
+
+                    except requests.exceptions.RequestException as e:
+                        print(f"Failed to download file: {e}")
 
         if "client_pub_key_file" not in opts or "client_priv_key_file" not in opts:
             if "client_cfg_dir" not in opts:
diff --git a/python/datafed_pkg/datafed/Config.py b/python/datafed_pkg/datafed/Config.py
index 1af86477f..f88a66aac 100644
--- a/python/datafed_pkg/datafed/Config.py
+++ b/python/datafed_pkg/datafed/Config.py
@@ -108,6 +108,17 @@
         ["--client-token"],
         "Client access token",
     ],
+    "allow_self_signed_certs": [
+        "client",
+        "allow_self_signed_certs",
+        "DATAFED_PYTHON_CLIENT_ALLOW_SELF_SIGNED_CERTS",
+        _OPT_BOOL,
+        ["--allow-self-signed-certs"],
+        (
+            "Allows client download from DATAFED_DOMAIN if it is using self signed"
+            " certs, needed when testing."
+        ),
+    ],
     "default_ep": [
         "general",
         "default_endpoint",
diff --git a/python/datafed_pkg/requirements.txt b/python/datafed_pkg/requirements.txt
index c928d7e3c..011143126 100644
--- a/python/datafed_pkg/requirements.txt
+++ b/python/datafed_pkg/requirements.txt
@@ -1,5 +1,6 @@
 protobuf>=4.21.1
 pyzmq>=16
 wget>=3
+requests>=2
 click>=7
 prompt_toolkit>=2
diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs
index 4e216affa..cbc028d73 100644
--- a/repository/docker/Dockerfile.gcs
+++ b/repository/docker/Dockerfile.gcs
@@ -36,6 +36,9 @@ ENV          DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs"
 # Value needed so tput command doesn't crash
 ENV                              TERM="xterm"
 ENV                    DATAFED_GCS_IP=""
+ENV      BUILD_WITH_METADATA_SERVICES="TRUE"
+ENV                    DATAFED_DOMAIN="localhost"
+ENV               DATAFED_REPO_DOMAIN="localhost"
 
 RUN mkdir -p ${BUILD_DIR}
 RUN mkdir -p ${DATAFED_INSTALL_PATH}/logs
@@ -98,6 +101,7 @@ COPY --from=dependencies /usr/bin/as   /usr/bin/as
 COPY --from=dependencies /usr/bin/ld   /usr/bin/ld
 COPY --from=dependencies /usr/bin/jq   /usr/bin/jq
 COPY --from=dependencies /usr/bin/sudo /usr/bin/sudo
+COPY --from=dependencies /usr/bin/host /usr/bin/host
 
 COPY --from=dependencies /etc/sudoers /etc/sudoers
 
@@ -142,8 +146,6 @@ COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/r
 
 USER root
 
-COPY ./repository/docker/000-default.conf    /etc/apache2/sites-available/000-default.conf
-RUN chmod 644 /etc/apache2/sites-available/000-default.conf
 # Remove --client-id from command because it was deprecated
 RUN sed -i '/--client-id/d' /entrypoint.sh
 
diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh
index b32549cc6..2680ae2c5 100755
--- a/repository/docker/entrypoint_authz.sh
+++ b/repository/docker/entrypoint_authz.sh
@@ -24,6 +24,47 @@ export GLOBUS_CLIENT_ID=$(jq -r .client < /opt/datafed/globus/client_cred.json)
 export GLOBUS_CLIENT_SECRET=$(jq -r .secret < /opt/datafed/globus/client_cred.json)
 export DEPLOYMENT_KEY=$(cat "$DEPLOYMENT_KEY_PATH"  )
 
+if [ "$BUILD_WITH_METADATA_SERVICES" == "TRUE" ]
+then
+
+cat <<EOF >> /etc/apache2/sites-available/000-default.conf
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
+# This block is needed if core web services are running on the same machine as
+# the GCS
+<VirtualHost *:443>
+		ServerName ${DATAFED_DOMAIN}
+ 
+    SSLEngine on
+    SSLCertificateFile /opt/datafed/keys/cert.crt
+    SSLCertificateKeyFile /opt/datafed/keys/cert.key
+ 
+    # SSL configuration
+    SSLProtocol TLSv1.2 TLSv1.3
+    SSLCipherSuite EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
+    SSLHonorCipherOrder on
+ 
+    # Proxy settings
+    ProxyPass / https://localhost:8080/
+    ProxyPassReverse / https://localhost:8080/
+    ProxyPreserveHost On
+    RequestHeader set X-Forwarded-Proto "https"
+ 
+    # Additional proxy SSL settings
+    SSLProxyEngine on
+    SSLProxyVerify none
+    SSLProxyCheckPeerCN off
+    SSLProxyCheckPeerName off
+    SSLProxyCheckPeerExpire off
+ 
+    SSLProxyVerifyDepth 2
+    SSLProxyCACertificateFile /opt/datafed/keys/cert.crt
+
+</VirtualHost>
+
+EOF
+fi
+
 if [ -n "$UID" ]; then
     echo "Switching datafed user to UID: ${UID}"
     usermod -u $UID datafed
@@ -164,9 +205,22 @@ GCS_CLI_CLIENT_SECRET="$GCS_CLI_CLIENT_SECRET" \
 DATAFED_REPO_USER="$DATAFED_REPO_USER" \
   python3 "${BUILD_DIR}/scripts/globus/create_guest_collection.py"
 
-"${BUILD_DIR}/scripts/globus/generate_repo_form.sh"
+"${BUILD_DIR}/scripts/globus/generate_repo_form.sh" -j -s
+
+# Why is this approach being used? 
+# 
+# Wild card expansion with *form was not working from within the script.
+find /opt/datafed/authz/ -name '*form.json' -or -name '*form.sh' | while read -r file; do
+    if [ -e "$file" ]; then
+        echo "Moving $file to /opt/datafed/globus/"
+        mv "$file" /opt/datafed/globus/
+    else
+        echo "No matching files found for pattern: $file"
+    fi
+done
 
 echo "Container is running."
 
 # Return to last file
 tail -f "${DATAFED_DEFAULT_LOG_PATH}/datafed-gsi-authz.log"
+
diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh
index e728d9d9d..2b6bc6206 100755
--- a/repository/docker/entrypoint_repo.sh
+++ b/repository/docker/entrypoint_repo.sh
@@ -46,4 +46,6 @@ fi
 
 # Allow the container to exist for a bit in case we need to jump in and look 
 # around
+echo "Container sleeping"
 sleep 10000
+echo "Container exiting after sleep"
diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp
index c58acb23a..36b33dbbf 100644
--- a/repository/server/RepoServer.cpp
+++ b/repository/server/RepoServer.cpp
@@ -177,7 +177,10 @@ void Server::checkServerVersion() {
     if (response.time_out) {
       DL_ERROR(msg_log_context,
                "Timeout waiting for response from core server: "
-                   << m_config.core_server);
+                   << m_config.core_server
+                   << " NOTE: The core server will be unable to communicate "
+                   << "with the repo server until it has been registered and"
+                   << " its public key is known.");
     } else if (response.error) {
       DL_ERROR(msg_log_context, "Error encountered waiting for core server: "
                                     << m_config.core_server << " msg "
diff --git a/scripts/compose_generate_env.sh b/scripts/compose_generate_env.sh
index 14751e546..f9c6e020b 100755
--- a/scripts/compose_generate_env.sh
+++ b/scripts/compose_generate_env.sh
@@ -110,12 +110,23 @@ then
 else
   local_DATAFED_COMPOSE_USER89_PASSWORD=$(printenv DATAFED_COMPOSE_USER89_PASSWORD)
 fi
+
+if [ -z "${DATAFED_COMPOSE_REPO_DOMAIN}" ]
+then
+  # Make the repo domain equivalent to the COMPOSE DOMAIN unless it is specified
+  # explicitly
+  local_DATAFED_COMPOSE_REPO_DOMAIN="${local_DATAFED_COMPOSE_DOMAIN}"
+else
+  local_DATAFED_COMPOSE_REPO_DOMAIN=$(printenv DATAFED_COMPOSE_REPO_DOMAIN)
+fi
+
 if [ -z "${DATAFED_COMPOSE_REPO_FORM_PATH}" ]
 then
   local_DATAFED_COMPOSE_REPO_FORM_PATH="" # Where the repo form is located also needed for testing
 else
   local_DATAFED_COMPOSE_REPO_FORM_PATH=$(printenv DATAFED_COMPOSE_REPO_FORM_PATH)
 fi
+
 if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_SECRET}" ]
 then
   local_DATAFED_COMPOSE_GLOBUS_APP_SECRET=""
@@ -275,6 +286,7 @@ DATAFED_HOST_DEPLOYMENT_KEY_PATH=${local_DATAFED_COMPOSE_HOST_DEPLOYMENT_KEY_PAT
 DATAFED_HOST_CRED_FILE_PATH=${local_DATAFED_HOST_CRED_FILE_PATH}
 DATAFED_GLOBUS_CONTROL_PORT=${local_DATAFED_GLOBUS_CONTROL_PORT}
 DATAFED_GLOBUS_SUBSCRIPTION=${local_DATAFED_GLOBUS_SUBSCRIPTION}
+DATAFED_REPO_DOMAIN=${local_DATAFED_COMPOSE_REPO_DOMAIN}
 EOF
 fi
 
diff --git a/scripts/container_stop.sh b/scripts/container_stop.sh
index 101881b8a..308485af2 100755
--- a/scripts/container_stop.sh
+++ b/scripts/container_stop.sh
@@ -11,7 +11,7 @@ Help()
 {
   echo "$(basename $0) Will stop containers."
   echo
-  echo "Syntax: $(basename $0) [-h|n]"
+  echo "Syntax: $(basename $0) [-h|n|t|p]"
   echo
   echo "Default:"
   echo "If no arguments are provided will stop all running containers."
@@ -20,15 +20,17 @@ Help()
   echo "-h, --help                     Print this help message"
   echo "-n, --name                     Remove this particular container provide the name"
   echo "-t, --tag                      Remove all containers with this particular image"
+  echo "-p, --prefix                   Only require the prefix to match and not"
+  echo "                               an exact match"
 } 
 
 local_NAME_FLAG_DETECTED="0"
 local_TAG_FLAG_DETECTED="0"
-
+local_EXACT_MATCH="TRUE"
 # Time in seconds
 SLEEP_TIME=30
 
-VALID_ARGS=$(getopt -o hn:t: --long 'help',name:,tag: -- "$@")
+VALID_ARGS=$(getopt -o hn:t:p --long 'help',name:,tag:,prefix -- "$@")
 if [[ $? -ne 0 ]]; then
       exit 1;
 fi
@@ -49,6 +51,10 @@ while [ : ]; do
         local_TAG_FLAG_DETECTED="1"
         shift 2
         ;;
+    -p | --prefix)
+        local_EXACT_MATCH="FALSE"
+        shift 1
+        ;;
     --) shift; 
         break 
         ;;
@@ -74,10 +80,19 @@ then
 
   for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do
     IMAGE="${CONTAINER_IMAGES[$i]}"
-    if [ "$IMAGE" == "$local_CONTAINER_TAG" ]
+    if [ "${local_EXACT_MATCH}" == "TRUE" ]
     then
-      echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE"
-      docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}"
+      if [ "$IMAGE" == "$local_CONTAINER_TAG" ]
+      then
+        echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE"
+        docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}"
+      fi
+    else
+      if [[ "$IMAGE" == "$local_CONTAINER_TAG"* ]]
+      then
+        echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE"
+        docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}"
+      fi
     fi
   done
 
@@ -86,10 +101,19 @@ then
 
   for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do
     NAME="${CONTAINER_NAMES[$i]}"
-    if [ "$NAME" == "$local_CONTAINER_NAME" ]
+    if [ "${local_EXACT_MATCH}" == "TRUE" ]
     then
-      echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}"
-      docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}"
+      if [ "$NAME" == "$local_CONTAINER_NAME" ]
+      then
+        echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}"
+        docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}"
+      fi
+    else
+      if [[ "$NAME" == "$local_CONTAINER_NAME"* ]]
+      then
+        echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}"
+        docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}"
+      fi
     fi
   done
 
@@ -117,10 +141,19 @@ then
 
   for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do
     IMAGE="${CONTAINER_IMAGES[$i]}"
-    if [ "$IMAGE" == "$local_CONTAINER_TAG" ]
+    if [ "${local_EXACT_MATCH}" == "TRUE" ]
     then
-      echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE"
-      exit 1
+      if [ "$IMAGE" == "$local_CONTAINER_TAG" ]
+      then
+        echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE"
+        exit 1
+      fi
+    else
+      if [[ "$IMAGE" == "$local_CONTAINER_TAG"* ]]
+      then
+        echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE"
+        exit 1
+      fi
     fi
   done
 
@@ -129,10 +162,19 @@ then
 
   for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do
     NAME="${CONTAINER_NAMES[$i]}"
-    if [ "$NAME" == "$local_CONTAINER_NAME" ]
+    if [ "${local_EXACT_MATCH}" == "TRUE" ]
     then
-      echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}"
-      exit 1
+      if [ "$NAME" == "$local_CONTAINER_NAME" ]
+      then
+        echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}"
+        exit 1
+      fi
+    else
+      if [[ "$NAME" == "$local_CONTAINER_NAME"* ]]
+      then
+        echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}"
+        exit 1
+      fi
     fi
   done
 
diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh
index ce6924ee9..223aae09d 100644
--- a/scripts/dependency_install_functions.sh
+++ b/scripts/dependency_install_functions.sh
@@ -420,9 +420,9 @@ install_foxx_cli() {
 }
 
 install_arangodb() {
-  curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key
+  curl -OL https://download.arangodb.com/arangodb311/DEBIAN/Release.key
   "$SUDO_CMD" apt-key add - < Release.key
-  echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | "$SUDO_CMD" tee /etc/apt/sources.list.d/arangodb.list
+  echo 'deb https://download.arangodb.com/arangodb311/DEBIAN/ /' | "$SUDO_CMD" tee /etc/apt/sources.list.d/arangodb.list
   "$SUDO_CMD" apt-get install apt-transport-https
   "$SUDO_CMD" apt-get update
   "$SUDO_CMD" apt-get install arangodb3
diff --git a/scripts/globus/generate_repo_form.sh b/scripts/globus/generate_repo_form.sh
index e279a1cce..7d49635e1 100755
--- a/scripts/globus/generate_repo_form.sh
+++ b/scripts/globus/generate_repo_form.sh
@@ -87,6 +87,16 @@ while [ : ]; do
   esac
 done
 
+function validate_domain() {
+  local DOMAIN="$1"
+
+  if host "$DOMAIN" | grep -q "has address"; then
+    echo "DEFINED"  
+  else
+    echo "UNDEFINED"
+  fi
+}
+
 
 public_key=$(cat ${DATAFED_INSTALL_PATH}/keys/datafed-repo-key.pub)
 
@@ -108,11 +118,25 @@ fi
 
 # Probably should grab this from the config file
 local_DATAFED_REPO_EGRESS_PORT="9000"
-repo_domain_name=$(domainname -A | awk '{print $1}')
+if [ -z "${DATAFED_REPO_DOMAIN}" ]
+then
+  repo_domain_name=""
+else
+  repo_domain_name=$(printenv DATAFED_REPO_DOMAIN)
+fi
+
+if [ "${repo_domain_name}" == "" ]
+then
+  repo_domain_name=$(domainname -A | awk '{print $1}')
+fi
 
-if [ -z "$repo_domain_name" ]
+local_DEFINED=$(validate_domain "$repo_domain_name")
+if [ "${local_DEFINED}" == "UNDEFINED" ] || [ -z "$repo_domain_name" ]
 then
-  echo "Unable to identify domain name of server."
+  echo "domain name (${repo_domain_name}) is ${local_DEFINED} using local IP."
+  local_address=$(hostname -I | awk '{print $1}')
+else
+  local_address="$repo_domain_name"
 fi
 
 if [ "$local_GENERATE_REPO_FORM_SCRIPT" = "TRUE" ]
@@ -120,13 +144,13 @@ then
   OUTPUT_SCRIPT_NAME="${DATAFED_REPO_ID_AND_DIR}-repo-form.sh"
   echo "Creating ${OUTPUT_SCRIPT_NAME} file"
   echo "export DATAFED_REPO_ID=\"$DATAFED_REPO_ID_AND_DIR\"" > ${OUTPUT_SCRIPT_NAME}
-  echo "export DATAFED_REPO_TITLE=\"\"" >> ${OUTPUT_SCRIPT_NAME}
-  echo "export DATAFED_REPO_DESCRIPTION=\"\"" >> ${OUTPUT_SCRIPT_NAME}
-  echo "export DATAFED_REPO_SERVER_ADDRESS=\"tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "export DATAFED_REPO_TITLE=\"$DATAFED_GCS_ROOT_NAME\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "export DATAFED_REPO_DESCRIPTION=\"Autogenerated\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "export DATAFED_REPO_SERVER_ADDRESS=\"tcp://$local_address:$local_DATAFED_REPO_EGRESS_PORT\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "export DATAFED_REPO_PUBLIC_KEY=\"$public_key\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "export DATAFED_REPO_ENDPOINT_UUID=\"$uuid_of_collection\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "export DATAFED_REPO_RELATIVE_PATH=\"${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR\"" >> ${OUTPUT_SCRIPT_NAME}
-  echo "export DATAFED_REPO_DOMAIN=\"\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "export DATAFED_REPO_DOMAIN=\"${repo_domain_name}\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "export DATAFED_REPO_EXPORT_PATH=\"\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "export DATAFED_REPO_CAPACITY=\"\"" >> ${OUTPUT_SCRIPT_NAME}
 fi
@@ -136,13 +160,13 @@ then
   OUTPUT_SCRIPT_NAME="${DATAFED_REPO_ID_AND_DIR}-repo-form.cfg"
   echo "Creating ${OUTPUT_SCRIPT_NAME} file"
   echo "id=\"$DATAFED_REPO_ID_AND_DIR\"" > ${OUTPUT_SCRIPT_NAME}
-  echo "title=\"\"" >> ${OUTPUT_SCRIPT_NAME}
-  echo "desc=\"\"" >> ${OUTPUT_SCRIPT_NAME}
-  echo "address=\"tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "title=\"$DATAFED_GCS_ROOT_NAME\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "desc=\"Autogenerated\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "address=\"tcp://$local_address:$local_DATAFED_REPO_EGRESS_PORT\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "pub_key=\"$public_key\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "endpoint=\"$uuid_of_collection\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "path=\"${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR\"" >> ${OUTPUT_SCRIPT_NAME}
-  echo "domain=\"\"" >> ${OUTPUT_SCRIPT_NAME}
+  echo "domain=\"${repo_domain_name}\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "exp_path=\"\"" >> ${OUTPUT_SCRIPT_NAME}
   echo "capacity=\"\"" >> ${OUTPUT_SCRIPT_NAME}
 fi
@@ -154,13 +178,13 @@ then
   echo "Creating ${OUTPUT_SCRIPT_NAME} script"
   echo "{" > ${OUTPUT_SCRIPT_NAME}
   echo "  \"id\": \"$DATAFED_REPO_ID_AND_DIR\"," >> ${OUTPUT_SCRIPT_NAME}
-  echo "  \"title\": \"\"," >> ${OUTPUT_SCRIPT_NAME}
-  echo "  \"desc\": \"\"," >> ${OUTPUT_SCRIPT_NAME}
-  echo "  \"address\": \"tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT\"," >> ${OUTPUT_SCRIPT_NAME}
+  echo "  \"title\": \"$DATAFED_GCS_ROOT_NAME\"," >> ${OUTPUT_SCRIPT_NAME}
+  echo "  \"desc\": \"Autogenerated\"," >> ${OUTPUT_SCRIPT_NAME}
+  echo "  \"address\": \"tcp://$local_address:$local_DATAFED_REPO_EGRESS_PORT\"," >> ${OUTPUT_SCRIPT_NAME}
   echo "  \"pub_key\": \"$public_key\"," >> ${OUTPUT_SCRIPT_NAME}
   echo "  \"endpoint\": \"$uuid_of_collection\"," >> ${OUTPUT_SCRIPT_NAME}
   echo "  \"path\": \"${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR\"," >> ${OUTPUT_SCRIPT_NAME}
-  echo "  \"domain\": \"\"," >> ${OUTPUT_SCRIPT_NAME}
+  echo "  \"domain\": \"${repo_domain_name}\"," >> ${OUTPUT_SCRIPT_NAME}
   echo "  \"exp_path\": \"\"," >> ${OUTPUT_SCRIPT_NAME}
   echo "  \"capacity\": 0," >> ${OUTPUT_SCRIPT_NAME}
   echo "  \"admins\": [\"\"]" >> ${OUTPUT_SCRIPT_NAME}
@@ -169,15 +193,15 @@ fi
 
 echo "DataFed Repo Form Registration Contents"
 echo "ID: $DATAFED_REPO_ID_AND_DIR"
-echo "Title: Whatever you want to call it"
-echo "Description: A description of the repository."
+echo "Title: $DATAFED_GCS_ROOT_NAME"
+echo "Description: Autogenerated"
 # Should be something like this: tcp://datafed-gcs-test.ornl.gov:9000
 # This is the domain name of the repository server
-echo "Srvr. Address: tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT"
+echo "Srvr. Address: tcp://$local_address:$local_DATAFED_REPO_EGRESS_PORT"
 echo "Public Key: $public_key"
 echo "End-point ID: $uuid_of_collection"
 echo "Path: ${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR"
-echo "Domain: "
+echo "Domain: ${repo_domain_name}"
 # I don't know what this is
 echo "Export Path: "
 echo "Capacity: The capacity of the repository"
diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh
index 68142b5a4..a63e37314 100755
--- a/scripts/install_authz_dependencies.sh
+++ b/scripts/install_authz_dependencies.sh
@@ -10,7 +10,7 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..)
 source "${PROJECT_ROOT}/scripts/utils.sh"
 source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
 
-packages=("libtool" "build-essential" "g++" "gcc" "autoconf"
+packages=("host" "libtool" "build-essential" "g++" "gcc" "autoconf"
   "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config"
   "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev" "python3-venv")
 pip_packages=("setuptools" "distro" "jwt" "globus_sdk")
diff --git a/scripts/install_foxx.sh b/scripts/install_foxx.sh
index 78d84414c..30dfed2b8 100755
--- a/scripts/install_foxx.sh
+++ b/scripts/install_foxx.sh
@@ -187,8 +187,11 @@ fi
 # syntax for the REST http endpoints with curl so we are going to try the node module
 
 # 1. Install nvm which will allow us to update node
+echo "Installing nvm"
 install_nvm
+echo "Installing node"
 install_node
+echo "Installing foxx_cli"
 install_foxx_cli
 
 FOXX_PREFIX=""


Test patch:
diff --git a/tests/end-to-end/CMakeLists.txt b/tests/end-to-end/CMakeLists.txt
index 9a379ad90..839931c74 100644
--- a/tests/end-to-end/CMakeLists.txt
+++ b/tests/end-to-end/CMakeLists.txt
@@ -9,6 +9,7 @@ if( ENABLE_END_TO_END_TESTS )
   add_test(NAME end_to_end_repo COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/test_api_repo.py")
   add_test(NAME end_to_end_alloc COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/test_api_alloc.py")
   add_test(NAME end_to_end_collection COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/test_api_collection.py")
+  add_test(NAME end_to_end_record COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/test_api_record.py")
 
   # Note because these tests are all using the same database we cannot run most of them concurrently
   # They must be run sequentially so that concurrent API calls do not create problems
@@ -24,5 +25,6 @@ if( ENABLE_END_TO_END_TESTS )
   set_tests_properties(end_to_end_alloc PROPERTIES FIXTURES_REQUIRED FIX_REPO)
   set_tests_properties(end_to_end_alloc PROPERTIES FIXTURES_SETUP FIX_ALLOC)
   set_tests_properties(end_to_end_collection PROPERTIES FIXTURES_REQUIRED FIX_ALLOC)
+  set_tests_properties(end_to_end_record PROPERTIES FIXTURES_REQUIRED FIX_ALLOC)
 
 endif()
diff --git a/tests/end-to-end/setUp.py b/tests/end-to-end/setUp.py
deleted file mode 100755
index 504bd8102..000000000
--- a/tests/end-to-end/setUp.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/python3
-
-import json
-import os
-import sys
-import time
-from datafed import version as df_ver
-
-path_of_file = os.path.abspath(__file__)
-current_folder = os.path.dirname(path_of_file)
-path_to_python_datafed_module = os.path.normpath(
-    current_folder + os.sep + ".." + os.sep + ".." + os.sep + "python/datafed_pkg"
-)
-sys.path.append(path_to_python_datafed_module)
-try:
-    from datafed.CommandLib import API
-except ImportError:
-    print(
-        "datafed was not found, make sure you are running script with "
-        "PYTHONPATH set to the location of the package in the datafed repo"
-    )
-    sys.exit(1)
-
-
-print(df_ver)
-
-opts = {"server_host": "datafed-server-test.ornl.gov"}
-_df_api = API(opts)
-
-username = "datafed89"
-password = os.environ.get("DATAFED_USER89_PASSWORD")
-_df_api.loginByPassword(username, password)
-
-path_to_repo_form = os.environ.get("DATAFED_REPO_FORM_PATH")
-
-_repo_form = {}
-with open(path_to_repo_form) as json_file:
-    _repo_form = json.load(json_file)
-
-# Create the repositories
-result = _df_api.repoCreate(
-    repo_id=_repo_form["id"],
-    title=_repo_form["title"],
-    desc=_repo_form["desc"],
-    domain=_repo_form["domain"],
-    capacity=_repo_form["capacity"],
-    pub_key=_repo_form["pub_key"],
-    address=_repo_form["address"],
-    endpoint=_repo_form["endpoint"],
-    path=_repo_form["path"],
-    exp_path=_repo_form["exp_path"],
-    admins=_repo_form["admins"],
-)
-
-
-result = _df_api.repoList(list_all=True)
-count = 0
-while len(result[0].repo) == 0:
-    time.sleep(1)
-    result = _df_api.repoList(list_all=True)
-    count = count + 1
-    if count > 3:
-        raise Exception("Setup failed with repo create")
-
-
-repo_id = _repo_form["id"]
-if not repo_id.startswith("repo/"):
-    repo_id = "repo/" + repo_id
-
-# Will return a task
-result = _df_api.repoAllocationCreate(
-    repo_id=repo_id, subject="datafed89", data_limit=1000000000, rec_limit=100
-)
-
-task_id = result[0].task[0].id
-
-# Check the status of the task
-task_result = _df_api.taskView(task_id)
-
-# If status is less than 3 it is in the works
-status = task_result[0].task[0].status
-count = 0
-while status < 3:
-    if count > 2:
-        print(task_result)
-        raise Exception(
-            "Something went wrong task was unable to complete, attempt to "
-            "create an allocation after 3 seconds failed, make sure all "
-            "services are running."
-        )
-    time.sleep(1)
-    task_result = _df_api.taskView(task_id)
-    status = task_result[0].task[0].status
-    count = count + 1
diff --git a/tests/end-to-end/setup.sh b/tests/end-to-end/setup.sh
index 8f748a7ec..2b9b55926 100755
--- a/tests/end-to-end/setup.sh
+++ b/tests/end-to-end/setup.sh
@@ -161,26 +161,25 @@ echo "IP is ${local_DATAFED_DATABASE_HOST}"
 echo "USER89 GLobud ID $DATAFED_USER89_GLOBUS_UUID"
 echo "Refresh is ${DATAFED_USER89_GLOBUS_REFRESH_TOKEN}"
 # Chreate user datafed89 who is admin
-HTTP_CODE=$( curl --user "${basic_auth}" -w "%{http_code}" -o /dev/null -X GET
-"http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed89&uuids=%5B\"${DATAFED_USER89_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER89_PASSWORD}&email=datafed89%40gmail.com&is_admin=true&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}" )
+HTTP_CODE=$( curl --user "${basic_auth}" -w "%{http_code}" -o /dev/null -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed89&uuids=%5B\"${DATAFED_USER89_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER89_PASSWORD}&email=datafed89%40gmail.com&is_admin=true&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}" )
 echo "HTTP_CODE: ${HTTP_CODE}"
 FIRST_INT=${HTTP_CODE:0:1}
 if [ "${FIRST_INT}" -ne "2" ]
 then
   response=$( curl --user "${basic_auth}" -X GET
-  "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed89&uuids=%5B\"${DATAFED_USER89_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER89_PASSWORD}&email=datafed89%40gmail.com&is_admin=true&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}" )
+  "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed89&uuids=%5B\"${DATAFED_USER89_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER89_PASSWORD}&email=datafed89%40gmail.com&is_admin=true&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}" )
   CODE=$(echo $response | jq .code )
   ERROR_MSG=$(echo $response | jq .errorMessage )
   echo "$ERROR_MSG"
   exit 1
 fi
 # Set globus tokens
-HTTP_CODE=$(curl --user "${basic_auth}" -w "%{http_code}" -o /dev/null  -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed89&access=${DATAFED_USER89_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER89_GLOBUS_REFRESH_TOKEN}&expires_in=1")
+HTTP_CODE=$(curl --user "${basic_auth}" -w "%{http_code}" -o /dev/null  -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed89&access=${DATAFED_USER89_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER89_GLOBUS_REFRESH_TOKEN}&expires_in=1")
 echo "HTTP_CODE: ${HTTP_CODE}"
 FIRST_INT=${HTTP_CODE:0:1}
 if [ "${FIRST_INT}" -ne "2" ]
 then
-  response=$(curl --user "${basic_auth}" --fail-early -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed89&access=${DATAFED_USER89_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER89_GLOBUS_REFRESH_TOKEN}&expires_in=1")
+  response=$(curl --user "${basic_auth}" --fail-early -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed89&access=${DATAFED_USER89_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER89_GLOBUS_REFRESH_TOKEN}&expires_in=1")
   CODE=$(echo $response | jq .code )
   ERROR_MSG=$(echo $response | jq .errorMessage )
   echo "$ERROR_MSG"
@@ -188,24 +187,24 @@ then
 fi
 
 # Create user datafed99 who is not admin
-HTTP_CODE=$(curl  --user "${basic_auth}"  -w "%{http_code}" -o /dev/null  -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed99&uuids=%5B\"${DATAFED_USER99_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER99_PASSWORD}&email=datafed99%40gmail.com&is_admin=false&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}")
+HTTP_CODE=$(curl  --user "${basic_auth}"  -w "%{http_code}" -o /dev/null  -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed99&uuids=%5B\"${DATAFED_USER99_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER99_PASSWORD}&email=datafed99%40gmail.com&is_admin=false&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}")
 echo "HTTP_CODE: ${HTTP_CODE}"
 FIRST_INT=${HTTP_CODE:0:1}
 if [ "${FIRST_INT}" -ne "2" ]
 then
-  response=$(curl  --user "${basic_auth}" --fail-early -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed99&uuids=%5B\"${DATAFED_USER99_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER99_PASSWORD}&email=datafed99%40gmail.com&is_admin=false&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}")
+  response=$(curl  --user "${basic_auth}" --fail-early -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/create?name=Data%20Fed&uid=datafed99&uuids=%5B\"${DATAFED_USER99_GLOBUS_UUID}\"%5D&password=${local_DATAFED_USER99_PASSWORD}&email=datafed99%40gmail.com&is_admin=false&secret=${DATAFED_ZEROMQ_SYSTEM_SECRET}")
   CODE=$(echo $response | jq .code )
   ERROR_MSG=$(echo $response | jq .errorMessage )
   echo "$ERROR_MSG"
   exit 1
 fi
 # Set globus tokens
-HTTP_CODE=$(curl --user "${basic_auth}"   -w "%{http_code}" -o /dev/null  -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed99&access=${DATAFED_USER99_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER99_GLOBUS_REFRESH_TOKEN}&expires_in=1")
+HTTP_CODE=$(curl --user "${basic_auth}"   -w "%{http_code}" -o /dev/null  -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed99&access=${DATAFED_USER99_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER99_GLOBUS_REFRESH_TOKEN}&expires_in=1")
 echo "HTTP_CODE: ${HTTP_CODE}"
 FIRST_INT=${HTTP_CODE:0:1}
 if [ "${FIRST_INT}" -ne "2" ]
 then
-  response=$(curl --user "${basic_auth}"  --fail-early -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed99&access=${DATAFED_USER99_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER99_GLOBUS_REFRESH_TOKEN}&expires_in=1")
+  response=$(curl --user "${basic_auth}"  --fail-early -X GET "http://${local_DATAFED_DATABASE_HOST}:${local_DATAFED_DATABASE_PORT}/_db/${local_DATABASE_NAME}/api/${local_FOXX_MAJOR_API_VERSION}/usr/token/set?client=u%2Fdatafed99&access=${DATAFED_USER99_GLOBUS_ACCESS_TOKEN}&refresh=${DATAFED_USER99_GLOBUS_REFRESH_TOKEN}&expires_in=1")
   CODE=$(echo $response | jq .code )
   ERROR_MSG=$(echo $response | jq .errorMessage )
   echo "$ERROR_MSG"
diff --git a/tests/end-to-end/test_api_alloc.py b/tests/end-to-end/test_api_alloc.py
index d687649b3..5b59659c4 100755
--- a/tests/end-to-end/test_api_alloc.py
+++ b/tests/end-to-end/test_api_alloc.py
@@ -34,7 +34,13 @@ def setUp(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
+
+        if datafed_domain is None:
+            print("DATAFED_DOMAIN must be set before the end-to-end tests can be run")
+            sys.exit(1)
+
         self._df_api = API(opts)
 
         username = "datafed89"
@@ -68,6 +74,15 @@ def setUp(self):
         with open(path_to_repo_form) as json_file:
             self._repo_form = json.load(json_file)
 
+        if len(self._repo_form["exp_path"]) == 0:
+            print(
+                "exp_path is empty, we will set it to / for the test. This is "
+                "cruft and should be removed anyway"
+            )
+            self._repo_form["exp_path"] = "/"
+
+        self._repo_form["admins"] = ["u/" + username]
+
         # Create the repositories
         print("Creating repository")
         result = self._df_api.repoCreate(
diff --git a/tests/end-to-end/test_api_collection.py b/tests/end-to-end/test_api_collection.py
index e46e66b50..ba1fcf8ac 100755
--- a/tests/end-to-end/test_api_collection.py
+++ b/tests/end-to-end/test_api_collection.py
@@ -38,7 +38,13 @@ def setUp(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
+
+        if datafed_domain is None:
+            print("DATAFED_DOMAIN must be set before the end-to-end tests can be run")
+            sys.exit(1)
+
         self._df_api = API(opts)
 
         self._username = "datafed89"
@@ -70,6 +76,15 @@ def setUp(self):
         with open(path_to_repo_form) as json_file:
             self._repo_form = json.load(json_file)
 
+        if len(self._repo_form["exp_path"]) == 0:
+            print(
+                "exp_path is empty, we will set it to / for the test. This is "
+                "cruft and should be removed anyway"
+            )
+            self._repo_form["exp_path"] = "/"
+
+        self._repo_form["admins"] = ["u/" + self._username]
+
         # Create the repositories
         print("Creating repo")
         result = self._df_api.repoCreate(
diff --git a/tests/end-to-end/test_api_context.py b/tests/end-to-end/test_api_context.py
index 0cd44e1ec..47ff13cdc 100755
--- a/tests/end-to-end/test_api_context.py
+++ b/tests/end-to-end/test_api_context.py
@@ -34,7 +34,13 @@ def setUp(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
+
+        if datafed_domain is None:
+            print("DATAFED_DOMAIN must be set before the end-to-end tests can be run")
+            sys.exit(1)
+
         self._df_api = API(opts)
 
         self._username = "datafed99"
@@ -52,6 +58,7 @@ def setUp(self):
             assert count < 3
 
     def test_context(self):
+        print("Running getContext()")
         self._df_api.getContext()
         self.assertEqual(self._df_api.getContext(), f"u/{self._username}")
 
diff --git a/tests/end-to-end/test_api_endpoint.py b/tests/end-to-end/test_api_endpoint.py
index 8d06c4617..66b473390 100755
--- a/tests/end-to-end/test_api_endpoint.py
+++ b/tests/end-to-end/test_api_endpoint.py
@@ -34,7 +34,13 @@ def setUp(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
+
+        if datafed_domain is None:
+            print("DATAFED_DOMAIN must be set before the end-to-end tests can be run")
+            sys.exit(1)
+
         self._df_api = API(opts)
 
         username = "datafed89"
diff --git a/tests/end-to-end/test_api_record.py b/tests/end-to-end/test_api_record.py
index 853ed69a9..020fc5937 100755
--- a/tests/end-to-end/test_api_record.py
+++ b/tests/end-to-end/test_api_record.py
@@ -39,7 +39,13 @@ def setUp(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
+
+        if datafed_domain is None:
+            print("DATAFED_DOMAIN must be set before the end-to-end tests can be run")
+            sys.exit(1)
+
         self._df_api = API(opts)
 
         self._username = "datafed89"
@@ -73,6 +79,15 @@ def setUp(self):
         with open(path_to_repo_form) as json_file:
             self._repo_form = json.load(json_file)
 
+        if len(self._repo_form["exp_path"]) == 0:
+            print(
+                "exp_path is empty, we will set it to / for the test. This is "
+                "cruft and should be removed anyway"
+            )
+            self._repo_form["exp_path"] = "/"
+
+        self._repo_form["admins"] = ["u/" + self._username]
+
         # Create the repositories
         result = self._df_api.repoCreate(
             repo_id=self._repo_form["id"],
@@ -171,16 +186,19 @@ def test_record_create_delete(self):
         status = task_result[0].task[0].status
         count = 0
         while status < 3:
-            if count > 10:
+            if count > 30:
                 break
-            time.sleep(1)
+            time.sleep(4)
             task_result = self._df_api.taskView(task_id)
+            print("task Result **************")
+            print(task_result)
             status = task_result[0].task[0].status
             count = count + 1
 
         # if status < 3:
         #    data_put_esnet_pass = True
 
+        print(f"Status is {status}")
         assert status == 3
         print(task_result)
         print(f"Status is {status}")
@@ -190,7 +208,7 @@ def test_record_create_delete(self):
         status = task_result[0].task[0].status
         count = 0
         while status < 3:
-            if count > 10:
+            if count > 20:
                 break
             time.sleep(1)
             task_result = self._df_api.taskView(task_id)
@@ -202,6 +220,7 @@ def test_record_create_delete(self):
         # Try data from a different location
 
     def tearDown(self):
+
         result = self._df_api.repoAllocationDelete(
             repo_id=self._repo_id, subject="datafed89"
         )
diff --git a/tests/end-to-end/test_api_repo.py b/tests/end-to-end/test_api_repo.py
index 71858dd8e..9f8989240 100755
--- a/tests/end-to-end/test_api_repo.py
+++ b/tests/end-to-end/test_api_repo.py
@@ -33,7 +33,8 @@ def setUp(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
         self._df_api = API(opts)
 
         username = "datafed89"
@@ -68,6 +69,15 @@ def setUp(self):
         with open(path_to_repo_form) as json_file:
             self._repo_form = json.load(json_file)
 
+        if len(self._repo_form["exp_path"]) == 0:
+            print(
+                "exp_path is empty, we will set it to / for the test. This is "
+                "cruft and should be removed anyway"
+            )
+            self._repo_form["exp_path"] = "/"
+
+        self._repo_form["admins"] = ["u/" + username]
+
     def test_repo_list(self):
         result = self._df_api.repoList(list_all=True)
         self.assertEqual(len(result[0].repo), 0)
diff --git a/tests/end-to-end/test_api_user_login.py b/tests/end-to-end/test_api_user_login.py
index c27152c84..571c56f89 100755
--- a/tests/end-to-end/test_api_user_login.py
+++ b/tests/end-to-end/test_api_user_login.py
@@ -33,15 +33,18 @@ def test_login_with_password(self):
 
         print(df_ver)
 
-        opts = {"server_host": "datafed-server-test.ornl.gov"}
+        datafed_domain = os.environ.get("DATAFED_DOMAIN")
+        opts = {"server_host": datafed_domain}
+        print("Creating API")
         df_api = API(opts)
-
+        print("API created")
         username = "datafed99"
         password = os.environ.get("DATAFED_USER99_PASSWORD")
 
         count = 0
         while True:
             try:
+                print(f"username: {username}, password: {password}")
                 df_api.loginByPassword(username, password)
                 break
             except BaseException: